From 065ed7d3d12f3e20bc79b7e8fc07ec637f1aace4 Mon Sep 17 00:00:00 2001 From: emileta Date: Mon, 27 Apr 2026 11:48:16 -0300 Subject: [PATCH 01/38] feat: add --verbose and --concise flags to agent publish authoring-bundle --- messages/agent.publish.authoring-bundle.md | 16 +++++ schemas/agent-publish-authoring__bundle.json | 25 ++++++++ .../agent/publish/authoring-bundle.ts | 62 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/messages/agent.publish.authoring-bundle.md b/messages/agent.publish.authoring-bundle.md index f5c40e83..dc4eab1b 100644 --- a/messages/agent.publish.authoring-bundle.md +++ b/messages/agent.publish.authoring-bundle.md @@ -20,6 +20,14 @@ This command uses the API name of the authoring bundle. <%= config.bin %> <%= command.id %> --api-name MyAuthoringbundle --target-org my-dev-org +- Publish with verbose output to see all retrieved and deployed metadata components: + + <%= config.bin %> <%= command.id %> --api-name MyAuthoringbundle --verbose + +- Publish with concise output showing only essential information: + + <%= config.bin %> <%= command.id %> --api-name MyAuthoringbundle --concise + # flags.api-name.summary API name of the authoring bundle you want to publish; if not specified, the command provides a list that you can choose from. @@ -32,6 +40,14 @@ API name of the authoring bundle to publish Don't retrieve the metadata associated with the agent to your DX project. +# flags.verbose.summary + +Display detailed output showing all metadata components retrieved and deployed during the publish process. + +# flags.concise.summary + +Display minimal output with only essential information about the publish operation. + # error.missingRequiredFlags Required flag(s) missing: %s. diff --git a/schemas/agent-publish-authoring__bundle.json b/schemas/agent-publish-authoring__bundle.json index 33b3777b..3a5df5c6 100644 --- a/schemas/agent-publish-authoring__bundle.json +++ b/schemas/agent-publish-authoring__bundle.json @@ -16,6 +16,31 @@ "items": { "type": "string" } + }, + "retrievedComponents": { + "type": "array", + "items": { + "type": "string" + } + }, + "deployedComponents": { + "type": "array", + "items": { + "type": "string" + } + }, + "summary": { + "type": "object", + "properties": { + "retrieved": { + "type": "number" + }, + "deployed": { + "type": "number" + } + }, + "required": ["retrieved", "deployed"], + "additionalProperties": false } }, "required": ["success"], diff --git a/src/commands/agent/publish/authoring-bundle.ts b/src/commands/agent/publish/authoring-bundle.ts index b7e3cf5d..8375b833 100644 --- a/src/commands/agent/publish/authoring-bundle.ts +++ b/src/commands/agent/publish/authoring-bundle.ts @@ -30,6 +30,12 @@ export type AgentPublishAuthoringBundleResult = { success: boolean; botDeveloperName?: string; errors?: string[]; + retrievedComponents?: string[]; + deployedComponents?: string[]; + summary?: { + retrieved: number; + deployed: number; + }; }; export default class AgentPublishAuthoringBundle extends SfCommand { @@ -48,6 +54,15 @@ export default class AgentPublishAuthoringBundle extends SfCommand({ stages: ['Validate Bundle', 'Publish Agent', 'Retrieve Metadata', 'Deploy Metadata'], @@ -137,11 +157,42 @@ export default class AgentPublishAuthoringBundle extends SfCommand { + const componentName = `${fp.type}${fp.fullName ? `:${fp.fullName}` : ''}`; + retrievedComponents.push(componentName); + }); + + // Display retrieved components based on output mode + if (outputMode === 'verbose' && !this.jsonEnabled()) { + this.log(`${EOL}Retrieved metadata components:`); + retrievedComponents.forEach((comp) => this.log(` • ${comp}`)); + } else if (outputMode === 'normal' && !this.jsonEnabled()) { + this.log(`${EOL}Retrieved ${retrievedComponents.length} metadata component(s)`); + } + return Promise.resolve(); }); Lifecycle.getInstance().on('scopedPostDeploy', (result: ScopedPostDeploy) => { if (result.deployResult.response.status === RequestStatus.Succeeded) { + // Capture deployed components + const deployedFiles = ensureArray(result.deployResult.response.details?.componentSuccesses ?? []); + deployedFiles.forEach((comp) => { + const componentName = `${String(comp.componentType)}${comp.fullName ? `:${String(comp.fullName)}` : ''}`; + deployedComponents.push(componentName); + }); + + // Display deployed components based on output mode + if (outputMode === 'verbose' && !this.jsonEnabled()) { + this.log(`${EOL}Deployed metadata components:`); + deployedComponents.forEach((comp) => this.log(` • ${comp}`)); + } else if (outputMode === 'normal' && !this.jsonEnabled()) { + this.log(`${EOL}Deployed ${deployedComponents.length} metadata component(s)`); + } + mso.stop(); } else { const deployResponse = result.deployResult.response; @@ -159,9 +210,20 @@ export default class AgentPublishAuthoringBundle extends SfCommand Date: Mon, 27 Apr 2026 12:23:03 -0300 Subject: [PATCH 02/38] chore: update command snapshot for --verbose and --concise flags --- command-snapshot.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/command-snapshot.json b/command-snapshot.json index 050e1343..2fb042a4 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -157,8 +157,8 @@ "alias": [], "command": "agent:publish:authoring-bundle", "flagAliases": [], - "flagChars": ["n", "o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "skip-retrieve", "target-org"], + "flagChars": ["n", "o", "v"], + "flags": ["api-name", "api-version", "concise", "flags-dir", "json", "skip-retrieve", "target-org", "verbose"], "plugin": "@salesforce/plugin-agent" }, { From 64f5452c351afae4f007046160181d75d3df5a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Tue, 28 Apr 2026 14:21:50 -0300 Subject: [PATCH 03/38] feat: add hidden --agent-json flag to agent preview start W-22203672 Adds a hidden --agent-json flag that accepts a file path to a pre-compiled AgentJSON, bypassing the compile step in ScriptAgent. Intended for internal developer testing. Co-Authored-By: Claude Sonnet 4.6 --- messages/agent.preview.start.md | 4 + src/commands/agent/preview/start.ts | 29 +++++- test/commands/agent/preview/start.test.ts | 105 ++++++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/messages/agent.preview.start.md b/messages/agent.preview.start.md index e6a04de3..7f2b8c61 100644 --- a/messages/agent.preview.start.md +++ b/messages/agent.preview.start.md @@ -23,6 +23,10 @@ API name of the activated published agent you want to preview. API name of the authoring bundle metadata component that contains the agent's Agent Script file. +# flags.agent-json.summary + +Path to a pre-compiled AgentJSON file to use instead of compiling the agent script. Intended for internal use and testing. + # flags.use-live-actions.summary Execute real actions in the org (Apex classes, flows, etc.). Required with --authoring-bundle. diff --git a/src/commands/agent/preview/start.ts b/src/commands/agent/preview/start.ts index 61a2e2dc..7d8c40fd 100644 --- a/src/commands/agent/preview/start.ts +++ b/src/commands/agent/preview/start.ts @@ -14,9 +14,10 @@ * limitations under the License. */ +import { readFile } from 'node:fs/promises'; import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; import { EnvironmentVariable, Lifecycle, Messages, SfError } from '@salesforce/core'; -import { Agent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; +import { Agent, ProductionAgent, ScriptAgent, type AgentJson } from '@salesforce/agents'; import { createCache, SessionType } from '../../../previewSessionStore.js'; import { COMPILATION_API_EXIT_CODES } from '../../../common.js'; @@ -68,6 +69,11 @@ export default class AgentPreviewStart extends SfCommand { @@ -87,11 +93,18 @@ export default class AgentPreviewStart extends SfCommand { + if (!filePath) return undefined; + try { + return JSON.parse(await readFile(filePath, 'utf-8')) as AgentJson; + } catch (error) { + throw new SfError( + `Failed to read or parse --agent-json file '${filePath}': ${SfError.wrap(error).message}`, + 'AgentJsonReadError' + ); + } +} diff --git a/test/commands/agent/preview/start.test.ts b/test/commands/agent/preview/start.test.ts index 6cee2026..d9c0276c 100644 --- a/test/commands/agent/preview/start.test.ts +++ b/test/commands/agent/preview/start.test.ts @@ -16,6 +16,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { expect } from 'chai'; import sinon from 'sinon'; @@ -76,6 +78,109 @@ describe('agent preview start', () => { $$.restore(); }); + describe('--agent-json flag', () => { + const mockAgentJson = { + schemaVersion: '1.0', + globalConfiguration: { + developerName: 'TestAgent', + label: 'Test Agent', + description: '', + enableEnhancedEventLogs: false, + agentType: 'AgentforceServiceAgent', + templateName: '', + defaultAgentUser: 'test@user.com', + defaultOutboundRouting: '', + contextVariables: [], + }, + agentVersion: { + developerName: null, + plannerType: 'ReAct', + systemMessages: [], + modalityParameters: { voice: {}, language: {} }, + additionalParameters: false, + company: 'Test', + role: 'Assistant', + stateVariables: [], + initialNode: 'start', + nodes: [], + knowledgeDefinitions: null, + }, + }; + let agentJsonInitStub: sinon.SinonStub; + let AgentPreviewStartWithFileFlag: any; + let tmpDir: string; + + beforeEach(async () => { + tmpDir = mkdtempSync(join(tmpdir(), 'agent-json-test-')); + + const mockPreview = { + setMockMode: $$.SANDBOX.stub(), + start: $$.SANDBOX.stub().resolves({ sessionId: 'test-session-id' }), + }; + class MockScriptAgent { + public preview = mockPreview; + public name = 'TestAgent'; + } + agentJsonInitStub = $$.SANDBOX.stub().resolves(new MockScriptAgent()); + + const mod = await esmock('../../../../src/commands/agent/preview/start.js', { + '@salesforce/agents': { + Agent: { init: agentJsonInitStub }, + ScriptAgent: MockScriptAgent, + ProductionAgent: class ProductionAgent {}, + }, + '../../../../src/previewSessionStore.js': { + createCache: $$.SANDBOX.stub().resolves(), + }, + }); + AgentPreviewStartWithFileFlag = mod.default; + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('reads the file and passes parsed agentJson to Agent.init', async () => { + const agentJsonPath = join(tmpDir, 'agent.json'); + writeFileSync(agentJsonPath, JSON.stringify(mockAgentJson)); + + await AgentPreviewStartWithFileFlag.run([ + '--authoring-bundle', + 'MyAgent', + '--simulate-actions', + '--target-org', + 'test@org.com', + '--agent-json', + agentJsonPath, + ]); + + expect(agentJsonInitStub.calledOnce).to.be.true; + const initArg = agentJsonInitStub.firstCall.args[0]; + expect(initArg).to.have.property('agentJson'); + expect(initArg.agentJson).to.deep.equal(mockAgentJson); + }); + + it('throws AgentJsonReadError when file contains invalid JSON', async () => { + const badJsonPath = join(tmpDir, 'bad.json'); + writeFileSync(badJsonPath, 'not-valid-json{{{'); + + try { + await AgentPreviewStartWithFileFlag.run([ + '--authoring-bundle', + 'MyAgent', + '--simulate-actions', + '--target-org', + 'test@org.com', + '--agent-json', + badJsonPath, + ]); + expect.fail('Should have thrown'); + } catch (error: unknown) { + expect((error as Error).message).to.include('Failed to read or parse --agent-json file'); + } + }); + }); + describe('setMockMode', () => { it('should call setMockMode with "Mock" when --simulate-actions is set', async () => { await AgentPreviewStart.run([ From 33487227a783064309705aa40aa8252d4137cf68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s?= <83368246+andresrivas-sf@users.noreply.github.com> Date: Tue, 28 Apr 2026 15:14:38 -0300 Subject: [PATCH 04/38] Update messages/agent.preview.start.md Co-authored-by: Juliet Shackell <63259011+jshackell-sfdc@users.noreply.github.com> --- messages/agent.preview.start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messages/agent.preview.start.md b/messages/agent.preview.start.md index 7f2b8c61..fd8356dc 100644 --- a/messages/agent.preview.start.md +++ b/messages/agent.preview.start.md @@ -25,7 +25,7 @@ API name of the authoring bundle metadata component that contains the agent's Ag # flags.agent-json.summary -Path to a pre-compiled AgentJSON file to use instead of compiling the agent script. Intended for internal use and testing. +Path to a pre-compiled AgentJSON file to use instead of compiling the Agent Script file. Intended for internal use and testing. # flags.use-live-actions.summary From 28fe2f1eddc66cbcd6397743bbb1b47fbcbc9d34 Mon Sep 17 00:00:00 2001 From: emileta Date: Tue, 28 Apr 2026 16:41:01 -0300 Subject: [PATCH 05/38] test: add NUTs for --verbose and --concise flags on agent publish authoring-bundle --- test/nuts/z2.agent.publish.nut.ts | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/nuts/z2.agent.publish.nut.ts b/test/nuts/z2.agent.publish.nut.ts index 133edb44..3ed76f01 100644 --- a/test/nuts/z2.agent.publish.nut.ts +++ b/test/nuts/z2.agent.publish.nut.ts @@ -218,4 +218,51 @@ describe('agent publish authoring-bundle NUTs', function () { expect(output).to.include('Deploy Metadata'); expect(output).to.include(`Agent Name: ${bundleApiName}`); }); + + it('should display detailed component listing with --verbose flag', async function () { + this.timeout(30 * 60 * 1000); + this.retries(1); + + const result = execCmd( + `agent publish authoring-bundle --api-name ${bundleApiName} --target-org ${getUsername()} --verbose`, + { ensureExitCode: 0 } + ); + + const output = result.shellOutput.stdout; + expect(output).to.include('Retrieved metadata components:'); + expect(output).to.include('Deployed metadata components:'); + expect(output).to.match(/•\s+\w/); + }); + + it('should include retrievedComponents and deployedComponents in JSON output with --verbose flag', async function () { + this.timeout(30 * 60 * 1000); + this.retries(1); + + const result = execCmd( + `agent publish authoring-bundle --api-name ${bundleApiName} --target-org ${getUsername()} --verbose --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result).to.be.ok; + expect(result?.success).to.be.true; + expect(result?.retrievedComponents).to.be.an('array'); + expect(result?.deployedComponents).to.be.an('array'); + expect(result?.retrievedComponents?.length).to.be.greaterThan(0); + expect(result?.deployedComponents?.length).to.be.greaterThan(0); + }); + + it('should suppress component counts and success message with --concise flag', async function () { + this.timeout(30 * 60 * 1000); + this.retries(1); + + const result = execCmd( + `agent publish authoring-bundle --api-name ${bundleApiName} --target-org ${getUsername()} --concise`, + { ensureExitCode: 0 } + ); + + const output = result.shellOutput.stdout; + expect(output).to.not.match(/Retrieved \d+ metadata component/); + expect(output).to.not.match(/Deployed \d+ metadata component/); + expect(output).to.not.include('published successfully'); + }); }); From a955e42d08630756461ccb4af073bcd84ff1a104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Tue, 28 Apr 2026 17:13:43 -0300 Subject: [PATCH 06/38] fix: address PR review comments on --agent-json flag W-22203672 - Apply jshackell's wording suggestion for flag summary - Add dependsOn authoring-bundle to --agent-json flag - Add telemetry for AgentJsonReadError - Add --agent-json flag to interactive agent preview command - Add NUT tests for --agent-json flag Co-Authored-By: Claude Sonnet 4.6 --- messages/agent.preview.md | 4 ++ src/commands/agent/preview.ts | 24 ++++++++- src/commands/agent/preview/start.ts | 2 + test/nuts/z3.agent.preview.nut.ts | 75 +++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 2 deletions(-) diff --git a/messages/agent.preview.md b/messages/agent.preview.md index d4d38ea8..59c0406b 100644 --- a/messages/agent.preview.md +++ b/messages/agent.preview.md @@ -39,6 +39,10 @@ Use real actions in the org; if not specified, preview uses AI to simulate (mock Enable Apex debug logging during the agent preview conversation. +# flags.agent-json.summary + +Path to a pre-compiled AgentJSON file to use instead of compiling the Agent Script file. Intended for internal use and testing. + # examples - Preview an agent by choosing from the list of available local Agent Script or published agents. If previewing a local Agent Script agent, use simulated mode. Use the org with alias "my-dev-org". diff --git a/src/commands/agent/preview.ts b/src/commands/agent/preview.ts index d46faa36..1938fd13 100644 --- a/src/commands/agent/preview.ts +++ b/src/commands/agent/preview.ts @@ -14,11 +14,12 @@ * limitations under the License. */ +import { readFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; import React from 'react'; import { render } from 'ink'; -import { Agent, AgentSource, PreviewableAgent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; +import { Agent, AgentSource, PreviewableAgent, ProductionAgent, ScriptAgent, type AgentJson } from '@salesforce/agents'; import { select } from '@inquirer/prompts'; import { Lifecycle, Messages, SfError } from '@salesforce/core'; import { AgentPreviewReact } from '../../components/agent-preview-react.js'; @@ -69,6 +70,12 @@ export default class AgentPreview extends SfCommand { summary: messages.getMessage('flags.use-live-actions.summary'), default: false, }), + 'agent-json': Flags.file({ + summary: messages.getMessage('flags.agent-json.summary'), + hidden: true, + exists: true, + dependsOn: ['authoring-bundle'], + }), }; public async run(): Promise { @@ -81,11 +88,24 @@ export default class AgentPreview extends SfCommand { const { 'api-name': apiNameOrId, 'use-live-actions': useLiveActions, 'authoring-bundle': aabName } = flags; const conn = flags['target-org'].getConnection(flags['api-version']); + let preloadedAgentJson: AgentJson | undefined; + if (flags['agent-json']) { + try { + preloadedAgentJson = JSON.parse(await readFile(flags['agent-json'], 'utf-8')) as AgentJson; + } catch (error) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_agent_json_read_failed' }); + throw new SfError( + `Failed to read or parse --agent-json file '${flags['agent-json']}': ${SfError.wrap(error).message}`, + 'AgentJsonReadError' + ); + } + } + let selectedAgent: ScriptAgent | ProductionAgent; if (aabName) { // user specified --authoring-bundle, use the API name directly - selectedAgent = await Agent.init({ connection: conn, project: this.project!, aabName }); + selectedAgent = await Agent.init({ connection: conn, project: this.project!, aabName, agentJson: preloadedAgentJson }); selectedAgent.preview.setMockMode(flags['use-live-actions'] ? 'Live Test' : 'Mock'); } else if (apiNameOrId) { selectedAgent = await Agent.init({ connection: conn, project: this.project!, apiNameOrId }); diff --git a/src/commands/agent/preview/start.ts b/src/commands/agent/preview/start.ts index 7d8c40fd..5b6b457b 100644 --- a/src/commands/agent/preview/start.ts +++ b/src/commands/agent/preview/start.ts @@ -73,6 +73,7 @@ export default class AgentPreviewStart extends SfCommand { + let tmpDir: string; + + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'agent-json-nut-')); + }); + + after(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('should start a preview session using a pre-compiled AgentJSON file', async function () { + this.timeout(5 * 60 * 1000); + + const bundleApiName = 'Willie_Resort_Manager'; + const targetOrg = getUsername(); + + // Compile agent JSON first via preview start (compile happens internally), then reuse it + // For the NUT we write a minimal valid agentJson extracted from a successful compile + const org = await Org.create({ aliasOrUsername: targetOrg }); + const conn = org.getConnection(); + const { ScriptAgent: ScriptAgentClass } = await import('@salesforce/agents'); + const { SfProject } = await import('@salesforce/core'); + const project = await SfProject.resolve(session.project.dir); + const scriptAgent = new ScriptAgentClass({ connection: conn, project, aabName: bundleApiName }); + await scriptAgent.compile(); + + // @ts-expect-error accessing private field for NUT purposes + const agentJson = scriptAgent.agentJson as object; + expect(agentJson).to.not.be.undefined; + + const agentJsonPath = join(tmpDir, 'compiled-agent.json'); + writeFileSync(agentJsonPath, JSON.stringify(agentJson)); + + const startResult = execCmd( + `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json ${agentJsonPath} --target-org ${targetOrg} --json`, + { cwd: session.project.dir } + ).jsonOutput?.result; + + expect(startResult?.sessionId).to.be.a('string'); + expect(startResult?.agentApiName).to.equal(bundleApiName); + + // Clean up session + execCmd( + `agent preview end --session-id ${startResult!.sessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, + { cwd: session.project.dir } + ); + }); + + it('should fail when --agent-json contains invalid JSON', () => { + const badJsonPath = join(tmpDir, 'bad.json'); + writeFileSync(badJsonPath, 'not-valid{{{'); + + const result = execCmd( + `agent preview start --authoring-bundle Willie_Resort_Manager --simulate-actions --agent-json ${badJsonPath} --target-org ${getUsername()} --json`, + { ensureExitCode: 1, cwd: session.project.dir } + ); + expect(JSON.stringify(result.shellOutput)).to.include('Failed to read or parse'); + }); + + it('should fail when --agent-json is used without --authoring-bundle', () => { + const agentJsonPath = join(tmpDir, 'any.json'); + writeFileSync(agentJsonPath, '{}'); + + execCmd( + `agent preview start --api-name Some_Agent --agent-json ${agentJsonPath} --target-org ${getUsername()} --json`, + { ensureExitCode: 2 } + ); + }); + }); + it('should fail when authoring bundle does not exist', async () => { const invalidBundle = 'NonExistent_Bundle'; execCmd( From 1b77efdf4dbd2461cff671cc64d764b31b993126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Wed, 29 Apr 2026 12:36:54 -0300 Subject: [PATCH 07/38] chore: update command-snapshot.json with agent-json flag W-22203672 Co-Authored-By: Claude Sonnet 4.6 --- command-snapshot.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/command-snapshot.json b/command-snapshot.json index 050e1343..d7eb3fec 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -92,6 +92,7 @@ "flagAliases": [], "flagChars": ["d", "n", "o", "x"], "flags": [ + "agent-json", "apex-debug", "api-name", "api-version", @@ -142,6 +143,7 @@ "flagAliases": [], "flagChars": ["n", "o"], "flags": [ + "agent-json", "api-name", "api-version", "authoring-bundle", From 0da59757d974c3008cc03529dd07bed177cc027f Mon Sep 17 00:00:00 2001 From: Esteban Romero Date: Thu, 30 Apr 2026 11:50:04 -0300 Subject: [PATCH 08/38] test: fix NUTs --- test/nuts/z2.agent.publish.nut.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/nuts/z2.agent.publish.nut.ts b/test/nuts/z2.agent.publish.nut.ts index 133edb44..4c3eb6a0 100644 --- a/test/nuts/z2.agent.publish.nut.ts +++ b/test/nuts/z2.agent.publish.nut.ts @@ -175,7 +175,7 @@ describe('agent publish authoring-bundle NUTs', function () { execCmd( `agent publish authoring-bundle --api-name ${invalidApiName} --target-org ${getUsername()} --json`, - { ensureExitCode: 2 } + { ensureExitCode: 1 } ); }); @@ -193,8 +193,8 @@ describe('agent publish authoring-bundle NUTs', function () { } catch (error) { const message = error instanceof Error ? error.message : String(error); // We assert both the "publish failed" prefix and that it looks like a compilation issue. - expect(message).to.include("SyntaxError: Unexpected 'syem'"); - expect(message).to.match(/Actual:\s*2\b|"exitCode"\s*:\s*2/); + expect(message).to.include('CompilationError: Invalid string'); + expect(message).to.match(/Actual:\s*2\b|"exitCode"\s*:\s*1/); } }); From 966347dbe67883b9c540db8fe9b7e79ade268422 Mon Sep 17 00:00:00 2001 From: Esteban Romero Date: Thu, 30 Apr 2026 12:26:37 -0300 Subject: [PATCH 09/38] docs: add error codes to command help --- src/commands/agent/publish/authoring-bundle.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/commands/agent/publish/authoring-bundle.ts b/src/commands/agent/publish/authoring-bundle.ts index b7e3cf5d..18180b69 100644 --- a/src/commands/agent/publish/authoring-bundle.ts +++ b/src/commands/agent/publish/authoring-bundle.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { EOL } from 'node:os'; -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; import { MultiStageOutput } from '@oclif/multi-stage-output'; import { Messages, Lifecycle, SfError } from '@salesforce/core'; import { Agent } from '@salesforce/agents'; @@ -38,6 +38,11 @@ export default class AgentPublishAuthoringBundle extends SfCommand Date: Thu, 30 Apr 2026 14:17:22 -0300 Subject: [PATCH 10/38] chore: bump @salesforce/agents to 1.3.0 W-22203672 Co-Authored-By: Claude Sonnet 4.6 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c6933ca8..e42cf769 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "@inquirer/prompts": "^7.10.1", "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.8.36", - "@salesforce/agents": "^1.2.0", + "@salesforce/agents": "^1.3.0", "@salesforce/core": "^8.28.3", "@salesforce/kit": "^3.2.6", "@salesforce/sf-plugins-core": "^12.2.6", diff --git a/yarn.lock b/yarn.lock index 45c4afb8..64405099 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1595,10 +1595,10 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@salesforce/agents@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-1.2.0.tgz#6f360f824b80bd66e8ba561f98fe52bc62e24ab9" - integrity sha512-gyF1xzJcEp3MuS8Apf1eUGUvy41zq7HQqKPEhDIU6aUiowoewW/RI9LcPB0K7LPrGCIiARq4TYKlxf452HASqA== +"@salesforce/agents@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-1.3.0.tgz#342f2790013978c6b8855df399bf58ad4b5e9aec" + integrity sha512-HuwVifaBXdPuabHrRrLdyuUDV600YDsuDPI0pxw6A/JCzCdwsOkExpzi/Zz7v8I7qnUGhX9uK1GSKsBLasSXSg== dependencies: "@salesforce/core" "^8.28.3" "@salesforce/kit" "^3.2.6" From d14d199125cb923e2b0208f619c76e74e2b6e5d4 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Thu, 30 Apr 2026 14:46:48 -0600 Subject: [PATCH 11/38] feat: add agent trace read command --- command-snapshot.json | 8 + messages/agent.trace.read.md | 171 +++++++++ schemas/agent-trace-read.json | 466 +++++++++++++++++++++++++ src/commands/agent/trace/read.ts | 429 +++++++++++++++++++++++ test/commands/agent/trace/read.test.ts | 462 ++++++++++++++++++++++++ 5 files changed, 1536 insertions(+) create mode 100644 messages/agent.trace.read.md create mode 100644 schemas/agent-trace-read.json create mode 100644 src/commands/agent/trace/read.ts create mode 100644 test/commands/agent/trace/read.test.ts diff --git a/command-snapshot.json b/command-snapshot.json index 3c8eb216..7ce251d6 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -257,6 +257,14 @@ "flags": ["agent", "flags-dir", "json", "no-prompt", "older-than", "session-id"], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:trace:read", + "flagAliases": [], + "flagChars": ["d", "f", "s", "t"], + "flags": ["dimension", "flags-dir", "format", "json", "session-id", "turn"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:validate:authoring-bundle", diff --git a/messages/agent.trace.read.md b/messages/agent.trace.read.md new file mode 100644 index 00000000..f6fb1de3 --- /dev/null +++ b/messages/agent.trace.read.md @@ -0,0 +1,171 @@ +# summary + +Read and analyze trace files from an agent preview session. + +# description + +Reads trace files recorded during an agent preview session and outputs them in one of three formats. + +**--format summary** (default): A per-turn narrative showing topic routing, actions executed, and the agent's response. Use this to quickly understand what happened in a session. + +**--format detail**: Diagnostic drill-down into a specific dimension (--dimension required). Filters output to only the trace steps relevant to that dimension, minimizing noise. + +**--format raw**: Unprocessed trace JSON. Use this as a fallback when the trace schema has changed or you need to perform custom analysis. + +Available dimensions for --format detail: actions, grounding, routing, errors. + +Use --turn N to scope output to a single conversation turn. + +# flags.session-id.summary + +Session ID to read traces for. + +# flags.format.summary + +Output format: summary (default), detail, or raw. Use detail with --dimension to drill into a specific aspect of the trace. + +# flags.dimension.summary + +Dimension to drill into when using --format detail. One of: actions, grounding, routing, errors. Required when --format is detail. + +# flags.turn.summary + +Scope output to this conversation turn number. + +# error.detailRequiresDimension + +--format detail requires --dimension. Specify one of: actions, grounding, routing, errors. + +# error.sessionNotFound + +Session '%s' was not found in the local session cache. Run "sf agent trace list" to see available sessions. + +# error.turnIndexNotFound + +No turn index found for session '%s'. Cannot filter by --turn without a turn index. + +# error.turnNotFound + +Turn %s was not found in session '%s'. + +# error.parseFailedAll + +Trace parsing failed for all files: %s. The trace schema may have changed. Try --format raw to access unprocessed trace data. + +# warn.dimensionIgnored + +--dimension is ignored when --format is '%s'. Use --format detail to drill into a dimension. + +# warn.parseFailed + +Trace parsing failed for some files (skipped): %s. Try --format raw to access unprocessed trace data. + +# output.empty + +No traces found for this session. + +# output.emptyDimension + +No '%s' data found in the traces for this session. + +# output.tableHeader.turn + +Turn + +# output.tableHeader.topic + +Topic + +# output.tableHeader.userInput + +User Input + +# output.tableHeader.agentResponse + +Agent Response + +# output.tableHeader.actionsExecuted + +Actions Executed + +# output.tableHeader.latencyMs + +Latency + +# output.tableHeader.error + +Error + +# output.tableHeader.action + +Action + +# output.tableHeader.input + +Input + +# output.tableHeader.output + +Output + +# output.tableHeader.prompt + +Prompt + +# output.tableHeader.response + +Response + +# output.tableHeader.intent + +Intent + +# output.tableHeader.fromTopic + +From Topic + +# output.tableHeader.toTopic + +To Topic + +# output.tableHeader.source + +Source + +# output.tableHeader.errorCode + +Error Code + +# output.tableHeader.message + +Message + +# examples + +- Show a session summary (all turns): + + <%= config.bin %> <%= command.id %> --session-id + +- Show summary for a single turn: + + <%= config.bin %> <%= command.id %> --session-id --turn 2 + +- Drill into action execution across all turns: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension actions + +- Drill into routing decisions for a specific turn: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension routing --turn 1 + +- Show all errors across the session: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension errors + +- Output raw trace JSON for custom parsing: + + <%= config.bin %> <%= command.id %> --session-id --format raw + +- Return results as JSON: + + <%= config.bin %> <%= command.id %> --session-id --json diff --git a/schemas/agent-trace-read.json b/schemas/agent-trace-read.json new file mode 100644 index 00000000..e8162963 --- /dev/null +++ b/schemas/agent-trace-read.json @@ -0,0 +1,466 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTraceReadResult", + "definitions": { + "AgentTraceReadResult": { + "type": "object", + "properties": { + "sessionId": { + "type": "string" + }, + "format": { + "type": "string", + "enum": ["summary", "detail", "raw"] + }, + "dimension": { + "$ref": "#/definitions/Dimension" + }, + "turns": { + "type": "array", + "items": { + "$ref": "#/definitions/TurnSummary" + } + }, + "detail": { + "type": "array", + "items": { + "$ref": "#/definitions/DimensionRow" + } + }, + "raw": { + "type": "array", + "items": { + "$ref": "#/definitions/PlannerResponse" + } + } + }, + "required": ["sessionId", "format"], + "additionalProperties": false + }, + "Dimension": { + "type": "string", + "enum": ["actions", "grounding", "routing", "errors"] + }, + "TurnSummary": { + "type": "object", + "properties": { + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "userInput": { + "type": "string" + }, + "agentResponse": { + "type": "string" + }, + "actionsExecuted": { + "type": "array", + "items": { + "type": "string" + } + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": ["string", "null"] + } + }, + "required": ["turn", "planId", "topic", "userInput", "agentResponse", "actionsExecuted", "latencyMs", "error"], + "additionalProperties": false + }, + "DimensionRow": { + "anyOf": [ + { + "$ref": "#/definitions/ActionsRow" + }, + { + "$ref": "#/definitions/GroundingRow" + }, + { + "$ref": "#/definitions/RoutingRow" + }, + { + "$ref": "#/definitions/ErrorsRow" + } + ] + }, + "ActionsRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "actions" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "action": { + "type": "string" + }, + "input": { + "type": "string" + }, + "output": { + "type": "string" + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": ["string", "null"] + } + }, + "required": ["dimension", "turn", "planId", "action", "input", "output", "latencyMs", "error"], + "additionalProperties": false + }, + "GroundingRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "grounding" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "response": { + "type": "string" + }, + "latencyMs": { + "type": "number" + } + }, + "required": ["dimension", "turn", "planId", "prompt", "response", "latencyMs"], + "additionalProperties": false + }, + "RoutingRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "routing" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "fromTopic": { + "type": "string" + }, + "toTopic": { + "type": "string" + }, + "intent": { + "type": "string" + } + }, + "required": ["dimension", "turn", "planId", "fromTopic", "toTopic", "intent"], + "additionalProperties": false + }, + "ErrorsRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "errors" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "source": { + "type": "string" + }, + "errorCode": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": ["dimension", "turn", "planId", "source", "errorCode", "message"], + "additionalProperties": false + }, + "PlannerResponse": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "PlanSuccessResponse" + }, + "planId": { + "type": "string" + }, + "sessionId": { + "type": "string" + }, + "intent": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "plan": { + "type": "array", + "items": { + "$ref": "#/definitions/PlanStep" + } + } + }, + "required": ["type", "planId", "sessionId", "intent", "topic", "plan"], + "additionalProperties": false + }, + "PlanStep": { + "anyOf": [ + { + "$ref": "#/definitions/UserInputStep" + }, + { + "$ref": "#/definitions/LLMExecutionStep" + }, + { + "$ref": "#/definitions/UpdateTopicStep" + }, + { + "$ref": "#/definitions/EventStep" + }, + { + "$ref": "#/definitions/FunctionStep" + }, + { + "$ref": "#/definitions/PlannerResponseStep" + } + ] + }, + "UserInputStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "UserInputStep" + }, + "message": { + "type": "string" + } + }, + "required": ["type", "message"], + "additionalProperties": false + }, + "LLMExecutionStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "LLMExecutionStep" + }, + "promptName": { + "type": "string" + }, + "promptContent": { + "type": "string" + }, + "promptResponse": { + "type": "string" + }, + "executionLatency": { + "type": "number" + }, + "startExecutionTime": { + "type": "number" + }, + "endExecutionTime": { + "type": "number" + } + }, + "required": [ + "type", + "promptName", + "promptContent", + "promptResponse", + "executionLatency", + "startExecutionTime", + "endExecutionTime" + ], + "additionalProperties": false + }, + "UpdateTopicStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "UpdateTopicStep" + }, + "topic": { + "type": "string" + }, + "description": { + "type": "string" + }, + "job": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": { + "type": "string" + } + }, + "availableFunctions": { + "type": "array", + "items": {} + } + }, + "required": ["type", "topic", "description", "job", "instructions", "availableFunctions"], + "additionalProperties": false + }, + "EventStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "EventStep" + }, + "eventName": { + "type": "string" + }, + "isError": { + "type": "boolean" + }, + "payload": { + "type": "object", + "properties": { + "oldTopic": { + "type": "string" + }, + "newTopic": { + "type": "string" + } + }, + "required": ["oldTopic", "newTopic"], + "additionalProperties": false + } + }, + "required": ["type", "eventName", "isError", "payload"], + "additionalProperties": false + }, + "FunctionStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "FunctionStep" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "input": { + "type": "object", + "additionalProperties": {} + }, + "output": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["name", "input", "output"], + "additionalProperties": false + }, + "executionLatency": { + "type": "number" + }, + "startExecutionTime": { + "type": "number" + }, + "endExecutionTime": { + "type": "number" + } + }, + "required": ["type", "function", "executionLatency", "startExecutionTime", "endExecutionTime"], + "additionalProperties": false + }, + "PlannerResponseStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "PlannerResponseStep" + }, + "message": { + "type": "string" + }, + "responseType": { + "type": "string" + }, + "isContentSafe": { + "type": "boolean" + }, + "safetyScore": { + "type": "object", + "properties": { + "safety_score": { + "type": "number" + }, + "category_scores": { + "type": "object", + "properties": { + "toxicity": { + "type": "number" + }, + "hate": { + "type": "number" + }, + "identity": { + "type": "number" + }, + "violence": { + "type": "number" + }, + "physical": { + "type": "number" + }, + "sexual": { + "type": "number" + }, + "profanity": { + "type": "number" + }, + "biased": { + "type": "number" + } + }, + "required": ["toxicity", "hate", "identity", "violence", "physical", "sexual", "profanity", "biased"], + "additionalProperties": false + } + }, + "required": ["safety_score", "category_scores"], + "additionalProperties": false + } + }, + "required": ["type", "message", "responseType", "isContentSafe", "safetyScore"], + "additionalProperties": false + } + } +} diff --git a/src/commands/agent/trace/read.ts b/src/commands/agent/trace/read.ts new file mode 100644 index 00000000..d3011b71 --- /dev/null +++ b/src/commands/agent/trace/read.ts @@ -0,0 +1,429 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; +import { Messages, SfError } from '@salesforce/core'; +import { + listCachedPreviewSessions, + listSessionTraces, + readSessionTrace, + readTurnIndex, + type PlannerResponse, + type PlanStep, + type FunctionStep, +} from '@salesforce/agents'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.trace.read'); + +export const DIMENSIONS = ['actions', 'grounding', 'routing', 'errors'] as const; +export type Dimension = (typeof DIMENSIONS)[number]; + +// FunctionStep in @salesforce/agents doesn't declare the optional errors field that the API returns +type FunctionStepWithErrors = FunctionStep & { + function: FunctionStep['function'] & { + errors?: Array<{ statusCode: string; message: string }>; + }; +}; + +export type TurnSummary = { + turn: number; + planId: string; + topic: string; + userInput: string; + agentResponse: string; + actionsExecuted: string[]; + latencyMs: number; + error: string | null; +}; + +export type ActionsRow = { + dimension: 'actions'; + turn: number; + planId: string; + action: string; + input: string; + output: string; + latencyMs: number; + error: string | null; +}; +export type GroundingRow = { + dimension: 'grounding'; + turn: number; + planId: string; + prompt: string; + response: string; + latencyMs: number; +}; +export type RoutingRow = { + dimension: 'routing'; + turn: number; + planId: string; + fromTopic: string; + toTopic: string; + intent: string; +}; +export type ErrorsRow = { + dimension: 'errors'; + turn: number; + planId: string; + source: string; + errorCode: string; + message: string; +}; +export type DimensionRow = ActionsRow | GroundingRow | RoutingRow | ErrorsRow; + +export type AgentTraceReadResult = { + sessionId: string; + format: 'summary' | 'detail' | 'raw'; + dimension?: Dimension; + turns?: TurnSummary[]; + detail?: DimensionRow[]; + raw?: PlannerResponse[]; +}; + +const isFunctionStep = (s: PlanStep): s is FunctionStep => s.type === 'FunctionStep'; +const asFunctionWithErrors = (s: FunctionStep): FunctionStepWithErrors => s as FunctionStepWithErrors; + +function summarizeTurn(turn: number, planId: string, trace: PlannerResponse): TurnSummary { + const plan = trace.plan; + const userInput = plan.find((s) => s.type === 'UserInputStep'); + const finalResponse = plan.find((s) => s.type === 'PlannerResponseStep'); + const functionSteps = plan.filter(isFunctionStep).map(asFunctionWithErrors); + + const errorStep = functionSteps.find((s) => s.function.errors?.length); + const errorMsg = errorStep?.function.errors?.[0]?.message ?? null; + const totalLatency = functionSteps.reduce((acc, s) => acc + (s.executionLatency ?? 0), 0); + + return { + turn, + planId, + topic: trace.topic, + userInput: userInput?.type === 'UserInputStep' ? userInput.message : '', + agentResponse: finalResponse?.type === 'PlannerResponseStep' ? finalResponse.message : '', + actionsExecuted: functionSteps.map((s) => s.function.name), + latencyMs: totalLatency, + error: errorMsg, + }; +} + +function extractActions(turn: number, planId: string, trace: PlannerResponse): ActionsRow[] { + return trace.plan + .filter(isFunctionStep) + .map(asFunctionWithErrors) + .map((step) => ({ + dimension: 'actions' as const, + turn, + planId, + action: step.function.name, + input: JSON.stringify(step.function.input), + output: JSON.stringify(step.function.output), + latencyMs: step.executionLatency, + error: step.function.errors?.length ? step.function.errors[0].message : null, + })); +} + +function extractGrounding(turn: number, planId: string, trace: PlannerResponse): GroundingRow[] { + return trace.plan + .filter((s): s is Extract => s.type === 'LLMExecutionStep') + .filter((s) => s.promptName.includes('React')) + .map((step) => ({ + dimension: 'grounding' as const, + turn, + planId, + prompt: step.promptName, + response: step.promptResponse.slice(0, 500), + latencyMs: step.executionLatency, + })); +} + +function extractRouting(turn: number, planId: string, trace: PlannerResponse): RoutingRow[] { + const topicStep = trace.plan.find((s) => s.type === 'UpdateTopicStep'); + const eventStep = trace.plan.find((s) => s.type === 'EventStep' && s.eventName === 'topicChangeEvent'); + const fromTopic = eventStep?.type === 'EventStep' ? eventStep.payload.oldTopic : 'null'; + const toTopic = topicStep?.type === 'UpdateTopicStep' ? topicStep.topic : trace.topic; + return [{ dimension: 'routing' as const, turn, planId, fromTopic, toTopic, intent: trace.intent }]; +} + +function extractErrors(turn: number, planId: string, trace: PlannerResponse): ErrorsRow[] { + const rows: ErrorsRow[] = []; + for (const step of trace.plan) { + if (step.type === 'FunctionStep') { + const errors = asFunctionWithErrors(step).function.errors ?? []; + for (const e of errors) { + rows.push({ + dimension: 'errors', + turn, + planId, + source: step.function.name, + errorCode: e.statusCode, + message: e.message, + }); + } + } + if (step.type === 'EventStep' && step.isError) { + rows.push({ + dimension: 'errors', + turn, + planId, + source: step.eventName, + errorCode: 'EVENT_ERROR', + message: JSON.stringify(step.payload), + }); + } + } + return rows; +} + +async function resolvePlanIds( + agentId: string, + sessionId: string, + turn: number | undefined +): Promise> { + const turnIndex = await readTurnIndex(agentId, sessionId); + + if (turn !== undefined) { + if (!turnIndex) { + throw new SfError(messages.getMessage('error.turnIndexNotFound', [sessionId]), 'TurnIndexNotFound'); + } + const entry = turnIndex.turns.find((t) => t.turn === turn && t.planId); + if (!entry?.planId) { + throw new SfError(messages.getMessage('error.turnNotFound', [turn, sessionId]), 'TurnNotFound'); + } + return [{ turn: entry.turn, planId: entry.planId }]; + } + + if (turnIndex) { + return turnIndex.turns.filter((t) => t.planId !== null).map((t) => ({ turn: t.turn, planId: t.planId! })); + } + + // Fall back to listing trace files when no turn index exists + const traceFiles = await listSessionTraces(agentId, sessionId); + return traceFiles.map((f, i) => ({ turn: i + 1, planId: f.planId })); +} + +async function readTraces( + agentId: string, + sessionId: string, + planIds: Array<{ turn: number; planId: string }> +): Promise<{ traces: Array<{ turn: number; planId: string; trace: PlannerResponse }>; failedFiles: string[] }> { + const traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> = []; + const failedFiles: string[] = []; + + for (const { turn, planId } of planIds) { + // eslint-disable-next-line no-await-in-loop + const trace = await readSessionTrace(agentId, sessionId, planId); + if (!trace?.plan || !Array.isArray(trace.plan)) { + failedFiles.push(planId); + continue; + } + traces.push({ turn, planId, trace }); + } + + return { traces, failedFiles }; +} + +function extractDimension(turn: number, planId: string, trace: PlannerResponse, dimension: Dimension): DimensionRow[] { + if (dimension === 'actions') return extractActions(turn, planId, trace); + if (dimension === 'grounding') return extractGrounding(turn, planId, trace); + if (dimension === 'routing') return extractRouting(turn, planId, trace); + return extractErrors(turn, planId, trace); +} + +export default class AgentTraceRead extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly requiresProject = true; + + public static readonly flags = { + 'session-id': Flags.string({ + summary: messages.getMessage('flags.session-id.summary'), + required: true, + char: 's', + }), + format: Flags.option({ + options: ['summary', 'detail', 'raw'] as const, + default: 'summary' as const, + summary: messages.getMessage('flags.format.summary'), + char: 'f', + })(), + dimension: Flags.option({ + options: DIMENSIONS, + summary: messages.getMessage('flags.dimension.summary'), + char: 'd', + })(), + turn: Flags.integer({ + summary: messages.getMessage('flags.turn.summary'), + char: 't', + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentTraceRead); + const sessionId = flags['session-id']; + + if (flags.format === 'detail' && !flags.dimension) { + throw new SfError(messages.getMessage('error.detailRequiresDimension'), 'MissingDimension'); + } + if (flags.dimension && flags.format !== 'detail') { + this.warn(messages.getMessage('warn.dimensionIgnored', [flags.format])); + } + + const agentId = await this.resolveAgentId(sessionId); + const planIds = await resolvePlanIds(agentId, sessionId, flags.turn); + + if (planIds.length === 0) { + this.log(messages.getMessage('output.empty')); + return { sessionId, format: flags.format, turns: [], detail: [], raw: [] }; + } + + const { traces, failedFiles } = await readTraces(agentId, sessionId, planIds); + + if (failedFiles.length > 0 && traces.length === 0) { + throw new SfError(messages.getMessage('error.parseFailedAll', [failedFiles.join(', ')]), 'TraceParseError'); + } + if (failedFiles.length > 0) { + this.warn(messages.getMessage('warn.parseFailed', [failedFiles.join(', ')])); + } + + return this.formatOutput(sessionId, flags.format, flags.dimension, traces); + } + + private async resolveAgentId(sessionId: string): Promise { + const cachedAgents = await listCachedPreviewSessions(this.project!); + const entry = cachedAgents.find((a) => a.sessions.some((s) => s.sessionId === sessionId)); + if (!entry) { + throw new SfError(messages.getMessage('error.sessionNotFound', [sessionId]), 'SessionNotFound'); + } + return entry.agentId; + } + + private formatOutput( + sessionId: string, + format: 'summary' | 'detail' | 'raw', + dimension: Dimension | undefined, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + if (format === 'raw') { + const raw = traces.map((t) => t.trace); + if (!this.jsonEnabled()) this.log(JSON.stringify(raw, null, 2)); + return { sessionId, format: 'raw', raw }; + } + + if (format === 'detail') { + return this.formatDetail(sessionId, dimension!, traces); + } + + return this.formatSummary(sessionId, traces); + } + + private formatDetail( + sessionId: string, + dimension: Dimension, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + const detail: DimensionRow[] = traces.flatMap(({ turn, planId, trace }) => + extractDimension(turn, planId, trace, dimension) + ); + + if (detail.length === 0) { + this.log(messages.getMessage('output.emptyDimension', [dimension])); + return { sessionId, format: 'detail', dimension, detail: [] }; + } + + if (!this.jsonEnabled()) { + this.renderDetailTable(dimension, detail); + } + + return { sessionId, format: 'detail', dimension, detail }; + } + + private renderDetailTable(dimension: Dimension, detail: DimensionRow[]): void { + if (dimension === 'actions') { + this.table({ + data: detail as ActionsRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'action', name: messages.getMessage('output.tableHeader.action') }, + { key: 'input', name: messages.getMessage('output.tableHeader.input') }, + { key: 'output', name: messages.getMessage('output.tableHeader.output') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + { key: 'error', name: messages.getMessage('output.tableHeader.error') }, + ], + }); + } else if (dimension === 'grounding') { + this.table({ + data: detail as GroundingRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'prompt', name: messages.getMessage('output.tableHeader.prompt') }, + { key: 'response', name: messages.getMessage('output.tableHeader.response') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + ], + }); + } else if (dimension === 'routing') { + this.table({ + data: detail as RoutingRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'intent', name: messages.getMessage('output.tableHeader.intent') }, + { key: 'fromTopic', name: messages.getMessage('output.tableHeader.fromTopic') }, + { key: 'toTopic', name: messages.getMessage('output.tableHeader.toTopic') }, + ], + }); + } else { + this.table({ + data: detail as ErrorsRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'source', name: messages.getMessage('output.tableHeader.source') }, + { key: 'errorCode', name: messages.getMessage('output.tableHeader.errorCode') }, + { key: 'message', name: messages.getMessage('output.tableHeader.message') }, + ], + }); + } + } + + private formatSummary( + sessionId: string, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + const turns: TurnSummary[] = traces.map(({ turn, planId, trace }) => summarizeTurn(turn, planId, trace)); + + if (!this.jsonEnabled()) { + this.table({ + data: turns.map((t) => ({ + ...t, + actionsExecuted: t.actionsExecuted.join(', ') || '—', + error: t.error ?? '—', + latencyMs: `${t.latencyMs}ms`, + })), + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'topic', name: messages.getMessage('output.tableHeader.topic') }, + { key: 'userInput', name: messages.getMessage('output.tableHeader.userInput') }, + { key: 'agentResponse', name: messages.getMessage('output.tableHeader.agentResponse') }, + { key: 'actionsExecuted', name: messages.getMessage('output.tableHeader.actionsExecuted') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + { key: 'error', name: messages.getMessage('output.tableHeader.error') }, + ], + }); + } + + return { sessionId, format: 'summary', turns }; + } +} diff --git a/test/commands/agent/trace/read.test.ts b/test/commands/agent/trace/read.test.ts new file mode 100644 index 00000000..4688308a --- /dev/null +++ b/test/commands/agent/trace/read.test.ts @@ -0,0 +1,462 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unnecessary-type-assertion */ + +import { join } from 'node:path'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; +import { TestContext } from '@salesforce/core/testSetup'; +import { SfProject } from '@salesforce/core'; + +const MOCK_PROJECT_DIR = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template'); + +const SESSION_ID = 'sess-abc'; +const AGENT_ID = 'AgentA'; +const PLAN_ID_1 = 'plan-1'; +const PLAN_ID_2 = 'plan-2'; + +const MOCK_CACHED_SESSIONS = [ + { + agentId: AGENT_ID, + displayName: 'My_Agent_A', + sessions: [{ sessionId: SESSION_ID, timestamp: '2026-04-07T17:00:00.000Z' }], + }, +]; + +const MOCK_TURN_INDEX = { + version: '1', + sessionId: SESSION_ID, + agentId: AGENT_ID, + created: '2026-04-07T17:00:00.000Z', + turns: [ + { + turn: 1, + timestamp: '2026-04-07T17:00:00.000Z', + role: 'user', + summary: 'Hi!', + summaryTruncated: false, + multiModal: null, + traceFile: `traces/${PLAN_ID_1}.json`, + planId: PLAN_ID_1, + }, + { + turn: 2, + timestamp: '2026-04-07T17:00:01.000Z', + role: 'user', + summary: "what's the weather", + summaryTruncated: false, + multiModal: null, + traceFile: `traces/${PLAN_ID_2}.json`, + planId: PLAN_ID_2, + }, + ], +}; + +// Simple off-topic trace (no actions, no errors) +const MOCK_TRACE_1 = { + type: 'PlanSuccessResponse', + planId: PLAN_ID_1, + sessionId: SESSION_ID, + intent: 'Off_Topic', + topic: 'Off_Topic', + plan: [ + { type: 'UserInputStep', message: 'Hi!' }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactTopicPrompt', + promptContent: 'classify...', + promptResponse: 'Off_Topic', + executionLatency: 460, + startExecutionTime: 1000, + endExecutionTime: 1460, + }, + { + type: 'UpdateTopicStep', + topic: 'Off_Topic', + description: 'Off topic', + job: 'redirect', + instructions: [], + availableFunctions: [], + }, + { + type: 'EventStep', + eventName: 'topicChangeEvent', + isError: false, + payload: { oldTopic: 'null', newTopic: 'Off_Topic' }, + }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactInitialPrompt', + promptContent: 'system...', + promptResponse: 'Hey there!', + executionLatency: 1637, + startExecutionTime: 1461, + endExecutionTime: 3098, + }, + { + type: 'PlannerResponseStep', + message: 'Hey there! How can I assist you today?', + responseType: 'Inform', + isContentSafe: true, + }, + ], +}; + +// Weather trace with action + error +const MOCK_TRACE_2 = { + type: 'PlanSuccessResponse', + planId: PLAN_ID_2, + sessionId: SESSION_ID, + intent: 'Local_Weather', + topic: 'Local_Weather', + plan: [ + { type: 'UserInputStep', message: "what's the weather" }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactTopicPrompt', + promptContent: 'classify...', + promptResponse: 'Local_Weather', + executionLatency: 572, + startExecutionTime: 2000, + endExecutionTime: 2572, + }, + { + type: 'UpdateTopicStep', + topic: 'Local_Weather', + description: 'Weather', + job: 'answer weather questions', + instructions: [], + availableFunctions: ['Check_Weather'], + }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactInitialPrompt', + promptContent: 'system...', + promptResponse: + '- id: call_xxx\n function:\n name: Check_Weather\n arguments: \'{"dateToCheck":"2025-08-18"}\'', + executionLatency: 748, + startExecutionTime: 2600, + endExecutionTime: 3348, + }, + { + type: 'FunctionStep', + function: { + name: 'Check_Weather', + input: { dateToCheck: '2025-08-18' }, + output: {}, + errors: [{ statusCode: 'UNKNOWN_EXCEPTION', message: 'Bad response: 404' }], + }, + executionLatency: 781, + startExecutionTime: 3350, + endExecutionTime: 4131, + }, + { + type: 'PlannerResponseStep', + message: "I'm having trouble accessing the weather.", + responseType: 'Inform', + isContentSafe: true, + }, + ], +}; + +describe('agent trace read', () => { + const $$ = new TestContext(); + let listCachedPreviewSessionsStub: sinon.SinonStub; + let listSessionTracesStub: sinon.SinonStub; + let readSessionTraceStub: sinon.SinonStub; + let readTurnIndexStub: sinon.SinonStub; + let AgentTraceRead: any; + + beforeEach(async () => { + listCachedPreviewSessionsStub = $$.SANDBOX.stub().resolves(MOCK_CACHED_SESSIONS); + listSessionTracesStub = $$.SANDBOX.stub().resolves([]); + readTurnIndexStub = $$.SANDBOX.stub().resolves(MOCK_TURN_INDEX); + readSessionTraceStub = $$.SANDBOX.stub(); + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_1).resolves(MOCK_TRACE_1); + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_2); + + const mod = await esmock('../../../../src/commands/agent/trace/read.js', { + '@salesforce/agents': { + listCachedPreviewSessions: listCachedPreviewSessionsStub, + listSessionTraces: listSessionTracesStub, + readSessionTrace: readSessionTraceStub, + readTurnIndex: readTurnIndexStub, + }, + }); + + AgentTraceRead = mod.default; + + $$.inProject(true); + const mockProject = { getPath: () => MOCK_PROJECT_DIR } as unknown as SfProject; + $$.SANDBOX.stub(SfProject, 'resolve').resolves(mockProject); + $$.SANDBOX.stub(SfProject, 'getInstance').returns(mockProject); + }); + + afterEach(() => { + $$.restore(); + }); + + describe('--format summary (default)', () => { + it('returns summary for all turns when no --turn specified', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.format).to.equal('summary'); + expect(result.turns).to.have.length(2); + }); + + it('each turn has required fields', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + const turn = result.turns[0]; + expect(turn).to.have.keys([ + 'turn', + 'planId', + 'topic', + 'userInput', + 'agentResponse', + 'actionsExecuted', + 'latencyMs', + 'error', + ]); + }); + + it('populates topic and userInput from trace', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[0].topic).to.equal('Off_Topic'); + expect(result.turns[0].userInput).to.equal('Hi!'); + }); + + it('lists executed actions', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[1].actionsExecuted).to.deep.equal(['Check_Weather']); + }); + + it('captures error from failed function step', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[1].error).to.equal('Bad response: 404'); + }); + + it('error is null when no function errors exist', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[0].error).to.be.null; + }); + + it('scopes to a single turn with --turn', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '1']); + expect(result.turns).to.have.length(1); + expect(result.turns![0].turn).to.equal(1); + }); + }); + + describe('--format detail', () => { + it('throws when --dimension is missing', async () => { + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'detail']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('--dimension'); + } + }); + + describe('--dimension actions', () => { + it('returns only action rows', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.format).to.equal('detail'); + expect(result.dimension).to.equal('actions'); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ action: 'Check_Weather' }); + }); + + it('includes error details in action row', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.detail![0].error).to.equal('Bad response: 404'); + }); + + it('returns empty when no actions exist', async () => { + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_1); + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.detail).to.deep.equal([]); + }); + }); + + describe('--dimension routing', () => { + it('returns routing rows for each turn', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'routing', + ]); + expect(result.detail).to.have.length(2); + expect(result.detail![0]).to.include({ fromTopic: 'null', toTopic: 'Off_Topic', intent: 'Off_Topic' }); + }); + + it('scopes to a single turn with --turn', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'routing', + '--turn', + '2', + ]); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ intent: 'Local_Weather' }); + }); + }); + + describe('--dimension errors', () => { + it('returns rows only for turns with errors', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'errors', + ]); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ source: 'Check_Weather', errorCode: 'UNKNOWN_EXCEPTION' }); + }); + + it('returns empty when no errors exist', async () => { + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_1); + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'errors', + ]); + expect(result.detail).to.deep.equal([]); + }); + }); + + describe('--dimension grounding', () => { + it('returns LLM execution steps with React prompts', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'grounding', + ]); + expect(result.detail!.length).to.be.greaterThan(0); + for (const row of result.detail!) { + expect((row as any).prompt).to.include('React'); + } + }); + }); + }); + + describe('--format raw', () => { + it('returns raw trace objects', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'raw']); + expect(result.format).to.equal('raw'); + expect(result.raw).to.have.length(2); + expect(result.raw![0].planId).to.equal(PLAN_ID_1); + }); + + it('raw output matches the full trace structure', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'raw']); + expect(result.raw![0]).to.deep.equal(MOCK_TRACE_1); + }); + }); + + describe('validation and error handling', () => { + it('throws when session is not found', async () => { + listCachedPreviewSessionsStub.resolves([]); + try { + await AgentTraceRead.run(['--session-id', 'no-such-session']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('no-such-session'); + } + }); + + it('throws when --turn is used but no turn index exists', async () => { + readTurnIndexStub.resolves(null); + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '1']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/turn index/i); + } + }); + + it('throws when --turn number does not exist in the index', async () => { + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '99']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/turn 99|not found/i); + } + }); + + it('throws when all trace files fail to parse', async () => { + readSessionTraceStub.resetBehavior(); + readSessionTraceStub.resolves(null); + try { + await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/Trace parsing failed|raw/i); + } + }); + + it('returns empty result when session has no trace files and no turn index', async () => { + readTurnIndexStub.resolves(null); + listSessionTracesStub.resolves([]); + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns).to.deep.equal([]); + }); + + it('falls back to listSessionTraces when no turn index exists', async () => { + readTurnIndexStub.resolves(null); + listSessionTracesStub.resolves([{ planId: PLAN_ID_1, path: '/path/plan-1.json', size: 1000, mtime: new Date() }]); + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns).to.have.length(1); + expect(result.turns![0].topic).to.equal('Off_Topic'); + }); + }); +}); From 4eca9784720573f2246098caf0ce53d55cf7924f Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Thu, 30 Apr 2026 15:14:39 -0600 Subject: [PATCH 12/38] feat: add NUTs for agent trace read --- test/nuts/z4.agent.trace.read.nut.ts | 177 +++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 test/nuts/z4.agent.trace.read.nut.ts diff --git a/test/nuts/z4.agent.trace.read.nut.ts b/test/nuts/z4.agent.trace.read.nut.ts new file mode 100644 index 00000000..2c6227f7 --- /dev/null +++ b/test/nuts/z4.agent.trace.read.nut.ts @@ -0,0 +1,177 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import type { AgentPreviewStartResult } from '../../src/commands/agent/preview/start.js'; +import type { AgentPreviewSendResult } from '../../src/commands/agent/preview/send.js'; +import type { AgentPreviewEndResult } from '../../src/commands/agent/preview/end.js'; +import type { AgentTraceReadResult } from '../../src/commands/agent/trace/read.js'; +import { getTestSession, getUsername } from './shared-setup.js'; + +describe('agent trace read', function () { + this.timeout(30 * 60 * 1000); + + let session: TestSession; + let sessionId: string; + const bundleApiName = 'Willie_Resort_Manager'; + + before(async function () { + this.timeout(30 * 60 * 1000); + session = await getTestSession(); + + // Start a preview session with two turns so there are traces to read + const targetOrg = getUsername(); + const startResult = execCmd( + `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + expect(startResult?.sessionId).to.be.a('string'); + sessionId = startResult!.sessionId; + + execCmd( + `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "What can you help me with?" --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ); + + execCmd( + `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "Tell me more" --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ); + + execCmd( + `agent preview end --session-id ${sessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ); + }); + + describe('--format summary (default)', () => { + it('returns a summary result for the session', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.format).to.equal('summary'); + expect(result?.sessionId).to.equal(sessionId); + expect(result?.turns).to.be.an('array').with.length.greaterThan(0); + }); + + it('each turn has the required summary fields', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + const turn = result!.turns![0]; + expect(turn).to.have.keys([ + 'turn', + 'planId', + 'topic', + 'userInput', + 'agentResponse', + 'actionsExecuted', + 'latencyMs', + 'error', + ]); + expect(turn.userInput).to.be.a('string').and.have.length.greaterThan(0); + expect(turn.agentResponse).to.be.a('string').and.have.length.greaterThan(0); + expect(turn.topic).to.be.a('string').and.have.length.greaterThan(0); + }); + + it('scopes to a single turn with --turn 1', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --turn 1 --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.turns).to.have.length(1); + expect(result?.turns![0].turn).to.equal(1); + }); + }); + + describe('--format detail', () => { + it('errors when --dimension is missing', () => { + execCmd(`agent trace read --session-id ${sessionId} --format detail --json`, { + ensureExitCode: 1, + cwd: session.project.dir, + }); + }); + + it('returns routing dimension rows', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension routing --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.dimension).to.equal('routing'); + expect(result?.detail).to.be.an('array').with.length.greaterThan(0); + const row = result!.detail![0] as { turn: number; intent: string; toTopic: string }; + expect(row.turn).to.be.a('number'); + expect(row.intent).to.be.a('string'); + expect(row.toTopic).to.be.a('string'); + }); + + it('returns grounding dimension rows', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension grounding --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.detail).to.be.an('array').with.length.greaterThan(0); + const row = result!.detail![0] as { prompt: string; latencyMs: number }; + expect(row.prompt).to.be.a('string').and.include('React'); + expect(row.latencyMs).to.be.a('number').and.greaterThanOrEqual(0); + }); + + it('returns actions dimension rows (may be empty for off-topic sessions)', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension actions --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.detail).to.be.an('array'); + }); + + it('returns errors dimension rows (may be empty for successful sessions)', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension errors --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.detail).to.be.an('array'); + }); + }); + + describe('--format raw', () => { + it('returns raw trace JSON', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --format raw --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.format).to.equal('raw'); + expect(result?.raw).to.be.an('array').with.length.greaterThan(0); + const trace = result!.raw![0] as { type: string; plan: unknown[] }; + expect(trace.type).to.equal('PlanSuccessResponse'); + expect(trace.plan).to.be.an('array').with.length.greaterThan(0); + }); + }); + + describe('error handling', () => { + it('errors when session is not found', () => { + execCmd('agent trace read --session-id no-such-session --json', { + ensureExitCode: 1, + cwd: session.project.dir, + }); + }); + }); +}); From ede549557a426bac8df7c7cbe43cecb2c863267c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Thu, 30 Apr 2026 19:41:30 -0300 Subject: [PATCH 13/38] test: use static fixture for --agent-json NUT to avoid live compile API call Co-Authored-By: Claude Sonnet 4.6 --- src/commands/agent/preview.ts | 10 ++++- src/commands/agent/preview/start.ts | 21 +++++---- test/nuts/fixtures/compiled-agent.json | 60 ++++++++++++++++++++++++++ test/nuts/z3.agent.preview.nut.ts | 26 ++++------- 4 files changed, 89 insertions(+), 28 deletions(-) create mode 100644 test/nuts/fixtures/compiled-agent.json diff --git a/src/commands/agent/preview.ts b/src/commands/agent/preview.ts index 1938fd13..51e37b82 100644 --- a/src/commands/agent/preview.ts +++ b/src/commands/agent/preview.ts @@ -105,8 +105,14 @@ export default class AgentPreview extends SfCommand { if (aabName) { // user specified --authoring-bundle, use the API name directly - selectedAgent = await Agent.init({ connection: conn, project: this.project!, aabName, agentJson: preloadedAgentJson }); - selectedAgent.preview.setMockMode(flags['use-live-actions'] ? 'Live Test' : 'Mock'); + const scriptAgent = await Agent.init({ + connection: conn, + project: this.project!, + aabName, + agentJson: preloadedAgentJson, + }); + scriptAgent.preview.setMockMode(flags['use-live-actions'] ? 'Live Test' : 'Mock'); + selectedAgent = scriptAgent; } else if (apiNameOrId) { selectedAgent = await Agent.init({ connection: conn, project: this.project!, apiNameOrId }); } else { diff --git a/src/commands/agent/preview/start.ts b/src/commands/agent/preview/start.ts index 5b6b457b..5b956696 100644 --- a/src/commands/agent/preview/start.ts +++ b/src/commands/agent/preview/start.ts @@ -99,14 +99,16 @@ export default class AgentPreviewStart extends SfCommand( `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json ${agentJsonPath} --target-org ${targetOrg} --json`, @@ -83,7 +73,9 @@ describe('agent preview', function () { // Clean up session execCmd( - `agent preview end --session-id ${startResult!.sessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, + `agent preview end --session-id ${ + startResult!.sessionId + } --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, { cwd: session.project.dir } ); }); From 8010feee11932d4030a6a325f48b5f7d1216adec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Thu, 30 Apr 2026 19:47:46 -0300 Subject: [PATCH 14/38] refactor: extract loadAgentJson to common, simplify NUT fixture path and instanceof narrowing Co-Authored-By: Claude Sonnet 4.6 --- src/commands/agent/preview.ts | 17 +++-------------- src/commands/agent/preview/start.ts | 21 +++------------------ src/common.ts | 18 ++++++++++++++++-- test/nuts/z3.agent.preview.nut.ts | 6 ++---- 4 files changed, 24 insertions(+), 38 deletions(-) diff --git a/src/commands/agent/preview.ts b/src/commands/agent/preview.ts index 51e37b82..2dee15ff 100644 --- a/src/commands/agent/preview.ts +++ b/src/commands/agent/preview.ts @@ -14,15 +14,15 @@ * limitations under the License. */ -import { readFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; import React from 'react'; import { render } from 'ink'; -import { Agent, AgentSource, PreviewableAgent, ProductionAgent, ScriptAgent, type AgentJson } from '@salesforce/agents'; +import { Agent, AgentSource, PreviewableAgent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; import { select } from '@inquirer/prompts'; import { Lifecycle, Messages, SfError } from '@salesforce/core'; import { AgentPreviewReact } from '../../components/agent-preview-react.js'; +import { loadAgentJson } from '../../common.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.preview'); @@ -88,18 +88,7 @@ export default class AgentPreview extends SfCommand { const { 'api-name': apiNameOrId, 'use-live-actions': useLiveActions, 'authoring-bundle': aabName } = flags; const conn = flags['target-org'].getConnection(flags['api-version']); - let preloadedAgentJson: AgentJson | undefined; - if (flags['agent-json']) { - try { - preloadedAgentJson = JSON.parse(await readFile(flags['agent-json'], 'utf-8')) as AgentJson; - } catch (error) { - await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_agent_json_read_failed' }); - throw new SfError( - `Failed to read or parse --agent-json file '${flags['agent-json']}': ${SfError.wrap(error).message}`, - 'AgentJsonReadError' - ); - } - } + const preloadedAgentJson = await loadAgentJson(flags['agent-json']); let selectedAgent: ScriptAgent | ProductionAgent; diff --git a/src/commands/agent/preview/start.ts b/src/commands/agent/preview/start.ts index 5b956696..86a4fed2 100644 --- a/src/commands/agent/preview/start.ts +++ b/src/commands/agent/preview/start.ts @@ -14,12 +14,11 @@ * limitations under the License. */ -import { readFile } from 'node:fs/promises'; import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; import { EnvironmentVariable, Lifecycle, Messages, SfError } from '@salesforce/core'; -import { Agent, ProductionAgent, ScriptAgent, type AgentJson } from '@salesforce/agents'; +import { Agent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; import { createCache, SessionType } from '../../../previewSessionStore.js'; -import { COMPILATION_API_EXIT_CODES } from '../../../common.js'; +import { COMPILATION_API_EXIT_CODES, loadAgentJson } from '../../../common.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.preview.start'); @@ -147,8 +146,7 @@ export default class AgentPreviewStart extends SfCommand { - if (!filePath) return undefined; - try { - return JSON.parse(await readFile(filePath, 'utf-8')) as AgentJson; - } catch (error) { - await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_start_agent_json_read_failed' }); - throw new SfError( - `Failed to read or parse --agent-json file '${filePath}': ${SfError.wrap(error).message}`, - 'AgentJsonReadError' - ); - } -} diff --git a/src/common.ts b/src/common.ts index 3f09486a..e6ece603 100644 --- a/src/common.ts +++ b/src/common.ts @@ -13,13 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { readFile } from 'node:fs/promises'; import { EOL } from 'node:os'; -import { SfError } from '@salesforce/core'; -import { CompilationError, COMPILATION_API_EXIT_CODES } from '@salesforce/agents'; +import { Lifecycle, SfError } from '@salesforce/core'; +import { CompilationError, COMPILATION_API_EXIT_CODES, type AgentJson } from '@salesforce/agents'; /** Re-export so consumers can use the library's exit code contract (404 → 2, 500 → 3). */ export { COMPILATION_API_EXIT_CODES }; +export async function loadAgentJson(filePath: string | undefined): Promise { + if (!filePath) return undefined; + try { + return JSON.parse(await readFile(filePath, 'utf-8')) as AgentJson; + } catch (error) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_agent_json_read_failed' }); + throw new SfError( + `Failed to read or parse --agent-json file '${filePath}': ${SfError.wrap(error).message}`, + 'AgentJsonReadError' + ); + } +} + /** * Utility function to generate SfError when there are agent compilation errors. * diff --git a/test/nuts/z3.agent.preview.nut.ts b/test/nuts/z3.agent.preview.nut.ts index 2d44efa5..ac768ea4 100644 --- a/test/nuts/z3.agent.preview.nut.ts +++ b/test/nuts/z3.agent.preview.nut.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { writeFileSync, rmSync, readFileSync } from 'node:fs'; +import { writeFileSync, rmSync } from 'node:fs'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -59,9 +59,7 @@ describe('agent preview', function () { // Use a static fixture instead of hitting the compile API, which avoids a // live network call that can fail independently of the feature under test. - const fixtureDir = join(fileURLToPath(import.meta.url), '..', 'fixtures'); - const agentJsonPath = join(tmpDir, 'compiled-agent.json'); - writeFileSync(agentJsonPath, readFileSync(join(fixtureDir, 'compiled-agent.json'))); + const agentJsonPath = join(fileURLToPath(import.meta.url), '..', 'fixtures', 'compiled-agent.json'); const startResult = execCmd( `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json ${agentJsonPath} --target-org ${targetOrg} --json`, From 090e128af75722d9f7f6ce89c9c082ae995b7865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Fri, 1 May 2026 10:33:17 -0300 Subject: [PATCH 15/38] fix: patch fixture defaultAgentUser so preview sessions API accepts it The static fixture had an empty defaultAgentUser, causing bypassUser to resolve to false and the preview sessions API to reject the request. Patch it at runtime with the real agent user from the shared test session, matching what the compile-then-start path does via string replacement. Co-Authored-By: Claude Sonnet 4.6 --- test/nuts/z3.agent.preview.nut.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/nuts/z3.agent.preview.nut.ts b/test/nuts/z3.agent.preview.nut.ts index ac768ea4..06cb4a61 100644 --- a/test/nuts/z3.agent.preview.nut.ts +++ b/test/nuts/z3.agent.preview.nut.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { writeFileSync, rmSync } from 'node:fs'; +import { writeFileSync, rmSync, readFileSync } from 'node:fs'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -26,7 +26,7 @@ import { Org } from '@salesforce/core'; import type { AgentPreviewStartResult } from '../../src/commands/agent/preview/start.js'; import type { AgentPreviewSendResult } from '../../src/commands/agent/preview/send.js'; import type { AgentPreviewEndResult } from '../../src/commands/agent/preview/end.js'; -import { getTestSession, getUsername } from './shared-setup.js'; +import { getAgentUsername, getTestSession, getUsername } from './shared-setup.js'; /* eslint-disable no-console */ describe('agent preview', function () { @@ -59,7 +59,14 @@ describe('agent preview', function () { // Use a static fixture instead of hitting the compile API, which avoids a // live network call that can fail independently of the feature under test. - const agentJsonPath = join(fileURLToPath(import.meta.url), '..', 'fixtures', 'compiled-agent.json'); + // Patch defaultAgentUser so bypassUser resolves correctly in the org. + const fixtureSource = join(fileURLToPath(import.meta.url), '..', 'fixtures', 'compiled-agent.json'); + const fixtureJson = JSON.parse(readFileSync(fixtureSource, 'utf-8')) as { + globalConfiguration: { defaultAgentUser: string }; + }; + fixtureJson.globalConfiguration.defaultAgentUser = getAgentUsername() ?? ''; + const agentJsonPath = join(tmpDir, 'compiled-agent.json'); + writeFileSync(agentJsonPath, JSON.stringify(fixtureJson)); const startResult = execCmd( `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json ${agentJsonPath} --target-org ${targetOrg} --json`, From 5dc2713f5ac649fee2e3f9619d4adaacbb428035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Fri, 1 May 2026 11:29:17 -0300 Subject: [PATCH 16/38] test: add ensureExitCode=0 to surface actual agent-json NUT failure Co-Authored-By: Claude Sonnet 4.6 --- test/nuts/z3.agent.preview.nut.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/nuts/z3.agent.preview.nut.ts b/test/nuts/z3.agent.preview.nut.ts index 06cb4a61..d11dbeed 100644 --- a/test/nuts/z3.agent.preview.nut.ts +++ b/test/nuts/z3.agent.preview.nut.ts @@ -68,10 +68,11 @@ describe('agent preview', function () { const agentJsonPath = join(tmpDir, 'compiled-agent.json'); writeFileSync(agentJsonPath, JSON.stringify(fixtureJson)); - const startResult = execCmd( - `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json ${agentJsonPath} --target-org ${targetOrg} --json`, - { cwd: session.project.dir } - ).jsonOutput?.result; + const startCmdResult = execCmd( + `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json "${agentJsonPath}" --target-org ${targetOrg} --json`, + { ensureExitCode: 0 } + ); + const startResult = startCmdResult.jsonOutput?.result; expect(startResult?.sessionId).to.be.a('string'); expect(startResult?.agentApiName).to.equal(bundleApiName); From 39b92a792e4a7577161432e0373487162d79fe8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Fri, 1 May 2026 12:16:59 -0300 Subject: [PATCH 17/38] fix: use valid agentType value in compiled-agent.json fixture The preview sessions API rejected "AgentforceAgent" with HTTP 400 "Define a valid value for ''agent_type''". Updated to "customer" which matches the AgentType union type in @salesforce/agents. Co-Authored-By: Claude Sonnet 4.6 --- test/nuts/fixtures/compiled-agent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nuts/fixtures/compiled-agent.json b/test/nuts/fixtures/compiled-agent.json index 7061f6b6..0de24e72 100644 --- a/test/nuts/fixtures/compiled-agent.json +++ b/test/nuts/fixtures/compiled-agent.json @@ -5,7 +5,7 @@ "label": "Willie Resort Manager", "description": "A resort manager agent for NUT testing", "enableEnhancedEventLogs": false, - "agentType": "AgentforceAgent", + "agentType": "customer", "templateName": "", "defaultAgentUser": "", "defaultOutboundRouting": "", From 39e157f4ada11489c3366bbefc08cfef466df7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Fri, 1 May 2026 13:29:28 -0300 Subject: [PATCH 18/38] fix: set agentType to AgentforceServiceAgent in compiled-agent fixture "customer" was also rejected by the preview sessions API. The unit test for --agent-json uses "AgentforceServiceAgent", which matches the value the compile API returns for service-type agents. Co-Authored-By: Claude Sonnet 4.6 --- test/nuts/fixtures/compiled-agent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nuts/fixtures/compiled-agent.json b/test/nuts/fixtures/compiled-agent.json index 0de24e72..9dd98f7c 100644 --- a/test/nuts/fixtures/compiled-agent.json +++ b/test/nuts/fixtures/compiled-agent.json @@ -5,7 +5,7 @@ "label": "Willie Resort Manager", "description": "A resort manager agent for NUT testing", "enableEnhancedEventLogs": false, - "agentType": "customer", + "agentType": "AgentforceServiceAgent", "templateName": "", "defaultAgentUser": "", "defaultOutboundRouting": "", From 94952aab888578de6f1c6d46a69dd325d5874a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Fri, 1 May 2026 13:33:21 -0300 Subject: [PATCH 19/38] fix: use EinsteinServiceAgent agentType in fixture; bump @salesforce/agents to 1.4.0 The preview sessions API rejected all prior agentType values. The bot-meta.xml for the test project's service agent uses "EinsteinServiceAgent", which matches what the compile API returns for this agent type. Also bumps @salesforce/agents to 1.4.0. Co-Authored-By: Claude Sonnet 4.6 --- package.json | 2 +- test/nuts/fixtures/compiled-agent.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index e42cf769..a9708024 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "@inquirer/prompts": "^7.10.1", "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.8.36", - "@salesforce/agents": "^1.3.0", + "@salesforce/agents": "^1.4.0", "@salesforce/core": "^8.28.3", "@salesforce/kit": "^3.2.6", "@salesforce/sf-plugins-core": "^12.2.6", diff --git a/test/nuts/fixtures/compiled-agent.json b/test/nuts/fixtures/compiled-agent.json index 9dd98f7c..f13ae3d3 100644 --- a/test/nuts/fixtures/compiled-agent.json +++ b/test/nuts/fixtures/compiled-agent.json @@ -5,7 +5,7 @@ "label": "Willie Resort Manager", "description": "A resort manager agent for NUT testing", "enableEnhancedEventLogs": false, - "agentType": "AgentforceServiceAgent", + "agentType": "EinsteinServiceAgent", "templateName": "", "defaultAgentUser": "", "defaultOutboundRouting": "", diff --git a/yarn.lock b/yarn.lock index 64405099..0ee8d258 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1595,10 +1595,10 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@salesforce/agents@^1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-1.3.0.tgz#342f2790013978c6b8855df399bf58ad4b5e9aec" - integrity sha512-HuwVifaBXdPuabHrRrLdyuUDV600YDsuDPI0pxw6A/JCzCdwsOkExpzi/Zz7v8I7qnUGhX9uK1GSKsBLasSXSg== +"@salesforce/agents@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-1.4.0.tgz#69c3e3e9d29b9596aadf99f94774c9c8713e34ca" + integrity sha512-NaiGUSjcyK0Wsy0n5pFuouHqOYH1KJEpgm605NqCaiOMAA3zn0pWZDHE9I75ymGqpvOXSHGDFNrUhKYPain9ZA== dependencies: "@salesforce/core" "^8.28.3" "@salesforce/kit" "^3.2.6" From 3d0ba3dbfa9037bddc05587a60ccf3763d8d073a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Fri, 1 May 2026 13:59:07 -0300 Subject: [PATCH 20/38] fix: stop patching defaultAgentUser in agent-json NUT Setting defaultAgentUser to a valid org user made bypassUser=true, which caused the preview sessions API to validate permission sets that aren't assigned, resulting in a 500. An empty defaultAgentUser keeps bypassUser=false and skips that check entirely. Co-Authored-By: Claude Sonnet 4.6 --- test/nuts/z3.agent.preview.nut.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/nuts/z3.agent.preview.nut.ts b/test/nuts/z3.agent.preview.nut.ts index d11dbeed..2f1085df 100644 --- a/test/nuts/z3.agent.preview.nut.ts +++ b/test/nuts/z3.agent.preview.nut.ts @@ -26,7 +26,7 @@ import { Org } from '@salesforce/core'; import type { AgentPreviewStartResult } from '../../src/commands/agent/preview/start.js'; import type { AgentPreviewSendResult } from '../../src/commands/agent/preview/send.js'; import type { AgentPreviewEndResult } from '../../src/commands/agent/preview/end.js'; -import { getAgentUsername, getTestSession, getUsername } from './shared-setup.js'; +import { getTestSession, getUsername } from './shared-setup.js'; /* eslint-disable no-console */ describe('agent preview', function () { @@ -59,14 +59,11 @@ describe('agent preview', function () { // Use a static fixture instead of hitting the compile API, which avoids a // live network call that can fail independently of the feature under test. - // Patch defaultAgentUser so bypassUser resolves correctly in the org. + // Leave defaultAgentUser empty so the library sends bypassUser=false, + // which skips the permission-set validation that causes a 500. const fixtureSource = join(fileURLToPath(import.meta.url), '..', 'fixtures', 'compiled-agent.json'); - const fixtureJson = JSON.parse(readFileSync(fixtureSource, 'utf-8')) as { - globalConfiguration: { defaultAgentUser: string }; - }; - fixtureJson.globalConfiguration.defaultAgentUser = getAgentUsername() ?? ''; const agentJsonPath = join(tmpDir, 'compiled-agent.json'); - writeFileSync(agentJsonPath, JSON.stringify(fixtureJson)); + writeFileSync(agentJsonPath, readFileSync(fixtureSource)); const startCmdResult = execCmd( `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json "${agentJsonPath}" --target-org ${targetOrg} --json`, From 833fbecc9c9ce14160ffafb21041be6b34028ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Fri, 1 May 2026 15:09:02 -0300 Subject: [PATCH 21/38] fix: compile agent in NUT instead of using static fixture A static fixture's agentVersion is rejected by the preview sessions API with a 500. Compile the agent at test time via Agent.init+compile(), write the real compiledArtifact to a temp file, and pass that as --agent-json. This tests the flag plumbing while ensuring the body is API-compatible. Co-Authored-By: Claude Sonnet 4.6 --- test/nuts/z3.agent.preview.nut.ts | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test/nuts/z3.agent.preview.nut.ts b/test/nuts/z3.agent.preview.nut.ts index 2f1085df..07581e2b 100644 --- a/test/nuts/z3.agent.preview.nut.ts +++ b/test/nuts/z3.agent.preview.nut.ts @@ -14,15 +14,14 @@ * limitations under the License. */ -import { writeFileSync, rmSync, readFileSync } from 'node:fs'; +import { writeFileSync, rmSync } from 'node:fs'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; import { expect } from 'chai'; import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import { Agent } from '@salesforce/agents'; -import { Org } from '@salesforce/core'; +import { Org, SfProject } from '@salesforce/core'; import type { AgentPreviewStartResult } from '../../src/commands/agent/preview/start.js'; import type { AgentPreviewSendResult } from '../../src/commands/agent/preview/send.js'; import type { AgentPreviewEndResult } from '../../src/commands/agent/preview/end.js'; @@ -57,13 +56,20 @@ describe('agent preview', function () { const bundleApiName = 'Willie_Resort_Manager'; const targetOrg = getUsername(); - // Use a static fixture instead of hitting the compile API, which avoids a - // live network call that can fail independently of the feature under test. - // Leave defaultAgentUser empty so the library sends bypassUser=false, - // which skips the permission-set validation that causes a 500. - const fixtureSource = join(fileURLToPath(import.meta.url), '..', 'fixtures', 'compiled-agent.json'); + // Compile the agent once to get a valid agentJson, write it to a temp file, + // then pass it back via --agent-json. This tests the flag plumbing (the + // command must accept a pre-compiled file and skip recompilation) while + // ensuring the fixture matches what the preview sessions API actually accepts. + const org = await Org.create({ aliasOrUsername: targetOrg }); + const conn = org.getConnection(); + const project = await SfProject.resolve(session.project.dir); + const agent = await Agent.init({ connection: conn, project, aabName: bundleApiName }); + const compileResult = await agent.compile(); + if (compileResult.status !== 'success' || !compileResult.compiledArtifact) { + throw new Error(`Compile failed: ${JSON.stringify(compileResult)}`); + } const agentJsonPath = join(tmpDir, 'compiled-agent.json'); - writeFileSync(agentJsonPath, readFileSync(fixtureSource)); + writeFileSync(agentJsonPath, JSON.stringify(compileResult.compiledArtifact)); const startCmdResult = execCmd( `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --agent-json "${agentJsonPath}" --target-org ${targetOrg} --json`, From 29d59f9163e3ef5b36ef6b262f2fbd5ea9fd48f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 May 2026 14:32:58 +0000 Subject: [PATCH 22/38] fix(deps): bump yaml from 2.8.3 to 2.8.4 Bumps [yaml](https://github.com/eemeli/yaml) from 2.8.3 to 2.8.4. - [Release notes](https://github.com/eemeli/yaml/releases) - [Commits](https://github.com/eemeli/yaml/compare/v2.8.3...v2.8.4) --- updated-dependencies: - dependency-name: yaml dependency-version: 2.8.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c6933ca8..e9422a78 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "ink-text-input": "^6.0.0", "inquirer-autocomplete-standalone": "^0.8.1", "react": "^18.3.1", - "yaml": "^2.8.3" + "yaml": "^2.8.4" }, "devDependencies": { "@oclif/plugin-command-snapshot": "^5.3.13", diff --git a/yarn.lock b/yarn.lock index 45c4afb8..8b52c3e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9073,10 +9073,10 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== -yaml@^2.5.1, yaml@^2.8.3: - version "2.8.3" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.3.tgz#a0d6bd2efb3dd03c59370223701834e60409bd7d" - integrity sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg== +yaml@^2.5.1, yaml@^2.8.3, yaml@^2.8.4: + version "2.8.4" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.4.tgz#4b5f411dd25f9544914d8673d4da7f29248e5e2e" + integrity sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog== yargs-parser@^18.1.2: version "18.1.3" From 144ad8e7a562e7107464347856d34f131c4f9193 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Sun, 3 May 2026 04:03:04 +0000 Subject: [PATCH 23/38] chore(release): 1.34.1 [skip ci] --- CHANGELOG.md | 6 ++++++ README.md | 38 +++++++++++++++++++------------------- package.json | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdcbf922..6dad7a28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## [1.34.1](https://github.com/salesforcecli/plugin-agent/compare/1.34.0...1.34.1) (2026-05-03) + +### Bug Fixes + +- **deps:** bump yaml from 2.8.3 to 2.8.4 ([29d59f9](https://github.com/salesforcecli/plugin-agent/commit/29d59f9163e3ef5b36ef6b262f2fbd5ea9fd48f8)) + # [1.34.0](https://github.com/salesforcecli/plugin-agent/compare/1.33.0...1.34.0) (2026-04-27) ### Bug Fixes diff --git a/README.md b/README.md index a02c36b4..527f2c64 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ EXAMPLES $ sf agent activate --api-name Resort_Manager --version 2 --target-org my-org ``` -_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/activate.ts)_ +_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/activate.ts)_ ## `sf agent create` @@ -193,7 +193,7 @@ EXAMPLES $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview ``` -_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/create.ts)_ +_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/create.ts)_ ## `sf agent deactivate` @@ -234,7 +234,7 @@ EXAMPLES $ sf agent deactivate --api-name Resort_Manager --target-org my-org ``` -_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/deactivate.ts)_ +_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/deactivate.ts)_ ## `sf agent generate agent-spec` @@ -341,7 +341,7 @@ EXAMPLES $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com ``` -_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/generate/agent-spec.ts)_ +_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/agent-spec.ts)_ ## `sf agent generate authoring-bundle` @@ -418,7 +418,7 @@ EXAMPLES other-package-dir/main/default --target-org my-dev-org ``` -_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/generate/authoring-bundle.ts)_ +_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/authoring-bundle.ts)_ ## `sf agent generate template` @@ -480,7 +480,7 @@ EXAMPLES my-package --source-org my-scratch-org ``` -_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/generate/template.ts)_ +_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/template.ts)_ ## `sf agent generate test-spec` @@ -545,7 +545,7 @@ EXAMPLES force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml ``` -_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/generate/test-spec.ts)_ +_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/test-spec.ts)_ ## `sf agent preview` @@ -618,7 +618,7 @@ EXAMPLES $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview ``` -_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/preview.ts)_ +_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview.ts)_ ## `sf agent preview end` @@ -673,7 +673,7 @@ EXAMPLES $ sf agent preview end --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/preview/end.ts)_ +_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/end.ts)_ ## `sf agent preview send` @@ -731,7 +731,7 @@ EXAMPLES $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/preview/send.ts)_ +_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/send.ts)_ ## `sf agent preview sessions` @@ -764,7 +764,7 @@ EXAMPLES $ sf agent preview sessions ``` -_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/preview/sessions.ts)_ +_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/sessions.ts)_ ## `sf agent preview start` @@ -829,7 +829,7 @@ EXAMPLES $ sf agent preview start --api-name My_Published_Agent ``` -_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/preview/start.ts)_ +_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/start.ts)_ ## `sf agent publish authoring-bundle` @@ -878,7 +878,7 @@ EXAMPLES $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --target-org my-dev-org ``` -_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/publish/authoring-bundle.ts)_ +_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/publish/authoring-bundle.ts)_ ## `sf agent test create` @@ -933,7 +933,7 @@ EXAMPLES $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview ``` -_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/test/create.ts)_ +_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/create.ts)_ ## `sf agent test list` @@ -968,7 +968,7 @@ EXAMPLES $ sf agent test list --target-org my-org ``` -_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/test/list.ts)_ +_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/list.ts)_ ## `sf agent test results` @@ -1034,7 +1034,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/test/results.ts)_ +_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/results.ts)_ ## `sf agent test resume` @@ -1107,7 +1107,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/test/resume.ts)_ +_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/resume.ts)_ ## `sf agent test run` @@ -1181,7 +1181,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/test/run.ts)_ +_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/run.ts)_ ## `sf agent validate authoring-bundle` @@ -1228,6 +1228,6 @@ EXAMPLES $ sf agent validate authoring-bundle --api-name MyAuthoringBundle --target-org my-dev-org ``` -_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.0/src/commands/agent/validate/authoring-bundle.ts)_ +_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/validate/authoring-bundle.ts)_ diff --git a/package.json b/package.json index e9422a78..c6bb16aa 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@salesforce/plugin-agent", "description": "Commands to interact with Salesforce agents", - "version": "1.34.0", + "version": "1.34.1", "author": "Salesforce", "bugs": "https://github.com/forcedotcom/cli/issues", "enableO11y": true, From 2dc17621eba0ce99d1a475d0aa95141c5d1e9b6a Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Mon, 4 May 2026 13:41:17 +0000 Subject: [PATCH 24/38] chore(release): 1.35.0 [skip ci] --- CHANGELOG.md | 16 ++++++++++++++++ README.md | 38 +++++++++++++++++++------------------- package.json | 2 +- yarn.lock | 2 +- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dad7a28..7ee9d721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# [1.35.0](https://github.com/salesforcecli/plugin-agent/compare/1.34.1...1.35.0) (2026-05-04) + +### Bug Fixes + +- address PR review comments on --agent-json flag ([a955e42](https://github.com/salesforcecli/plugin-agent/commit/a955e42d08630756461ccb4af073bcd84ff1a104)) +- compile agent in NUT instead of using static fixture ([833fbec](https://github.com/salesforcecli/plugin-agent/commit/833fbecc9c9ce14160ffafb21041be6b34028ed1)) +- patch fixture defaultAgentUser so preview sessions API accepts it ([090e128](https://github.com/salesforcecli/plugin-agent/commit/090e128af75722d9f7f6ce89c9c082ae995b7865)) +- set agentType to AgentforceServiceAgent in compiled-agent fixture ([39e157f](https://github.com/salesforcecli/plugin-agent/commit/39e157f4ada11489c3366bbefc08cfef466df7ce)) +- stop patching defaultAgentUser in agent-json NUT ([3d0ba3d](https://github.com/salesforcecli/plugin-agent/commit/3d0ba3dbfa9037bddc05587a60ccf3763d8d073a)) +- use EinsteinServiceAgent agentType in fixture; bump @salesforce/agents to 1.4.0 ([94952aa](https://github.com/salesforcecli/plugin-agent/commit/94952aab888578de6f1c6d46a69dd325d5874a65)) +- use valid agentType value in compiled-agent.json fixture ([39b92a7](https://github.com/salesforcecli/plugin-agent/commit/39b92a792e4a7577161432e0373487162d79fe8c)) + +### Features + +- add hidden --agent-json flag to agent preview start ([64f5452](https://github.com/salesforcecli/plugin-agent/commit/64f5452c351afae4f007046160181d75d3df5a60)) + ## [1.34.1](https://github.com/salesforcecli/plugin-agent/compare/1.34.0...1.34.1) (2026-05-03) ### Bug Fixes diff --git a/README.md b/README.md index 527f2c64..568177b2 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ EXAMPLES $ sf agent activate --api-name Resort_Manager --version 2 --target-org my-org ``` -_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/activate.ts)_ +_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/activate.ts)_ ## `sf agent create` @@ -193,7 +193,7 @@ EXAMPLES $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview ``` -_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/create.ts)_ +_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/create.ts)_ ## `sf agent deactivate` @@ -234,7 +234,7 @@ EXAMPLES $ sf agent deactivate --api-name Resort_Manager --target-org my-org ``` -_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/deactivate.ts)_ +_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/deactivate.ts)_ ## `sf agent generate agent-spec` @@ -341,7 +341,7 @@ EXAMPLES $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com ``` -_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/agent-spec.ts)_ +_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/agent-spec.ts)_ ## `sf agent generate authoring-bundle` @@ -418,7 +418,7 @@ EXAMPLES other-package-dir/main/default --target-org my-dev-org ``` -_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/authoring-bundle.ts)_ +_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/authoring-bundle.ts)_ ## `sf agent generate template` @@ -480,7 +480,7 @@ EXAMPLES my-package --source-org my-scratch-org ``` -_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/template.ts)_ +_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/template.ts)_ ## `sf agent generate test-spec` @@ -545,7 +545,7 @@ EXAMPLES force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml ``` -_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/generate/test-spec.ts)_ +_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/test-spec.ts)_ ## `sf agent preview` @@ -618,7 +618,7 @@ EXAMPLES $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview ``` -_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview.ts)_ +_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview.ts)_ ## `sf agent preview end` @@ -673,7 +673,7 @@ EXAMPLES $ sf agent preview end --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/end.ts)_ +_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/end.ts)_ ## `sf agent preview send` @@ -731,7 +731,7 @@ EXAMPLES $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/send.ts)_ +_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/send.ts)_ ## `sf agent preview sessions` @@ -764,7 +764,7 @@ EXAMPLES $ sf agent preview sessions ``` -_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/sessions.ts)_ +_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/sessions.ts)_ ## `sf agent preview start` @@ -829,7 +829,7 @@ EXAMPLES $ sf agent preview start --api-name My_Published_Agent ``` -_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/preview/start.ts)_ +_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/start.ts)_ ## `sf agent publish authoring-bundle` @@ -878,7 +878,7 @@ EXAMPLES $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --target-org my-dev-org ``` -_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/publish/authoring-bundle.ts)_ +_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/publish/authoring-bundle.ts)_ ## `sf agent test create` @@ -933,7 +933,7 @@ EXAMPLES $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview ``` -_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/create.ts)_ +_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/create.ts)_ ## `sf agent test list` @@ -968,7 +968,7 @@ EXAMPLES $ sf agent test list --target-org my-org ``` -_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/list.ts)_ +_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/list.ts)_ ## `sf agent test results` @@ -1034,7 +1034,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/results.ts)_ +_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/results.ts)_ ## `sf agent test resume` @@ -1107,7 +1107,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/resume.ts)_ +_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/resume.ts)_ ## `sf agent test run` @@ -1181,7 +1181,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/test/run.ts)_ +_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/run.ts)_ ## `sf agent validate authoring-bundle` @@ -1228,6 +1228,6 @@ EXAMPLES $ sf agent validate authoring-bundle --api-name MyAuthoringBundle --target-org my-dev-org ``` -_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.34.1/src/commands/agent/validate/authoring-bundle.ts)_ +_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/validate/authoring-bundle.ts)_ diff --git a/package.json b/package.json index 9ed09bc9..ec3766b6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@salesforce/plugin-agent", "description": "Commands to interact with Salesforce agents", - "version": "1.34.1", + "version": "1.35.0", "author": "Salesforce", "bugs": "https://github.com/forcedotcom/cli/issues", "enableO11y": true, diff --git a/yarn.lock b/yarn.lock index e513db1b..ebe6fc51 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1595,7 +1595,7 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@salesforce/agents@1.4.0": +"@salesforce/agents@^1.4.0": version "1.4.0" resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-1.4.0.tgz#69c3e3e9d29b9596aadf99f94774c9c8713e34ca" integrity sha512-NaiGUSjcyK0Wsy0n5pFuouHqOYH1KJEpgm605NqCaiOMAA3zn0pWZDHE9I75ymGqpvOXSHGDFNrUhKYPain9ZA== From e2670cf7449b835c852c4985df2b236e05707fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Tue, 5 May 2026 13:46:30 -0300 Subject: [PATCH 25/38] feat(agent/test/run-eval): promote command to GA and delegate logic to @salesforce/agents W-22203426 Removes local copies of evalNormalizer, evalFormatter, and yamlSpecTranslator now that they live in the shared library. Rewrites run-eval.ts as a thin CLI shell that imports normalizePayload, translateTestSpec, resolveAgent, executeBatches, buildResultSummary, and formatResults from @salesforce/agents. Promotes the command from state='beta'/hidden to state='ga'. Co-Authored-By: Claude Sonnet 4.6 --- src/commands/agent/test/run-eval.ts | 155 +------ src/evalFormatter.ts | 355 --------------- src/evalNormalizer.ts | 500 ---------------------- src/yamlSpecTranslator.ts | 265 ------------ test/commands/agent/test/run-eval.test.ts | 360 ++++++++++++++++ test/evalFormatter.test.ts | 2 +- test/evalNormalizer.test.ts | 2 +- test/yamlSpecTranslator.test.ts | 2 +- 8 files changed, 381 insertions(+), 1260 deletions(-) delete mode 100644 src/evalFormatter.ts delete mode 100644 src/evalNormalizer.ts delete mode 100644 src/yamlSpecTranslator.ts create mode 100644 test/commands/agent/test/run-eval.test.ts diff --git a/src/commands/agent/test/run-eval.ts b/src/commands/agent/test/run-eval.ts index d2569526..d0187566 100644 --- a/src/commands/agent/test/run-eval.ts +++ b/src/commands/agent/test/run-eval.ts @@ -16,11 +16,22 @@ import { readFile } from 'node:fs/promises'; import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; -import { EnvironmentVariable, Messages, Org, SfError } from '@salesforce/core'; -import { type EvalPayload, normalizePayload, splitIntoBatches } from '../../../evalNormalizer.js'; -import { type EvalApiResponse, formatResults, type ResultFormat } from '../../../evalFormatter.js'; +import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; +import { + type EvalPayload, + normalizePayload, + splitIntoBatches, + type EvalApiResponse, + formatResults, + type ResultFormat, + isYamlTestSpec, + parseTestSpec, + translateTestSpec, + resolveAgent, + executeBatches, + buildResultSummary, +} from '@salesforce/agents'; import { resultFormatFlag } from '../../../flags.js'; -import { isYamlTestSpec, parseTestSpec, translateTestSpec } from '../../../yamlSpecTranslator.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run-eval'); @@ -30,132 +41,11 @@ export type RunEvalResult = { summary: { passed: number; failed: number; scored: number; errors: number }; }; -// --- Standalone helper functions --- - -type ApiHeaders = { - orgId: string; - userId: string; - instanceUrl: string; -}; - -async function getApiHeaders(org: Org): Promise { - const conn = org.getConnection(); - const userInfo = await conn.request<{ user_id: string }>(`${conn.instanceUrl}/services/oauth2/userinfo`); - - return { - orgId: org.getOrgId(), - userId: userInfo.user_id, - instanceUrl: conn.instanceUrl, - }; -} - -async function callEvalApi(org: Org, payload: EvalPayload, headers: ApiHeaders): Promise<{ results?: unknown[] }> { - const conn = org.getConnection(); - - return conn.request<{ results?: unknown[] }>({ - url: 'https://api.salesforce.com/einstein/evaluation/v1/tests', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-sfdc-core-tenant-id': `core/prod/${headers.orgId}`, - 'x-org-id': headers.orgId, - 'x-sfdc-core-instance-url': headers.instanceUrl, - 'x-sfdc-user-id': headers.userId, - 'x-client-feature-id': 'AIPlatformEvaluation', - 'x-sfdc-app-context': 'EinsteinGPT', - }, - body: JSON.stringify(payload), - }); -} - -async function resolveAgent(org: Org, apiName: string): Promise<{ agentId: string; versionId: string }> { - const conn = org.getConnection(); - - // Escape single quotes to prevent SOQL injection - const escapedApiName = apiName.replace(/'/g, "\\'"); - - const botResult = await conn.query<{ Id: string }>( - `SELECT Id FROM BotDefinition WHERE DeveloperName = '${escapedApiName}'` - ); - if (!botResult.records.length) { - throw messages.createError('error.agentNotFound', [apiName]); - } - const agentId = botResult.records[0].Id; - - // Filter to published/active versions only - const versionResult = await conn.query<{ Id: string }>( - `SELECT Id FROM BotVersion WHERE BotDefinitionId = '${agentId}' ORDER BY VersionNumber DESC LIMIT 1` - ); - if (!versionResult.records.length) { - throw messages.createError('error.agentVersionNotFound', [apiName]); - } - const versionId = versionResult.records[0].Id; - - return { agentId, versionId }; -} - -async function executeBatches( - org: Org, - batches: Array, - log: (msg: string) => void -): Promise { - // Pre-calculate headers once to avoid redundant API calls - const headers = await getApiHeaders(org); - - // Execute all batches in parallel for better performance - if (batches.length > 1) { - log(messages.getMessage('info.batchProgress', [batches.length, batches.length, 'total'])); - } - - const batchPromises = batches.map(async (batch) => { - const batchPayload: EvalPayload = { tests: batch }; - const resultObj = await callEvalApi(org, batchPayload, headers); - return resultObj.results ?? []; - }); - - const batchResults = await Promise.all(batchPromises); - return batchResults.flat(); -} - -function buildResultSummary(mergedResponse: EvalApiResponse): { - summary: RunEvalResult['summary']; - testSummaries: RunEvalResult['tests']; -} { - const summary = { passed: 0, failed: 0, scored: 0, errors: 0 }; - const testSummaries: Array<{ id: string; status: string; evaluations: unknown[]; outputs: unknown[] }> = []; - - for (const testResult of mergedResponse.results ?? []) { - const tr = testResult as Record; - const testId = (tr.id as string) ?? 'unknown'; - const evalResults = (tr.evaluation_results as Array>) ?? []; - const testErrors = (tr.errors as unknown[]) ?? []; - - const passed = evalResults.filter((e) => e.is_pass === true).length; - const failed = evalResults.filter((e) => e.is_pass === false).length; - const scored = evalResults.filter((e) => e.score != null && e.is_pass == null).length; - - summary.passed += passed; - summary.failed += failed; - summary.scored += scored; - summary.errors += testErrors.length; - - testSummaries.push({ - id: testId, - status: failed > 0 || testErrors.length > 0 ? 'failed' : 'passed', - evaluations: evalResults, - outputs: (tr.outputs as unknown[]) ?? [], - }); - } - - return { summary, testSummaries }; -} - export default class AgentTestRunEval extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); - public static state = 'beta'; - public static readonly hidden = true; + public static state = 'ga'; public static readonly envVariablesSection = toHelpSection( 'ENVIRONMENT VARIABLES', @@ -207,14 +97,10 @@ export default class AgentTestRunEval extends SfCommand { // If spec looks like it might be a file path (not parseable content), read the file try { - // Try to detect if it's actual content vs a file path - // If it's a valid YAML/JSON, it's content; otherwise treat as file path if (!isYamlTestSpec(rawContent)) { JSON.parse(rawContent); } - // If we got here, it's valid content } catch { - // Not valid content, must be a file path - read it try { rawContent = await readFile(flags.spec, 'utf-8'); } catch (e) { @@ -228,17 +114,14 @@ export default class AgentTestRunEval extends SfCommand { let agentApiName = flags['api-name']; if (isYamlTestSpec(rawContent)) { - // YAML TestSpec detected — translate to EvalPayload const spec = parseTestSpec(rawContent); payload = translateTestSpec(spec); - // Auto-infer api-name from subjectName if not explicitly provided if (!agentApiName) { agentApiName = spec.subjectName; this.log(messages.getMessage('info.yamlDetected', [spec.subjectName, spec.testCases.length.toString()])); } } else { - // JSON EvalPayload (original behavior) try { payload = JSON.parse(rawContent) as EvalPayload; } catch (e) { @@ -303,16 +186,14 @@ export default class AgentTestRunEval extends SfCommand { const mergedResponse: EvalApiResponse = { results: allResults as EvalApiResponse['results'] }; - // 9. Format output + // 8. Format output const resultFormat = (flags['result-format'] ?? 'human') as ResultFormat; const formatted = formatResults(mergedResponse, resultFormat); this.log(formatted); - // 10. Build structured result for --json + // 9. Build structured result for --json const { summary, testSummaries } = buildResultSummary(mergedResponse); - // Set exit code to 1 only for execution errors (tests couldn't run) - // Test failures (assertions failed) are business logic and should not affect exit code if (summary.errors > 0) { process.exitCode = 1; } diff --git a/src/evalFormatter.ts b/src/evalFormatter.ts deleted file mode 100644 index 2b965c7e..00000000 --- a/src/evalFormatter.ts +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Copyright 2026, Salesforce, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export type ResultFormat = 'human' | 'json' | 'junit' | 'tap'; - -type EvalOutput = { - type?: string; - id?: string; - session_id?: string; - response?: unknown; -}; - -type EvalResult = { - id?: string; - score?: number | null; - is_pass?: boolean | null; - actual_value?: string; - expected_value?: string; - error_message?: string; -}; - -type TestError = { - id?: string; - error_message?: string; -}; - -type TestResult = { - id?: string; - outputs?: EvalOutput[]; - evaluation_results?: EvalResult[]; - errors?: TestError[]; -}; - -export type EvalApiResponse = { - results?: TestResult[]; -}; - -export function formatResults(results: EvalApiResponse, format: ResultFormat): string { - switch (format) { - case 'human': - return formatHuman(results); - case 'json': - return JSON.stringify(results, null, 2); - case 'junit': - return formatJunit(results); - case 'tap': - return formatTap(results); - default: - return formatHuman(results); - } -} - -// --- formatHuman helpers --- - -function formatOutputLines(outputs: EvalOutput[]): string[] { - const lines: string[] = []; - - for (const output of outputs) { - const stepType = output.type ?? ''; - const stepId = output.id ?? ''; - - if (stepType === 'agent.create_session') { - const sessionId = output.session_id ?? 'N/A'; - lines.push(`- **Create Session**: ${sessionId}`); - } else if (stepType === 'agent.send_message') { - let agentMsg = output.response; - if (agentMsg !== null && typeof agentMsg === 'object' && !Array.isArray(agentMsg)) { - const msgObj = agentMsg as Record; - const msgs = msgObj.messages as Array> | undefined; - agentMsg = msgs?.[0]?.message ?? String(agentMsg); - } - const msgStr = String(agentMsg ?? ''); - const displayMsg = msgStr.length > 200 ? msgStr.substring(0, 200) + '...' : msgStr; - lines.push(`- **Agent Response** (${stepId}): ${displayMsg}`); - } else if (stepType === 'agent.get_state') { - const respData = output.response; - if (respData !== null && typeof respData === 'object') { - const resp = respData as Record; - const planner = resp.planner_response as Record | undefined; - const lastExec = planner?.lastExecution as Record | undefined; - const topic = lastExec?.topic ?? 'N/A'; - const latency = lastExec?.latency ?? 'N/A'; - lines.push(`- **Topic Selected**: ${String(topic)}`); - lines.push(`- **Response Latency**: ${String(latency)}ms`); - } else { - lines.push(`- **State**: ${String(respData).substring(0, 200)}`); - } - } - } - - return lines; -} - -function formatEvaluationTable(evalResults: EvalResult[]): string[] { - const lines: string[] = []; - - if (evalResults.length > 0) { - lines.push('### Evaluation Results\n'); - lines.push('| Metric | Score | Pass | Actual | Expected |'); - lines.push('|--------|-------|------|--------|----------|'); - - for (const evalR of evalResults) { - const metricId = evalR.id ?? 'unknown'; - const score = evalR.score; - const scoreStr = score != null ? score.toFixed(3) : 'N/A'; - const isPass = evalR.is_pass; - const passStr = isPass === true ? 'PASS' : isPass === false ? 'FAIL' : 'N/A'; - const actual = String(evalR.actual_value ?? '').substring(0, 60); - const expected = String(evalR.expected_value ?? '').substring(0, 60); - const error = evalR.error_message; - - if (error) { - lines.push(`| ${metricId} | ERROR | - | ${error.substring(0, 80)} | - |`); - } else { - lines.push(`| ${metricId} | ${scoreStr} | ${passStr} | ${actual} | ${expected} |`); - } - } - - lines.push(''); - } - - return lines; -} - -function formatErrorLines(errors: TestError[]): string[] { - const lines: string[] = []; - - if (errors.length > 0) { - lines.push('### Errors\n'); - for (const error of errors) { - const errorId = error.id ?? 'unknown'; - const errorMsg = error.error_message ?? String(error); - lines.push(`- **${errorId}**: ${errorMsg}`); - } - lines.push(''); - } - - return lines; -} - -function formatTestSummaryLines(evalResults: EvalResult[], errors: TestError[]): string[] { - const lines: string[] = []; - - const totalEvals = evalResults.length; - const passed = evalResults.filter((e) => e.is_pass === true).length; - const failed = evalResults.filter((e) => e.is_pass === false).length; - const scored = evalResults.filter((e) => e.score != null && e.is_pass == null).length; - - lines.push(`**Summary**: ${totalEvals} evaluations`); - if (passed || failed) { - lines.push(` - Passed: ${passed}, Failed: ${failed}`); - } - if (scored) { - lines.push(` - Scored (no threshold): ${scored}`); - } - if (errors.length > 0) { - lines.push(` - Errors: ${errors.length}`); - } - lines.push(''); - - return lines; -} - -function formatHuman(results: EvalApiResponse): string { - const lines: string[] = ['# Agent Evaluation Results\n']; - - for (const testResult of results.results ?? []) { - const testId = testResult.id ?? 'unknown'; - const errors = testResult.errors ?? []; - const evalResults = testResult.evaluation_results ?? []; - const outputs = testResult.outputs ?? []; - - lines.push(`## Test: ${testId}\n`); - - lines.push(...formatOutputLines(outputs)); - lines.push(''); - lines.push(...formatEvaluationTable(evalResults)); - lines.push(...formatErrorLines(errors)); - lines.push(...formatTestSummaryLines(evalResults, errors)); - } - - return lines.join('\n'); -} - -function formatJunit(results: EvalApiResponse): string { - const allTests: Array<{ - name: string; - classname: string; - failed: boolean; - errored: boolean; - message: string; - score: string; - }> = []; - - for (const testResult of results.results ?? []) { - const testId = testResult.id ?? 'unknown'; - - for (const evalR of testResult.evaluation_results ?? []) { - const stepId = evalR.id ?? 'unknown'; - const name = `${testId}.${stepId}`; - const score = evalR.score; - const isPass = evalR.is_pass; - const error = evalR.error_message; - - allTests.push({ - name, - classname: 'agent-eval-labs', - failed: isPass === false, - errored: !!error, - message: error - ? error - : isPass === false - ? `Expected ${String(evalR.expected_value ?? '')} but got ${String(evalR.actual_value ?? '')}` - : '', - score: score != null ? score.toFixed(3) : 'N/A', - }); - } - - for (const err of testResult.errors ?? []) { - const stepId = err.id ?? 'unknown'; - allTests.push({ - name: `${testId}.${stepId}`, - classname: 'agent-eval-labs', - failed: false, - errored: true, - message: err.error_message ?? 'Unknown error', - score: 'N/A', - }); - } - } - - const totalTests = allTests.length; - const failures = allTests.filter((t) => t.failed).length; - const errors = allTests.filter((t) => t.errored).length; - - const lines: string[] = [ - '', - '', - ` `, - ]; - - for (const tc of allTests) { - lines.push(` `); - if (tc.errored) { - lines.push(` ${escapeXml(tc.message)}`); - } else if (tc.failed) { - lines.push(` Score: ${tc.score}`); - } - lines.push(' '); - } - - lines.push(' '); - lines.push(''); - - return lines.join('\n'); -} - -// --- formatTap helpers --- - -type TapEntry = { - ok: boolean; - name: string; - score: string; - expected?: string; - actual?: string; - error?: string; -}; - -function buildTapEntries(results: EvalApiResponse): TapEntry[] { - const entries: TapEntry[] = []; - - for (const testResult of results.results ?? []) { - const testId = testResult.id ?? 'unknown'; - - for (const evalR of testResult.evaluation_results ?? []) { - const stepId = evalR.id ?? 'unknown'; - const name = `${testId}.${stepId}`; - const score = evalR.score; - const isPass = evalR.is_pass; - const error = evalR.error_message; - - entries.push({ - ok: isPass !== false && !error, - name, - score: score != null ? score.toFixed(3) : 'N/A', - expected: evalR.expected_value != null ? String(evalR.expected_value) : undefined, - actual: evalR.actual_value != null ? String(evalR.actual_value) : undefined, - error: error ?? undefined, - }); - } - - for (const err of testResult.errors ?? []) { - const stepId = err.id ?? 'unknown'; - entries.push({ - ok: false, - name: `${testId}.${stepId}`, - score: 'N/A', - error: err.error_message ?? 'Unknown error', - }); - } - } - - return entries; -} - -function formatTap(results: EvalApiResponse): string { - const entries = buildTapEntries(results); - - const lines: string[] = ['TAP version 13', `1..${entries.length}`]; - - for (let i = 0; i < entries.length; i++) { - const e = entries[i]; - const num = i + 1; - const prefix = e.ok ? 'ok' : 'not ok'; - lines.push(`${prefix} ${num} - ${e.name} (score: ${e.score})`); - - if (!e.ok) { - lines.push(' ---'); - if (e.expected !== undefined) { - lines.push(` expected: "${e.expected}"`); - } - if (e.actual !== undefined) { - lines.push(` actual: "${e.actual}"`); - } - if (e.error) { - lines.push(` error: "${e.error}"`); - } - lines.push(' ...'); - } - } - - return lines.join('\n'); -} - -function escapeXml(str: string): string { - return str - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); -} diff --git a/src/evalNormalizer.ts b/src/evalNormalizer.ts deleted file mode 100644 index f9901cd1..00000000 --- a/src/evalNormalizer.ts +++ /dev/null @@ -1,500 +0,0 @@ -/* - * Copyright 2026, Salesforce, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* eslint-disable camelcase */ - -// --- Types --- - -export type EvalPayload = { - tests: EvalTest[]; -}; - -export type EvalTest = { - id: string; - steps: EvalStep[]; -}; - -export type EvalStep = { - [key: string]: unknown; - type: string; - id: string; -}; - -// --- Evaluator classification --- - -const SCORING_EVALUATORS = new Set([ - 'evaluator.text_alignment', - 'evaluator.hallucination_detection', - 'evaluator.citation_recall', - 'evaluator.answer_faithfulness', -]); - -const ASSERTION_EVALUATORS = new Set(['evaluator.string_assertion', 'evaluator.json_assertion']); - -const DEFAULT_METRIC_NAMES: Record = { - 'evaluator.text_alignment': 'base.cosine_similarity', - 'evaluator.hallucination_detection': 'hallucination_detection', - 'evaluator.citation_recall': 'citation_recall', - 'evaluator.answer_faithfulness': 'answer_faithfulness', -}; - -const SCORING_VALID_FIELDS = new Set([ - 'type', - 'id', - 'generated_output', - 'reference_answer', - 'metric_name', - 'threshold', -]); - -const ASSERTION_VALID_FIELDS = new Set([ - 'type', - 'id', - 'actual', - 'expected', - 'operator', - 'threshold', - 'json_path', - 'json_schema', - 'metric_name', -]); - -const VALID_AGENT_FIELDS: Record> = { - 'agent.create_session': new Set([ - 'type', - 'id', - 'agent_id', - 'agent_version_id', - 'use_agent_api', - 'planner_id', - 'state', - 'setupSessionContext', - 'context_variables', - ]), - 'agent.send_message': new Set(['type', 'id', 'session_id', 'utterance']), - 'agent.get_state': new Set(['type', 'id', 'session_id']), -}; - -// --- Auto-correction maps --- - -const AGENT_CORRECTIONS: Record = { - agentId: 'agent_id', - agentVersionId: 'agent_version_id', - sessionId: 'session_id', - text: 'utterance', - message: 'utterance', - input: 'utterance', - prompt: 'utterance', - user_message: 'utterance', - userMessage: 'utterance', -}; - -const EVALUATOR_CORRECTIONS: Record = { - subject: 'actual', - expectedValue: 'expected', - expected_value: 'expected', - actualValue: 'actual', - actual_value: 'actual', - assertionType: 'operator', - assertion_type: 'operator', - comparator: 'operator', -}; - -// --- camelCase alias maps for agent.create_session --- - -const AGENT_FIELD_ALIASES: Record = { - useAgentApi: 'use_agent_api', - plannerId: 'planner_id', - plannerDefinitionId: 'planner_id', - planner_definition_id: 'planner_id', - planner_version_id: 'planner_id', - plannerVersionId: 'planner_id', -}; - -// --- Scoring evaluator field aliases --- - -const SCORING_FIELD_ALIASES: Record = { - actual: 'generated_output', - expected: 'reference_answer', - actual_value: 'generated_output', - expected_value: 'reference_answer', - actual_output: 'generated_output', - expected_output: 'reference_answer', - response: 'generated_output', - ground_truth: 'reference_answer', -}; - -// --- Assertion evaluator field aliases --- - -const ASSERTION_FIELD_ALIASES: Record = { - actual_value: 'actual', - expected_value: 'expected', - generated_output: 'actual', - reference_answer: 'expected', - actual_output: 'actual', - expected_output: 'expected', - response: 'actual', - ground_truth: 'expected', -}; - -// --- MCP shorthand field mapping --- - -// MCP uses `field: "gs1.planner_state.topic"` — map to Eval API `actual` with correct JSONPath -const MCP_FIELD_MAP: Record = { - 'planner_state.topic': 'response.planner_response.lastExecution.topic', - 'planner_state.invokedActions': 'response.planner_response.lastExecution.invokedActions', - 'planner_state.actionsSequence': 'response.planner_response.lastExecution.invokedActions', - response: 'response', - 'response.messages': 'response', -}; - -// --- Main entry point --- - -/** - * Apply all normalizations to a test payload. - * Passes run in order: mcp-shorthand -> auto-correct -> camelCase -> evaluator fields -> shorthand refs -> defaults -> strip. - */ -export function normalizePayload(payload: EvalPayload): EvalPayload { - const normalized: EvalPayload = { - tests: payload.tests.map((test) => { - let steps = [...test.steps]; - steps = normalizeMcpShorthand(steps); - steps = autoCorrectFields(steps); - steps = normalizeCamelCase(steps); - steps = normalizeEvaluatorFields(steps); - steps = convertShorthandRefs(steps); - steps = injectDefaults(steps); - steps = stripUnrecognizedFields(steps); - return { ...test, steps }; - }), - }; - return normalized; -} - -// --- Individual normalization passes --- - -/** - * Convert MCP shorthand format to raw Eval API format. - * MCP uses type="evaluator" + evaluator_type, raw API uses type="evaluator.xxx". - * Also maps `field` to `actual` with proper JSONPath and auto-generates missing `id` fields. - */ -export function normalizeMcpShorthand(steps: EvalStep[]): EvalStep[] { - let evalCounter = 0; - - return steps.map((step) => { - const evaluator_type = step.evaluator_type as string | undefined; - - // Only applies to MCP shorthand: type="evaluator" with evaluator_type field - if (step.type !== 'evaluator' || !evaluator_type) return step; - - const normalized = { ...step }; - - // Merge type: "evaluator" + evaluator_type: "xxx" → type: "evaluator.xxx" - normalized.type = `evaluator.${evaluator_type}`; - delete normalized.evaluator_type; - - // Convert `field` to `actual` with proper shorthand ref format - if ('field' in normalized) { - if (!('actual' in normalized)) { - const fieldValue = normalized.field as string; - - // Parse "gs1.planner_state.topic" → stepId="gs1", fieldPath="planner_state.topic" - const dotIdx = fieldValue.indexOf('.'); - if (dotIdx > 0) { - const stepId = fieldValue.substring(0, dotIdx); - const fieldPath = fieldValue.substring(dotIdx + 1); - const mappedPath = MCP_FIELD_MAP[fieldPath] ?? fieldPath; - normalized.actual = `{${stepId}.${mappedPath}}`; - } else { - normalized.actual = fieldValue; - } - } - delete normalized.field; - } - - // Auto-generate id if missing - if (!normalized.id || normalized.id === '') { - normalized.id = `eval_${evalCounter}`; - } - evalCounter++; - - return normalized as EvalStep; - }); -} - -/** - * Auto-correct common field name mistakes. - * Maps wrong field names to correct ones (agentId->agent_id, text->utterance, etc.) - */ -export function autoCorrectFields(steps: EvalStep[]): EvalStep[] { - return steps.map((step) => { - const corrected = { ...step }; - const stepType = corrected.type ?? ''; - - if (stepType.startsWith('agent.')) { - for (const [wrong, correct] of Object.entries(AGENT_CORRECTIONS)) { - if (wrong in corrected && !(correct in corrected)) { - corrected[correct] = corrected[wrong]; - delete corrected[wrong]; - } - } - } else if (stepType.startsWith('evaluator.')) { - for (const [wrong, correct] of Object.entries(EVALUATOR_CORRECTIONS)) { - if (wrong in corrected && !(correct in corrected)) { - corrected[correct] = corrected[wrong]; - delete corrected[wrong]; - } - } - } - - return corrected as EvalStep; - }); -} - -/** - * Normalize camelCase agent field names to snake_case. - * useAgentApi->use_agent_api, plannerDefinitionId->planner_id, etc. - */ -export function normalizeCamelCase(steps: EvalStep[]): EvalStep[] { - return steps.map((step) => { - if (step.type !== 'agent.create_session') return step; - - const normalized = { ...step }; - for (const [alias, canonical] of Object.entries(AGENT_FIELD_ALIASES)) { - if (alias in normalized) { - if (!(canonical in normalized)) { - normalized[canonical] = normalized[alias]; - } - delete normalized[alias]; - } - } - return normalized as EvalStep; - }); -} - -/** - * Apply field aliases: remap alias keys to canonical keys, removing duplicates. - */ -function applyFieldAliases(step: EvalStep, aliases: Record): void { - for (const [alias, canonical] of Object.entries(aliases)) { - if (alias in step && !(canonical in step)) { - step[canonical] = step[alias]; - delete step[alias]; - } else if (alias in step && canonical in step) { - delete step[alias]; - } - } -} - -/** - * Normalize a scoring evaluator step (field aliases + metric_name injection). - */ -function normalizeScoringEvaluator(normalized: EvalStep, evalType: string): void { - applyFieldAliases(normalized, SCORING_FIELD_ALIASES); - - // Auto-inject or correct metric_name - if (!('metric_name' in normalized)) { - const defaultMetric = DEFAULT_METRIC_NAMES[evalType]; - if (defaultMetric) { - normalized.metric_name = defaultMetric; - } - } else if (normalized.metric_name === evalType.split('.')[1]) { - const defaultMetric = DEFAULT_METRIC_NAMES[evalType]; - if (defaultMetric) { - normalized.metric_name = defaultMetric; - } - } -} - -/** - * Normalize an assertion evaluator step (field aliases + operator lowercase + metric_name). - */ -function normalizeAssertionEvaluator(normalized: EvalStep, evalType: string): void { - applyFieldAliases(normalized, ASSERTION_FIELD_ALIASES); - - // Auto-lowercase operator - if ('operator' in normalized && typeof normalized.operator === 'string') { - normalized.operator = normalized.operator.toLowerCase(); - } - - // Auto-inject metric_name for assertion evaluators - if (!('metric_name' in normalized)) { - normalized.metric_name = evalType.split('.')[1]; - } -} - -/** - * Normalize evaluator field names based on evaluator category. - * Maps actual/expected <-> generated_output/reference_answer. - * Also auto-lowercases operator values and auto-injects metric_name. - */ -export function normalizeEvaluatorFields(steps: EvalStep[]): EvalStep[] { - return steps.map((step) => { - const evalType = step.type ?? ''; - if (!evalType.startsWith('evaluator.')) return step; - - const normalized = { ...step }; - - if (SCORING_EVALUATORS.has(evalType)) { - normalizeScoringEvaluator(normalized, evalType); - } else if (ASSERTION_EVALUATORS.has(evalType)) { - normalizeAssertionEvaluator(normalized, evalType); - } - // Don't inject metric_name for unknown evaluator types to avoid API validation errors - // Unknown evaluators like bot_response_rating and planner_topic_assertion don't use metric_name - - return normalized as EvalStep; - }); -} - -/** - * Convert {step_id.field} shorthand references to JSONPath $.outputs[N].field. - * Builds step_id->index mapping from non-evaluator steps. - */ -export function convertShorthandRefs(steps: EvalStep[]): EvalStep[] { - // Build step_id -> output-array index mapping - const stepIdToIdx: Record = {}; - let outputIdx = 0; - for (const step of steps) { - const sid = step.id; - const stype = step.type ?? ''; - if (sid && !stype.startsWith('evaluator.')) { - stepIdToIdx[sid] = outputIdx; - outputIdx += 1; - } - } - - const refPattern = /\{([^}]+)\}/g; - - function replaceValue(value: unknown): unknown { - if (typeof value !== 'string') return value; - - return value.replace(refPattern, (match, ref: string) => { - const dotIdx = ref.indexOf('.'); - if (dotIdx < 0) return match; - - const sid = ref.substring(0, dotIdx); - let field = ref.substring(dotIdx + 1); - - if (!(sid in stepIdToIdx)) return match; - - const idx = stepIdToIdx[sid]; - - // Normalize legacy nested-response path to flat response - if (field.startsWith('response.messages')) { - field = 'response'; - } - - return `$.outputs[${idx}].${field}`; - }); - } - - return steps.map((step) => { - const newStep: Record = {}; - for (const [key, val] of Object.entries(step)) { - if (typeof val === 'string') { - newStep[key] = replaceValue(val); - } else if (val !== null && typeof val === 'object' && !Array.isArray(val)) { - const newObj: Record = {}; - for (const [k, v] of Object.entries(val as Record)) { - newObj[k] = typeof v === 'string' ? replaceValue(v) : v; - } - newStep[key] = newObj; - } else if (Array.isArray(val)) { - newStep[key] = (val as unknown[]).map((item: unknown) => - typeof item === 'string' ? replaceValue(item) : item - ); - } else { - newStep[key] = val; - } - } - return newStep as EvalStep; - }); -} - -/** - * Inject default values: - * - use_agent_api=true on agent.create_session if neither use_agent_api nor planner_id present - */ -export function injectDefaults(steps: EvalStep[]): EvalStep[] { - return steps.map((step) => { - if (step.type === 'agent.create_session') { - if (!('use_agent_api' in step) && !('planner_id' in step)) { - return { ...step, use_agent_api: true }; - } - } - return step; - }); -} - -/** - * Strip unrecognized fields from steps based on type-specific whitelists. - */ -export function stripUnrecognizedFields(steps: EvalStep[]): EvalStep[] { - return steps.map((step) => { - const stepType = step.type ?? ''; - - // Agent steps - if (stepType in VALID_AGENT_FIELDS) { - const validFields = VALID_AGENT_FIELDS[stepType]; - const stripped: Record = {}; - for (const [key, val] of Object.entries(step)) { - if (validFields.has(key)) { - stripped[key] = val; - } - } - return stripped as EvalStep; - } - - // Scoring evaluators - if (SCORING_EVALUATORS.has(stepType)) { - const stripped: Record = {}; - for (const [key, val] of Object.entries(step)) { - if (SCORING_VALID_FIELDS.has(key)) { - stripped[key] = val; - } - } - return stripped as EvalStep; - } - - // Assertion evaluators - if (ASSERTION_EVALUATORS.has(stepType)) { - const stripped: Record = {}; - for (const [key, val] of Object.entries(step)) { - if (ASSERTION_VALID_FIELDS.has(key)) { - stripped[key] = val; - } - } - return stripped as EvalStep; - } - - // Unknown types: don't strip (to avoid breaking future evaluator types) - return step; - }); -} - -// --- Batch splitting --- - -/** - * Split tests array into chunks of batchSize. - */ -export function splitIntoBatches(tests: EvalTest[], batchSize: number): EvalTest[][] { - const batches: EvalTest[][] = []; - for (let i = 0; i < tests.length; i += batchSize) { - batches.push(tests.slice(i, i + batchSize)); - } - return batches; -} diff --git a/src/yamlSpecTranslator.ts b/src/yamlSpecTranslator.ts deleted file mode 100644 index b19015f2..00000000 --- a/src/yamlSpecTranslator.ts +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Copyright 2026, Salesforce, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* eslint-disable camelcase */ - -import { parse as parseYaml } from 'yaml'; -import type { TestSpec, TestCase } from '@salesforce/agents'; -import type { EvalPayload, EvalTest, EvalStep } from './evalNormalizer.js'; - -// --- JSONPath mappings from org model to Eval API refs --- - -const ACTUAL_PATH_MAP: Record = { - '$.generatedData.outcome': '{sm.response}', - '$.generatedData.topic': '{gs.response.planner_response.lastExecution.topic}', - '$.generatedData.invokedActions': '{gs.response.planner_response.lastExecution.invokedActions}', - '$.generatedData.actionsSequence': '{gs.response.planner_response.lastExecution.invokedActions}', -}; - -// --- Custom evaluation name to evaluator type mapping --- - -const CUSTOM_EVAL_TYPE_MAP: Record = { - string_comparison: 'evaluator.string_assertion', - numeric_comparison: 'evaluator.numeric_assertion', -}; - -// JSONPaths that require the get_state step -const PLANNER_PATHS = new Set([ - '$.generatedData.topic', - '$.generatedData.invokedActions', - '$.generatedData.actionsSequence', -]); - -// --- Public API --- - -/** - * Returns true if the content looks like a YAML TestSpec (has testCases + subjectName). - * Returns false for JSON EvalPayload, invalid content, or YAML missing required fields. - */ -export function isYamlTestSpec(content: string): boolean { - try { - const parsed: unknown = parseYaml(content); - if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { - return false; - } - const obj = parsed as Record; - return Array.isArray(obj.testCases) && typeof obj.subjectName === 'string'; - } catch { - return false; - } -} - -/** - * Parse a YAML string into a TestSpec. - * Throws if the content is not valid YAML or is missing required fields. - */ -export function parseTestSpec(content: string): TestSpec { - const parsed: unknown = parseYaml(content); - if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error('Invalid TestSpec: expected a YAML object'); - } - const obj = parsed as Record; - if (!Array.isArray(obj.testCases)) { - throw new Error('Invalid TestSpec: missing testCases array'); - } - if (typeof obj.subjectName !== 'string') { - throw new Error('Invalid TestSpec: missing subjectName'); - } - if (typeof obj.name !== 'string') { - throw new Error('Invalid TestSpec: missing name'); - } - return parsed as TestSpec; -} - -/** - * Translate a full TestSpec into an EvalPayload. - */ -export function translateTestSpec(spec: TestSpec): EvalPayload { - return { - tests: spec.testCases.map((tc, idx) => translateTestCase(tc, idx, spec.name)), - }; -} - -/** - * Translate a single TestCase into an EvalTest with ordered steps. - */ -export function translateTestCase(testCase: TestCase, index: number, specName?: string): EvalTest { - const id = specName ? `${specName}_case_${index}` : `test_case_${index}`; - const steps: EvalStep[] = []; - - // 1. agent.create_session - const createSessionStep: EvalStep = { - type: 'agent.create_session', - id: 'cs', - use_agent_api: true, - }; - - if (testCase.contextVariables && testCase.contextVariables.length > 0) { - // Validate for duplicate names - const names = testCase.contextVariables.map((cv) => cv.name); - const duplicates = names.filter((name, idx) => names.indexOf(name) !== idx); - if (duplicates.length > 0) { - throw new Error( - `Duplicate contextVariable names found in test case ${index}: ${[...new Set(duplicates)].join( - ', ' - )}. Each contextVariable name must be unique.` - ); - } - - createSessionStep.context_variables = Object.fromEntries( - testCase.contextVariables.map((cv) => [cv.name, cv.value]) - ); - } - - steps.push(createSessionStep); - - // 2. Conversation history — only user messages become send_message steps - let historyIdx = 0; - if (testCase.conversationHistory) { - for (const entry of testCase.conversationHistory) { - if (entry.role === 'user') { - steps.push({ - type: 'agent.send_message', - id: `history_${historyIdx}`, - session_id: '{cs.session_id}', - utterance: entry.message, - }); - historyIdx++; - } - } - } - - // 3. Test utterance - steps.push({ - type: 'agent.send_message', - id: 'sm', - session_id: '{cs.session_id}', - utterance: testCase.utterance, - }); - - // 4. Determine if get_state is needed - const needsGetState = needsPlannerState(testCase); - if (needsGetState) { - steps.push({ - type: 'agent.get_state', - id: 'gs', - session_id: '{cs.session_id}', - }); - } - - // 5. Evaluators - if (testCase.expectedTopic !== undefined) { - steps.push({ - type: 'evaluator.planner_topic_assertion', - id: 'check_topic', - expected: testCase.expectedTopic, - actual: '{gs.response.planner_response.lastExecution.topic}', - operator: 'contains', - }); - } - - if (testCase.expectedActions !== undefined && testCase.expectedActions.length > 0) { - steps.push({ - type: 'evaluator.planner_actions_assertion', - id: 'check_actions', - expected: testCase.expectedActions, - actual: '{gs.response.planner_response.lastExecution.invokedActions}', - operator: 'includes_items', - }); - } - - if (testCase.expectedOutcome !== undefined) { - steps.push({ - type: 'evaluator.bot_response_rating', - id: 'check_outcome', - utterance: testCase.utterance, - expected: testCase.expectedOutcome, - actual: '{sm.response}', - threshold: 3.0, - }); - } - - if (testCase.customEvaluations) { - testCase.customEvaluations.forEach((customEval, customIdx) => { - const step = translateCustomEvaluation(customEval, customIdx); - steps.push(step); - }); - } - - return { id, steps }; -} - -// --- Internal helpers --- - -/** - * Determine whether the get_state step is needed for this test case. - */ -function needsPlannerState(testCase: TestCase): boolean { - if (testCase.expectedTopic !== undefined) return true; - if (testCase.expectedActions !== undefined && testCase.expectedActions.length > 0) return true; - - if (testCase.customEvaluations) { - for (const customEval of testCase.customEvaluations) { - for (const param of customEval.parameters) { - if (param.name === 'actual' && PLANNER_PATHS.has(param.value)) { - return true; - } - } - } - } - - return false; -} - -/** - * Translate a single customEvaluation entry into an EvalStep. - */ -function translateCustomEvaluation( - customEval: NonNullable[number], - index: number -): EvalStep { - const evalType = CUSTOM_EVAL_TYPE_MAP[customEval.name] ?? `evaluator.${customEval.name}`; - - let operator = ''; - let actual = ''; - let expected = ''; - - for (const param of customEval.parameters) { - if (param.name === 'operator') { - operator = param.value; - } else if (param.name === 'actual') { - actual = mapActualPath(param.value); - } else if (param.name === 'expected') { - expected = param.value; - } - } - - return { - type: evalType, - id: `custom_${index}`, - operator, - actual, - expected, - }; -} - -/** - * Map an org-model JSONPath to the Eval API shorthand ref. - * Unknown paths are returned as-is. - */ -function mapActualPath(path: string): string { - return ACTUAL_PATH_MAP[path] ?? path; -} diff --git a/test/commands/agent/test/run-eval.test.ts b/test/commands/agent/test/run-eval.test.ts new file mode 100644 index 00000000..cccef486 --- /dev/null +++ b/test/commands/agent/test/run-eval.test.ts @@ -0,0 +1,360 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, + @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, camelcase */ + +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; +import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; + +// ─── Shared fixtures ───────────────────────────────────────────────────────── + +const EVAL_PAYLOAD = JSON.stringify({ + tests: [ + { + id: 'test-topic-routing', + steps: [ + { type: 'agent.create_session', id: 'session' }, + { + type: 'agent.send_message', + id: 'msg1', + session_id: '{session.session_id}', + utterance: 'What is the weather?', + }, + { type: 'agent.get_state', id: 'state1', session_id: '{session.session_id}' }, + { + type: 'evaluator.planner_topic_assertion', + id: 'check-topic', + actual: '{state1.response.planner_response.lastExecution.topic}', + expected: 'Weather_and_Temperature_Information', + operator: 'equals', + }, + ], + }, + ], +}); + +const YAML_SPEC = ` +name: Weather_Test +description: Test weather agent +subjectType: AGENT +subjectName: Local_Info_Agent +testCases: + - utterance: 'What is the weather?' + expectedTopic: Weather_and_Temperature_Information + expectedActions: [] + expectedOutcome: 'The agent should provide weather information' +`; + +const MOCK_API_RESULTS = [ + { + id: 'test-topic-routing', + evaluation_results: [{ id: 'check-topic', is_pass: true }], + errors: [], + outputs: [], + }, +]; + +// ─── Test suite ────────────────────────────────────────────────────────────── + +describe('agent test run-eval command', () => { + const $$ = new TestContext(); + let testOrg: MockTestOrgData; + let tmpDir: string; + + // Stubs for @salesforce/agents exports + let isYamlTestSpecStub: sinon.SinonStub; + let parseTestSpecStub: sinon.SinonStub; + let translateTestSpecStub: sinon.SinonStub; + let normalizePayloadStub: sinon.SinonStub; + let splitIntoBatchesStub: sinon.SinonStub; + let resolveAgentStub: sinon.SinonStub; + let executeBatchesStub: sinon.SinonStub; + let buildResultSummaryStub: sinon.SinonStub; + let formatResultsStub: sinon.SinonStub; + + let AgentTestRunEval: any; + + beforeEach(async () => { + testOrg = new MockTestOrgData(); + await $$.stubAuths(testOrg); + + tmpDir = mkdtempSync(join(tmpdir(), 'run-eval-test-')); + + // Default stub implementations + isYamlTestSpecStub = sinon.stub().returns(false); + parseTestSpecStub = sinon.stub().returns({ + name: 'Weather_Test', + subjectName: 'Local_Info_Agent', + testCases: [{ utterance: 'What is the weather?' }], + }); + translateTestSpecStub = sinon.stub().returns(JSON.parse(EVAL_PAYLOAD)); + normalizePayloadStub = sinon.stub().callsFake((p: unknown) => p); + splitIntoBatchesStub = sinon.stub().callsFake((tests: unknown[]) => [tests]); + resolveAgentStub = sinon.stub().resolves({ agentId: 'bot-001', versionId: 'ver-001' }); + executeBatchesStub = sinon.stub().resolves(MOCK_API_RESULTS); + buildResultSummaryStub = sinon.stub().returns({ + summary: { passed: 1, failed: 0, scored: 0, errors: 0 }, + testSummaries: [{ id: 'test-topic-routing', status: 'passed', evaluations: [], outputs: [] }], + }); + formatResultsStub = sinon.stub().returns('# Agent Evaluation Results'); + + const mod = await esmock('../../../../src/commands/agent/test/run-eval.js', { + '@salesforce/agents': { + isYamlTestSpec: isYamlTestSpecStub, + parseTestSpec: parseTestSpecStub, + translateTestSpec: translateTestSpecStub, + normalizePayload: normalizePayloadStub, + splitIntoBatches: splitIntoBatchesStub, + resolveAgent: resolveAgentStub, + executeBatches: executeBatchesStub, + buildResultSummary: buildResultSummaryStub, + formatResults: formatResultsStub, + }, + }); + + AgentTestRunEval = mod.default; + }); + + afterEach(() => { + sinon.restore(); + $$.restore(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + // ─── State ───────────────────────────────────────────────────────────────── + + describe('command metadata', () => { + it('is marked as GA state', () => { + expect(AgentTestRunEval.state).to.equal('ga'); + }); + + it('is not hidden', () => { + expect(AgentTestRunEval.hidden).to.not.equal(true); + }); + }); + + // ─── JSON payload path ───────────────────────────────────────────────────── + + describe('JSON payload', () => { + it('runs with an inline JSON string', async () => { + const result = await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--target-org', testOrg.username]); + + expect(result.summary.passed).to.equal(1); + expect(result.tests).to.have.length(1); + expect(result.tests[0].status).to.equal('passed'); + }); + + it('reads the spec from a file when the string is not valid JSON', async () => { + const specFile = join(tmpDir, 'payload.json'); + writeFileSync(specFile, EVAL_PAYLOAD, 'utf-8'); + + const result = await AgentTestRunEval.run(['--spec', specFile, '--target-org', testOrg.username]); + + expect(result.summary.passed).to.equal(1); + }); + + it('throws SpecFileNotFound (exit 2) when the file does not exist', async () => { + try { + await AgentTestRunEval.run(['--spec', '/nonexistent/path.json', '--target-org', testOrg.username]); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.exitCode).to.equal(2); + expect(err.name).to.equal('SpecFileNotFound'); + } + }); + + it('calls normalizePayload by default', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--target-org', testOrg.username]); + + expect(normalizePayloadStub.calledOnce).to.be.true; + }); + + it('skips normalizePayload when --no-normalize is set', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--no-normalize', '--target-org', testOrg.username]); + + expect(normalizePayloadStub.called).to.be.false; + }); + + it('resolves agent IDs when --api-name is provided', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--api-name', 'My_Agent', '--target-org', testOrg.username]); + + expect(resolveAgentStub.calledOnceWith(sinon.match.any, 'My_Agent')).to.be.true; + }); + + it('throws AgentNotFound (exit 2) when resolveAgent fails', async () => { + resolveAgentStub.rejects(new Error('not found')); + + try { + await AgentTestRunEval.run([ + '--spec', + EVAL_PAYLOAD, + '--api-name', + 'Missing_Agent', + '--target-org', + testOrg.username, + ]); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.exitCode).to.equal(2); + expect(err.name).to.equal('AgentNotFound'); + } + }); + + it('throws TestExecutionFailed (exit 4) when executeBatches fails', async () => { + executeBatchesStub.rejects(new Error('API down')); + + try { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--target-org', testOrg.username]); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.exitCode).to.equal(4); + expect(err.name).to.equal('TestExecutionFailed'); + } + }); + }); + + // ─── YAML spec path ──────────────────────────────────────────────────────── + + describe('YAML spec', () => { + beforeEach(() => { + isYamlTestSpecStub.returns(true); + }); + + it('runs with an inline YAML string', async () => { + const result = await AgentTestRunEval.run(['--spec', YAML_SPEC, '--target-org', testOrg.username]); + + expect(parseTestSpecStub.calledOnce).to.be.true; + expect(translateTestSpecStub.calledOnce).to.be.true; + expect(result.summary.passed).to.equal(1); + }); + + it('auto-infers api-name from subjectName when --api-name is omitted', async () => { + await AgentTestRunEval.run(['--spec', YAML_SPEC, '--target-org', testOrg.username]); + + // resolveAgent should be called with the subjectName from the parsed spec + expect(resolveAgentStub.calledOnceWith(sinon.match.any, 'Local_Info_Agent')).to.be.true; + }); + + it('prefers explicit --api-name over auto-inferred subjectName', async () => { + await AgentTestRunEval.run([ + '--spec', + YAML_SPEC, + '--api-name', + 'Override_Agent', + '--target-org', + testOrg.username, + ]); + + expect(resolveAgentStub.calledOnceWith(sinon.match.any, 'Override_Agent')).to.be.true; + }); + + it('reads YAML spec from a file', async () => { + const specFile = join(tmpDir, 'spec.yaml'); + writeFileSync(specFile, YAML_SPEC, 'utf-8'); + + // isYamlTestSpec returns false for the file path string, true for the file content + isYamlTestSpecStub.onFirstCall().returns(false).onSecondCall().returns(true); + + const result = await AgentTestRunEval.run(['--spec', specFile, '--target-org', testOrg.username]); + + expect(result.summary.passed).to.equal(1); + }); + }); + + // ─── Batch size ──────────────────────────────────────────────────────────── + + describe('batch size', () => { + it('clamps --batch-size to maximum of 5', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--batch-size', '99', '--target-org', testOrg.username]); + + const batchSize = splitIntoBatchesStub.firstCall.args[1] as number; + expect(batchSize).to.equal(5); + }); + + it('clamps --batch-size to minimum of 1', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--batch-size', '0', '--target-org', testOrg.username]); + + const batchSize = splitIntoBatchesStub.firstCall.args[1] as number; + expect(batchSize).to.equal(1); + }); + + it('passes through a valid --batch-size unchanged', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--batch-size', '3', '--target-org', testOrg.username]); + + const batchSize = splitIntoBatchesStub.firstCall.args[1] as number; + expect(batchSize).to.equal(3); + }); + }); + + // ─── Result format ───────────────────────────────────────────────────────── + + describe('result format', () => { + it('defaults to human format', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--target-org', testOrg.username]); + + expect(formatResultsStub.calledOnceWith(sinon.match.any, 'human')).to.be.true; + }); + + it('passes --result-format tap to formatResults', async () => { + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--result-format', 'tap', '--target-org', testOrg.username]); + + expect(formatResultsStub.calledOnceWith(sinon.match.any, 'tap')).to.be.true; + }); + + it('passes --result-format junit to formatResults', async () => { + await AgentTestRunEval.run([ + '--spec', + EVAL_PAYLOAD, + '--result-format', + 'junit', + '--target-org', + testOrg.username, + ]); + + expect(formatResultsStub.calledOnceWith(sinon.match.any, 'junit')).to.be.true; + }); + }); + + // ─── Exit code behaviour ─────────────────────────────────────────────────── + + describe('exit code', () => { + it('sets process.exitCode to 1 when summary contains errors', async () => { + buildResultSummaryStub.returns({ + summary: { passed: 0, failed: 0, scored: 0, errors: 2 }, + testSummaries: [{ id: 'test-1', status: 'failed', evaluations: [], outputs: [] }], + }); + + const originalExitCode = process.exitCode; + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--target-org', testOrg.username]); + + expect(process.exitCode).to.equal(1); + process.exitCode = originalExitCode; + }); + + it('does not set process.exitCode when there are no errors', async () => { + process.exitCode = undefined; + await AgentTestRunEval.run(['--spec', EVAL_PAYLOAD, '--target-org', testOrg.username]); + + expect(process.exitCode).to.be.undefined; + }); + }); +}); diff --git a/test/evalFormatter.test.ts b/test/evalFormatter.test.ts index b4b85a58..822b93eb 100644 --- a/test/evalFormatter.test.ts +++ b/test/evalFormatter.test.ts @@ -17,7 +17,7 @@ /* eslint-disable camelcase */ import { expect } from 'chai'; -import { formatResults, type EvalApiResponse } from '../src/evalFormatter.js'; +import { formatResults, type EvalApiResponse } from '@salesforce/agents'; const MOCK_RESPONSE: EvalApiResponse = { results: [ diff --git a/test/evalNormalizer.test.ts b/test/evalNormalizer.test.ts index 3ddab444..c4857d36 100644 --- a/test/evalNormalizer.test.ts +++ b/test/evalNormalizer.test.ts @@ -29,7 +29,7 @@ import { splitIntoBatches, type EvalStep, type EvalPayload, -} from '../src/evalNormalizer.js'; +} from '@salesforce/agents'; describe('evalNormalizer', () => { describe('normalizeMcpShorthand', () => { diff --git a/test/yamlSpecTranslator.test.ts b/test/yamlSpecTranslator.test.ts index 40914c67..1341fd77 100644 --- a/test/yamlSpecTranslator.test.ts +++ b/test/yamlSpecTranslator.test.ts @@ -18,7 +18,7 @@ import { expect } from 'chai'; import type { TestCase } from '@salesforce/agents'; -import { isYamlTestSpec, parseTestSpec, translateTestCase, translateTestSpec } from '../src/yamlSpecTranslator.js'; +import { isYamlTestSpec, parseTestSpec, translateTestCase, translateTestSpec } from '@salesforce/agents'; describe('yamlSpecTranslator', () => { describe('isYamlTestSpec', () => { From f48d2b0d064cdc2c778e97be66f4edf7cf78738d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Tue, 5 May 2026 17:13:08 -0300 Subject: [PATCH 26/38] fix(agent/test/run-eval): address pre-PR review findings - Remove state='ga' (GA commands omit state to avoid oclif help regression) - Remove unused --wait flag - Wrap parseTestSpec/translateTestSpec in try/catch for named SfError - Validate test.steps is an array before injection loop - Hoist isYamlTestSpec to avoid double-call - Extract resolveAndInjectAgent helper to stay within complexity limit - Fix test stubs to use $$.SANDBOX, remove sinon.restore(), fix describe label Co-Authored-By: Claude Sonnet 4.6 --- messages/agent.test.run-eval.md | 4 - src/commands/agent/test/run-eval.ts | 119 +++++++++++----------- test/commands/agent/test/run-eval.test.ts | 31 +++--- 3 files changed, 74 insertions(+), 80 deletions(-) diff --git a/messages/agent.test.run-eval.md b/messages/agent.test.run-eval.md index 304ab4bd..2350c84b 100644 --- a/messages/agent.test.run-eval.md +++ b/messages/agent.test.run-eval.md @@ -20,10 +20,6 @@ Path to test spec file (YAML or JSON). Supports reading from stdin when piping c Agent DeveloperName (also called API name) to resolve agent_id and agent_version_id. Auto-inferred from the YAML spec's subjectName. -# flags.wait.summary - -Number of minutes to wait for results. - # flags.result-format.summary Format of the agent test results. diff --git a/src/commands/agent/test/run-eval.ts b/src/commands/agent/test/run-eval.ts index d0187566..c31b3815 100644 --- a/src/commands/agent/test/run-eval.ts +++ b/src/commands/agent/test/run-eval.ts @@ -41,11 +41,35 @@ export type RunEvalResult = { summary: { passed: number; failed: number; scored: number; errors: number }; }; +async function resolveAndInjectAgent( + org: Parameters[0], + agentApiName: string, + payload: EvalPayload +): Promise { + let agentId: string; + let versionId: string; + try { + ({ agentId, versionId } = await resolveAgent(org, agentApiName)); + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError(`Agent '${agentApiName}' not found.`, 'AgentNotFound', [], 2, wrapped); + } + for (const test of payload.tests) { + for (const step of test.steps) { + if (step.type === 'agent.create_session') { + // eslint-disable-next-line camelcase + step.agent_id = agentId; + // eslint-disable-next-line camelcase + step.agent_version_id = versionId; + } + } + } +} + export default class AgentTestRunEval extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); - public static state = 'ga'; public static readonly envVariablesSection = toHelpSection( 'ENVIRONMENT VARIABLES', @@ -72,11 +96,6 @@ export default class AgentTestRunEval extends SfCommand { char: 'n', summary: messages.getMessage('flags.api-name.summary'), }), - wait: Flags.integer({ - char: 'w', - default: 10, - summary: messages.getMessage('flags.wait.summary'), - }), 'result-format': resultFormatFlag(), 'batch-size': Flags.integer({ default: 5, @@ -94,32 +113,37 @@ export default class AgentTestRunEval extends SfCommand { // 1. Get spec content (from file or stdin via allowStdin) let rawContent = flags.spec; + let isYaml = isYamlTestSpec(rawContent); - // If spec looks like it might be a file path (not parseable content), read the file - try { - if (!isYamlTestSpec(rawContent)) { - JSON.parse(rawContent); - } - } catch { + if (!isYaml) { try { - rawContent = await readFile(flags.spec, 'utf-8'); - } catch (e) { - const wrapped = SfError.wrap(e); - throw new SfError(`Spec file not found: ${flags.spec}`, 'SpecFileNotFound', [], 2, wrapped); + JSON.parse(rawContent); + } catch { + try { + rawContent = await readFile(flags.spec, 'utf-8'); + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError(`Spec file not found: ${flags.spec}`, 'SpecFileNotFound', [], 2, wrapped); + } + isYaml = isYamlTestSpec(rawContent); } } // 2. Detect format and parse - let payload: EvalPayload; + let payload!: EvalPayload; let agentApiName = flags['api-name']; - if (isYamlTestSpec(rawContent)) { - const spec = parseTestSpec(rawContent); - payload = translateTestSpec(spec); + if (isYaml) { + try { + const spec = parseTestSpec(rawContent); + payload = translateTestSpec(spec); - if (!agentApiName) { - agentApiName = spec.subjectName; - this.log(messages.getMessage('info.yamlDetected', [spec.subjectName, spec.testCases.length.toString()])); + if (!agentApiName) { + agentApiName = spec.subjectName; + this.log(messages.getMessage('info.yamlDetected', [spec.subjectName, spec.testCases.length.toString()])); + } + } catch (e) { + throw messages.createError('error.invalidPayload', [(e as Error).message]); } } else { try { @@ -133,29 +157,15 @@ export default class AgentTestRunEval extends SfCommand { throw messages.createError('error.invalidPayload', ['missing or empty "tests" array']); } - // 3. If --api-name (or auto-inferred from YAML), resolve IDs and inject - if (agentApiName) { - let agentId; - let versionId; - try { - const resolved = await resolveAgent(org, agentApiName); - agentId = resolved.agentId; - versionId = resolved.versionId; - } catch (e) { - const wrapped = SfError.wrap(e); - throw new SfError(`Agent '${agentApiName}' not found.`, 'AgentNotFound', [], 2, wrapped); + for (const test of payload.tests) { + if (!Array.isArray(test.steps)) { + throw messages.createError('error.invalidPayload', [`test '${test.id}' has missing or invalid 'steps' array`]); } + } - for (const test of payload.tests) { - for (const step of test.steps) { - if (step.type === 'agent.create_session') { - // eslint-disable-next-line camelcase - step.agent_id = agentId; - // eslint-disable-next-line camelcase - step.agent_version_id = versionId; - } - } - } + // 3. If --api-name (or auto-inferred from YAML), resolve IDs and inject + if (agentApiName) { + await resolveAndInjectAgent(org, agentApiName, payload); } // 4. Normalize payload unless --no-normalize @@ -163,17 +173,12 @@ export default class AgentTestRunEval extends SfCommand { payload = normalizePayload(payload); } - // 5. Clamp batch size + // 5. Clamp batch size and split into batches const batchSize = Math.min(Math.max(flags['batch-size'], 1), 5); - - // 6. Split into batches const batches = splitIntoBatches(payload.tests, batchSize); - // 7. Execute batches - let allResults; - try { - allResults = await executeBatches(org, batches, (msg) => this.log(msg)); - } catch (e) { + // 6. Execute batches + const allResults = await executeBatches(org, batches, (msg) => this.log(msg)).catch((e) => { const wrapped = SfError.wrap(e); throw new SfError( `Failed to execute tests: ${wrapped.message}`, @@ -182,16 +187,14 @@ export default class AgentTestRunEval extends SfCommand { 4, wrapped ); - } + }); const mergedResponse: EvalApiResponse = { results: allResults as EvalApiResponse['results'] }; - // 8. Format output - const resultFormat = (flags['result-format'] ?? 'human') as ResultFormat; - const formatted = formatResults(mergedResponse, resultFormat); - this.log(formatted); + // 7. Format output + this.log(formatResults(mergedResponse, (flags['result-format'] ?? 'human') as ResultFormat)); - // 9. Build structured result for --json + // 8. Build structured result for --json const { summary, testSummaries } = buildResultSummary(mergedResponse); if (summary.errors > 0) { diff --git a/test/commands/agent/test/run-eval.test.ts b/test/commands/agent/test/run-eval.test.ts index cccef486..8be229b9 100644 --- a/test/commands/agent/test/run-eval.test.ts +++ b/test/commands/agent/test/run-eval.test.ts @@ -14,8 +14,7 @@ * limitations under the License. */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, - @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, camelcase */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, camelcase */ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -75,7 +74,7 @@ const MOCK_API_RESULTS = [ // ─── Test suite ────────────────────────────────────────────────────────────── -describe('agent test run-eval command', () => { +describe('agent test run-eval', () => { const $$ = new TestContext(); let testOrg: MockTestOrgData; let tmpDir: string; @@ -100,22 +99,22 @@ describe('agent test run-eval command', () => { tmpDir = mkdtempSync(join(tmpdir(), 'run-eval-test-')); // Default stub implementations - isYamlTestSpecStub = sinon.stub().returns(false); - parseTestSpecStub = sinon.stub().returns({ + isYamlTestSpecStub = $$.SANDBOX.stub().returns(false); + parseTestSpecStub = $$.SANDBOX.stub().returns({ name: 'Weather_Test', subjectName: 'Local_Info_Agent', testCases: [{ utterance: 'What is the weather?' }], }); - translateTestSpecStub = sinon.stub().returns(JSON.parse(EVAL_PAYLOAD)); - normalizePayloadStub = sinon.stub().callsFake((p: unknown) => p); - splitIntoBatchesStub = sinon.stub().callsFake((tests: unknown[]) => [tests]); - resolveAgentStub = sinon.stub().resolves({ agentId: 'bot-001', versionId: 'ver-001' }); - executeBatchesStub = sinon.stub().resolves(MOCK_API_RESULTS); - buildResultSummaryStub = sinon.stub().returns({ + translateTestSpecStub = $$.SANDBOX.stub().returns(JSON.parse(EVAL_PAYLOAD)); + normalizePayloadStub = $$.SANDBOX.stub().callsFake((p: unknown) => p); + splitIntoBatchesStub = $$.SANDBOX.stub().callsFake((tests: unknown[]) => [tests]); + resolveAgentStub = $$.SANDBOX.stub().resolves({ agentId: 'bot-001', versionId: 'ver-001' }); + executeBatchesStub = $$.SANDBOX.stub().resolves(MOCK_API_RESULTS); + buildResultSummaryStub = $$.SANDBOX.stub().returns({ summary: { passed: 1, failed: 0, scored: 0, errors: 0 }, testSummaries: [{ id: 'test-topic-routing', status: 'passed', evaluations: [], outputs: [] }], }); - formatResultsStub = sinon.stub().returns('# Agent Evaluation Results'); + formatResultsStub = $$.SANDBOX.stub().returns('# Agent Evaluation Results'); const mod = await esmock('../../../../src/commands/agent/test/run-eval.js', { '@salesforce/agents': { @@ -135,7 +134,6 @@ describe('agent test run-eval command', () => { }); afterEach(() => { - sinon.restore(); $$.restore(); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -143,11 +141,8 @@ describe('agent test run-eval command', () => { // ─── State ───────────────────────────────────────────────────────────────── describe('command metadata', () => { - it('is marked as GA state', () => { - expect(AgentTestRunEval.state).to.equal('ga'); - }); - - it('is not hidden', () => { + it('is not in beta or hidden state', () => { + expect(AgentTestRunEval.state).to.not.equal('beta'); expect(AgentTestRunEval.hidden).to.not.equal(true); }); }); From 7f9c3533bbb497ff1847c048a47fb8641f0b2519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Rivas?= Date: Tue, 5 May 2026 17:14:08 -0300 Subject: [PATCH 27/38] fix(agent/test/run-eval): update command snapshot to reflect --wait removal Co-Authored-By: Claude Sonnet 4.6 --- command-snapshot.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/command-snapshot.json b/command-snapshot.json index f48911bf..9f84671b 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -228,7 +228,7 @@ "alias": [], "command": "agent:test:run-eval", "flagAliases": [], - "flagChars": ["n", "o", "s", "w"], + "flagChars": ["n", "o", "s"], "flags": [ "api-name", "api-version", @@ -238,8 +238,7 @@ "no-normalize", "result-format", "spec", - "target-org", - "wait" + "target-org" ], "plugin": "@salesforce/plugin-agent" }, From 8a5b531745f6494f53301e960b6570dc61844e0e Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Thu, 7 May 2026 15:28:35 +0000 Subject: [PATCH 28/38] chore(release): 1.36.0 [skip ci] --- CHANGELOG.md | 6 ++++++ README.md | 51 +++++++++++++++++++++++++++++++-------------------- package.json | 2 +- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee9d721..704ba27d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# [1.36.0](https://github.com/salesforcecli/plugin-agent/compare/1.35.0...1.36.0) (2026-05-07) + +### Features + +- add --verbose and --concise flags to agent publish authoring-bundle ([065ed7d](https://github.com/salesforcecli/plugin-agent/commit/065ed7d3d12f3e20bc79b7e8fc07ec637f1aace4)) + # [1.35.0](https://github.com/salesforcecli/plugin-agent/compare/1.34.1...1.35.0) (2026-05-04) ### Bug Fixes diff --git a/README.md b/README.md index 568177b2..6226fbb9 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ EXAMPLES $ sf agent activate --api-name Resort_Manager --version 2 --target-org my-org ``` -_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/activate.ts)_ +_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/activate.ts)_ ## `sf agent create` @@ -193,7 +193,7 @@ EXAMPLES $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview ``` -_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/create.ts)_ +_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/create.ts)_ ## `sf agent deactivate` @@ -234,7 +234,7 @@ EXAMPLES $ sf agent deactivate --api-name Resort_Manager --target-org my-org ``` -_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/deactivate.ts)_ +_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/deactivate.ts)_ ## `sf agent generate agent-spec` @@ -341,7 +341,7 @@ EXAMPLES $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com ``` -_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/agent-spec.ts)_ +_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/agent-spec.ts)_ ## `sf agent generate authoring-bundle` @@ -418,7 +418,7 @@ EXAMPLES other-package-dir/main/default --target-org my-dev-org ``` -_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/authoring-bundle.ts)_ +_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/authoring-bundle.ts)_ ## `sf agent generate template` @@ -480,7 +480,7 @@ EXAMPLES my-package --source-org my-scratch-org ``` -_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/template.ts)_ +_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/template.ts)_ ## `sf agent generate test-spec` @@ -545,7 +545,7 @@ EXAMPLES force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml ``` -_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/generate/test-spec.ts)_ +_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/test-spec.ts)_ ## `sf agent preview` @@ -618,7 +618,7 @@ EXAMPLES $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview ``` -_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview.ts)_ +_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview.ts)_ ## `sf agent preview end` @@ -673,7 +673,7 @@ EXAMPLES $ sf agent preview end --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/end.ts)_ +_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/end.ts)_ ## `sf agent preview send` @@ -731,7 +731,7 @@ EXAMPLES $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/send.ts)_ +_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/send.ts)_ ## `sf agent preview sessions` @@ -764,7 +764,7 @@ EXAMPLES $ sf agent preview sessions ``` -_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/sessions.ts)_ +_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/sessions.ts)_ ## `sf agent preview start` @@ -829,7 +829,7 @@ EXAMPLES $ sf agent preview start --api-name My_Published_Agent ``` -_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/preview/start.ts)_ +_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/start.ts)_ ## `sf agent publish authoring-bundle` @@ -838,14 +838,17 @@ Publish an authoring bundle to your org, which results in a new agent or a new v ``` USAGE $ sf agent publish authoring-bundle -o [--json] [--flags-dir ] [--api-version ] [-n ] - [--skip-retrieve] + [--skip-retrieve] [-v | --concise] FLAGS -n, --api-name= API name of the authoring bundle you want to publish; if not specified, the command provides a list that you can choose from. -o, --target-org= (required) Username or alias of the target org. Not required if the `target-org` configuration variable is already set. + -v, --verbose Display detailed output showing all metadata components retrieved and deployed during the + publish process. --api-version= Override the api version used for api requests made by this command + --concise Display minimal output with only essential information about the publish operation. --skip-retrieve Don't retrieve the metadata associated with the agent to your DX project. GLOBAL FLAGS @@ -876,9 +879,17 @@ EXAMPLES Publish an authoring bundle with API name MyAuthoringBundle to the org with alias "my-dev-org": $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --target-org my-dev-org + + Publish with verbose output to see all retrieved and deployed metadata components: + + $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --verbose + + Publish with concise output showing only essential information: + + $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --concise ``` -_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/publish/authoring-bundle.ts)_ +_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/publish/authoring-bundle.ts)_ ## `sf agent test create` @@ -933,7 +944,7 @@ EXAMPLES $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview ``` -_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/create.ts)_ +_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/test/create.ts)_ ## `sf agent test list` @@ -968,7 +979,7 @@ EXAMPLES $ sf agent test list --target-org my-org ``` -_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/list.ts)_ +_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/test/list.ts)_ ## `sf agent test results` @@ -1034,7 +1045,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/results.ts)_ +_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/test/results.ts)_ ## `sf agent test resume` @@ -1107,7 +1118,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/resume.ts)_ +_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/test/resume.ts)_ ## `sf agent test run` @@ -1181,7 +1192,7 @@ FLAG DESCRIPTIONS expression when using custom evaluations. ``` -_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/test/run.ts)_ +_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/test/run.ts)_ ## `sf agent validate authoring-bundle` @@ -1228,6 +1239,6 @@ EXAMPLES $ sf agent validate authoring-bundle --api-name MyAuthoringBundle --target-org my-dev-org ``` -_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.35.0/src/commands/agent/validate/authoring-bundle.ts)_ +_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/validate/authoring-bundle.ts)_ diff --git a/package.json b/package.json index ec3766b6..6dc9eb73 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@salesforce/plugin-agent", "description": "Commands to interact with Salesforce agents", - "version": "1.35.0", + "version": "1.36.0", "author": "Salesforce", "bugs": "https://github.com/forcedotcom/cli/issues", "enableO11y": true, From 9f32ecb00c0f85a37baf67c4d3900b2b48692858 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Thu, 7 May 2026 13:01:09 -0600 Subject: [PATCH 29/38] Merge pull request #407 from salesforcecli/wr/ngt Wr/ngt @W-22143479@ @W-22143478@ --- command-snapshot.json | 14 +- messages/agent.test.list.md | 2 +- messages/shared.md | 8 + package.json | 4 +- schemas/agent-activate.json | 7 +- schemas/agent-create.json | 47 +- schemas/agent-deactivate.json | 7 +- schemas/agent-generate-agent__spec.json | 22 +- schemas/agent-generate-authoring__bundle.json | 8 +- schemas/agent-generate-template.json | 7 +- schemas/agent-preview-end.json | 7 +- schemas/agent-preview-send.json | 8 +- schemas/agent-preview-sessions.json | 13 +- schemas/agent-preview-start.json | 7 +- schemas/agent-preview.json | 2 +- schemas/agent-test-create.json | 7 +- schemas/agent-test-list.json | 2 +- schemas/agent-test-results.json | 121 +++- schemas/agent-test-resume.json | 123 +++- schemas/agent-test-run.json | 123 +++- schemas/agent-test-run__eval.json | 21 +- schemas/agent-validate-authoring__bundle.json | 6 +- src/agentTestCache.ts | 9 +- src/commands/agent/test/list.ts | 8 +- src/commands/agent/test/results.ts | 16 +- src/commands/agent/test/resume.ts | 32 +- src/commands/agent/test/run.ts | 49 +- src/flags.ts | 40 +- src/handleTestResults.ts | 184 +++++- src/testRunnerFactory.ts | 43 ++ src/testStages.ts | 7 +- test/common.test.ts | 141 +++++ test/nuts/z4.agent.test.AFS.nut.ts | 256 ++++++++ test/testRunnerFactory.test.ts | 128 ++++ yarn.lock | 582 ++++++++++-------- 35 files changed, 1702 insertions(+), 359 deletions(-) create mode 100644 src/testRunnerFactory.ts create mode 100644 test/common.test.ts create mode 100644 test/nuts/z4.agent.test.AFS.nut.ts create mode 100644 test/testRunnerFactory.test.ts diff --git a/command-snapshot.json b/command-snapshot.json index f48911bf..ab354f76 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -184,7 +184,17 @@ "command": "agent:test:results", "flagAliases": [], "flagChars": ["d", "i", "o"], - "flags": ["api-version", "flags-dir", "job-id", "json", "output-dir", "result-format", "target-org", "verbose"], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "verbose" + ], "plugin": "@salesforce/plugin-agent" }, { @@ -200,6 +210,7 @@ "output-dir", "result-format", "target-org", + "test-runner", "use-most-recent", "verbose", "wait" @@ -219,6 +230,7 @@ "output-dir", "result-format", "target-org", + "test-runner", "verbose", "wait" ], diff --git a/messages/agent.test.list.md b/messages/agent.test.list.md index 3df6961e..6166ee28 100644 --- a/messages/agent.test.list.md +++ b/messages/agent.test.list.md @@ -4,7 +4,7 @@ List the available agent tests in your org. # description -The command outputs a table with the name (API name) of each test along with its unique ID and the date it was created in the org. +The command outputs a table with the name (API name) of each test along with its unique ID, type ('agentforce-studio' or 'testing-center'), and the date it was created in the org. # examples diff --git a/messages/shared.md b/messages/shared.md index e007fadc..173e30cf 100644 --- a/messages/shared.md +++ b/messages/shared.md @@ -20,6 +20,14 @@ When enabled, includes detailed generated data (such as invoked actions) in the The generated data is in JSON format and includes the Apex classes or Flows that were invoked, the Salesforce objects that were touched, and so on. Use the JSON structure of this information to build the test case JSONPath expression when using custom evaluations. +# flags.test-runner.summary + +Explicitly specify which test runner to use (agentforce-studio or testing-center). + +# flags.test-runner.description + +By default, the command automatically detects which test runner to use based on the test definition metadata type in your org. Use this flag to explicitly specify the runner type. 'agentforce-studio' uses AiTestingDefinition metadata. 'testing-center' uses AiEvaluationDefinition metadata. + # error.invalidAgentType agentType must be either "customer" or "internal". Found: [%s] diff --git a/package.json b/package.json index 6dc9eb73..ba41f9c0 100644 --- a/package.json +++ b/package.json @@ -12,11 +12,11 @@ "@inquirer/prompts": "^7.10.1", "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.8.36", - "@salesforce/agents": "^1.4.0", + "@salesforce/agents": "^1.5.1", "@salesforce/core": "^8.28.3", "@salesforce/kit": "^3.2.6", "@salesforce/sf-plugins-core": "^12.2.6", - "@salesforce/source-deploy-retrieve": "^12.32.8", + "@salesforce/source-deploy-retrieve": "^12.35.3", "@salesforce/types": "^1.7.1", "ansis": "^3.3.2", "fast-xml-parser": "^5.7.1", diff --git a/schemas/agent-activate.json b/schemas/agent-activate.json index 5827721b..0422b6e7 100644 --- a/schemas/agent-activate.json +++ b/schemas/agent-activate.json @@ -12,8 +12,11 @@ "type": "number" } }, - "required": ["success", "version"], + "required": [ + "success", + "version" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-create.json b/schemas/agent-create.json index a6172a99..e9232db8 100644 --- a/schemas/agent-create.json +++ b/schemas/agent-create.json @@ -28,7 +28,11 @@ "type": "string" } }, - "required": ["botId", "botVersionId", "plannerId"], + "required": [ + "botId", + "botVersionId", + "plannerId" + ], "additionalProperties": false, "description": "If the agent was created with saveAgent=true, these are the IDs that make up an agent; Bot, BotVersion, and GenAiPlanner metadata." }, @@ -78,7 +82,11 @@ "type": "string" } }, - "required": ["inputName", "inputDataType", "inputDescription"], + "required": [ + "inputName", + "inputDataType", + "inputDescription" + ], "additionalProperties": false }, "minItems": 1, @@ -99,14 +107,24 @@ "type": "string" } }, - "required": ["outputName", "outputDataType", "outputDescription"], + "required": [ + "outputName", + "outputDataType", + "outputDescription" + ], "additionalProperties": false }, "minItems": 1, "maxItems": 1 } }, - "required": ["actionName", "exampleOutput", "actionDescription", "inputs", "outputs"], + "required": [ + "actionName", + "exampleOutput", + "actionDescription", + "inputs", + "outputs" + ], "additionalProperties": false }, "minItems": 1, @@ -122,7 +140,13 @@ "type": "string" } }, - "required": ["scope", "topic", "actions", "instructions", "classificationDescription"], + "required": [ + "scope", + "topic", + "actions", + "instructions", + "classificationDescription" + ], "additionalProperties": false }, "minItems": 1, @@ -135,11 +159,18 @@ } } }, - "required": ["agentDescription", "topics", "sampleUtterances"], + "required": [ + "agentDescription", + "topics", + "sampleUtterances" + ], "additionalProperties": false } }, - "required": ["agentDefinition", "isSuccess"] + "required": [ + "agentDefinition", + "isSuccess" + ] } } -} +} \ No newline at end of file diff --git a/schemas/agent-deactivate.json b/schemas/agent-deactivate.json index 5827721b..0422b6e7 100644 --- a/schemas/agent-deactivate.json +++ b/schemas/agent-deactivate.json @@ -12,8 +12,11 @@ "type": "number" } }, - "required": ["success", "version"], + "required": [ + "success", + "version" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-generate-agent__spec.json b/schemas/agent-generate-agent__spec.json index d18950f0..d389f667 100644 --- a/schemas/agent-generate-agent__spec.json +++ b/schemas/agent-generate-agent__spec.json @@ -47,7 +47,14 @@ "type": "string" } }, - "required": ["agentType", "companyDescription", "companyName", "isSuccess", "role", "topics"] + "required": [ + "agentType", + "companyDescription", + "companyName", + "isSuccess", + "role", + "topics" + ] }, "DraftAgentTopics": { "type": "array", @@ -61,7 +68,10 @@ "type": "string" } }, - "required": ["name", "description"], + "required": [ + "name", + "description" + ], "additionalProperties": false }, "minItems": 1, @@ -69,7 +79,11 @@ }, "AgentType": { "type": "string", - "enum": ["customer", "internal", "AGENT"] + "enum": [ + "customer", + "internal", + "AGENT" + ] } } -} +} \ No newline at end of file diff --git a/schemas/agent-generate-authoring__bundle.json b/schemas/agent-generate-authoring__bundle.json index 60ba9c5e..58c56df2 100644 --- a/schemas/agent-generate-authoring__bundle.json +++ b/schemas/agent-generate-authoring__bundle.json @@ -15,8 +15,12 @@ "type": "string" } }, - "required": ["agentPath", "metaXmlPath", "outputDir"], + "required": [ + "agentPath", + "metaXmlPath", + "outputDir" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-generate-template.json b/schemas/agent-generate-template.json index a7ea28da..a847bcad 100644 --- a/schemas/agent-generate-template.json +++ b/schemas/agent-generate-template.json @@ -12,8 +12,11 @@ "type": "string" } }, - "required": ["genAiPlannerBundlePath", "botTemplatePath"], + "required": [ + "genAiPlannerBundlePath", + "botTemplatePath" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-preview-end.json b/schemas/agent-preview-end.json index 9ce8737d..84c0bc25 100644 --- a/schemas/agent-preview-end.json +++ b/schemas/agent-preview-end.json @@ -12,8 +12,11 @@ "type": "string" } }, - "required": ["sessionId", "tracesPath"], + "required": [ + "sessionId", + "tracesPath" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-preview-send.json b/schemas/agent-preview-send.json index a731d993..df699672 100644 --- a/schemas/agent-preview-send.json +++ b/schemas/agent-preview-send.json @@ -27,8 +27,12 @@ "type": "string" } }, - "required": ["messages", "agentApiName", "sessionId"], + "required": [ + "messages", + "agentApiName", + "sessionId" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-preview-sessions.json b/schemas/agent-preview-sessions.json index 168f2e48..b4d0babb 100644 --- a/schemas/agent-preview-sessions.json +++ b/schemas/agent-preview-sessions.json @@ -23,13 +23,20 @@ "$ref": "#/definitions/SessionType" } }, - "required": ["agentId", "sessionId"], + "required": [ + "agentId", + "sessionId" + ], "additionalProperties": false } }, "SessionType": { "type": "string", - "enum": ["simulated", "live", "published"] + "enum": [ + "simulated", + "live", + "published" + ] } } -} +} \ No newline at end of file diff --git a/schemas/agent-preview-start.json b/schemas/agent-preview-start.json index b57d7a7d..a0d49890 100644 --- a/schemas/agent-preview-start.json +++ b/schemas/agent-preview-start.json @@ -12,8 +12,11 @@ "type": "string" } }, - "required": ["sessionId", "agentApiName"], + "required": [ + "sessionId", + "agentApiName" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-preview.json b/schemas/agent-preview.json index 617d82f7..feb4e009 100644 --- a/schemas/agent-preview.json +++ b/schemas/agent-preview.json @@ -6,4 +6,4 @@ "type": "null" } } -} +} \ No newline at end of file diff --git a/schemas/agent-test-create.json b/schemas/agent-test-create.json index e9d1333b..67f7dbeb 100644 --- a/schemas/agent-test-create.json +++ b/schemas/agent-test-create.json @@ -12,8 +12,11 @@ "type": "string" } }, - "required": ["path", "contents"], + "required": [ + "path", + "contents" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-test-list.json b/schemas/agent-test-list.json index 08d26e14..900974a3 100644 --- a/schemas/agent-test-list.json +++ b/schemas/agent-test-list.json @@ -57,4 +57,4 @@ "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-test-results.json b/schemas/agent-test-results.json index 50332090..522e3175 100644 --- a/schemas/agent-test-results.json +++ b/schemas/agent-test-results.json @@ -3,7 +3,14 @@ "$ref": "#/definitions/AgentTestResultsResult", "definitions": { "AgentTestResultsResult": { - "$ref": "#/definitions/AgentTestResultsResponse" + "anyOf": [ + { + "$ref": "#/definitions/AgentTestResultsResponse" + }, + { + "$ref": "#/definitions/AgentforceStudioTestResultsResponse" + } + ] }, "AgentTestResultsResponse": { "type": "object", @@ -30,12 +37,23 @@ } } }, - "required": ["status", "startTime", "subjectName", "testCases"], + "required": [ + "status", + "startTime", + "subjectName", + "testCases" + ], "additionalProperties": false }, "TestStatus": { "type": "string", - "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR", "TERMINATED"] + "enum": [ + "NEW", + "IN_PROGRESS", + "COMPLETED", + "ERROR", + "TERMINATED" + ] }, "TestCaseResult": { "type": "object", @@ -56,7 +74,9 @@ "type": "string" } }, - "required": ["utterance"], + "required": [ + "utterance" + ], "additionalProperties": false }, "generatedData": { @@ -81,7 +101,13 @@ "type": "string" } }, - "required": ["actionsSequence", "invokedActions", "sessionId", "outcome", "topic"], + "required": [ + "actionsSequence", + "invokedActions", + "sessionId", + "outcome", + "topic" + ], "additionalProperties": false }, "testResults": { @@ -102,12 +128,22 @@ "type": "number" }, "result": { - "type": ["null", "string"], - "enum": [null, "PASS", "FAILURE"] + "type": [ + "null", + "string" + ], + "enum": [ + null, + "PASS", + "FAILURE" + ] }, "metricLabel": { "type": "string", - "enum": ["Accuracy", "Precision"] + "enum": [ + "Accuracy", + "Precision" + ] }, "metricExplainability": { "type": "string" @@ -146,8 +182,73 @@ "type": "number" } }, - "required": ["status", "startTime", "inputs", "generatedData", "testResults", "testNumber"], + "required": [ + "status", + "startTime", + "inputs", + "generatedData", + "testResults", + "testNumber" + ], + "additionalProperties": false + }, + "AgentforceStudioTestResultsResponse": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "testCases": { + "type": "array", + "items": { + "$ref": "#/definitions/AgentforceStudioTestCaseResult" + } + } + }, + "required": [ + "status", + "testCases" + ], + "additionalProperties": false + }, + "AgentforceStudioTestCaseResult": { + "type": "object", + "properties": { + "subjectResponse": { + "type": "string" + }, + "testNumber": { + "type": "number" + }, + "testScorerResults": { + "type": "array", + "items": { + "$ref": "#/definitions/TestScorerResult" + } + } + }, + "required": [ + "subjectResponse", + "testNumber", + "testScorerResults" + ], + "additionalProperties": false + }, + "TestScorerResult": { + "type": "object", + "properties": { + "scorerName": { + "type": "string" + }, + "scorerResponse": { + "type": "string" + } + }, + "required": [ + "scorerName", + "scorerResponse" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-test-resume.json b/schemas/agent-test-resume.json index 3b73ce7d..db4dd9d6 100644 --- a/schemas/agent-test-resume.json +++ b/schemas/agent-test-resume.json @@ -33,7 +33,10 @@ } } }, - "required": ["runId", "status"] + "required": [ + "runId", + "status" + ] }, { "type": "object", @@ -64,13 +67,49 @@ } } }, - "required": ["runId", "startTime", "status", "subjectName", "testCases"] + "required": [ + "runId", + "startTime", + "status", + "subjectName", + "testCases" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "type": "string", + "const": "COMPLETED" + }, + "runId": { + "type": "string" + }, + "testCases": { + "type": "array", + "items": { + "$ref": "#/definitions/AgentforceStudioTestCaseResult" + } + } + }, + "required": [ + "runId", + "status", + "testCases" + ] } ] }, "TestStatus": { "type": "string", - "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR", "TERMINATED"] + "enum": [ + "NEW", + "IN_PROGRESS", + "COMPLETED", + "ERROR", + "TERMINATED" + ] }, "TestCaseResult": { "type": "object", @@ -91,7 +130,9 @@ "type": "string" } }, - "required": ["utterance"], + "required": [ + "utterance" + ], "additionalProperties": false }, "generatedData": { @@ -116,7 +157,13 @@ "type": "string" } }, - "required": ["actionsSequence", "invokedActions", "sessionId", "outcome", "topic"], + "required": [ + "actionsSequence", + "invokedActions", + "sessionId", + "outcome", + "topic" + ], "additionalProperties": false }, "testResults": { @@ -137,12 +184,22 @@ "type": "number" }, "result": { - "type": ["null", "string"], - "enum": [null, "PASS", "FAILURE"] + "type": [ + "null", + "string" + ], + "enum": [ + null, + "PASS", + "FAILURE" + ] }, "metricLabel": { "type": "string", - "enum": ["Accuracy", "Precision"] + "enum": [ + "Accuracy", + "Precision" + ] }, "metricExplainability": { "type": "string" @@ -181,8 +238,54 @@ "type": "number" } }, - "required": ["status", "startTime", "inputs", "generatedData", "testResults", "testNumber"], + "required": [ + "status", + "startTime", + "inputs", + "generatedData", + "testResults", + "testNumber" + ], + "additionalProperties": false + }, + "AgentforceStudioTestCaseResult": { + "type": "object", + "properties": { + "subjectResponse": { + "type": "string" + }, + "testNumber": { + "type": "number" + }, + "testScorerResults": { + "type": "array", + "items": { + "$ref": "#/definitions/TestScorerResult" + } + } + }, + "required": [ + "subjectResponse", + "testNumber", + "testScorerResults" + ], + "additionalProperties": false + }, + "TestScorerResult": { + "type": "object", + "properties": { + "scorerName": { + "type": "string" + }, + "scorerResponse": { + "type": "string" + } + }, + "required": [ + "scorerName", + "scorerResponse" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-test-run.json b/schemas/agent-test-run.json index 3b73ce7d..db4dd9d6 100644 --- a/schemas/agent-test-run.json +++ b/schemas/agent-test-run.json @@ -33,7 +33,10 @@ } } }, - "required": ["runId", "status"] + "required": [ + "runId", + "status" + ] }, { "type": "object", @@ -64,13 +67,49 @@ } } }, - "required": ["runId", "startTime", "status", "subjectName", "testCases"] + "required": [ + "runId", + "startTime", + "status", + "subjectName", + "testCases" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "type": "string", + "const": "COMPLETED" + }, + "runId": { + "type": "string" + }, + "testCases": { + "type": "array", + "items": { + "$ref": "#/definitions/AgentforceStudioTestCaseResult" + } + } + }, + "required": [ + "runId", + "status", + "testCases" + ] } ] }, "TestStatus": { "type": "string", - "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR", "TERMINATED"] + "enum": [ + "NEW", + "IN_PROGRESS", + "COMPLETED", + "ERROR", + "TERMINATED" + ] }, "TestCaseResult": { "type": "object", @@ -91,7 +130,9 @@ "type": "string" } }, - "required": ["utterance"], + "required": [ + "utterance" + ], "additionalProperties": false }, "generatedData": { @@ -116,7 +157,13 @@ "type": "string" } }, - "required": ["actionsSequence", "invokedActions", "sessionId", "outcome", "topic"], + "required": [ + "actionsSequence", + "invokedActions", + "sessionId", + "outcome", + "topic" + ], "additionalProperties": false }, "testResults": { @@ -137,12 +184,22 @@ "type": "number" }, "result": { - "type": ["null", "string"], - "enum": [null, "PASS", "FAILURE"] + "type": [ + "null", + "string" + ], + "enum": [ + null, + "PASS", + "FAILURE" + ] }, "metricLabel": { "type": "string", - "enum": ["Accuracy", "Precision"] + "enum": [ + "Accuracy", + "Precision" + ] }, "metricExplainability": { "type": "string" @@ -181,8 +238,54 @@ "type": "number" } }, - "required": ["status", "startTime", "inputs", "generatedData", "testResults", "testNumber"], + "required": [ + "status", + "startTime", + "inputs", + "generatedData", + "testResults", + "testNumber" + ], + "additionalProperties": false + }, + "AgentforceStudioTestCaseResult": { + "type": "object", + "properties": { + "subjectResponse": { + "type": "string" + }, + "testNumber": { + "type": "number" + }, + "testScorerResults": { + "type": "array", + "items": { + "$ref": "#/definitions/TestScorerResult" + } + } + }, + "required": [ + "subjectResponse", + "testNumber", + "testScorerResults" + ], + "additionalProperties": false + }, + "TestScorerResult": { + "type": "object", + "properties": { + "scorerName": { + "type": "string" + }, + "scorerResponse": { + "type": "string" + } + }, + "required": [ + "scorerName", + "scorerResponse" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-test-run__eval.json b/schemas/agent-test-run__eval.json index f4094406..798f7f2c 100644 --- a/schemas/agent-test-run__eval.json +++ b/schemas/agent-test-run__eval.json @@ -25,7 +25,12 @@ "items": {} } }, - "required": ["id", "status", "evaluations", "outputs"], + "required": [ + "id", + "status", + "evaluations", + "outputs" + ], "additionalProperties": false } }, @@ -45,12 +50,20 @@ "type": "number" } }, - "required": ["passed", "failed", "scored", "errors"], + "required": [ + "passed", + "failed", + "scored", + "errors" + ], "additionalProperties": false } }, - "required": ["tests", "summary"], + "required": [ + "tests", + "summary" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/schemas/agent-validate-authoring__bundle.json b/schemas/agent-validate-authoring__bundle.json index 196ee052..57c40f53 100644 --- a/schemas/agent-validate-authoring__bundle.json +++ b/schemas/agent-validate-authoring__bundle.json @@ -15,8 +15,10 @@ } } }, - "required": ["success"], + "required": [ + "success" + ], "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/src/agentTestCache.ts b/src/agentTestCache.ts index bdb17c1d..62d7ca28 100644 --- a/src/agentTestCache.ts +++ b/src/agentTestCache.ts @@ -16,6 +16,7 @@ import { Global, SfError, TTLConfig } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; +import type { TestRunnerType } from '@salesforce/agents'; type ResultFormat = 'json' | 'human' | 'junit' | 'tap'; @@ -24,6 +25,7 @@ type CacheContents = { name: string; outputDir?: string; resultFormat?: ResultFormat; + runnerType?: TestRunnerType; }; export class AgentTestCache extends TTLConfig { @@ -45,11 +47,12 @@ export class AgentTestCache extends TTLConfig runId: string, name: string, outputDir?: string, - resultFormat?: ResultFormat + resultFormat?: ResultFormat, + runnerType?: TestRunnerType ): Promise { if (!runId) throw new SfError('runId is required to create a cache entry'); - this.set(runId, { runId, name, outputDir, resultFormat }); + this.set(runId, { runId, name, outputDir, resultFormat, runnerType }); await this.write(); } @@ -70,7 +73,7 @@ export class AgentTestCache extends TTLConfig public useIdOrMostRecent( runId: string | undefined, useMostRecent: boolean - ): { runId: string; name?: string; outputDir?: string; resultFormat?: ResultFormat } { + ): { runId: string; name?: string; outputDir?: string; resultFormat?: ResultFormat; runnerType?: TestRunnerType } { if (runId && useMostRecent) { throw new SfError('Cannot specify both a runId and use most recent flag'); } diff --git a/src/commands/agent/test/list.ts b/src/commands/agent/test/list.ts index b4382b7e..f52687f1 100644 --- a/src/commands/agent/test/list.ts +++ b/src/commands/agent/test/list.ts @@ -60,11 +60,17 @@ export default class AgentTestList extends SfCommand { ); } + const rows = results.map((r) => ({ + ...r, + type: r.type === 'AiEvaluationDefinition' ? 'testing-center' : 'agentforce-studio', + })); + this.table({ - data: results, + data: rows, columns: [ { key: 'fullName', name: 'API Name' }, { key: 'id', name: 'Id' }, + { key: 'type', name: 'Type' }, { key: 'createdDate', name: 'Created Date' }, ], sort: { fullName: 'asc' }, diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index 69cfe77e..0e74219f 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -16,14 +16,15 @@ import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; -import { AgentTester, AgentTestResultsResponse } from '@salesforce/agents'; -import { resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js'; +import { AgentTestResultsResponse, AgentforceStudioTestResultsResponse } from '@salesforce/agents'; +import { resultFormatFlag, testOutputDirFlag, testRunnerFlag, verboseFlag } from '../../../flags.js'; import { handleTestResults } from '../../../handleTestResults.js'; +import { createTestRunner } from '../../../testRunnerFactory.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.results'); -export type AgentTestResultsResult = AgentTestResultsResponse; +export type AgentTestResultsResult = AgentTestResultsResponse | AgentforceStudioTestResultsResponse; export default class AgentTestResults extends SfCommand { public static readonly summary = messages.getMessage('summary'); @@ -51,13 +52,20 @@ export default class AgentTestResults extends SfCommand }), 'result-format': resultFormatFlag(), 'output-dir': testOutputDirFlag(), + 'test-runner': testRunnerFlag, verbose: verboseFlag, }; public async run(): Promise { const { flags } = await this.parse(AgentTestResults); - const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); + const connection = flags['target-org'].getConnection(flags['api-version']); + const { runner: agentTester } = await createTestRunner( + connection, + flags['test-runner'], + undefined, + flags['job-id'] + ); let response; try { diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index bca01e77..8cdce823 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -16,12 +16,18 @@ import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; -import { AgentTester } from '@salesforce/agents'; import { CLIError } from '@oclif/core/errors'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; -import { AgentTestRunResult, resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js'; +import { + AgentTestRunResult, + resultFormatFlag, + testOutputDirFlag, + testRunnerFlag, + verboseFlag, +} from '../../../flags.js'; import { handleTestResults } from '../../../handleTestResults.js'; +import { createTestRunner } from '../../../testRunnerFactory.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.resume'); @@ -65,6 +71,7 @@ export default class AgentTestResume extends SfCommand { }), 'result-format': resultFormatFlag(), 'output-dir': testOutputDirFlag(), + 'test-runner': testRunnerFlag, verbose: verboseFlag, }; @@ -78,6 +85,7 @@ export default class AgentTestResume extends SfCommand { let runId; let outputDir; let resultFormat; + let cachedRunnerType; try { const cacheEntry = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); @@ -85,6 +93,7 @@ export default class AgentTestResume extends SfCommand { runId = cacheEntry.runId; outputDir = cacheEntry.outputDir; resultFormat = cacheEntry.resultFormat; + cachedRunnerType = cacheEntry.runnerType; } catch (e) { const wrapped = SfError.wrap(e); @@ -105,7 +114,15 @@ export default class AgentTestResume extends SfCommand { jsonEnabled: this.jsonEnabled(), }); this.mso.start({ id: runId }); - const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); + + // Use explicit flag > cached runner type > ID prefix detection > org metadata query + const connection = flags['target-org'].getConnection(flags['api-version']); + const { runner: agentTester } = await createTestRunner( + connection, + flags['test-runner'] ?? cachedRunnerType, + name, + runId + ); let completed; let response; @@ -139,12 +156,17 @@ export default class AgentTestResume extends SfCommand { // Set exit code to 1 only for execution errors (tests couldn't run properly) // Test assertion failures are business logic and should not affect exit code - if (response?.testCases.some((tc) => tc.status === 'ERROR')) { + // Only applicable to legacy responses (Agentforce Studio doesn't have test case status) + if ( + response && + 'subjectName' in response && + response.testCases.some((tc) => 'status' in tc && tc.status === 'ERROR') + ) { process.exitCode = 1; } // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - return { ...response!, runId, status: 'COMPLETED' }; + return { ...response!, runId, status: 'COMPLETED' } as AgentTestRunResult; } protected catch(error: Error | SfError | CLIError): Promise { diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 330223c6..75c90444 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -16,21 +16,24 @@ import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; -import { AgentTester, AgentTestStartResponse } from '@salesforce/agents'; +import { AgentTestStartResponse, AgentforceStudioTestStartResponse } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; import { CLIError } from '@oclif/core/errors'; import { AgentTestRunResult, FlaggablePrompt, makeFlags, - promptForAiEvaluationDefinitionApiName, + promptForTestDefinitionApiName, resultFormatFlag, testOutputDirFlag, + testRunnerFlag, verboseFlag, + type TestDefinitionSelection, } from '../../../flags.js'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; import { handleTestResults } from '../../../handleTestResults.js'; +import { createTestRunner } from '../../../testRunnerFactory.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); @@ -88,6 +91,7 @@ export default class AgentTestRun extends SfCommand { }), 'result-format': resultFormatFlag(), 'output-dir': testOutputDirFlag(), + 'test-runner': testRunnerFlag, verbose: verboseFlag, }; @@ -101,14 +105,26 @@ export default class AgentTestRun extends SfCommand { throw messages.createError('error.missingRequiredFlags', ['api-name']); } - const apiName = - flags['api-name'] ?? (await promptForAiEvaluationDefinitionApiName(FLAGGABLE_PROMPTS['api-name'], connection)); + let apiName: string; + let promptedTestRunner: TestDefinitionSelection['testRunner']; + if (flags['api-name']) { + apiName = flags['api-name']; + } else { + const selection = await promptForTestDefinitionApiName(FLAGGABLE_PROMPTS['api-name'], connection); + apiName = selection.apiName; + promptedTestRunner = selection.testRunner; + } this.mso = new TestStages({ title: `Agent Test Run: ${apiName}`, jsonEnabled: this.jsonEnabled() }); this.mso.start(); - const agentTester = new AgentTester(connection); - let response: AgentTestStartResponse; + const { runner: agentTester, type: runnerType } = await createTestRunner( + connection, + flags['test-runner'] ?? promptedTestRunner, + apiName + ); + + let response: AgentTestStartResponse | AgentforceStudioTestStartResponse; try { response = await agentTester.start(apiName); } catch (e) { @@ -117,7 +133,8 @@ export default class AgentTestRun extends SfCommand { // Check for test definition not found if ( wrapped.message.includes('Invalid AiEvalDefinitionVersion identifier') || - wrapped.message.toLowerCase().includes('not found') + wrapped.message.toLowerCase().includes('not found') || + wrapped.message.includes('Failed to run test suite') ) { throw new SfError( `Test definition '${apiName}' not found.`, @@ -135,7 +152,13 @@ export default class AgentTestRun extends SfCommand { this.mso.update({ id: response.runId }); const agentTestCache = await AgentTestCache.create(); - await agentTestCache.createCacheEntry(response.runId, apiName, flags['output-dir'], flags['result-format']); + await agentTestCache.createCacheEntry( + response.runId, + apiName, + flags['output-dir'], + flags['result-format'], + runnerType + ); if (flags.wait?.minutes) { let completed; @@ -170,12 +193,16 @@ export default class AgentTestRun extends SfCommand { // Set exit code to 1 only for execution errors (tests couldn't run properly) // Test assertion failures are business logic and should not affect exit code - if (detailsResponse?.testCases.some((tc) => tc.status === 'ERROR')) { + // Only applicable to legacy responses (Agentforce Studio doesn't have test case status) + if ( + detailsResponse && + 'subjectName' in detailsResponse && + detailsResponse.testCases.some((tc) => 'status' in tc && tc.status === 'ERROR') + ) { process.exitCode = 1; } - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - return { ...detailsResponse!, status: 'COMPLETED', runId: response.runId }; + return { ...detailsResponse, status: 'COMPLETED', runId: response.runId } as AgentTestRunResult; } else { this.mso.stop(); this.log( diff --git a/src/flags.ts b/src/flags.ts index bc627118..1b6a337c 100644 --- a/src/flags.ts +++ b/src/flags.ts @@ -68,6 +68,12 @@ export const verboseFlag = Flags.boolean({ description: messages.getMessage('flags.verbose.description'), }); +export const testRunnerFlag = Flags.custom<'agentforce-studio' | 'testing-center'>({ + options: ['agentforce-studio', 'testing-center'], + summary: messages.getMessage('flags.test-runner.summary'), + description: messages.getMessage('flags.test-runner.description'), +})(); + function validateInput(input: string, validate: (input: string) => boolean | string): never | string { const result = validate(input); if (typeof result === 'string') throw new Error(result); @@ -128,12 +134,33 @@ export function traverseForFiles(dirOrDirs: string | string[], suffixes: string[ return results; } -export const promptForAiEvaluationDefinitionApiName = async ( +export type TestDefinitionSelection = { + apiName: string; + testRunner?: 'agentforce-studio' | 'testing-center'; +}; + +export const promptForTestDefinitionApiName = async ( flagDef: FlaggablePrompt, connection: Connection -): Promise => { +): Promise => { const aiDefFiles = await AgentTest.list(connection); + const duplicateNames = new Set( + aiDefFiles + .filter((o, i) => aiDefFiles.some((other, j) => i !== j && o.fullName === other.fullName)) + .map((o) => o.fullName) + ); + + // Map each entry to a value that encodes the runner type for duplicates + const choicesByValue = new Map( + aiDefFiles.map((o) => { + const runner: 'agentforce-studio' | 'testing-center' = + o.type === 'AiEvaluationDefinition' ? 'testing-center' : 'agentforce-studio'; + const valueKey = duplicateNames.has(o.fullName) ? `${o.fullName}::${runner}` : o.fullName; + return [valueKey, { apiName: o.fullName, testRunner: duplicateNames.has(o.fullName) ? runner : undefined }]; + }) + ); + let id: NodeJS.Timeout; const timeout = new Promise((_, reject) => { id = setTimeout(() => { @@ -146,16 +173,19 @@ export const promptForAiEvaluationDefinitionApiName = async ( message: flagDef.promptMessage ?? flagDef.message, // eslint-disable-next-line @typescript-eslint/require-await source: async (input) => { - const arr = aiDefFiles.map((o) => ({ name: o.fullName, value: o.fullName })); + const arr = [...choicesByValue.entries()].map(([valueKey, { apiName, testRunner }]) => ({ + name: testRunner ? `${apiName} (${testRunner})` : apiName, + value: valueKey, + })); if (!input) return arr; return arr.filter((o) => o.name.includes(input)); }, }), timeout, - ]).then((result) => { + ]).then((valueKey) => { clearTimeout(id); - return result as string; + return choicesByValue.get(valueKey as string) ?? { apiName: valueKey as string }; }); }; diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index e859037c..2055a174 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -16,11 +16,24 @@ import { join } from 'node:path'; import { stripVTControlCharacters } from 'node:util'; import { writeFile, mkdir } from 'node:fs/promises'; -import { AgentTestResultsResponse, convertTestResultsToFormat, humanFriendlyName, metric } from '@salesforce/agents'; +import { + AgentTestResultsResponse, + AgentforceStudioTestResultsResponse, + convertTestResultsToFormat, + humanFriendlyName, + metric, +} from '@salesforce/agents'; +import { XMLBuilder } from 'fast-xml-parser'; import { Ux } from '@salesforce/sf-plugins-core/Ux'; import { ux as ocux } from '@oclif/core'; import ansis from 'ansis'; +type TestResultsResponse = AgentTestResultsResponse | AgentforceStudioTestResultsResponse; + +function isLegacyResponse(response: TestResultsResponse): response is AgentTestResultsResponse { + return 'subjectName' in response; +} + async function writeFileToDir(outputDir: string, fileName: string, content: string): Promise { // if directory doesn't exist, create it await mkdir(outputDir, { recursive: true }); @@ -55,11 +68,6 @@ export function readableTime(time: number, decimalPlaces = 2): string { return '< 1s'; } - // if time < 1000ms, return time in ms - if (time < 1000) { - return `${time}ms`; - } - // if time < 60s, return time in seconds if (time < 60_000) { return `${truncate(time / 1000, decimalPlaces)}s`; @@ -78,6 +86,148 @@ export function readableTime(time: number, decimalPlaces = 2): string { return `${hours}h ${minutes}m`; } +type ParsedScorerResponse = { + status?: string; + score?: number; + reasoning?: string; + actualValue?: string; + expectedValue?: string; +}; + +function parseScorerResponse(raw: string): ParsedScorerResponse { + try { + return JSON.parse(raw) as ParsedScorerResponse; + } catch { + return {}; + } +} + +function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { + const ux = new Ux(); + const tables: string[] = []; + + for (const testCase of results.testCases) { + let userInput = ''; + try { + const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; + userInput = parsed.userInput ?? ''; + } catch { + // ignore + } + + const scorerRows = testCase.testScorerResults.map((scorer) => { + const parsed = parseScorerResponse(scorer.scorerResponse); + return { + scorer: scorer.scorerName, + result: parsed.status === 'PASS' ? ansis.green('Pass') : ansis.red('Fail'), + expected: parsed.expectedValue ?? '', + actual: parsed.actualValue ?? '', + reasoning: parsed.reasoning ?? '', + }; + }); + + tables.push( + ux.makeTable({ + title: `${ansis.bold(`Test Case #${testCase.testNumber}`)}\n${ansis.dim('User Input')}: ${userInput}`, + overflow: 'wrap', + columns: [ + { key: 'scorer', name: 'Scorer' }, + { key: 'result', name: 'Result' }, + { key: 'expected', name: 'Expected', width: '25%' }, + { key: 'actual', name: 'Actual', width: '25%' }, + { key: 'reasoning', name: 'Reasoning', width: '35%' }, + ], + data: scorerRows, + width: '100%', + }) + ); + tables.push('\n'); + } + + const totalCases = results.testCases.length; + const passCases = results.testCases.filter((tc) => + tc.testScorerResults.every((s) => parseScorerResponse(s.scorerResponse).status === 'PASS') + ).length; + + const summary = makeSimpleTable( + { + Status: results.status, + 'Total Test Cases': String(totalCases), + 'Passing Test Cases': String(passCases), + 'Failing Test Cases': String(totalCases - passCases), + }, + ansis.bold.blue('Test Results') + ); + + return tables.join('') + `\n${summary}\n`; +} + +function junitFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { + const builder = new XMLBuilder({ format: true, attributeNamePrefix: '$', ignoreAttributes: false }); + const testCount = results.testCases.length; + const failureCount = results.testCases.filter((tc) => + tc.testScorerResults.some((s) => parseScorerResponse(s.scorerResponse).status !== 'PASS') + ).length; + + const suites = builder.build({ + testsuites: { + $name: 'AgentforceStudioTest', + $tests: testCount, + $failures: failureCount, + property: [{ $name: 'status', $value: results.status }], + testsuite: results.testCases.map((tc) => ({ + $name: tc.testNumber, + $assertions: tc.testScorerResults.length, + failure: tc.testScorerResults + .map((s) => { + const parsed = parseScorerResponse(s.scorerResponse); + if (parsed.status !== 'PASS') { + return { $message: parsed.reasoning ?? 'Unknown error', $name: s.scorerName }; + } + }) + .filter(Boolean), + })), + }, + }); + + return `\n${suites}`.trim(); +} + +function tapFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { + const lines: string[] = []; + let expectationCount = 0; + + for (const tc of results.testCases) { + for (const scorer of tc.testScorerResults) { + const parsed = parseScorerResponse(scorer.scorerResponse); + const pass = parsed.status === 'PASS'; + expectationCount++; + lines.push(`${pass ? 'ok' : 'not ok'} ${expectationCount} ${tc.testNumber}.${scorer.scorerName}`); + if (!pass) { + lines.push(' ---'); + lines.push(` message: ${parsed.reasoning ?? 'Unknown error'}`); + lines.push(` scorer: ${scorer.scorerName}`); + lines.push(` actual: ${parsed.actualValue ?? ''}`); + lines.push(` expected: ${parsed.expectedValue ?? ''}`); + lines.push(' ...'); + } + } + } + + return `TAP version 13\n1..${expectationCount}\n${lines.join('\n')}`; +} + +function convertAgentforceStudioTestResultsToFormat(results: AgentforceStudioTestResultsResponse, format: 'json' | 'junit' | 'tap'): string { + switch (format) { + case 'json': + return JSON.stringify(results, null, 2); + case 'junit': + return junitFormatAgentforceStudio(results); + case 'tap': + return tapFormatAgentforceStudio(results); + } +} + export function humanFormat(results: AgentTestResultsResponse, verbose = false): string { const ux = new Ux(); @@ -227,7 +377,7 @@ export async function handleTestResults({ }: { id: string; format: 'human' | 'json' | 'junit' | 'tap'; - results: AgentTestResultsResponse | undefined; + results: TestResultsResponse | undefined; jsonEnabled: boolean; outputDir?: string; verbose?: boolean; @@ -239,6 +389,26 @@ export async function handleTestResults({ const ux = new Ux({ jsonEnabled }); + if (!isLegacyResponse(results)) { + const ngtFormatConfig = { + human: { ext: 'txt', label: 'human-readable', get: () => humanFormatAgentforceStudio(results), strip: true }, + json: { ext: 'json', label: 'JSON', get: () => convertAgentforceStudioTestResultsToFormat(results, 'json'), strip: false }, + junit: { ext: 'xml', label: 'JUnit', get: () => convertAgentforceStudioTestResultsToFormat(results, 'junit'), strip: false }, + tap: { ext: 'txt', label: 'TAP', get: () => convertAgentforceStudioTestResultsToFormat(results, 'tap'), strip: false }, + } as const; + const cfg = ngtFormatConfig[format]; + const formatted = cfg.get(); + if (outputDir) { + const file = `test-result-${id}.${cfg.ext}`; + await writeFileToDir(outputDir, file, cfg.strip ? stripVTControlCharacters(formatted) : formatted); + ux.log(`Created ${cfg.label} file at ${join(outputDir, file)}`); + } else { + ux.log(formatted); + } + return; + } + + // Legacy response formatting if (format === 'human') { const formatted = humanFormat(results, verbose); if (outputDir) { diff --git a/src/testRunnerFactory.ts b/src/testRunnerFactory.ts new file mode 100644 index 00000000..28c5d127 --- /dev/null +++ b/src/testRunnerFactory.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Connection, SfError } from '@salesforce/core'; +import { createAgentTester, AgentTester, AgentforceStudioTester, type TestRunnerType } from '@salesforce/agents'; + +export type TestRunnerInstance = AgentTester | AgentforceStudioTester; + +export async function createTestRunner( + connection: Connection, + explicitType?: TestRunnerType, + testDefinitionName?: string, + runId?: string +): Promise<{ runner: TestRunnerInstance; type: TestRunnerType }> { + try { + return await createAgentTester(connection, { explicitType, runId, testDefinitionName }); + } catch (e) { + const wrapped = SfError.wrap(e); + if (wrapped.name === 'AmbiguousTestDefinition') { + throw new SfError( + wrapped.message, + wrapped.name, + ['Use --test-runner to explicitly specify the runner type (agentforce-studio or testing-center)'], + undefined, + wrapped + ); + } + throw wrapped; + } +} diff --git a/src/testStages.ts b/src/testStages.ts index 5eff2def..958e37cb 100644 --- a/src/testStages.ts +++ b/src/testStages.ts @@ -16,10 +16,11 @@ import { colorize } from '@oclif/core/ux'; import { MultiStageOutput } from '@oclif/multi-stage-output'; -import { AgentTestResultsResponse, AgentTester } from '@salesforce/agents'; +import { AgentTestResultsResponse, AgentforceStudioTestResultsResponse } from '@salesforce/agents'; import { Lifecycle } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; import { Ux } from '@salesforce/sf-plugins-core'; +import type { TestRunnerInstance } from './testRunnerFactory.js'; type Data = { id: string; @@ -86,10 +87,10 @@ export class TestStages { } public async poll( - agentTester: AgentTester, + agentTester: TestRunnerInstance, id: string, wait: Duration - ): Promise<{ completed: boolean; response?: AgentTestResultsResponse }> { + ): Promise<{ completed: boolean; response?: AgentTestResultsResponse | AgentforceStudioTestResultsResponse }> { this.mso.skipTo('Polling for Test Results'); const lifecycle = Lifecycle.getInstance(); lifecycle.on( diff --git a/test/common.test.ts b/test/common.test.ts new file mode 100644 index 00000000..25ec52bb --- /dev/null +++ b/test/common.test.ts @@ -0,0 +1,141 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { SfError } from '@salesforce/core'; +import type { CompilationError } from '@salesforce/agents'; +import { throwAgentCompilationError, COMPILATION_API_EXIT_CODES } from '../src/common.js'; + +describe('common', () => { + describe('COMPILATION_API_EXIT_CODES', () => { + it('should re-export COMPILATION_API_EXIT_CODES from @salesforce/agents', () => { + expect(COMPILATION_API_EXIT_CODES).to.be.an('object'); + }); + }); + + describe('throwAgentCompilationError', () => { + it('should throw SfError with unknown error message when given empty array', () => { + try { + throwAgentCompilationError([]); + expect.fail('Expected error to be thrown'); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + expect((e as SfError).name).to.equal('CompileAgentScriptError'); + expect((e as SfError).message).to.equal('Unknown compilation error occurred'); + expect((e as SfError).exitCode).to.equal(1); + } + }); + + it('should throw SfError with formatted error message for single error', () => { + const errors: CompilationError[] = [ + { + errorType: 'SyntaxError', + description: 'Unexpected token', + lineStart: 5, + colStart: 10, + lineEnd: 5, + colEnd: 15, + }, + ]; + + try { + throwAgentCompilationError(errors); + expect.fail('Expected error to be thrown'); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + expect((e as SfError).message).to.equal('SyntaxError: Unexpected token [Ln 5, Col 10]'); + expect((e as SfError).exitCode).to.equal(1); + } + }); + + it('should join multiple errors with EOL separator', () => { + const errors: CompilationError[] = [ + { + errorType: 'SyntaxError', + description: 'Unexpected token', + lineStart: 5, + colStart: 10, + lineEnd: 5, + colEnd: 15, + }, + { + errorType: 'TypeError', + description: 'Cannot read property', + lineStart: 12, + colStart: 3, + lineEnd: 12, + colEnd: 20, + }, + ]; + + try { + throwAgentCompilationError(errors); + expect.fail('Expected error to be thrown'); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + const msg = (e as SfError).message; + expect(msg).to.include('SyntaxError: Unexpected token [Ln 5, Col 10]'); + expect(msg).to.include('TypeError: Cannot read property [Ln 12, Col 3]'); + } + }); + + it('should always set exitCode to 1', () => { + const errors: CompilationError[] = [ + { errorType: 'AnyError', description: 'Something failed', lineStart: 1, colStart: 1, lineEnd: 1, colEnd: 5 }, + ]; + + try { + throwAgentCompilationError(errors); + expect.fail('Expected error to be thrown'); + } catch (e) { + expect((e as SfError).exitCode).to.equal(1); + } + }); + + it('should always set error name to CompileAgentScriptError', () => { + try { + throwAgentCompilationError([]); + expect.fail('Expected error to be thrown'); + } catch (e) { + expect((e as SfError).name).to.equal('CompileAgentScriptError'); + } + }); + + it('should include errors array in data for non-empty input', () => { + const errors: CompilationError[] = [ + { errorType: 'SyntaxError', description: 'Bad token', lineStart: 1, colStart: 1, lineEnd: 1, colEnd: 5 }, + ]; + + try { + throwAgentCompilationError(errors); + expect.fail('Expected error to be thrown'); + } catch (e) { + const sfErr = e as SfError; + expect(sfErr.data).to.deep.equal({ errors }); + } + }); + + it('should include empty array in data for empty input', () => { + try { + throwAgentCompilationError([]); + expect.fail('Expected error to be thrown'); + } catch (e) { + const sfErr = e as SfError; + expect(sfErr.data).to.deep.equal([]); + } + }); + }); +}); diff --git a/test/nuts/z4.agent.test.AFS.nut.ts b/test/nuts/z4.agent.test.AFS.nut.ts new file mode 100644 index 00000000..186f2f1d --- /dev/null +++ b/test/nuts/z4.agent.test.AFS.nut.ts @@ -0,0 +1,256 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { writeFileSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { expect } from 'chai'; +import { execCmd, Duration, TestSession } from '@salesforce/cli-plugins-testkit'; +import { ComponentSetBuilder } from '@salesforce/source-deploy-retrieve'; +import { Agent } from '@salesforce/agents'; +import { Org } from '@salesforce/core'; +import { AgentTestCache } from '../../src/agentTestCache.js'; +import type { AgentTestListResult } from '../../src/commands/agent/test/list.js'; +import type { AgentTestResultsResult } from '../../src/commands/agent/test/results.js'; +import type { AgentTestRunResult } from '../../src/flags.js'; +import { getTestSession, getUsername } from './shared-setup.js'; + +/* eslint-disable no-console */ + +// Agentforce Studio (AiTestingDefinition) NUTs. +// Depends on z2 having published a Test_Agent_* agent. The before() hook discovers that +// agent, writes an AiTestingDefinition with the correct subjectName, and deploys it. +describe('agent test (agentforce-studio)', function () { + this.timeout(30 * 60 * 1000); + + let session: TestSession; + let afsTestName: string; + + before(async function () { + this.timeout(30 * 60 * 1000); + session = await getTestSession(); + + const org = await Org.create({ aliasOrUsername: getUsername() }); + const connection = org.getConnection(); + + // Find the agent published in z2 + const publishedAgent = (await Agent.listRemote(connection)).find((a) => a.DeveloperName?.startsWith('Test_Agent_')); + if (!publishedAgent?.DeveloperName) { + throw new Error('No published Test_Agent_* found — ensure z2.agent.publish.nut runs first'); + } + + const agentName = publishedAgent.DeveloperName; + afsTestName = `${agentName}_AFS_Test`; + console.log(`Using agent '${agentName}', test definition '${afsTestName}'`); + + // Write AiTestingDefinition metadata file + const metaDir = join(session.project.dir, 'force-app', 'main', 'default', 'aiTestingDefinitions'); + mkdirSync(metaDir, { recursive: true }); + + const metaXml = ` + + AFS NUT test for ${agentName} + ${afsTestName} + ${agentName} + AGENT + v1 + + + Hi, can you tell me what your return or refund policy is? Please include citations in the answer, and use this citations URL: https://help.example.com/citations. + + 1 + + GeneralFAQ + topic_sequence_match + + + ['AnswerQuestionsWithKnowledge'] + action_sequence_match + + + I can help with that. Here's what I found in our knowledge base about the return/refund policy. + bot_response_rating + + + conciseness + + + coherence + + + output_latency_milliseconds + + + completeness + + + + + Hey, I need help with something important—can you take care of it for me? + + 2 + + ambiguous_question + topic_sequence_match + + + conciseness + + + coherence + + + output_latency_milliseconds + + + completeness + + + +`; + writeFileSync(join(metaDir, `${afsTestName}.aiTestingDefinition-meta.xml`), metaXml, 'utf8'); + console.log(`Wrote AiTestingDefinition metadata to ${metaDir}`); + + // Deploy the definition + const cs = await ComponentSetBuilder.build({ + sourcepath: [metaDir], + }); + const deploy = await cs.deploy({ usernameOrConnection: getUsername() }); + const deployResult = await deploy.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(10) }); + if (!deployResult.response.success) { + const failures = deployResult.response.details?.componentFailures; + const msgs = (Array.isArray(failures) ? failures : failures ? [failures] : []) + .map((f) => `${f.problemType}: ${f.problem}`) + .join('\n'); + throw new Error(`Deploy of AiTestingDefinition failed:\n${msgs}`); + } + console.log(`Deployed AiTestingDefinition '${afsTestName}'`); + }); + + // Set by the run test, consumed by the results tests (Mocha runs describes sequentially) + let completedRunId: string; + + describe('agent test list', () => { + it('should include the AFS test definition in list', async () => { + const result = execCmd(`agent test list --target-org ${getUsername()} --json`, { + ensureExitCode: 0, + }).jsonOutput?.result; + expect(result).to.be.ok; + const afsDefs = result?.filter((r) => r.type?.includes('AiTestingDefinition')); + expect(afsDefs?.length).to.be.greaterThanOrEqual(1); + expect(afsDefs?.some((r) => r.fullName === afsTestName)).to.be.true; + }); + }); + + describe('agent test run', () => { + it('should run with --wait, auto-detect agentforce-studio, and return AFS result shape', function () { + this.timeout(30 * 60 * 1000); + const output = execCmd( + `agent test run --api-name ${afsTestName} --target-org ${getUsername()} --wait 10 --json`, + { ensureExitCode: 0 } + ).jsonOutput; + + expect(output?.result.status).to.equal('COMPLETED'); + expect(output?.result.runId.startsWith('3A2')).to.be.true; + const result = output?.result as AgentTestRunResult & { testCases?: unknown[] }; + expect(result?.testCases).to.be.an('array'); + expect(result).to.not.have.property('subjectName'); + + completedRunId = output!.result.runId; + }); + }); + + describe('agent test results', () => { + it('should fetch AFS results by job ID (json)', async () => { + const output = execCmd( + `agent test results --job-id ${completedRunId} --target-org ${getUsername()} --json`, + { ensureExitCode: 0 } + ).jsonOutput; + + const result = output?.result as { status: string; testCases?: unknown[] }; + expect(result?.status).to.be.a('string'); + expect(result?.testCases).to.be.an('array'); + expect(output?.result).to.not.have.property('subjectName'); + }); + + it('should support human result format', () => { + const output = execCmd( + `agent test results --job-id ${completedRunId} --result-format human --target-org ${getUsername()}`, + { ensureExitCode: 0 } + ); + expect(output.shellOutput.stdout).to.be.a('string').with.length.greaterThan(0); + }); + + it('should support junit result format', () => { + const output = execCmd( + `agent test results --job-id ${completedRunId} --result-format junit --target-org ${getUsername()}`, + { ensureExitCode: 0 } + ); + expect(output.shellOutput.stdout).to.include(' { + const output = execCmd( + `agent test results --job-id ${completedRunId} --result-format tap --target-org ${getUsername()}`, + { ensureExitCode: 0 } + ); + expect(output.shellOutput.stdout).to.include('TAP version 13'); + }); + }); + + describe('agent test resume', () => { + it('should start async then resume by job ID, and support --use-most-recent', async () => { + // Clear any stale entries before the run + const cacheBefore = await AgentTestCache.create(); + cacheBefore.clear(); + await cacheBefore.write(); + + // One async start covers both resume paths + const runResult = execCmd( + `agent test run --api-name ${afsTestName} --target-org ${getUsername()} --json`, + { ensureExitCode: 0 } + ).jsonOutput; + + expect(runResult?.result.runId.startsWith('3A2')).to.be.true; + expect(runResult?.result.status).to.equal('NEW'); + + // Re-read from disk — the run command wrote the cache entry in a subprocess + const cache = await AgentTestCache.create(); + expect(cache.resolveFromCache().runnerType).to.equal('agentforce-studio'); + + const output = execCmd( + `agent test resume --job-id ${runResult?.result.runId} --target-org ${getUsername()} --json`, + { ensureExitCode: 0 } + ).jsonOutput; + + expect(output?.result.status).to.equal('COMPLETED'); + expect(output?.result.runId.startsWith('3A2')).to.be.true; + + // Re-read from disk — resume removes the entry in a subprocess + const cacheAfter = await AgentTestCache.create(); + expect(() => cacheAfter.resolveFromCache()).to.throw('Could not find a runId to resume'); + }); + }); + + describe('error handling', () => { + it('should return exit code 2 for a non-existent AFS test definition', () => { + execCmd( + `agent test run --api-name NonExistent_AFS_Test_XYZ --test-runner agentforce-studio --target-org ${getUsername()} --json`, + { ensureExitCode: 2 } + ); + }); + }); +}); diff --git a/test/testRunnerFactory.test.ts b/test/testRunnerFactory.test.ts new file mode 100644 index 00000000..1c1ebb96 --- /dev/null +++ b/test/testRunnerFactory.test.ts @@ -0,0 +1,128 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; +import { SfError } from '@salesforce/core'; +import type { Connection } from '@salesforce/core'; +import type { TestRunnerType } from '@salesforce/agents'; +import type { createTestRunner as CreateTestRunnerFn } from '../src/testRunnerFactory.js'; + +type MockConnection = Pick; +const makeMockConnection = (): MockConnection => ({ instanceUrl: 'https://test.salesforce.com' }); + +describe('testRunnerFactory', () => { + let createAgentTesterStub: sinon.SinonStub; + let createTestRunner: typeof CreateTestRunnerFn; + + beforeEach(async () => { + createAgentTesterStub = sinon.stub(); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + const { createTestRunner: fn } = await esmock('../src/testRunnerFactory.js', { + '@salesforce/agents': { + createAgentTester: createAgentTesterStub, + }, + }); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createTestRunner = fn as typeof CreateTestRunnerFn; + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('argument passthrough', () => { + it('passes explicitType, runId, and testDefinitionName to createAgentTester', async () => { + const mockResult = { runner: {}, type: 'agentforce-studio' as TestRunnerType }; + createAgentTesterStub.resolves(mockResult); + const connection = makeMockConnection() as Connection; + + await createTestRunner(connection, 'agentforce-studio', 'myTest', '3A2xxx'); + + expect( + createAgentTesterStub.calledOnceWith(connection, { + explicitType: 'agentforce-studio', + runId: '3A2xxx', + testDefinitionName: 'myTest', + }) + ).to.be.true; + }); + + it('passes undefined fields when not provided', async () => { + createAgentTesterStub.resolves({ runner: {}, type: 'testing-center' as TestRunnerType }); + const connection = makeMockConnection() as Connection; + + await createTestRunner(connection); + + expect( + createAgentTesterStub.calledOnceWith(connection, { + explicitType: undefined, + runId: undefined, + testDefinitionName: undefined, + }) + ).to.be.true; + }); + + it('returns the result from createAgentTester', async () => { + const mockRunner = { poll: sinon.stub() }; + const mockResult = { runner: mockRunner, type: 'agentforce-studio' as TestRunnerType }; + createAgentTesterStub.resolves(mockResult); + const connection = makeMockConnection() as Connection; + + const result = await createTestRunner(connection, 'agentforce-studio'); + + expect(result.runner).to.equal(mockRunner); + expect(result.type).to.equal('agentforce-studio'); + }); + }); + + describe('AmbiguousTestDefinition error handling', () => { + it('re-throws with --test-runner action hint', async () => { + const original = new SfError('MySuite exists in both metadata types', 'AmbiguousTestDefinition'); + createAgentTesterStub.rejects(original); + const connection = makeMockConnection() as Connection; + + try { + await createTestRunner(connection, undefined, 'MySuite'); + expect.fail('Expected error was not thrown'); + } catch (err) { + expect(err).to.be.instanceOf(SfError); + const sfErr = err as SfError; + expect(sfErr.name).to.equal('AmbiguousTestDefinition'); + expect(sfErr.actions).to.include( + 'Use --test-runner to explicitly specify the runner type (agentforce-studio or testing-center)' + ); + expect(sfErr.cause).to.equal(original); + } + }); + + it('passes through non-AmbiguousTestDefinition errors unchanged', async () => { + const original = new SfError('Network error', 'NetworkError'); + createAgentTesterStub.rejects(original); + const connection = makeMockConnection() as Connection; + + try { + await createTestRunner(connection, undefined, 'MySuite'); + expect.fail('Expected error was not thrown'); + } catch (err) { + expect(err).to.equal(original); + } + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index ebe6fc51..e4568dd1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -186,13 +186,13 @@ "@smithy/util-waiter" "^4.2.13" tslib "^2.6.2" -"@aws-sdk/core@^3.973.20", "@aws-sdk/core@^3.973.23", "@aws-sdk/core@^3.974.6": - version "3.974.6" - resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.974.6.tgz#c945af325d56b122cd75e21d3d0d9c759f803843" - integrity sha512-8Vu7zGxu+39ChR/s5J7nXBw3a2kMHAi0OfKT8ohgTVjX0qYed/8mIfdBb638oBmKrWCwwKjYAM5J/4gMJ8nAJA== +"@aws-sdk/core@^3.973.20", "@aws-sdk/core@^3.973.23", "@aws-sdk/core@^3.974.8": + version "3.974.8" + resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.974.8.tgz#cdd51195a31322f1e429e66919eb18da8944c081" + integrity sha512-njR2qoG6ZuB0kvAS2FyICsFZJ6gmCcf2X/7JcD14sUvGDm26wiZ5BrA6LOiUxKFEF+IVe7kdroxyE00YlkiYsw== dependencies: "@aws-sdk/types" "^3.973.8" - "@aws-sdk/xml-builder" "^3.972.20" + "@aws-sdk/xml-builder" "^3.972.22" "@smithy/core" "^3.23.17" "@smithy/node-config-provider" "^4.3.14" "@smithy/property-provider" "^4.2.14" @@ -202,7 +202,7 @@ "@smithy/types" "^4.14.1" "@smithy/util-base64" "^4.3.2" "@smithy/util-middleware" "^4.2.14" - "@smithy/util-retry" "^4.3.5" + "@smithy/util-retry" "^4.3.6" "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" @@ -214,23 +214,23 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/credential-provider-env@^3.972.32": - version "3.972.32" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.32.tgz#bb6421c71de2f5dab2a996c48ad39a3b646b1b28" - integrity sha512-7vA4GHg8NSmQxquJHSBcSM3RgB4ZaaRi6u4+zGFKOmOH6aqlgr2Sda46clkZDYzlirgfY96w15Zj0jh6PT48ng== +"@aws-sdk/credential-provider-env@^3.972.34": + version "3.972.34" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.34.tgz#9d420adf02e7604094a641ae613a353aa86e1b83" + integrity sha512-XT0jtf8Fw9JE6ppsQeoNnZRiG+jqRixMT1v1ZR17G60UvVdsQmTG8nbEyHuEPfMxDXEhfdARaM/XiEhca4lGHQ== dependencies: - "@aws-sdk/core" "^3.974.6" + "@aws-sdk/core" "^3.974.8" "@aws-sdk/types" "^3.973.8" "@smithy/property-provider" "^4.2.14" "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/credential-provider-http@^3.972.34": - version "3.972.34" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.34.tgz#f0f9eecf93371e42217ad4382904f2d0dd2c22cd" - integrity sha512-vBrhWujFCLp1u8ptJRWYlipMutzPptb8pDQ00rKVH9q67T7rGd3VTWIj63aKrlLuY6qSsw1Rt5F/D/7wnNgryA== +"@aws-sdk/credential-provider-http@^3.972.36": + version "3.972.36" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.36.tgz#842268559da2ffc5855cde1e90e7302d53639c08" + integrity sha512-DPoGWfy7J7RKxvbf5kOKIGQkD2ek3dbKgzKIGrnLuvZBz5myU+Im/H6pmc14QcnFbqHMqxvtWSgRDSJW3qXLQg== dependencies: - "@aws-sdk/core" "^3.974.6" + "@aws-sdk/core" "^3.974.8" "@aws-sdk/types" "^3.973.8" "@smithy/fetch-http-handler" "^5.3.17" "@smithy/node-http-handler" "^4.6.1" @@ -241,19 +241,19 @@ "@smithy/util-stream" "^4.5.25" tslib "^2.6.2" -"@aws-sdk/credential-provider-ini@^3.972.36": - version "3.972.36" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.36.tgz#d5ad2c627766990811967b82b996bfa29020fd2a" - integrity sha512-FBHyCmV8EB0gUvh1d+CZm87zt2PrdC7OyWexLRoH3I5zWSOUGa+9t58Y5jbxRfwUp3AWpHAFvKY6YzgR845sVA== - dependencies: - "@aws-sdk/core" "^3.974.6" - "@aws-sdk/credential-provider-env" "^3.972.32" - "@aws-sdk/credential-provider-http" "^3.972.34" - "@aws-sdk/credential-provider-login" "^3.972.36" - "@aws-sdk/credential-provider-process" "^3.972.32" - "@aws-sdk/credential-provider-sso" "^3.972.36" - "@aws-sdk/credential-provider-web-identity" "^3.972.36" - "@aws-sdk/nested-clients" "^3.997.4" +"@aws-sdk/credential-provider-ini@^3.972.38": + version "3.972.38" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.38.tgz#e20955fdfe4a88149b20dc7e25a517542e1dfd9f" + integrity sha512-oDzUBu2MGJFgoar05sPMCwSrhw44ASyccrHzj66vO69OZqi7I6hZZxXfuPLC8OCzW7C+sU+bI73XHij41yekgQ== + dependencies: + "@aws-sdk/core" "^3.974.8" + "@aws-sdk/credential-provider-env" "^3.972.34" + "@aws-sdk/credential-provider-http" "^3.972.36" + "@aws-sdk/credential-provider-login" "^3.972.38" + "@aws-sdk/credential-provider-process" "^3.972.34" + "@aws-sdk/credential-provider-sso" "^3.972.38" + "@aws-sdk/credential-provider-web-identity" "^3.972.38" + "@aws-sdk/nested-clients" "^3.997.6" "@aws-sdk/types" "^3.973.8" "@smithy/credential-provider-imds" "^4.2.14" "@smithy/property-provider" "^4.2.14" @@ -261,13 +261,13 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/credential-provider-login@^3.972.36": - version "3.972.36" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.36.tgz#21dacb3ac08b3c0da5fbc8b0b3b69bd574765fad" - integrity sha512-IFap01lJKxQc0C/OHmZwZQr/cKq0DhrcmKedRrdnnl42D+P0SImnnnWQjv07uIPqpEdtqmkPXb9TiPYTU+prxQ== +"@aws-sdk/credential-provider-login@^3.972.38": + version "3.972.38" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.38.tgz#278437712c02a3ad1785f70c93b4f591cb3f6491" + integrity sha512-g1NosS8qe4OF++G2UFCM5ovSkgipC7YYor5KCWatG0UoMSO5YFj9C8muePlyVmOBV/WTI16Jo3/s1NUo/o1Bww== dependencies: - "@aws-sdk/core" "^3.974.6" - "@aws-sdk/nested-clients" "^3.997.4" + "@aws-sdk/core" "^3.974.8" + "@aws-sdk/nested-clients" "^3.997.6" "@aws-sdk/types" "^3.973.8" "@smithy/property-provider" "^4.2.14" "@smithy/protocol-http" "^5.3.14" @@ -276,16 +276,16 @@ tslib "^2.6.2" "@aws-sdk/credential-provider-node@^3.972.21", "@aws-sdk/credential-provider-node@^3.972.24": - version "3.972.37" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.37.tgz#5545396a3248c74568ab49d2397fc790b1f70241" - integrity sha512-/WFixFAAiw8WpmjZcI0l4t3DerXLmVinOIfuotmRZnu2qmsFPoqqmstASz0z8bi1pGdFXzeLzf6bwucM3mZcUQ== - dependencies: - "@aws-sdk/credential-provider-env" "^3.972.32" - "@aws-sdk/credential-provider-http" "^3.972.34" - "@aws-sdk/credential-provider-ini" "^3.972.36" - "@aws-sdk/credential-provider-process" "^3.972.32" - "@aws-sdk/credential-provider-sso" "^3.972.36" - "@aws-sdk/credential-provider-web-identity" "^3.972.36" + version "3.972.39" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.39.tgz#71f87848b7615dda4f31a57b113be9666e4bbd1a" + integrity sha512-HEswDQyxUtadoZ/bJsPPENHg7R0Lzym5LuMksJeHvqhCOpP+rtkDLKI4/ZChH4w3cf5kG8n6bZuI8PzajoiqMg== + dependencies: + "@aws-sdk/credential-provider-env" "^3.972.34" + "@aws-sdk/credential-provider-http" "^3.972.36" + "@aws-sdk/credential-provider-ini" "^3.972.38" + "@aws-sdk/credential-provider-process" "^3.972.34" + "@aws-sdk/credential-provider-sso" "^3.972.38" + "@aws-sdk/credential-provider-web-identity" "^3.972.38" "@aws-sdk/types" "^3.973.8" "@smithy/credential-provider-imds" "^4.2.14" "@smithy/property-provider" "^4.2.14" @@ -293,39 +293,39 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/credential-provider-process@^3.972.32": - version "3.972.32" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.32.tgz#c2f5a401859637c64126826d595bb523ac1acb7d" - integrity sha512-uZp4tlGbpczV8QxmtIwOpSkcyGtBRR8/T4BAumRKfAt1nwCig3FSCZvrKl6ARDIDVRYn5p2oRcAsfFR01EgMGA== +"@aws-sdk/credential-provider-process@^3.972.34": + version "3.972.34" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.34.tgz#c964275be1a528ac73ade6d98c309fb6b7cdfb68" + integrity sha512-T3IFs4EVmVi1dVN5RciFnklCANSzvrQd/VuHY9ThHSQmYkTogjcGkoJEr+oNUPQZnso52183088NqysMPji1/Q== dependencies: - "@aws-sdk/core" "^3.974.6" + "@aws-sdk/core" "^3.974.8" "@aws-sdk/types" "^3.973.8" "@smithy/property-provider" "^4.2.14" "@smithy/shared-ini-file-loader" "^4.4.9" "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/credential-provider-sso@^3.972.36": - version "3.972.36" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.36.tgz#210e14d5e53f7398ae6ee0833f85dd5a0219372f" - integrity sha512-DsLr0UHMyKzRJKe2bjlwU8q1cfoXg8TIJKV/xwvnalAemiZLOZunFzj/whGnFDZIBVLdnbLiwv5SvRf1+CSwkg== +"@aws-sdk/credential-provider-sso@^3.972.38": + version "3.972.38" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.38.tgz#ec754bfecb2426a3307e19ef7e6c6b6438a327c6" + integrity sha512-5ZxG+t0+3Q3QPh8KEjX6syskhgNf7I0MN7oGioTf6Lm1NTjfP7sIcYGNsthXC2qR8vcD3edNZwCr2ovfSSWuRA== dependencies: - "@aws-sdk/core" "^3.974.6" - "@aws-sdk/nested-clients" "^3.997.4" - "@aws-sdk/token-providers" "3.1038.0" + "@aws-sdk/core" "^3.974.8" + "@aws-sdk/nested-clients" "^3.997.6" + "@aws-sdk/token-providers" "3.1041.0" "@aws-sdk/types" "^3.973.8" "@smithy/property-provider" "^4.2.14" "@smithy/shared-ini-file-loader" "^4.4.9" "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/credential-provider-web-identity@^3.972.36": - version "3.972.36" - resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.36.tgz#40005a41380668bbbc4c9fe6d73dd13cac64e6e0" - integrity sha512-uzrURO7frJhHQVVNR5zBJcCYeMYflmXcWBK1+MiBym2Dfjh6nXATrMixrmGZi+97Q7ETZ+y/4lUwAy0Nfnznjw== +"@aws-sdk/credential-provider-web-identity@^3.972.38": + version "3.972.38" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.38.tgz#149951ef6e12db5292118e8ed5d95133c24ad719" + integrity sha512-lYHFF30DGI20jZcYX8cm6Ns0V7f1dDN6g/MBDLTyD/5iw+bXs3yBr2iAiHDkx4RFU5JgsnZvCHYKiRVPRdmOgw== dependencies: - "@aws-sdk/core" "^3.974.6" - "@aws-sdk/nested-clients" "^3.997.4" + "@aws-sdk/core" "^3.974.8" + "@aws-sdk/nested-clients" "^3.997.6" "@aws-sdk/types" "^3.973.8" "@smithy/property-provider" "^4.2.14" "@smithy/shared-ini-file-loader" "^4.4.9" @@ -356,14 +356,14 @@ tslib "^2.6.2" "@aws-sdk/middleware-flexible-checksums@^3.974.3": - version "3.974.14" - resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.14.tgz#b8bb019df5ade60293d1c20a905cca86dd999b28" - integrity sha512-mhTO3amGzYv/DQNbbqZo6UkHquBHlEEVRZwXmjeRqLmy1l9z3xCiFzglPL7n9JpVc2DZc9kjaraAn3JQrueZbw== + version "3.974.16" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.16.tgz#89b78cb0ad389aba7d12d060f46017e1fa3784a9" + integrity sha512-6ru8doI0/XzszqLIPXf0E/V7HhAw1Pu94010XCKYtBUfD0LxF0BuOzrUf8OQGR6j2o6wgKTHUniOmndQycHwCA== dependencies: "@aws-crypto/crc32" "5.2.0" "@aws-crypto/crc32c" "5.2.0" "@aws-crypto/util" "5.2.0" - "@aws-sdk/core" "^3.974.6" + "@aws-sdk/core" "^3.974.8" "@aws-sdk/crc64-nvme" "^3.972.7" "@aws-sdk/types" "^3.973.8" "@smithy/is-array-buffer" "^4.2.2" @@ -414,12 +414,12 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/middleware-sdk-s3@^3.972.23", "@aws-sdk/middleware-sdk-s3@^3.972.35": - version "3.972.35" - resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.35.tgz#8cbf56d3c31a0ce31a09d1fd56a266b75cf1d28a" - integrity sha512-lLppaNTAz+wNgLdi4FtHzrlwrGF0ODTnBWHBaFg85SKs0eJ+M+tP5ifrA8f/0lNd+Ak3MC1NGC6RavV3ny4HTg== +"@aws-sdk/middleware-sdk-s3@^3.972.23", "@aws-sdk/middleware-sdk-s3@^3.972.37": + version "3.972.37" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.37.tgz#82ef4953cddd3373d2942d07a5d2baf443bbf3ea" + integrity sha512-Km7M+i8DrLArVzrid1gfxeGhYHBd3uxvE77g0s5a52zPSVosxzQBnJ0gwWb6NIp/DOk8gsBMhi7V+cpJG0ndTA== dependencies: - "@aws-sdk/core" "^3.974.6" + "@aws-sdk/core" "^3.974.8" "@aws-sdk/types" "^3.973.8" "@aws-sdk/util-arn-parser" "^3.972.3" "@smithy/core" "^3.23.17" @@ -443,38 +443,38 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/middleware-user-agent@^3.972.21", "@aws-sdk/middleware-user-agent@^3.972.24", "@aws-sdk/middleware-user-agent@^3.972.36": - version "3.972.36" - resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.36.tgz#76fd80d86940c5daf7bf91ebf95966a19c73d870" - integrity sha512-O2beToxguBvrZFFZ+fFgPbbae8MvyIBjQ6lImee4APHEXXNAD5ZJ2ayLF1mb7rsKw86TM81y5czg82bZncjSjg== +"@aws-sdk/middleware-user-agent@^3.972.21", "@aws-sdk/middleware-user-agent@^3.972.24", "@aws-sdk/middleware-user-agent@^3.972.38": + version "3.972.38" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.38.tgz#626d9a2499f5a6398a4db917abeeaac14b54c6cb" + integrity sha512-iz+B29TXcAZsJpwB+AwG/TTGA5l/VnmMZ2UxtiySOZjI6gCdmviXPwdgzcmuazMy16rXoPY4mYCGe7zdNKfx5A== dependencies: - "@aws-sdk/core" "^3.974.6" + "@aws-sdk/core" "^3.974.8" "@aws-sdk/types" "^3.973.8" "@aws-sdk/util-endpoints" "^3.996.8" "@smithy/core" "^3.23.17" "@smithy/protocol-http" "^5.3.14" "@smithy/types" "^4.14.1" - "@smithy/util-retry" "^4.3.5" + "@smithy/util-retry" "^4.3.6" tslib "^2.6.2" -"@aws-sdk/nested-clients@^3.997.4": - version "3.997.4" - resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.4.tgz#674c14256b870fab0a1c0a0567be8e1c9026fdee" - integrity sha512-4Sf+WY1lMJzXlw5MiyCMe/UzdILCwvuaHThbqMXS6dfh9gZy3No360I42RXquOI/ULUOhWy2HCyU0Fp20fQGPQ== +"@aws-sdk/nested-clients@^3.997.6": + version "3.997.6" + resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.6.tgz#17433cfac2160ec620a14cbff9d2b33675712cae" + integrity sha512-WBDnqatJl+kGObpfmfSxqnXeYTu3Me8wx8WCtvoxX3pfWrrTv8I4WTMSSs7PZqcRcVh8WeUKMgGFjMG+52SR1w== dependencies: "@aws-crypto/sha256-browser" "5.2.0" "@aws-crypto/sha256-js" "5.2.0" - "@aws-sdk/core" "^3.974.6" + "@aws-sdk/core" "^3.974.8" "@aws-sdk/middleware-host-header" "^3.972.10" "@aws-sdk/middleware-logger" "^3.972.10" "@aws-sdk/middleware-recursion-detection" "^3.972.11" - "@aws-sdk/middleware-user-agent" "^3.972.36" + "@aws-sdk/middleware-user-agent" "^3.972.38" "@aws-sdk/region-config-resolver" "^3.972.13" - "@aws-sdk/signature-v4-multi-region" "^3.996.23" + "@aws-sdk/signature-v4-multi-region" "^3.996.25" "@aws-sdk/types" "^3.973.8" "@aws-sdk/util-endpoints" "^3.996.8" "@aws-sdk/util-user-agent-browser" "^3.972.10" - "@aws-sdk/util-user-agent-node" "^3.973.22" + "@aws-sdk/util-user-agent-node" "^3.973.24" "@smithy/config-resolver" "^4.4.17" "@smithy/core" "^3.23.17" "@smithy/fetch-http-handler" "^5.3.17" @@ -482,7 +482,7 @@ "@smithy/invalid-dependency" "^4.2.14" "@smithy/middleware-content-length" "^4.2.14" "@smithy/middleware-endpoint" "^4.4.32" - "@smithy/middleware-retry" "^4.5.6" + "@smithy/middleware-retry" "^4.5.7" "@smithy/middleware-serde" "^4.2.20" "@smithy/middleware-stack" "^4.2.14" "@smithy/node-config-provider" "^4.3.14" @@ -498,7 +498,7 @@ "@smithy/util-defaults-mode-node" "^4.2.54" "@smithy/util-endpoints" "^3.4.2" "@smithy/util-middleware" "^4.2.14" - "@smithy/util-retry" "^4.3.5" + "@smithy/util-retry" "^4.3.6" "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" @@ -513,25 +513,25 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/signature-v4-multi-region@^3.996.11", "@aws-sdk/signature-v4-multi-region@^3.996.23": - version "3.996.23" - resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.23.tgz#a275afc0d1727c4fe9cd67c965071a27c5f0b87c" - integrity sha512-wBbys3Y53Ikly556vyADurKpYQHXS7Jjaskbz+Ga9PZCz7PB/9f3VdKbDlz7dqIzn+xwz7L/a6TR4iXcOi8IRw== +"@aws-sdk/signature-v4-multi-region@^3.996.11", "@aws-sdk/signature-v4-multi-region@^3.996.25": + version "3.996.25" + resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.25.tgz#b50651b7e4f9c82482416caa9953ad17645d4a2d" + integrity sha512-+CMIt3e1VzlklAECmG+DtP1sV8iKq25FuA0OKpnJ4KA0kxUtd7CgClY7/RU6VzJBQwbN4EJ9Ue6plvqx1qGadw== dependencies: - "@aws-sdk/middleware-sdk-s3" "^3.972.35" + "@aws-sdk/middleware-sdk-s3" "^3.972.37" "@aws-sdk/types" "^3.973.8" "@smithy/protocol-http" "^5.3.14" "@smithy/signature-v4" "^5.3.14" "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@aws-sdk/token-providers@3.1038.0": - version "3.1038.0" - resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1038.0.tgz#9f0583bb79bc798f8f33876ff6fce3072dfca760" - integrity sha512-Qniru+9oGGb/HNK/gGZWbV3jsD0k71ngE7qMQ/x6gYNYLd2EOwHCS6E2E6jfkaqO4i0d+nNKmfRy8bNcshKdGQ== +"@aws-sdk/token-providers@3.1041.0": + version "3.1041.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1041.0.tgz#f3f068010780fc85fc4a7faa6a080cfb8afd73a4" + integrity sha512-Th7kPI6YPtvJUcdznooXJMy+9rQWjmEF81LxaJssngBzuysK4a/x+l8kjm1zb7nYsUPbndnBdUnwng/3PLvtGw== dependencies: - "@aws-sdk/core" "^3.974.6" - "@aws-sdk/nested-clients" "^3.997.4" + "@aws-sdk/core" "^3.974.8" + "@aws-sdk/nested-clients" "^3.997.6" "@aws-sdk/types" "^3.973.8" "@smithy/property-provider" "^4.2.14" "@smithy/shared-ini-file-loader" "^4.4.9" @@ -581,22 +581,22 @@ bowser "^2.11.0" tslib "^2.6.2" -"@aws-sdk/util-user-agent-node@^3.973.10", "@aws-sdk/util-user-agent-node@^3.973.22", "@aws-sdk/util-user-agent-node@^3.973.7": - version "3.973.22" - resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.22.tgz#efbbdb819cb58ac23e5e6787c437242f9931a565" - integrity sha512-YTYqTmOUrwbm1h99Ee4y/mVYpFRl0oSO/amtP5cc1BZZWdaAVWs9zj3TkyRHWvR9aI/ZS8m3mS6awXtYUlWyaw== +"@aws-sdk/util-user-agent-node@^3.973.10", "@aws-sdk/util-user-agent-node@^3.973.24", "@aws-sdk/util-user-agent-node@^3.973.7": + version "3.973.24" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.24.tgz#cf44a63b92adfecaeb8cb9f948b390456310566a" + integrity sha512-ZWwlkjcIp7cEL8ZfTpTAPNkwx25p7xol0xlKoWVVf22+nsjwmLcHYtTPjIV1cSpmB/b6DaK4cb1fSkvCXHgRdw== dependencies: - "@aws-sdk/middleware-user-agent" "^3.972.36" + "@aws-sdk/middleware-user-agent" "^3.972.38" "@aws-sdk/types" "^3.973.8" "@smithy/node-config-provider" "^4.3.14" "@smithy/types" "^4.14.1" "@smithy/util-config-provider" "^4.2.2" tslib "^2.6.2" -"@aws-sdk/xml-builder@^3.972.20": - version "3.972.20" - resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.972.20.tgz#d5f41188072ff6ae9e3e794a3ec62f3c82b9ff28" - integrity sha512-MDcUfroaMAnDAHn29vN781t0wudR8zjfgg+r3s5otx8TJXFWg01NZB7HvHkBbOf7UUmKEwIZf5kHxiaVUgwjlQ== +"@aws-sdk/xml-builder@^3.972.22": + version "3.972.22" + resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.972.22.tgz#1e44ca9fd9c3fdc3d9af9540ced024f34cfc60b2" + integrity sha512-PMYKKtJd70IsSG0yHrdAbxBr+ZWBKLvzFZfD3/urxgf6hXVMzuU5M+3MJ5G67RpOmLBu1fAUN65SbWuKUCOlAA== dependencies: "@nodable/entities" "2.1.0" "@smithy/types" "^4.14.1" @@ -618,9 +618,9 @@ picocolors "^1.1.1" "@babel/compat-data@^7.28.6": - version "7.29.0" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.29.0.tgz#00d03e8c0ac24dd9be942c5370990cbe1f17d88d" - integrity sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg== + version "7.29.3" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.29.3.tgz#e3f5347f0589596c91d227ccb6a541d37fb1307b" + integrity sha512-LIVqM46zQWZhj17qA8wb4nW/ixr2y1Nw+r1etiAWgRM6U1IqP+LNhL1yg440jYZR72jCWcWbLWzIosH+uP1fqg== "@babel/core@^7.23.9": version "7.29.0" @@ -711,9 +711,9 @@ "@babel/types" "^7.29.0" "@babel/parser@^7.23.9", "@babel/parser@^7.28.6", "@babel/parser@^7.29.0": - version "7.29.2" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.29.2.tgz#58bd50b9a7951d134988a1ae177a35ef9a703ba1" - integrity sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA== + version "7.29.3" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.29.3.tgz#116f70a77958307fceac27747573032f8a62f88e" + integrity sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA== dependencies: "@babel/types" "^7.29.0" @@ -1042,6 +1042,11 @@ resolved "https://registry.yarnpkg.com/@inquirer/ansi/-/ansi-1.0.2.tgz#674a4c4d81ad460695cb2a1fc69d78cd187f337e" integrity sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ== +"@inquirer/ansi@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@inquirer/ansi/-/ansi-2.0.5.tgz#7b7e121f6a0c40128711daf20325e6ff2cdff8b7" + integrity sha512-doc2sWgJpbFQ64UflSVd17ibMGDuxO1yKgOgLMwavzESnXjFWJqUeG8saYosqKpHp4kWiM5x1nXvEjbpx90gzw== + "@inquirer/checkbox@^4.3.2": version "4.3.2" resolved "https://registry.yarnpkg.com/@inquirer/checkbox/-/checkbox-4.3.2.tgz#e1483e6519d6ffef97281a54d2a5baa0d81b3f3b" @@ -1053,7 +1058,7 @@ "@inquirer/type" "^3.0.10" yoctocolors-cjs "^2.1.3" -"@inquirer/confirm@^3.1.22", "@inquirer/confirm@^3.2.0": +"@inquirer/confirm@^3.1.22": version "3.2.0" resolved "https://registry.yarnpkg.com/@inquirer/confirm/-/confirm-3.2.0.tgz#6af1284670ea7c7d95e3f1253684cfbd7228ad6a" integrity sha512-oOIwPs0Dvq5220Z8lGL/6LHRTEr9TgLHmiI99Rj1PJ1p1czTys+olrgBqZk4E2qC0YTzeHprxSQmoHioVdJ7Lw== @@ -1069,6 +1074,14 @@ "@inquirer/core" "^10.3.2" "@inquirer/type" "^3.0.10" +"@inquirer/confirm@^6.0.12": + version "6.0.12" + resolved "https://registry.yarnpkg.com/@inquirer/confirm/-/confirm-6.0.12.tgz#7a317aec813214cec2f5339b9fa0926c20bf0dbe" + integrity sha512-h9FgGun3QwVYNj5TWIZZ+slii73bMoBFjPfVIGtnFuL4t8gBiNDV9PcSfIzkuxvgquJKt9nr1QzszpBzTbH8Og== + dependencies: + "@inquirer/core" "^11.1.9" + "@inquirer/type" "^4.0.5" + "@inquirer/core@^10.3.2": version "10.3.2" resolved "https://registry.yarnpkg.com/@inquirer/core/-/core-10.3.2.tgz#535979ff3ff4fe1e7cc4f83e2320504c743b7e20" @@ -1083,6 +1096,19 @@ wrap-ansi "^6.2.0" yoctocolors-cjs "^2.1.3" +"@inquirer/core@^11.1.9": + version "11.1.9" + resolved "https://registry.yarnpkg.com/@inquirer/core/-/core-11.1.9.tgz#97f099f5217f50f168c12db00ac07f51ab550fbb" + integrity sha512-BDE4fG22uYh1bGSifcj7JSx119TVYNViMhMu85usp4Fswrzh6M0DV3yld64jA98uOAa2GSQ4Bg4bZRm2d2cwSg== + dependencies: + "@inquirer/ansi" "^2.0.5" + "@inquirer/figures" "^2.0.5" + "@inquirer/type" "^4.0.5" + cli-width "^4.1.0" + fast-wrap-ansi "^0.2.0" + mute-stream "^3.0.0" + signal-exit "^4.1.0" + "@inquirer/core@^3.1.1": version "3.1.2" resolved "https://registry.yarnpkg.com/@inquirer/core/-/core-3.1.2.tgz#d9691e6ffae85935685641550b8370ab7e599caa" @@ -1151,6 +1177,11 @@ resolved "https://registry.yarnpkg.com/@inquirer/figures/-/figures-1.0.15.tgz#dbb49ed80df11df74268023b496ac5d9acd22b3a" integrity sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g== +"@inquirer/figures@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@inquirer/figures/-/figures-2.0.5.tgz#d12f4889ac6ea7731ebc52bd9c066ca51d8fdee7" + integrity sha512-NsSs4kzfm12lNetHwAn3GEuH317IzpwrMCbOuMIVytpjnJ90YYHNwdRgYGuKmVxwuIqSgqk3M5qqQt1cDk0tGQ== + "@inquirer/input@^2.2.4": version "2.3.0" resolved "https://registry.yarnpkg.com/@inquirer/input/-/input-2.3.0.tgz#9b99022f53780fecc842908f3f319b52a5a16865" @@ -1193,6 +1224,15 @@ "@inquirer/core" "^10.3.2" "@inquirer/type" "^3.0.10" +"@inquirer/password@^5.0.12": + version "5.0.12" + resolved "https://registry.yarnpkg.com/@inquirer/password/-/password-5.0.12.tgz#c1dcf197258a8cfba800325b78287fa55a5a3ef8" + integrity sha512-CBh7YHju623lxJRcAOo498ZUwIuMy63bqW/vVq0tQAZVv+lkWlHkP9ealYE1utWSisEShY5VMdzIXRmyEODzcQ== + dependencies: + "@inquirer/ansi" "^2.0.5" + "@inquirer/core" "^11.1.9" + "@inquirer/type" "^4.0.5" + "@inquirer/prompts@^7.10.1": version "7.10.1" resolved "https://registry.yarnpkg.com/@inquirer/prompts/-/prompts-7.10.1.tgz#e1436c0484cf04c22548c74e2cd239e989d5f847" @@ -1269,6 +1309,11 @@ resolved "https://registry.yarnpkg.com/@inquirer/type/-/type-3.0.10.tgz#11ed564ec78432a200ea2601a212d24af8150d50" integrity sha512-BvziSRxfz5Ov8ch0z/n3oijRSEcEsHnhggm4xFZe93DHcUCTlutlq9Ox4SVENAfcRD22UQq7T/atg9Wr3k09eA== +"@inquirer/type@^4.0.5": + version "4.0.5" + resolved "https://registry.yarnpkg.com/@inquirer/type/-/type-4.0.5.tgz#a02d90e5da8a36dce27ac8e7237a50c99a9003a3" + integrity sha512-aetVUNeKNc/VriqXlw1NRSW0zhMBB0W4bNbWRJgzRl/3d0QNDQFfk0GO5SDdtjMZVg6o8ZKEiadd7SCCzoOn5Q== + "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" @@ -1455,10 +1500,10 @@ wordwrap "^1.0.0" wrap-ansi "^7.0.0" -"@oclif/core@^4", "@oclif/core@^4.0.27", "@oclif/core@^4.10.6", "@oclif/core@^4.5.2": - version "4.10.6" - resolved "https://registry.yarnpkg.com/@oclif/core/-/core-4.10.6.tgz#233d66284d8c7c8162c9d437754503734069dd85" - integrity sha512-ySCOYnPKZE3KACT1V9It99hWG9b8E5MpagbRdWxPNRO3beMqmbr4SLUQoFtZ9XRtW++kks1ZVwZOdpnR8rpb9A== +"@oclif/core@^4", "@oclif/core@^4.0.27", "@oclif/core@^4.11.0": + version "4.11.1" + resolved "https://registry.yarnpkg.com/@oclif/core/-/core-4.11.1.tgz#f72284b4e14c4a3f47a9952ed2234f000fc23536" + integrity sha512-+N5yqeoOKPnT0p+ZJiNutMILsZukZrEpsVup24XERla594EkGSWS9tiCqRfvzr1xfvf/AhM9pb0yPaf8L3Y9Uw== dependencies: ansi-escapes "^4.3.2" ansis "^3.17.0" @@ -1480,9 +1525,9 @@ wrap-ansi "^7.0.0" "@oclif/multi-stage-output@^0.8.36": - version "0.8.37" - resolved "https://registry.yarnpkg.com/@oclif/multi-stage-output/-/multi-stage-output-0.8.37.tgz#f705a1af7f9201cc6ab1797ddd53ea9eff8a4ce8" - integrity sha512-szads7f+FV3i1JGifGOAKpTaTwy70+gBjjfmrHavPIeahQog1ehyIP0L7v4jZnj29EZtUbNmxUpKPJbrCAsJDQ== + version "0.8.38" + resolved "https://registry.yarnpkg.com/@oclif/multi-stage-output/-/multi-stage-output-0.8.38.tgz#7c5a55afc3126cd023e4001015287f4d51390a15" + integrity sha512-+1CV2VP+keL1NfcGJNPONEVEimq5JvaVUgSqU6YzW6FOZjh0zvkbVQr2krE0twwhYwI0J8SkQLwssR1pfpLKnw== dependencies: "@oclif/core" "^4" "@types/react" "^18.3.12" @@ -1493,9 +1538,9 @@ wrap-ansi "^9.0.2" "@oclif/plugin-command-snapshot@^5.3.13": - version "5.3.16" - resolved "https://registry.yarnpkg.com/@oclif/plugin-command-snapshot/-/plugin-command-snapshot-5.3.16.tgz#d4d45298a8be71c20ba8b6a850edc82b31000184" - integrity sha512-pwTKWQRDrK9eOz1VVUTKm9q3tKf9Kihunu3R53LfcPXPpEhIJ6N+2zmCYqeuifrvjBqLT1VhAKpVgTfHXxURHg== + version "5.3.17" + resolved "https://registry.yarnpkg.com/@oclif/plugin-command-snapshot/-/plugin-command-snapshot-5.3.17.tgz#441c6cdb0c236462405a4fb78fd44871d1a54160" + integrity sha512-lzjs9x6rxXo+EBFSvL96NytY1uXzPbmRn22GNYbtYhY1oSRmOedihcYboQwVPP+V2d6BfPdbxxWzz3Rc6hoHGg== dependencies: "@oclif/core" "^4" ansis "^3.17.0" @@ -1508,26 +1553,26 @@ ts-json-schema-generator "^1.5.1" "@oclif/plugin-help@^6.2.38": - version "6.2.45" - resolved "https://registry.yarnpkg.com/@oclif/plugin-help/-/plugin-help-6.2.45.tgz#26bdb2df07881e2147e735bbdd32b6d797ee9f52" - integrity sha512-avWOKYmjANtyu8ipju/kopIIrSrbS/scJjiZTpBp/HKEHNm46v5riOo5LQj6MZ4bYJVQEoyHPg/2Seig5Ilkjw== + version "6.2.46" + resolved "https://registry.yarnpkg.com/@oclif/plugin-help/-/plugin-help-6.2.46.tgz#2bd7a7ec4706b0e00f037f60d0fa08d60bb54161" + integrity sha512-KmuMFt/fURCVxor0rrRjEqs2nLN0Y3ixcixo/M5VjKcN920gbuw5T+AF23FBeyUDuW/Dg79YPcTWy/Rtz0Dg/A== dependencies: "@oclif/core" "^4" "@oclif/plugin-not-found@^3.2.76": - version "3.2.81" - resolved "https://registry.yarnpkg.com/@oclif/plugin-not-found/-/plugin-not-found-3.2.81.tgz#bd48e6103be81e612a1ec42f8dcdab22598f5f0b" - integrity sha512-M88tLONBH36hLAbkFbmCo1hoZPSdU5l8Px1xEIlIgSmGMam+CoAzx4kGqpLbokgfpaHeP8/Jx3QJ18u9ef/2Qw== + version "3.2.82" + resolved "https://registry.yarnpkg.com/@oclif/plugin-not-found/-/plugin-not-found-3.2.82.tgz#5ec97fceac671f8198e08acaec64961df2517cf6" + integrity sha512-6heNFE2gadcDYijWy4XJc6ZLzPd1qKe0i8sb8uyrR3mX0o5IFA+5KSAx/BFBkGS8j/tKOsCYvvmMKVdReeb1Gg== dependencies: "@inquirer/prompts" "^7.10.1" - "@oclif/core" "^4.10.6" + "@oclif/core" "^4.11.0" ansis "^3.17.0" fast-levenshtein "^3.0.0" "@oclif/plugin-warn-if-update-available@^3.1.57": - version "3.1.61" - resolved "https://registry.yarnpkg.com/@oclif/plugin-warn-if-update-available/-/plugin-warn-if-update-available-3.1.61.tgz#0a5588f4899c565a2b899fbe220f244b33e263ac" - integrity sha512-4XcrTxcCs+brR/eZ0BPeuiREiH3USlJiaHbUqPhnIBuyxhhUSYVd8ZO6s5MQN7AXJq4SMQ+B5zLaHq+ep/afIw== + version "3.1.62" + resolved "https://registry.yarnpkg.com/@oclif/plugin-warn-if-update-available/-/plugin-warn-if-update-available-3.1.62.tgz#29329c7f58285cc54290acf330b759c1bfc3a564" + integrity sha512-g1tOOf9tJ3RE4dqhUynw3TH8Gea78IkzG9hq2lcUJ5wIdOSzcp8+3SWVzzpKfHgwBGgFupmj8peCjypybJVIrg== dependencies: "@oclif/core" "^4" ansis "^3.17.0" @@ -1536,10 +1581,10 @@ lodash "^4.18.1" registry-auth-token "^5.1.1" -"@oclif/table@^0.5.0": - version "0.5.4" - resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.5.4.tgz#a6b9845c50bcaeb15d7cfcb1dee36cc8078446e6" - integrity sha512-tIW7JTfO/t19cfOZofxEi16GAV12osvuSwdDHQ6ltIYtNeDyT8vJqdqo5NmyKNNwUy6V1DoGsVhJtPTFabR4hg== +"@oclif/table@^0.5.6": + version "0.5.6" + resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.5.6.tgz#cc22cc3f620c9f23bdb734b0dd6f97cdda4a0f28" + integrity sha512-W0SlIIkcqYxRSbZVw+VnFPT6hStjm5OoMduli35569OXezBPS92yNsoIA9jAuAt3lVIGlnUIetnAUUjs+yLHKg== dependencies: "@types/react" "^18.3.12" change-case "^5.4.4" @@ -1548,6 +1593,7 @@ natural-orderby "^3.0.2" object-hash "^3.0.0" react "^18.3.1" + string-width "^8.2.1" strip-ansi "^7.1.2" wrap-ansi "^9.0.2" @@ -1595,25 +1641,25 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@salesforce/agents@^1.4.0": - version "1.4.0" - resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-1.4.0.tgz#69c3e3e9d29b9596aadf99f94774c9c8713e34ca" - integrity sha512-NaiGUSjcyK0Wsy0n5pFuouHqOYH1KJEpgm605NqCaiOMAA3zn0pWZDHE9I75ymGqpvOXSHGDFNrUhKYPain9ZA== +"@salesforce/agents@^1.5.1": + version "1.5.1" + resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-1.5.1.tgz#cdb8a1a814edc175d7dd51da50cdcb32b3d95bdb" + integrity sha512-Y5aaMmNP06ykuG84bfShExrrpKz0DEP8GE6CQFLGbfYvhPZApTdluG+OBIqwyj38dY9gb4V5oLXy8cKI+jxYYg== dependencies: - "@salesforce/core" "^8.28.3" + "@salesforce/core" "^8.29.0" "@salesforce/kit" "^3.2.6" - "@salesforce/source-deploy-retrieve" "^12.32.7" + "@salesforce/source-deploy-retrieve" "^12.35.1" "@salesforce/types" "^1.7.1" - fast-xml-parser "^5.6.0" + fast-xml-parser "^5.7.2" nock "^13.5.6" - yaml "^2.8.3" + yaml "^2.8.4" "@salesforce/cli-plugins-testkit@^5.3.41": - version "5.3.54" - resolved "https://registry.yarnpkg.com/@salesforce/cli-plugins-testkit/-/cli-plugins-testkit-5.3.54.tgz#636bed66cb28f94c3d415277d1796f5407a6313f" - integrity sha512-OT4S2Wstg2gqWWbhG4q+/AioIWpU1BEsC+KBIBtw1Wckktxv6B/SWlpgIrCz6nQAPwwDpY1sNeGLxpDsNUKVzw== + version "5.3.55" + resolved "https://registry.yarnpkg.com/@salesforce/cli-plugins-testkit/-/cli-plugins-testkit-5.3.55.tgz#9230cb131950c60334b250940826ad20fe029023" + integrity sha512-NzV5WWHJDoybEtHVeTOQt/P/VizWsYwhiyAMU98NG+xAdQVBYyTx/2NhRTgnatxqHyNkYKwPMIJJQ9FLvHpU1A== dependencies: - "@salesforce/core" "^8.28.4" + "@salesforce/core" "^8.29.0" "@salesforce/kit" "^3.2.6" "@salesforce/ts-types" "^2.0.11" "@types/shelljs" "^0.10.0" @@ -1624,10 +1670,10 @@ strip-ansi "6.0.1" ts-retry-promise "^0.8.1" -"@salesforce/core@^8.18.7", "@salesforce/core@^8.23.1", "@salesforce/core@^8.28.3", "@salesforce/core@^8.28.4", "@salesforce/core@^8.5.1": - version "8.28.4" - resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-8.28.4.tgz#98920d62c2b2a7fa025af38c74bdf6687d91b0a8" - integrity sha512-XQ0BBSetdW9cu36pu8ig5ZBX3oAbDSSH4djHkSU3iAzjLdTygEPIBVtKNeyj1++GmCy0rRiLr/yqoFYr21+GuQ== +"@salesforce/core@^8.23.1", "@salesforce/core@^8.28.3", "@salesforce/core@^8.28.4", "@salesforce/core@^8.29.0", "@salesforce/core@^8.5.1": + version "8.29.0" + resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-8.29.0.tgz#d75d1afe06962e10c466bdf670fa8512bd683882" + integrity sha512-q6xDNLPbbZW1n4X4YK1iM8jZvwvJRiwbJxdeF5iHuETxmMka16FoCVi+WziK/Rh5EP0yW08FYyiynwPlgz5RBw== dependencies: "@jsforce/jsforce-node" "^3.10.13" "@salesforce/kit" "^3.2.4" @@ -1694,9 +1740,9 @@ "@salesforce/ts-types" "^2.0.12" "@salesforce/plugin-command-reference@^3.1.81": - version "3.1.94" - resolved "https://registry.yarnpkg.com/@salesforce/plugin-command-reference/-/plugin-command-reference-3.1.94.tgz#909d363aba0224b189cf2ff6fd42b82cf9a62efe" - integrity sha512-4BWmOfpvqXYYRVet7f4AEsC0IxsY3gE6Dc/gHIv+FxqhOmgi9giZOXiTP2J7PtapGdGvCCnj8sQnnsLjd/mgEw== + version "3.1.95" + resolved "https://registry.yarnpkg.com/@salesforce/plugin-command-reference/-/plugin-command-reference-3.1.95.tgz#b194e9c08e582dd445de1b4aae7f0ba02c7f6a39" + integrity sha512-K0BQ6f/fPzPOHMf9mpKKxx6x9IbGmUNmKkVNql9zQwXmmfNZoDfozv2EREfijZwgECiIKuFkwB1PjoQkkWyIPQ== dependencies: "@oclif/core" "^4" "@salesforce/core" "^8.28.4" @@ -1731,27 +1777,27 @@ terminal-link "^3.0.0" "@salesforce/sf-plugins-core@^12.2.6": - version "12.2.6" - resolved "https://registry.yarnpkg.com/@salesforce/sf-plugins-core/-/sf-plugins-core-12.2.6.tgz#856303e786f5fac1c6aa6dcd42f282cb1ac703e8" - integrity sha512-EDKE72f/gGk9vL7KI9wsFO5wl/jFVvA2l5XBGR+6sJ1+FTMUbTGRMLObkkYJACk7bKvUFx00xdur2R6K8SWNvg== - dependencies: - "@inquirer/confirm" "^3.2.0" - "@inquirer/password" "^2.2.0" - "@oclif/core" "^4.5.2" - "@oclif/table" "^0.5.0" - "@salesforce/core" "^8.18.7" + version "12.2.13" + resolved "https://registry.yarnpkg.com/@salesforce/sf-plugins-core/-/sf-plugins-core-12.2.13.tgz#7dd7e9ad19c8279c97d020727d7acbcf17a5877b" + integrity sha512-Ug+CIQ7yLZVqdmSlgFPCnkzFXz5gzCR/l5hnMoVDrgNdI/PmCHAx2ZS0WsM3xkebkkOHtWFIZQ9ARip7wAbROw== + dependencies: + "@inquirer/confirm" "^6.0.12" + "@inquirer/password" "^5.0.12" + "@oclif/core" "^4.11.0" + "@oclif/table" "^0.5.6" + "@salesforce/core" "^8.29.0" "@salesforce/kit" "^3.2.3" "@salesforce/ts-types" "^2.0.12" ansis "^3.3.2" cli-progress "^3.12.0" terminal-link "^3.0.0" -"@salesforce/source-deploy-retrieve@^12.32.7", "@salesforce/source-deploy-retrieve@^12.32.8": - version "12.34.5" - resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.34.5.tgz#c02cd88f6c673c75b0d15bf06c8dafc85d06a004" - integrity sha512-iL+656HXNGFMv0DTPF4MKCaJ3de9qGyIzOrOyZ/CL1l0CYcgp2YHrLJuSaCKujul0ymceUGcAptSsZcSVWhjqw== +"@salesforce/source-deploy-retrieve@^12.35.1", "@salesforce/source-deploy-retrieve@^12.35.3": + version "12.35.3" + resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.35.3.tgz#d32453d746408435e9420c53f4948b7aa5ebdb8a" + integrity sha512-Mf5As7bQytwf+zdzHKEFUJrcbyOcMNHZX9cYrt5lLn59pciH814Nzq7kwmtIdXesbxsx95cLQD7OX0MImpe18g== dependencies: - "@salesforce/core" "^8.28.4" + "@salesforce/core" "^8.29.0" "@salesforce/kit" "^3.2.4" "@salesforce/ts-types" "^2.0.12" "@salesforce/types" "^1.6.0" @@ -2090,10 +2136,10 @@ "@smithy/util-middleware" "^4.2.14" tslib "^2.6.2" -"@smithy/middleware-retry@^4.4.42", "@smithy/middleware-retry@^4.4.44", "@smithy/middleware-retry@^4.5.6": - version "4.5.6" - resolved "https://registry.yarnpkg.com/@smithy/middleware-retry/-/middleware-retry-4.5.6.tgz#8f3857c2e654a03a8aaccf8b267aa4d9c2ad1046" - integrity sha512-5zhmo2AkstmM/RMKYP0NHfmuYWBR+/umlmSuALgajLxf0X0rLE6d17MfzTxpzkILWVhwvCJkCyPH0AfMlbaucQ== +"@smithy/middleware-retry@^4.4.42", "@smithy/middleware-retry@^4.4.44", "@smithy/middleware-retry@^4.5.7": + version "4.5.7" + resolved "https://registry.yarnpkg.com/@smithy/middleware-retry/-/middleware-retry-4.5.7.tgz#a2da0c472d631ee408ff566186c99571b3efb70b" + integrity sha512-bRt6ZImqVSeTk39Nm81K20ObIiAZ3WefY7G6+iz/0tZjs4dgRRjvRX2sgsH+zi6iDCRR/aQvQofLKxxz4rPBZg== dependencies: "@smithy/core" "^3.23.17" "@smithy/node-config-provider" "^4.3.14" @@ -2102,7 +2148,7 @@ "@smithy/smithy-client" "^4.12.13" "@smithy/types" "^4.14.1" "@smithy/util-middleware" "^4.2.14" - "@smithy/util-retry" "^4.3.5" + "@smithy/util-retry" "^4.3.6" "@smithy/uuid" "^1.1.2" tslib "^2.6.2" @@ -2328,10 +2374,10 @@ "@smithy/types" "^4.14.1" tslib "^2.6.2" -"@smithy/util-retry@^4.2.12", "@smithy/util-retry@^4.3.5": - version "4.3.5" - resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-4.3.5.tgz#7ca07446905188bbbe7f0bbd171cc7682f461103" - integrity sha512-h1IJsbgMDA+jaTjrco/JsyfWOgHRJBv8myB1y4AEI2fjIzD6ktZ7pFAyTw+gwN9GKIAygvC6db0mq0j8N2rFOg== +"@smithy/util-retry@^4.2.12", "@smithy/util-retry@^4.3.6": + version "4.3.8" + resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-4.3.8.tgz#7f904ed8e5bad2b5f2e6aa1e193db2b46b2c57df" + integrity sha512-LUIxbTBi+OpvXpg91poGA6BdyoleMDLnfXjVDqyi2RvZmTveY5loE/FgYUBCR5LU2BThW2SoZRh8dTIIy38IPw== dependencies: "@smithy/service-error-classification" "^4.3.1" "@smithy/types" "^4.14.1" @@ -2688,9 +2734,9 @@ integrity sha512-iZqi+Ds1y4EDYUtlOOC+aUmxnE9xS/yCigkjA7XpTKV6nCBd3Hp/PRGGmdwnfkV2ThMyYldP1wRpm/id99spTQ== "@typescript-eslint/types@^8.56.0": - version "8.59.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-8.59.1.tgz#c1d014d3f03a97e0113a8899fc9d4e45a7fb0ca9" - integrity sha512-ZDCjgccSdYPw5Bxh+my4Z0lJU96ZDN7jbBzvmEn0FZx3RtU1C7VWl6NbDx94bwY3V5YsgwRzJPOgeY2Q/nLG8A== + version "8.59.2" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-8.59.2.tgz#01caabcd7e4715c33ad5e11cab260829714d6b9c" + integrity sha512-e82GVOE8Ps3E++Egvb6Y3Dw0S10u8NkQ9KXmtRhCWJJ8kDhOJTvtMAWnFL16kB1583goCWXsr0NieKCZMs2/0Q== "@typescript-eslint/typescript-estree@6.21.0": version "6.21.0" @@ -2760,9 +2806,9 @@ eslint-visitor-keys "^3.4.3" "@ungap/structured-clone@^1.0.0", "@ungap/structured-clone@^1.2.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.3.0.tgz#d06bbb384ebcf6c505fde1c3d0ed4ddffe0aaff8" - integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g== + version "1.3.1" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.3.1.tgz#0e8f34854df7966b09304a18e808b23997bb9fc1" + integrity sha512-mUFwbeTqrVgDQxFveS+df2yfap6iuP20NAKAsBt5jDEoOTDew+zwLAOilHCeQJOVSvmgCX4ogqIrA0mnyr08yQ== JSONStream@^1.3.5: version "1.3.5" @@ -3119,14 +3165,14 @@ base64url@^3.0.1: integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A== baseline-browser-mapping@^2.10.12: - version "2.10.23" - resolved "https://registry.yarnpkg.com/baseline-browser-mapping/-/baseline-browser-mapping-2.10.23.tgz#3a1a55d1a691a8c8d74688af7f1fd17eac23c184" - integrity sha512-xwVXGqevyKPsiuQdLj+dZMVjidjJV508TBqexND5HrF89cGdCYCJFB3qhcxRHSeMctdCfbR1jrxBajhDy7o29g== + version "2.10.27" + resolved "https://registry.yarnpkg.com/baseline-browser-mapping/-/baseline-browser-mapping-2.10.27.tgz#fee941c2a0b42cdf83c6427e4c830b1d0bdab2c3" + integrity sha512-zEs/ufmZoUd7WftKpKyXaT6RFxpQ5Qm9xytKRHvJfxFV9DFJkZph9RvJ1LcOUi0Z1ZVijMte65JbILeV+8QQEA== basic-ftp@^5.0.2: - version "5.3.0" - resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.3.0.tgz#88f057d1ba8442643c505c4c83bbaa4442b15cfd" - integrity sha512-5K9eNNn7ywHPsYnFwjKgYH8Hf8B5emh7JKcPaVjjrMJFQQwGpwowEnZNEtHs7DfR7hCZsmaK3VA4HUK0YarT+w== + version "5.3.1" + resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.3.1.tgz#3148ee9af43c0522514a4f973fecb1d3cbb6d71e" + integrity sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw== binary-extensions@^2.0.0: version "2.3.0" @@ -3318,9 +3364,9 @@ camelcase@^6.0.0: integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== caniuse-lite@^1.0.30001782: - version "1.0.30001791" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001791.tgz#dfb93d85c40ad380c57123e72e10f3c575786b51" - integrity sha512-yk0l/YSrOnFZk3UROpDLQD9+kC1l4meK/wed583AXrzoarMGJcbRi2Q4RaUYbKxYAsZ8sWmaSa/DsLmdBeI1vQ== + version "1.0.30001792" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001792.tgz#ca8bb9be244835a335e2018272ce7223691873c5" + integrity sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw== capital-case@^1.0.4: version "1.0.4" @@ -3974,9 +4020,9 @@ ejs@^3.1.10: jake "^10.8.5" electron-to-chromium@^1.5.328: - version "1.5.344" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.344.tgz#6437cc08a7d9b914a98120e182f37793c9eaffd4" - integrity sha512-4MxfbmNDm+KPh066EZy+eUnkcDPcZ35wNmOWzFuh/ijvHsve6kbLTLURy88uCNK5FbpN+yk2nQY6BYh1GEt+wg== + version "1.5.352" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.352.tgz#0b57303cf654d7e4353edf01abe1ca55e5136063" + integrity sha512-9wHk8x6dyuimoe18EdiDPWKExNdxYqo4fn4FwOVVper6RxT3cmpBwBkWWfSOCYJjQdIco/nPhJhNLmn4Ufg1Yg== emoji-regex-xs@^1.0.0: version "1.0.0" @@ -4428,9 +4474,9 @@ eslint@^8.56.0: text-table "^0.2.0" esmock@^2.7.3: - version "2.7.3" - resolved "https://registry.yarnpkg.com/esmock/-/esmock-2.7.3.tgz#25d8fd57b9608f9430185c501e7dab91fb1247bc" - integrity sha512-/M/YZOjgyLaVoY6K83pwCsGE1AJQnj4S4GyXLYgi/Y79KL8EeW6WU7Rmjc89UO7jv6ec8+j34rKeWOfiLeEu0A== + version "2.7.4" + resolved "https://registry.yarnpkg.com/esmock/-/esmock-2.7.4.tgz#3ed0bd041e8990c070ab20de06b6d2c3335a1c78" + integrity sha512-HznWDUaqlMU3N2Ef0RW9u649Nj6ydZhZVXXcN/IUY/CKRSl5X2K0jGXRWNeOw00ZasE9l6QYbvt1XE8+Tulgrw== espree@^10.4.0: version "10.4.0" @@ -4567,19 +4613,38 @@ fast-safe-stringify@^2.1.1: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== +fast-string-truncated-width@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz#23afe0da67d752ca0727538f1e6967759728ce49" + integrity sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g== + +fast-string-width@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/fast-string-width/-/fast-string-width-3.0.2.tgz#16dbabb491ce5585b5ecb675b65c165d71688eeb" + integrity sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg== + dependencies: + fast-string-truncated-width "^3.0.2" + fast-uri@^3.0.1: - version "3.1.0" - resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.1.0.tgz#66eecff6c764c0df9b762e62ca7edcfb53b4edfa" - integrity sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA== + version "3.1.2" + resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.1.2.tgz#8af3d4fc9d3e71b11572cc2673b514a7d1a8c8ec" + integrity sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ== -fast-xml-builder@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz#50188e1452a5fa095f415d3e63dcac0a1dbcbf11" - integrity sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA== +fast-wrap-ansi@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/fast-wrap-ansi/-/fast-wrap-ansi-0.2.0.tgz#c0ae3f3982d061c3d657ec927196fbb47e22fe64" + integrity sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w== + dependencies: + fast-string-width "^3.0.2" + +fast-xml-builder@^1.1.5, fast-xml-builder@^1.1.7: + version "1.1.9" + resolved "https://registry.yarnpkg.com/fast-xml-builder/-/fast-xml-builder-1.1.9.tgz#96bf8de1e3a5f560149b6092844db4e6fd0ee38f" + integrity sha512-jcyKVSEX13iseJqg7n/KWw+xnu/7fdrZ333Fac54KjHDIELVCfDDJXYIm6DTJ0Su4gSzrhqiK0DzY/wZbF40mw== dependencies: path-expression-matcher "^1.1.3" -fast-xml-parser@5.7.2, fast-xml-parser@^5.6.0, fast-xml-parser@^5.7.1, fast-xml-parser@^5.7.2: +fast-xml-parser@5.7.2: version "5.7.2" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.7.2.tgz#fecd0b054c6c132fc03dab994a413da781e0eb9f" integrity sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w== @@ -4589,6 +4654,16 @@ fast-xml-parser@5.7.2, fast-xml-parser@^5.6.0, fast-xml-parser@^5.7.1, fast-xml- path-expression-matcher "^1.5.0" strnum "^2.2.3" +fast-xml-parser@^5.7.1, fast-xml-parser@^5.7.2: + version "5.7.3" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.7.3.tgz#309b04b08d835defc62ab657a0bb340c0e0fbe6a" + integrity sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg== + dependencies: + "@nodable/entities" "^2.1.0" + fast-xml-builder "^1.1.7" + path-expression-matcher "^1.5.0" + strnum "^2.2.3" + fastest-levenshtein@^1.0.7: version "1.0.16" resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz#210e61b6ff181de91ea9b3d1b84fdedd47e034e5" @@ -4764,9 +4839,9 @@ fromentries@^1.2.0: integrity sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg== fs-extra@^11.0.0: - version "11.3.4" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.3.4.tgz#ab6934eca8bcf6f7f6b82742e33591f86301d6fc" - integrity sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA== + version "11.3.5" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.3.5.tgz#07a44eff40bea53e719909a532f91a23bf0769ff" + integrity sha512-eKpRKAovdpZtR1WopLHxlBWvAgPny3c4gX1G5Jhwmmw4XJj0ifSD5qB5TOo8hmA0wlRKDAOAhEE1yVPgs6Fgcg== dependencies: graceful-fs "^4.2.0" jsonfile "^6.0.1" @@ -4839,7 +4914,7 @@ get-caller-file@^2.0.1, get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-east-asian-width@^1.0.0, get-east-asian-width@^1.3.1: +get-east-asian-width@^1.0.0, get-east-asian-width@^1.3.1, get-east-asian-width@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz#ce7008fe345edcf5497a6f557cfa54bc318a9ce7" integrity sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA== @@ -5015,9 +5090,9 @@ globals@^13.19.0: type-fest "^0.20.2" globals@^17.3.0: - version "17.5.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-17.5.0.tgz#a82c641d898f8dfbe0e81f66fdff7d0de43f88c6" - integrity sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g== + version "17.6.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-17.6.0.tgz#0f0be018d5cca8690e6375ead1f65c4bb96191fc" + integrity sha512-sepffkT8stwnIYbsMBpoCHJuJM5l98FUF2AnE07hfvE0m/qp3R586hw4jF4uadbhvg1ooIdzuu7CsfD2jzCaNA== globalthis@^1.0.4: version "1.0.4" @@ -5166,7 +5241,7 @@ hasha@^5.0.0: is-stream "^2.0.0" type-fest "^0.8.0" -hasown@^2.0.2: +hasown@^2.0.2, hasown@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.3.tgz#5e5c2b15b60370a4c7930c383dfb76bf17bc403c" integrity sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg== @@ -5466,10 +5541,10 @@ interpret@^1.0.0: resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== -ip-address@^10.0.1: - version "10.1.1" - resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-10.1.1.tgz#a7614252413e3751b841aaffba939090d2c4c37b" - integrity sha512-1FMu8/N15Ck1BL551Jf42NYIoin2unWjLQ2Fze/DXryJRl5twqtwNHlO39qERGbIOcKYWHdgRryhOC+NG4eaLw== +ip-address@^10.1.1: + version "10.2.0" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-10.2.0.tgz#805fc178b20c518bd4c8548b24fe30892d7f3206" + integrity sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA== is-array-buffer@^3.0.4, is-array-buffer@^3.0.5: version "3.0.5" @@ -5531,11 +5606,11 @@ is-callable@^1.2.7: integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== is-core-module@^2.16.1, is-core-module@^2.5.0: - version "2.16.1" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.16.1.tgz#2a98801a849f43e2add644fbb6bc6229b19a4ef4" - integrity sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w== + version "2.16.2" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.16.2.tgz#3e07450a8080ebce3fbf0cac494f4d2ab324e082" + integrity sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA== dependencies: - hasown "^2.0.2" + hasown "^2.0.3" is-data-view@^1.0.1, is-data-view@^1.0.2: version "1.0.2" @@ -6304,9 +6379,9 @@ lru-cache@^10.0.1, lru-cache@^10.2.0: integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== lru-cache@^11.0.0: - version "11.3.5" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.3.5.tgz#29047d348c0b2793e3112a01c739bb7c6d855637" - integrity sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw== + version "11.3.6" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.3.6.tgz#f0306ad6e9f0a5dc25b16aeba4e8f57b7ec2df55" + integrity sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A== lru-cache@^5.1.1: version "5.1.1" @@ -6647,6 +6722,11 @@ mute-stream@^2.0.0: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-2.0.0.tgz#a5446fc0c512b71c83c44d908d5c7b7b4c493b2b" integrity sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA== +mute-stream@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-3.0.0.tgz#cd8014dd2acb72e1e91bb67c74f0019e620ba2d1" + integrity sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw== + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -8032,11 +8112,11 @@ socks-proxy-agent@^8.0.5: socks "^2.8.3" socks@^2.8.3: - version "2.8.7" - resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.7.tgz#e2fb1d9a603add75050a2067db8c381a0b5669ea" - integrity sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A== + version "2.8.8" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.8.tgz#23bef6d02748eac847ad75610deb6c472554c67a" + integrity sha512-NlGELfPrgX2f1TAAcz0WawlLn+0r3FyhhCRpFFK2CemXenPYvzMWWZINv3eDNo9ucdwme7oCHRY0Jnbs4aIkog== dependencies: - ip-address "^10.0.1" + ip-address "^10.1.1" smart-buffer "^4.2.0" sonic-boom@^4.0.1: @@ -8207,6 +8287,14 @@ string-width@^7.0.0, string-width@^7.2.0: get-east-asian-width "^1.0.0" strip-ansi "^7.1.0" +string-width@^8.2.1: + version "8.2.1" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-8.2.1.tgz#165089cfa527cc88fbc23dd73313f5e334af1ea1" + integrity sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA== + dependencies: + get-east-asian-width "^1.5.0" + strip-ansi "^7.1.2" + string.prototype.matchall@^4.0.12: version "4.0.12" resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.12.tgz#6c88740e49ad4956b1332a911e949583a275d4c0" @@ -8337,9 +8425,9 @@ strip-json-comments@^3.1.1: integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== strnum@^2.2.3: - version "2.2.3" - resolved "https://registry.yarnpkg.com/strnum/-/strnum-2.2.3.tgz#0119fce02749a11bb126a4d686ac5dbdf6e57586" - integrity sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg== + version "2.3.0" + resolved "https://registry.yarnpkg.com/strnum/-/strnum-2.3.0.tgz#81bfbfef53db8c3217ea62a98c026886ec4a2761" + integrity sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q== supports-color@^7, supports-color@^7.0.0, supports-color@^7.1.0: version "7.2.0" @@ -8432,17 +8520,17 @@ tinyglobby@^0.2.14, tinyglobby@^0.2.9: fdir "^6.5.0" picomatch "^4.0.4" -tldts-core@^7.0.28: - version "7.0.28" - resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-7.0.28.tgz#28c256edae2ed177b2a8338a51caf81d41580ecf" - integrity sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ== +tldts-core@^7.0.30: + version "7.0.30" + resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-7.0.30.tgz#c495dba27778f2220bea94f3f6399005c7aca61c" + integrity sha512-uiHN8PIB1VmWyS98eZYja4xzlYqeFZVjb4OuYlJQnZAuJhMw4PbKQOKgHKhBdJR3FE/t5mUQ1Kd80++B+qhD1Q== tldts@^7.0.5: - version "7.0.28" - resolved "https://registry.yarnpkg.com/tldts/-/tldts-7.0.28.tgz#5a5bb26ef3f70008d88c6e53ff58cd59ed8d4c68" - integrity sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw== + version "7.0.30" + resolved "https://registry.yarnpkg.com/tldts/-/tldts-7.0.30.tgz#497cea8d610953222f9dcb3ceb07c7207efcd816" + integrity sha512-ELrFxuqsDdHUwoh0XxDbxuLD3Wnz49Z57IFvTtvWy1hJdcMZjXLIuonjilCiWHlT2GbE4Wlv1wKVTzDFnXH1aw== dependencies: - tldts-core "^7.0.28" + tldts-core "^7.0.30" to-regex-range@^5.0.1: version "5.0.1" @@ -9170,9 +9258,9 @@ yoga-wasm-web@~0.3.3: integrity sha512-N+d4UJSJbt/R3wqY7Coqs5pcV0aUj2j9IaQ3rNj9bVCLld8tTGKRa2USARjnvZJWVx1NDmQev8EknoczaOQDOA== zod@^4.1.12: - version "4.3.6" - resolved "https://registry.yarnpkg.com/zod/-/zod-4.3.6.tgz#89c56e0aa7d2b05107d894412227087885ab112a" - integrity sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg== + version "4.4.3" + resolved "https://registry.yarnpkg.com/zod/-/zod-4.4.3.tgz#b680f172885d18bbebf21a834ea25e55a1bbf356" + integrity sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ== zwitch@^2.0.4: version "2.0.4" From ae5534ad4c826b699743e1453b238f982b12fee2 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Thu, 7 May 2026 13:34:07 -0600 Subject: [PATCH 30/38] test: expect oclif parse exit code 2 for invalid-input NUTs Six NUT cases asserted ensureExitCode: 1 but the commands fail at oclif flag-parse time (Flags.file({ exists: true }), parse callbacks that throw, validate callbacks, and missing required flags), which exits with code 2. Update the assertions to match. Co-Authored-By: Claude Opus 4.7 --- test/nuts/agent.generate.template.nut.ts | 2 +- test/nuts/agent.generate.test-spec.nut.ts | 4 ++-- test/nuts/agent.test.create.nut.ts | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/nuts/agent.generate.template.nut.ts b/test/nuts/agent.generate.template.nut.ts index c4931835..c2462a27 100644 --- a/test/nuts/agent.generate.template.nut.ts +++ b/test/nuts/agent.generate.template.nut.ts @@ -92,7 +92,7 @@ describe('agent generate template NUTs', function () { execCmd( `agent generate template --agent-file ${invalidAgentFile} --agent-version ${agentVersion} --output-dir ${outputDir} --json`, - { ensureExitCode: 1 } + { ensureExitCode: 2 } ); }); diff --git a/test/nuts/agent.generate.test-spec.nut.ts b/test/nuts/agent.generate.test-spec.nut.ts index e0ae08b3..1ae339f3 100644 --- a/test/nuts/agent.generate.test-spec.nut.ts +++ b/test/nuts/agent.generate.test-spec.nut.ts @@ -58,7 +58,7 @@ describe('agent generate test-spec NUTs', function () { const outputFile = join(session.project.dir, 'specs', genUniqueString('testSpec_%s.yaml')); execCmd(`agent generate test-spec --from-definition ${invalidFile} --output-file ${outputFile} --force-overwrite`, { - ensureExitCode: 1, + ensureExitCode: 2, }); }); @@ -75,7 +75,7 @@ describe('agent generate test-spec NUTs', function () { const outputFile = join(session.project.dir, 'specs', genUniqueString('testSpec_%s.yaml')); execCmd(`agent generate test-spec --from-definition ${invalidFile} --output-file ${outputFile} --force-overwrite`, { - ensureExitCode: 1, + ensureExitCode: 2, }); }); }); diff --git a/test/nuts/agent.test.create.nut.ts b/test/nuts/agent.test.create.nut.ts index ab5c05e5..3a6bfe79 100644 --- a/test/nuts/agent.test.create.nut.ts +++ b/test/nuts/agent.test.create.nut.ts @@ -58,14 +58,14 @@ describe('agent test create', function () { const normalizedInvalidSpecPath = normalize(invalidSpecPath).replace(/\\/g, '/'); execCmd( `agent test create --api-name "${testApiName}" --spec "${normalizedInvalidSpecPath}" --target-org ${getUsername()} --preview --json`, - { ensureExitCode: 1 } + { ensureExitCode: 2 } ); }); it('should fail when required flags are missing in JSON mode', async () => { // Missing --api-name execCmd(`agent test create --target-org ${getUsername()} --json`, { - ensureExitCode: 1, + ensureExitCode: 2, }); // Missing --spec @@ -73,7 +73,7 @@ describe('agent test create', function () { execCmd( `agent test create --api-name "${testApiName}" --target-org ${getUsername()} --json`, { - ensureExitCode: 1, + ensureExitCode: 2, } ); }); From 9ea48eeae8e324ac4280a92cc53fad2d3ffaf8ff Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Thu, 7 May 2026 14:12:42 -0600 Subject: [PATCH 31/38] fix: use nonZero ensureExitCode for invalid-input NUTs The exit code for these oclif parse failures differs by sf-plugins-core version: 12.2.6 (bundled in CLI 2.135.0) emits 2, while 12.2.13 (used in plugin-agent's own CI) catches the parse error and re-emits exit 1. Use 'nonZero' so the assertion is correct in both environments. Co-Authored-By: Claude Opus 4.7 --- test/nuts/agent.generate.template.nut.ts | 2 +- test/nuts/agent.generate.test-spec.nut.ts | 4 ++-- test/nuts/agent.test.create.nut.ts | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/nuts/agent.generate.template.nut.ts b/test/nuts/agent.generate.template.nut.ts index c2462a27..3ca561f9 100644 --- a/test/nuts/agent.generate.template.nut.ts +++ b/test/nuts/agent.generate.template.nut.ts @@ -92,7 +92,7 @@ describe('agent generate template NUTs', function () { execCmd( `agent generate template --agent-file ${invalidAgentFile} --agent-version ${agentVersion} --output-dir ${outputDir} --json`, - { ensureExitCode: 2 } + { ensureExitCode: 'nonZero' } ); }); diff --git a/test/nuts/agent.generate.test-spec.nut.ts b/test/nuts/agent.generate.test-spec.nut.ts index 1ae339f3..524bc3c8 100644 --- a/test/nuts/agent.generate.test-spec.nut.ts +++ b/test/nuts/agent.generate.test-spec.nut.ts @@ -58,7 +58,7 @@ describe('agent generate test-spec NUTs', function () { const outputFile = join(session.project.dir, 'specs', genUniqueString('testSpec_%s.yaml')); execCmd(`agent generate test-spec --from-definition ${invalidFile} --output-file ${outputFile} --force-overwrite`, { - ensureExitCode: 2, + ensureExitCode: 'nonZero', }); }); @@ -75,7 +75,7 @@ describe('agent generate test-spec NUTs', function () { const outputFile = join(session.project.dir, 'specs', genUniqueString('testSpec_%s.yaml')); execCmd(`agent generate test-spec --from-definition ${invalidFile} --output-file ${outputFile} --force-overwrite`, { - ensureExitCode: 2, + ensureExitCode: 'nonZero', }); }); }); diff --git a/test/nuts/agent.test.create.nut.ts b/test/nuts/agent.test.create.nut.ts index 3a6bfe79..13adf4d4 100644 --- a/test/nuts/agent.test.create.nut.ts +++ b/test/nuts/agent.test.create.nut.ts @@ -58,14 +58,14 @@ describe('agent test create', function () { const normalizedInvalidSpecPath = normalize(invalidSpecPath).replace(/\\/g, '/'); execCmd( `agent test create --api-name "${testApiName}" --spec "${normalizedInvalidSpecPath}" --target-org ${getUsername()} --preview --json`, - { ensureExitCode: 2 } + { ensureExitCode: 'nonZero' } ); }); it('should fail when required flags are missing in JSON mode', async () => { // Missing --api-name execCmd(`agent test create --target-org ${getUsername()} --json`, { - ensureExitCode: 2, + ensureExitCode: 'nonZero', }); // Missing --spec @@ -73,7 +73,7 @@ describe('agent test create', function () { execCmd( `agent test create --api-name "${testApiName}" --target-org ${getUsername()} --json`, { - ensureExitCode: 2, + ensureExitCode: 'nonZero', } ); }); From 76324bd3463a7fd81c4171eb86d6b8e80b0e55f4 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Thu, 7 May 2026 20:48:29 +0000 Subject: [PATCH 32/38] chore(release): 1.36.1 [skip ci] --- CHANGELOG.md | 6 ++++ README.md | 79 ++++++++++++++++++++++++++++++++++++---------------- package.json | 2 +- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 704ba27d..9eb1fcab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## [1.36.1](https://github.com/salesforcecli/plugin-agent/compare/1.36.0...1.36.1) (2026-05-07) + +### Bug Fixes + +- use nonZero ensureExitCode for invalid-input NUTs ([9ea48ee](https://github.com/salesforcecli/plugin-agent/commit/9ea48eeae8e324ac4280a92cc53fad2d3ffaf8ff)) + # [1.36.0](https://github.com/salesforcecli/plugin-agent/compare/1.35.0...1.36.0) (2026-05-07) ### Features diff --git a/README.md b/README.md index 6226fbb9..27dd2587 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ EXAMPLES $ sf agent activate --api-name Resort_Manager --version 2 --target-org my-org ``` -_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/activate.ts)_ +_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/activate.ts)_ ## `sf agent create` @@ -193,7 +193,7 @@ EXAMPLES $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview ``` -_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/create.ts)_ +_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/create.ts)_ ## `sf agent deactivate` @@ -234,7 +234,7 @@ EXAMPLES $ sf agent deactivate --api-name Resort_Manager --target-org my-org ``` -_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/deactivate.ts)_ +_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/deactivate.ts)_ ## `sf agent generate agent-spec` @@ -341,7 +341,7 @@ EXAMPLES $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com ``` -_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/agent-spec.ts)_ +_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/agent-spec.ts)_ ## `sf agent generate authoring-bundle` @@ -418,7 +418,7 @@ EXAMPLES other-package-dir/main/default --target-org my-dev-org ``` -_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/authoring-bundle.ts)_ +_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/authoring-bundle.ts)_ ## `sf agent generate template` @@ -480,7 +480,7 @@ EXAMPLES my-package --source-org my-scratch-org ``` -_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/template.ts)_ +_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/template.ts)_ ## `sf agent generate test-spec` @@ -545,7 +545,7 @@ EXAMPLES force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml ``` -_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/generate/test-spec.ts)_ +_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/test-spec.ts)_ ## `sf agent preview` @@ -618,7 +618,7 @@ EXAMPLES $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview ``` -_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview.ts)_ +_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview.ts)_ ## `sf agent preview end` @@ -673,7 +673,7 @@ EXAMPLES $ sf agent preview end --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/end.ts)_ +_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/end.ts)_ ## `sf agent preview send` @@ -731,7 +731,7 @@ EXAMPLES $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent ``` -_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/send.ts)_ +_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/send.ts)_ ## `sf agent preview sessions` @@ -764,7 +764,7 @@ EXAMPLES $ sf agent preview sessions ``` -_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/sessions.ts)_ +_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/sessions.ts)_ ## `sf agent preview start` @@ -829,7 +829,7 @@ EXAMPLES $ sf agent preview start --api-name My_Published_Agent ``` -_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/preview/start.ts)_ +_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/start.ts)_ ## `sf agent publish authoring-bundle` @@ -889,7 +889,7 @@ EXAMPLES $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --concise ``` -_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/publish/authoring-bundle.ts)_ +_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/publish/authoring-bundle.ts)_ ## `sf agent test create` @@ -944,7 +944,7 @@ EXAMPLES $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview ``` -_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/test/create.ts)_ +_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/test/create.ts)_ ## `sf agent test list` @@ -966,8 +966,8 @@ GLOBAL FLAGS DESCRIPTION List the available agent tests in your org. - The command outputs a table with the name (API name) of each test along with its unique ID and the date it was created - in the org. + The command outputs a table with the name (API name) of each test along with its unique ID, type ('agentforce-studio' + or 'testing-center'), and the date it was created in the org. EXAMPLES List the agent tests in your default org: @@ -979,7 +979,7 @@ EXAMPLES $ sf agent test list --target-org my-org ``` -_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.0/src/commands/agent/test/list.ts)_ +_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/test/list.ts)_ ## `sf agent test results` @@ -988,7 +988,7 @@ Get the results of a completed agent test run. ``` USAGE $ sf agent test results -o -i [--json] [--flags-dir ] [--api-version ] [--result-format - json|human|junit|tap] [-d ] [--verbose] + json|human|junit|tap] [-d ] [--test-runner agentforce-studio|testing-center] [--verbose] FLAGS -d, --output-dir= Directory to write the agent test results into. @@ -998,6 +998,8 @@ FLAGS --api-version= Override the api version used for api requests made by this command --result-format=