diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04639c09..4420d9c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,18 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] + # Split the NUTs across jobs so each group runs on its own runner (its own + # egress IP). The `login rate exceeded` OAuth cap is per-IP, and running the + # whole suite serially on one IP exhausts the budget before run-eval. run-eval + # is isolated on its own job for maximum headroom; the z0-z5 pipeline stays in + # one job because its tests share one org in create->...->trace order. + command: + - 'yarn test:nuts:run-eval' + - 'yarn test:nuts:test' + - 'yarn test:nuts:generate' + - 'yarn test:nuts:z' + - 'yarn test:nuts:adl-mcp' fail-fast: false with: os: ${{ matrix.os }} + command: ${{ matrix.command }} diff --git a/README.md b/README.md index 69129dc5..53b78b6c 100644 --- a/README.md +++ b/README.md @@ -2498,3 +2498,5 @@ EXAMPLES _See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/2.2.3/src/commands/agent/validate/authoring-bundle.ts)_ + + diff --git a/package.json b/package.json index f63aa1a1..a6f3f40e 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,11 @@ "prepare": "sf-install", "test": "wireit", "test:nuts": "nyc mocha \"test/nuts/**/*.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:run-eval": "nyc mocha \"test/nuts/agent.test.run-eval.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:test": "nyc mocha \"test/nuts/agent.test.nut.ts\" \"test/nuts/agent.test.create.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:generate": "nyc mocha \"test/nuts/agent.activate.nut.ts\" \"test/nuts/agent.generate.*.nut.ts\" \"test/nuts/template.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:z": "nyc mocha \"test/nuts/z*.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", + "test:nuts:adl-mcp": "nyc mocha \"test/nuts/agent.adl.nut.ts\" \"test/nuts/agent.mcp.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit", "test:only": "wireit", "version": "oclif readme" }, diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index 8c73b42c..3533be5b 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -29,55 +29,65 @@ describe('agent test run-eval', function () { const yamlSpecPath = join(mockProjectDir, 'eval-test-spec.yaml'); const yamlWithContextPath = join(mockProjectDir, 'eval-with-context.yaml'); + // The JSON-payload run-eval is expensive (~18 min) and hammers the shared admin user's + // OAuth refresh rate limit when invoked repeatedly. Run it exactly once here and share the + // parsed result across every test that asserts against it, so a rate-limit failure surfaces + // once rather than per-test. + let jsonResult: RunEvalResult; + before(async function () { this.timeout(30 * 60 * 1000); // 30 minutes for setup await getTestSession(); + + const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; + const jsonExec = execCmd(jsonCommand, { ensureExitCode: 0 }); + expect(jsonExec.jsonOutput?.result).to.be.ok; + jsonResult = jsonExec.jsonOutput!.result; }); describe('run-eval with JSON file', () => { it('should run evaluation with JSON payload file', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); - expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); - expect(result.jsonOutput?.result.summary).to.be.ok; - expect(result.jsonOutput?.result.summary.passed).to.be.a('number'); - expect(result.jsonOutput?.result.summary.failed).to.be.a('number'); - expect(result.jsonOutput?.result.summary.scored).to.be.a('number'); - expect(result.jsonOutput?.result.summary.errors).to.be.a('number'); + expect(jsonResult).to.be.ok; + expect(jsonResult.tests).to.be.an('array'); + expect(jsonResult.tests.length).to.be.greaterThan(0); + expect(jsonResult.summary).to.be.ok; + expect(jsonResult.summary.passed).to.be.a('number'); + expect(jsonResult.summary.failed).to.be.a('number'); + expect(jsonResult.summary.scored).to.be.a('number'); + expect(jsonResult.summary.errors).to.be.a('number'); }); it('should run evaluation with normalized payload', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result.tests[0]).to.be.ok; - expect(result.jsonOutput?.result.tests[0].id).to.equal('test-topic-routing'); - expect(result.jsonOutput?.result.tests[0].status).to.be.oneOf(['passed', 'failed']); - expect(result.jsonOutput?.result.tests[0].evaluations).to.be.an('array'); + expect(jsonResult.tests[0]).to.be.ok; + expect(jsonResult.tests[0].id).to.equal('test-topic-routing'); + expect(jsonResult.tests[0].status).to.be.oneOf(['passed', 'failed']); + expect(jsonResult.tests[0].evaluations).to.be.an('array'); }); }); describe('run-eval with YAML file', () => { - it('should run evaluation with YAML test spec file', async () => { + // The YAML-spec run-eval is run once and shared across the two tests that assert against it, + // for the same OAuth-rate-limit reason as the JSON-payload run above. + let yamlResult: RunEvalResult; + + before(() => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); + const output = execCmd(command, { ensureExitCode: 0 }); + expect(output.jsonOutput?.result).to.be.ok; + yamlResult = output.jsonOutput!.result; + }); - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); - expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); - expect(result.jsonOutput?.result.summary).to.be.ok; + it('should run evaluation with YAML test spec file', async () => { + expect(yamlResult).to.be.ok; + expect(yamlResult.tests).to.be.an('array'); + expect(yamlResult.tests.length).to.be.greaterThan(0); + expect(yamlResult.summary).to.be.ok; }); it('should auto-infer agent name from YAML subjectName', async () => { - const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - // Should succeed without explicit --api-name flag - expect(result.jsonOutput?.result).to.be.ok; - expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(yamlResult).to.be.ok; + expect(yamlResult.tests).to.be.an('array'); }); it('should handle YAML spec with contextVariables', async () => { @@ -173,11 +183,8 @@ describe('agent test run-eval', function () { describe('run-eval output structure', () => { it('should include test summaries with correct structure', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - expect(result.jsonOutput?.result.tests).to.be.an('array'); - const firstTest = result.jsonOutput?.result.tests[0]; + expect(jsonResult.tests).to.be.an('array'); + const firstTest = jsonResult.tests[0]; expect(firstTest).to.have.property('id'); expect(firstTest).to.have.property('status'); expect(firstTest).to.have.property('evaluations'); @@ -187,10 +194,7 @@ describe('agent test run-eval', function () { }); it('should include summary with all metrics', async () => { - const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); - - const summary = result.jsonOutput?.result.summary; + const summary = jsonResult.summary; expect(summary).to.have.property('passed'); expect(summary).to.have.property('failed'); expect(summary).to.have.property('scored');