From fbb0ceeabdb20848694c36b3bf3de9131d24c810 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 7 Aug 2026 18:01:40 +0000 Subject: [PATCH] feat(eval): add read-only batch-evaluation CLI (get, list) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `agentcore eval batch-evaluation get` and `list` (read-only slice of the imperative batch-evaluation commands). - get --id [--disable-cw-results]: returns the service job and, by default, the per-session results read from its CloudWatch output stream once terminal. getBatchEvaluation returns { detail, resultsError? } — a CloudWatch read failure is surfaced (io.warn on stderr), never hides the job status. - list [--max-results/--next-token]: service-side pagination, page returned verbatim (no client-side filtering). Core: - CloudWatch Logs client wired into AwsClients (logs() accessor + factory). - core/batchEvaluationResults.tsx: reads + parses the result stream, following GetLogEvents pagination to completion; uses the service-returned log group/stream from outputConfig (no derived name). Parser keeps evaluation level + session/trace/span scope (the old CLI flattened them); field names verified against a recorded live stream. - io.warn() helper for user-facing stderr advisories. Tests: parser unit + real recorded-event replay, handler command-flow, io.warn. Verified live against the exploratory account. --- bun.lock | 53 ++ package.json | 3 +- src/core/batchEvaluationResults.test.ts | 167 ++++ src/core/batchEvaluationResults.tsx | 117 +++ src/core/core.test.ts | 12 + src/core/datasetDownload.test.ts | 2 +- src/core/eval.tsx | 77 ++ src/core/factories.tsx | 11 +- src/core/index.tsx | 24 +- src/core/types.tsx | 6 + ...tchEvaluationCommand.1e79922919dcb237.json | 72 ++ ...tchEvaluationCommand.94c2cede863d8fbd.json | 6 + .../GetLogEventsCommand.1bfa6f1de138b98c.json | 96 +++ .../GetLogEventsCommand.5bd3c931772927ac.json | 5 + .../GetLogEventsCommand.f3f79313bdec0f23.json | 5 + ...chEvaluationsCommand.23f97c9dcdd6350b.json | 747 ++++++++++++++++++ .../__fixtures__/get-disable-cw.golden.json | 64 ++ .../__fixtures__/get.golden.json | 228 ++++++ .../__fixtures__/list.golden.json | 547 +++++++++++++ .../batch-evaluation.fixture.test.tsx | 109 +++ .../batch-evaluation.test.tsx | 162 ++++ .../eval/batch-evaluation/get/index.tsx | 60 ++ src/handlers/eval/batch-evaluation/index.tsx | 16 + .../eval/batch-evaluation/list/index.tsx | 23 + .../eval/dataset/dataset.fixture.test.tsx | 4 +- .../eval/evaluator/evaluator.test.tsx | 4 +- src/handlers/eval/index.tsx | 4 +- .../eval/online-eval/online-eval.test.tsx | 4 +- src/handlers/eval/types.tsx | 62 ++ src/handlers/gateway/gateway.fixture.test.tsx | 4 +- src/handlers/harness/harness.test.tsx | 4 +- src/handlers/identity/identity.test.tsx | 4 +- .../oauth2.fixture.test.tsx | 4 +- src/handlers/memory/memory.test.tsx | 4 +- src/handlers/runtime/runtime.test.tsx | 4 +- src/index.ts | 8 +- src/io/index.ts | 1 + src/io/warn.test.ts | 16 + src/io/warn.ts | 10 + src/testing/TestCoreClient.tsx | 82 ++ src/testing/fixtures.tsx | 16 +- 41 files changed, 2830 insertions(+), 17 deletions(-) create mode 100644 src/core/batchEvaluationResults.test.ts create mode 100644 src/core/batchEvaluationResults.tsx create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.1e79922919dcb237.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.94c2cede863d8fbd.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.1bfa6f1de138b98c.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.5bd3c931772927ac.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.f3f79313bdec0f23.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/get-disable-cw.golden.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/get.golden.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/list.golden.json create mode 100644 src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx create mode 100644 src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx create mode 100644 src/handlers/eval/batch-evaluation/get/index.tsx create mode 100644 src/handlers/eval/batch-evaluation/index.tsx create mode 100644 src/handlers/eval/batch-evaluation/list/index.tsx create mode 100644 src/io/warn.test.ts create mode 100644 src/io/warn.ts diff --git a/bun.lock b/bun.lock index a61c220ed..e07d4358a 100644 --- a/bun.lock +++ b/bun.lock @@ -7,6 +7,7 @@ "dependencies": { "@aws-sdk/client-bedrock-agentcore": "^3.1092.0", "@aws-sdk/client-bedrock-agentcore-control": "^3.1079.0", + "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", "@aws-sdk/client-iam": "^3.1080.0", "@smithy/core": "3.29.3", "@tanstack/react-query": "^5.101.2", @@ -45,6 +46,8 @@ "@aws-sdk/client-bedrock-agentcore-control": ["@aws-sdk/client-bedrock-agentcore-control@3.1087.0", "", { "dependencies": { "@aws-sdk/core": "^3.975.2", "@aws-sdk/credential-provider-node": "^3.972.68", "@aws-sdk/types": "^3.974.1", "@smithy/core": "^3.29.3", "@smithy/fetch-http-handler": "^5.6.5", "@smithy/node-http-handler": "^4.9.5", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-ZCWITtXgDZG1g8NfTZVMTIwH/NR9eifXRW1g5+d4Jb93MmwbJWLWRSd0xqxgywMay0sqhNOKXHE4nQRIf9glag=="], + "@aws-sdk/client-cloudwatch-logs": ["@aws-sdk/client-cloudwatch-logs@3.1104.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/credential-provider-node": "^3.972.78", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/fetch-http-handler": "^5.6.13", "@smithy/node-http-handler": "^4.9.13", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-cetLaPXxPgAVjVyC+dvLFaoasgt1/SX2RhjhEBTepTX8tX+WMD1pWoyRn9Uk2XYzz3RODdavekmue66r52S7nw=="], + "@aws-sdk/client-iam": ["@aws-sdk/client-iam@3.1087.0", "", { "dependencies": { "@aws-sdk/core": "^3.975.2", "@aws-sdk/credential-provider-node": "^3.972.68", "@aws-sdk/types": "^3.974.1", "@smithy/core": "^3.29.3", "@smithy/fetch-http-handler": "^5.6.5", "@smithy/node-http-handler": "^4.9.5", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-S2UBBQnPMTREF52WPz7yabvags3rPYLOPwq8LOPNJIekyAIqUwtGN46z2tqGNr+NyGz9cpEI+3brJgkWKQGNIQ=="], "@aws-sdk/core": ["@aws-sdk/core@3.976.0", "", { "dependencies": { "@aws-sdk/types": "^3.974.2", "@aws-sdk/xml-builder": "^3.972.36", "@aws/lambda-invoke-store": "^0.3.0", "@smithy/core": "^3.29.4", "@smithy/signature-v4": "^5.6.5", "@smithy/types": "^4.16.1", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-0cjRaEdlVoOrsNb9pP5q1Syyc8pXw5xSj2Np2ryReRTr9FppIIRVSdZK4lbnfmc2Hvgux/xBOUU6baB7z8//uA=="], @@ -357,6 +360,16 @@ "@aws-sdk/client-bedrock-agentcore-control/@smithy/node-http-handler": ["@smithy/node-http-handler@4.9.5", "", { "dependencies": { "@smithy/core": "^3.29.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-bNqdxTQTxmLbomSmlkZFz8L6B/feQ2HHzw4L2zY7Ecp2XffYAZq2uzdWDdxJHJFbEvqd+SRuluJso0P8+xPdbw=="], + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/core": ["@aws-sdk/core@3.977.6", "", { "dependencies": { "@aws-sdk/types": "^3.974.2", "@aws-sdk/xml-builder": "^3.972.37", "@aws/lambda-invoke-store": "^0.3.0", "@smithy/core": "^3.31.1", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.16.1", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-QiaJV4/zDrB4ZY2mfeSXSzSTc36W16sZXcGz+SPFk0CJ26gziO0cS+4LjJUMAbdeeBOvS0k0Aq1cZpfGdUXxSw=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.78", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.67", "@aws-sdk/credential-provider-http": "^3.972.69", "@aws-sdk/credential-provider-ini": "^3.973.12", "@aws-sdk/credential-provider-process": "^3.972.67", "@aws-sdk/credential-provider-sso": "^3.973.11", "@aws-sdk/credential-provider-web-identity": "^3.972.73", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-OgPAnfvbGAMWac6yvxJ1ihslrvDpPVwR68D2csospdNCCyPvHk9JLzYKwz48SNiS1T2znDwHauywRKRFfpyYng=="], + + "@aws-sdk/client-cloudwatch-logs/@smithy/core": ["@smithy/core@3.31.1", "", { "dependencies": { "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-CyogUINxvi7C7LDsh8Syo6hVJOT9ckz4rG8dRZfTJ8r91HkMY59PnNooaj7WcHyxEkxPfBAmbgztZU+xTo76lg=="], + + "@aws-sdk/client-cloudwatch-logs/@smithy/fetch-http-handler": ["@smithy/fetch-http-handler@5.6.13", "", { "dependencies": { "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-4fW86pEUOMbrD5nkbyl/tTvPHHWJFbuB2odl6ps9lWfHoXf9HWh3Q/Smh59qH1g7+c/BSZghX6bbUk4gsiMs8A=="], + + "@aws-sdk/client-cloudwatch-logs/@smithy/node-http-handler": ["@smithy/node-http-handler@4.9.13", "", { "dependencies": { "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-Nmd/Nl35zfYrd+a6OO2cDJb3GPh9bgTjIUhcM+JFfjpp8/osCgboDV5nCT1I01Pv6R13eSKDKLSoVa5ZB6Zsfw=="], + "@aws-sdk/client-iam/@aws-sdk/core": ["@aws-sdk/core@3.975.2", "", { "dependencies": { "@aws-sdk/types": "^3.974.1", "@aws-sdk/xml-builder": "^3.972.35", "@aws/lambda-invoke-store": "^0.3.0", "@smithy/core": "^3.29.3", "@smithy/signature-v4": "^5.6.3", "@smithy/types": "^4.16.1", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-iyeXwziyjJpixq5OmhsIyrSWx8vwcI7gDo4yRUC3EP7NQtOo9iAJiIEc3G+/HkhtNXqOhofiCK7Lc34Sq+fJWg=="], "@aws-sdk/client-iam/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.68", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.58", "@aws-sdk/credential-provider-http": "^3.972.60", "@aws-sdk/credential-provider-ini": "^3.973.2", "@aws-sdk/credential-provider-process": "^3.972.58", "@aws-sdk/credential-provider-sso": "^3.973.2", "@aws-sdk/credential-provider-web-identity": "^3.972.64", "@aws-sdk/types": "^3.974.1", "@smithy/core": "^3.29.3", "@smithy/credential-provider-imds": "^4.4.7", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-4akjzW9CjorByYfqXBXmYUh/h7Io3U4DtVgGGh9TQraZ7ZlyJqNyHwDRGiUFnHD+BTOeTbCesCa4sJaK7BGZ7A=="], @@ -397,6 +410,24 @@ "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@smithy/credential-provider-imds": ["@smithy/credential-provider-imds@4.4.8", "", { "dependencies": { "@smithy/core": "^3.29.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-q9J7JTiXrAhB8sDp4px97uEPT7CwKH61Co78grdNQvU8QZAdiuaSRhP0tUVf2ogy36RZTrlMU1rBmDEH+cnkiA=="], + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/core/@aws-sdk/xml-builder": ["@aws-sdk/xml-builder@3.972.37", "", { "dependencies": { "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-zKq4HQum8JwDyEuyfuI4bbiAcU0KxP6qy+9PR/IsR92IyE/DaBAikzAS50tjxip4bqIIANpCcG+Yyj6CVhXupg=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/core/@smithy/signature-v4": ["@smithy/signature-v4@5.6.12", "", { "dependencies": { "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.67", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-rcIpk5kxUqDaaNa6Xk23pQ6ViY7jlqzmfFWCahQcBT97ddXaXYYwzCen9Tz1Jvo6aJft6wDl5bN44/Jw5B4oLA=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.69", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/fetch-http-handler": "^5.6.13", "@smithy/node-http-handler": "^4.9.13", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-nggwJtZ4eeNsUw5IeWBMXsi1ryct5idi0K+/SCRF3kybLubOMaNTb3XCihXpWMiVpyzyPeIrl0zTkzhBH9porA=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.973.12", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/credential-provider-env": "^3.972.67", "@aws-sdk/credential-provider-http": "^3.972.69", "@aws-sdk/credential-provider-login": "^3.972.74", "@aws-sdk/credential-provider-process": "^3.972.67", "@aws-sdk/credential-provider-sso": "^3.973.11", "@aws-sdk/credential-provider-web-identity": "^3.972.73", "@aws-sdk/nested-clients": "^3.997.41", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-pNEf/OeyN5X3VmLKlgSO6TqaWmW10CvI3TfwL1XhsuhYjSLT2VDaxFnCPHnOeQXSaFisMX4jNhpETriqN8DOmg=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.67", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-IlUEejorGTWKb4/Dm7K5Yw4QxUmXLThLhrvBmzVBqZFTbW72cv9LTcITmo1dsnYriALE4h68mOq4LB99x6sQ7Q=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.973.11", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/nested-clients": "^3.997.41", "@aws-sdk/token-providers": "3.1103.0", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-gAQBkBZxUB84d71+pPcI9L+jh2ujhuAVxc/4FgGiWFDjkPBlMKxzd5XDtkSXTFX8Ro7ansnT88+XadasxMeCRw=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.73", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/nested-clients": "^3.997.41", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-SnlEmQa6SjOgs6iOPLUQl1Eyq4AKiAdPQlkOhFhqNfDtDCwibMGvL6QlkSmf3o6vAUSImzdPCxowT5dfQUZP1A=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@smithy/credential-provider-imds": ["@smithy/credential-provider-imds@4.4.16", "", { "dependencies": { "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-QfuLWAkLzptffFW980AFeHZFdqds2B64rpEd3uJ6lgs3xVn9QegGMUgUcj+4d7dRrAsya3r58ZKpku97WcFb4w=="], + "@aws-sdk/client-iam/@aws-sdk/core/@aws-sdk/xml-builder": ["@aws-sdk/xml-builder@3.972.35", "", { "dependencies": { "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-pXzaWe3evZhjxDXAlMnqISe/XefTCGwBJG4nFTXaWSgAnMkqPEhxEPqJNhhpGesEvKFhvNpnozJJ4GTL11bRYw=="], "@aws-sdk/client-iam/@aws-sdk/core/@smithy/signature-v4": ["@smithy/signature-v4@5.6.4", "", { "dependencies": { "@smithy/core": "^3.29.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ=="], @@ -431,6 +462,16 @@ "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.32", "", { "dependencies": { "@aws-sdk/core": "^3.975.2", "@aws-sdk/signature-v4-multi-region": "^3.996.40", "@aws-sdk/types": "^3.974.1", "@smithy/core": "^3.29.3", "@smithy/fetch-http-handler": "^5.6.5", "@smithy/node-http-handler": "^4.9.5", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-6Yj2fr9XF67cndITea48rchTdVr3VGx6PN47bIKNinJAjLkmaIlz/4EBPCgJ8UmhVopiXmeAuPLI3+DXDDbMhQ=="], + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.74", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/nested-clients": "^3.997.41", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-0AQfDcf99TNmqVKv0owHrw/TQs6i4ZE5t9qmz6NvO53bE/sA/tpXhXL9AAcEP1qHc6Zzjd1UMb69+/9zdhvY3g=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.41", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/signature-v4-multi-region": "^3.996.43", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/fetch-http-handler": "^5.6.13", "@smithy/node-http-handler": "^4.9.13", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-RDHqPGQWlF6tatA/Tp3rg6oIwtgN9IVderxE+9av2Y93Dfyu+mO1hZ5Bu2jpfZg2rwdNbsssnwM+sLafIczMlQ=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.41", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/signature-v4-multi-region": "^3.996.43", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/fetch-http-handler": "^5.6.13", "@smithy/node-http-handler": "^4.9.13", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-RDHqPGQWlF6tatA/Tp3rg6oIwtgN9IVderxE+9av2Y93Dfyu+mO1hZ5Bu2jpfZg2rwdNbsssnwM+sLafIczMlQ=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1103.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/nested-clients": "^3.997.41", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-N4wy26MNn31ItGVHYHPrEuCIFY4MBBjC+C5v1lJKqIUSA7OZBdhleCY53zCCrXn27hsk7YNOaTuhQu807S4AfQ=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.41", "", { "dependencies": { "@aws-sdk/core": "^3.977.6", "@aws-sdk/signature-v4-multi-region": "^3.996.43", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.31.1", "@smithy/fetch-http-handler": "^5.6.13", "@smithy/node-http-handler": "^4.9.13", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-RDHqPGQWlF6tatA/Tp3rg6oIwtgN9IVderxE+9av2Y93Dfyu+mO1hZ5Bu2jpfZg2rwdNbsssnwM+sLafIczMlQ=="], + "@aws-sdk/client-iam/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.64", "", { "dependencies": { "@aws-sdk/core": "^3.975.2", "@aws-sdk/nested-clients": "^3.997.32", "@aws-sdk/types": "^3.974.1", "@smithy/core": "^3.29.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-YQoSI4d6kXvoenoG/0Jv/PqaAuukHzGmGXGyHBQYeEUNsYovlNAn/Sw1wp/WQbhcQ3HsEMGgjEahvD3igz6ecQ=="], "@aws-sdk/client-iam/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.32", "", { "dependencies": { "@aws-sdk/core": "^3.975.2", "@aws-sdk/signature-v4-multi-region": "^3.996.40", "@aws-sdk/types": "^3.974.1", "@smithy/core": "^3.29.3", "@smithy/fetch-http-handler": "^5.6.5", "@smithy/node-http-handler": "^4.9.5", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-6Yj2fr9XF67cndITea48rchTdVr3VGx6PN47bIKNinJAjLkmaIlz/4EBPCgJ8UmhVopiXmeAuPLI3+DXDDbMhQ=="], @@ -451,6 +492,12 @@ "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.40", "", { "dependencies": { "@aws-sdk/types": "^3.974.1", "@smithy/signature-v4": "^5.6.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-wrGZ/authosokclY1DXsiWT/1WjfCI22FuZGgdcilF+XLTXs5dCjAtiFYSPsEToZkbm3Lj2YP8PoWg0yoMNu0g=="], + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.43", "", { "dependencies": { "@aws-sdk/types": "^3.974.2", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.43", "", { "dependencies": { "@aws-sdk/types": "^3.974.2", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.43", "", { "dependencies": { "@aws-sdk/types": "^3.974.2", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA=="], + "@aws-sdk/client-iam/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.40", "", { "dependencies": { "@aws-sdk/types": "^3.974.1", "@smithy/signature-v4": "^5.6.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-wrGZ/authosokclY1DXsiWT/1WjfCI22FuZGgdcilF+XLTXs5dCjAtiFYSPsEToZkbm3Lj2YP8PoWg0yoMNu0g=="], "@aws-sdk/client-iam/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.40", "", { "dependencies": { "@aws-sdk/types": "^3.974.1", "@smithy/signature-v4": "^5.6.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-wrGZ/authosokclY1DXsiWT/1WjfCI22FuZGgdcilF+XLTXs5dCjAtiFYSPsEToZkbm3Lj2YP8PoWg0yoMNu0g=="], @@ -463,6 +510,12 @@ "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.4", "", { "dependencies": { "@smithy/core": "^3.29.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ=="], + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.12", "", { "dependencies": { "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.12", "", { "dependencies": { "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ=="], + + "@aws-sdk/client-cloudwatch-logs/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.12", "", { "dependencies": { "@smithy/core": "^3.31.1", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ=="], + "@aws-sdk/client-iam/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.4", "", { "dependencies": { "@smithy/core": "^3.29.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ=="], "@aws-sdk/client-iam/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.4", "", { "dependencies": { "@smithy/core": "^3.29.3", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ=="], diff --git a/package.json b/package.json index c2fa1da53..fd8af96a2 100644 --- a/package.json +++ b/package.json @@ -51,11 +51,13 @@ "dependencies": { "@aws-sdk/client-bedrock-agentcore": "^3.1092.0", "@aws-sdk/client-bedrock-agentcore-control": "^3.1079.0", + "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", "@aws-sdk/client-iam": "^3.1080.0", "@smithy/core": "3.29.3", "@tanstack/react-query": "^5.101.2", "cli-truncate": "^6.1.1", "commander": "^15.0.0", + "handlebars": "^4.7.9", "ink": "^7.1.0", "ink-scroll-view": "^0.3.7", "lodash": "^4.18.1", @@ -65,7 +67,6 @@ "string-width": "^8.2.2", "winston": "^3.19.0", "winston-daily-rotate-file": "^5.0.0", - "handlebars": "^4.7.9", "zod": "^4.4.3" } } diff --git a/src/core/batchEvaluationResults.test.ts b/src/core/batchEvaluationResults.test.ts new file mode 100644 index 000000000..b1e5a7b00 --- /dev/null +++ b/src/core/batchEvaluationResults.test.ts @@ -0,0 +1,167 @@ +import { test, expect } from "bun:test"; +import type { CloudWatchLogsClient, OutputLogEvent } from "@aws-sdk/client-cloudwatch-logs"; +import { createSilentLogger } from "../testing"; +import { + isTerminalStatus, + parseEvaluationLogEvent, + readEvaluationResults, +} from "./batchEvaluationResults"; + +// fakeLogs returns a CloudWatchLogsClient that serves `events` as a single page, +// then signals exhaustion by echoing the same nextForwardToken on the next call — +// exactly how GetLogEvents ends pagination. Records the tokens it was called with. +function fakeLogs(events: OutputLogEvent[]): CloudWatchLogsClient { + let served = false; + return { + send: async () => { + if (!served) { + served = true; + return { events, nextForwardToken: "t-end" }; + } + return { events: [], nextForwardToken: "t-end" }; // token unchanged → done + }, + } as unknown as CloudWatchLogsClient; +} + +// fakePagedLogs serves each element of `pages` on successive calls, advancing the +// forward token per page and repeating the last token once to end. Captures every +// nextToken the caller sent, so a test can assert the loop paged correctly. +function fakePagedLogs(pages: OutputLogEvent[][]): { + client: CloudWatchLogsClient; + tokens: (string | undefined)[]; +} { + const tokens: (string | undefined)[] = []; + let call = 0; + const client = { + send: async (command: { input: { nextToken?: string } }) => { + tokens.push(command.input.nextToken); + const i = call++; + if (i < pages.length) return { events: pages[i], nextForwardToken: `t-${i}` }; + return { events: [], nextForwardToken: `t-${pages.length - 1}` }; // repeat last → done + }, + } as unknown as CloudWatchLogsClient; + return { client, tokens }; +} + +// A realistic stream shaped after the real `gen_ai.evaluation.result` records +// (see the recorded fixture below): the level is +// attributes["aws.bedrock_agentcore.evaluation_level"] (Title-case), session.id +// sits under attributes, and the trace id is the top-level camelCase `traceId`. +// One SESSION-level and one TRACE-level record, plus a non-JSON control line. +const EVENTS: OutputLogEvent[] = [ + { + message: JSON.stringify({ + attributes: { + "gen_ai.evaluation.name": "Builtin.Helpfulness", + "aws.bedrock_agentcore.evaluation_level": "Session", + "session.id": "session-orders-123", + "gen_ai.evaluation.score.value": 5, + "gen_ai.evaluation.score.label": "helpful", + "gen_ai.evaluation.explanation": "Directly answered with tracking detail.", + }, + }), + }, + { + message: JSON.stringify({ + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + attributes: { + "gen_ai.evaluation.name": "Builtin.Faithfulness", + "aws.bedrock_agentcore.evaluation_level": "Trace", + "session.id": "session-orders-123", + "gen_ai.evaluation.score.value": 4, + "gen_ai.evaluation.score.label": "faithful", + "gen_ai.evaluation.explanation": "Grounded in the tool result.", + }, + }), + }, + { message: "AWS log control line, not JSON" }, +]; + +test("isTerminalStatus recognizes the terminal arm only", () => { + for (const s of ["COMPLETED", "COMPLETED_WITH_ERRORS", "FAILED", "STOPPED"]) { + expect(isTerminalStatus(s)).toBe(true); + } + for (const s of ["IN_PROGRESS", "PENDING", "STOPPING", "DELETING", undefined]) { + expect(isTerminalStatus(s)).toBe(false); + } +}); + +test("readEvaluationResults keeps level + scope so sessions and traces are distinguishable", async () => { + const results = await readEvaluationResults(fakeLogs(EVENTS), "lg", "ls", createSilentLogger()); + + // The non-JSON control line is skipped; the two evaluation records parse. + expect(results).toHaveLength(2); + expect(results[0]).toMatchObject({ + evaluatorId: "Builtin.Helpfulness", + level: "Session", + sessionId: "session-orders-123", + score: 5, + label: "helpful", + }); + expect(results[0]?.traceId).toBeUndefined(); + expect(results[1]).toMatchObject({ + evaluatorId: "Builtin.Faithfulness", + level: "Trace", + sessionId: "session-orders-123", + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }); + expect(results.map((r) => r.level)).toEqual(["Session", "Trace"]); +}); + +test("readEvaluationResults follows pagination until the forward token stops advancing", async () => { + const page = (name: string): OutputLogEvent => ({ + message: JSON.stringify({ + attributes: { + "gen_ai.evaluation.name": name, + "aws.bedrock_agentcore.evaluation_level": "Trace", + "session.id": "s1", + }, + }), + }); + const { client, tokens } = fakePagedLogs([ + [page("Builtin.Correctness")], + [page("Builtin.Helpfulness")], + [page("Builtin.Faithfulness")], + ]); + + const results = await readEvaluationResults(client, "lg", "ls", createSilentLogger()); + + // All three pages' records are collected. + expect(results.map((r) => r.evaluatorId)).toEqual([ + "Builtin.Correctness", + "Builtin.Helpfulness", + "Builtin.Faithfulness", + ]); + // First call has no token; later calls carry the prior page's forward token; a + // final call detects the repeated token and stops. + expect(tokens).toEqual([undefined, "t-0", "t-1", "t-2"]); +}); + +// Real-log-shape validation lives in the fixture-backed command-flow test +// (batch-evaluation.fixture.test.tsx), where RECORD=1 captures a live GetLogEvents +// response and matchGolden pins the parsed output. This file stays a pure unit +// test over inline synthetic events, matching the rest of src/core. + +test("readEvaluationResults skips lines without an evaluation name", async () => { + const results = await readEvaluationResults( + fakeLogs([ + { message: JSON.stringify({ attributes: { "some.other.metric": 1 } }) }, + { message: "" }, + { message: undefined }, + ]), + "lg", + "ls", + createSilentLogger(), + ); + expect(results).toEqual([]); +}); + +test("parseEvaluationLogEvent warns on and skips an unparseable line", () => { + const warnings: string[] = []; + const logger = createSilentLogger(); + logger.warn = (...msgs: string[]) => warnings.push(msgs.join(" ")); + + expect(parseEvaluationLogEvent("AWS log control line, not JSON", logger)).toBeNull(); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("unparseable"); +}); diff --git a/src/core/batchEvaluationResults.tsx b/src/core/batchEvaluationResults.tsx new file mode 100644 index 000000000..b576c6aff --- /dev/null +++ b/src/core/batchEvaluationResults.tsx @@ -0,0 +1,117 @@ +import { GetLogEventsCommand, type CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import type { BatchEvaluationResultEntry } from "../handlers/eval/types"; +import type { Logger } from "../logging"; + +// Per-session batch-evaluation result retrieval, mirroring +// core/onlineEvalExecutionRole.tsx's pattern: a self-contained module that takes +// an injected AWS client (here CloudWatchLogsClient) and owns one slice of Core's +// behavior. A completed batch evaluation writes each score as an OTel-shaped log +// record to a per-job CloudWatch stream; this module reads that stream and parses +// the records. EvalClient calls readEvaluationResults with the client from +// `this.clients.logs(...)` and the log group + stream from the job's outputConfig. + +// Terminal batch-evaluation statuses — after these, results are final and worth +// retrieving. Mirrors the AgentCore BatchEvaluationStatus enum's terminal arm. +const TERMINAL_STATUSES = new Set(["COMPLETED", "COMPLETED_WITH_ERRORS", "FAILED", "STOPPED"]); + +export function isTerminalStatus(status?: string): boolean { + return !!status && TERMINAL_STATUSES.has(status); +} + +// GetLogEvents returns at most 1 MB / 10,000 events per call, so a job with many +// results spans multiple pages. This caps the page loop as a safety valve against +// a non-advancing token (see below); at 10k events/page it allows ~1M results, +// far beyond the 500-session job limit. +const MAX_RESULT_PAGES = 100; + +// readEvaluationResults reads and parses the per-session/-trace/-tool scores from +// a completed batch evaluation's CloudWatch result stream, following pagination to +// completion. The caller supplies the log group and stream name from the job's +// GetBatchEvaluation outputConfig (the service-selected values — we do not derive +// the stream name, since its format is not part of the SDK contract). +export async function readEvaluationResults( + logs: CloudWatchLogsClient, + logGroupName: string, + logStreamName: string, + logger: Logger, +): Promise { + const results: BatchEvaluationResultEntry[] = []; + + // Page forward from the head. GetLogEvents echoes the input token back as + // nextForwardToken once the stream is exhausted, so the loop ends when the + // token stops advancing. startFromHead is only honored on the first call (no + // token); subsequent calls are positioned by the token. + let token: string | undefined; + for (let page = 0; page < MAX_RESULT_PAGES; page++) { + const response = await logs.send( + new GetLogEventsCommand({ + logGroupName, + logStreamName, + startFromHead: true, + nextToken: token, + }), + ); + + for (const event of response.events ?? []) { + if (!event.message) continue; + const entry = parseEvaluationLogEvent(event.message, logger); + if (entry) results.push(entry); + } + + const next = response.nextForwardToken; + if (!next || next === token) return results; // exhausted: token stopped advancing + token = next; + } + + logger.warn( + `stopped reading batch-evaluation results after ${MAX_RESULT_PAGES} pages; results may be truncated`, + ); + return results; +} + +// parseEvaluationLogEvent turns one CloudWatch result-log message into a result +// entry, or null for non-JSON / non-evaluation lines (log control lines, blank +// messages). AgentCore emits each score as an OTel-shaped log record named +// `gen_ai.evaluation.result`: the `gen_ai.evaluation.*` scores and the +// `session.id` live under `attributes`, the trace id is the top-level `traceId`, +// and the level is `attributes["aws.bedrock_agentcore.evaluation_level"]` (e.g. +// "Trace" / "Session"). Field names verified against a recorded result stream +// (src/core/__fixtures__/batch-eval-result-log-events.json). We keep `level` and +// the ids — the old CLI dropped them, flattening every level into one list. +export function parseEvaluationLogEvent( + message: string, + logger: Logger, +): BatchEvaluationResultEntry | null { + let parsed: Record; + try { + parsed = JSON.parse(message) as Record; + } catch { + // Swallow rather than throw: CloudWatch result streams interleave non-JSON + // control lines with the evaluation records, so one unparseable line is + // expected noise — failing here would drop every result for the job over it. + // Warn (not silent) so a systematic format change is still visible in logs. + logger.warn("skipping unparseable batch-evaluation result log line"); + return null; + } + const attrs = (parsed["attributes"] ?? {}) as Record; + const evaluatorId = attrs["gen_ai.evaluation.name"] as string | undefined; + if (!evaluatorId) return null; + + const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined); + // Keys are read verbatim as the recorded stream emits them: scores, level, + // session id, and tool name under `attributes`; traceId/spanId at the top + // level. If AgentCore renames one, the corresponding field goes undefined and + // the fixture-replay test fails — the signal to update the key here. + return { + evaluatorId, + level: str(attrs["aws.bedrock_agentcore.evaluation_level"]), + sessionId: str(attrs["session.id"]), + traceId: str(parsed["traceId"]), + spanId: str(parsed["spanId"]), + toolName: str(attrs["gen_ai.tool.name"]), + score: attrs["gen_ai.evaluation.score.value"] as number | undefined, + label: str(attrs["gen_ai.evaluation.score.label"]), + explanation: str(attrs["gen_ai.evaluation.explanation"]), + error: str(attrs["gen_ai.evaluation.error"]), + }; +} diff --git a/src/core/core.test.ts b/src/core/core.test.ts index 4b51b7674..81ba600d8 100644 --- a/src/core/core.test.ts +++ b/src/core/core.test.ts @@ -4,6 +4,7 @@ import { type BedrockAgentCoreControlClient, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { IAMClient } from "@aws-sdk/client-iam"; +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import { GetEventCommand, GetMemoryRecordCommand, @@ -76,6 +77,9 @@ function fakeData(config: ClientConfig): BedrockAgentCoreClient { function fakeIam(config: ClientConfig): IAMClient { return { config, kind: "iam" } as unknown as IAMClient; } +function fakeLogs(config: ClientConfig): CloudWatchLogsClient { + return { config, kind: "logs" } as unknown as CloudWatchLogsClient; +} function coreWithDataSend( send: (command: unknown, options: unknown) => Promise, @@ -86,6 +90,7 @@ function coreWithDataSend( createDataClient: (config) => ({ config, kind: "data", send }) as unknown as BedrockAgentCoreClient, createIamClient: fakeIam, + createLogsClient: fakeLogs, logger, }); } @@ -121,6 +126,7 @@ function customJwtCore( }, }) as unknown as BedrockAgentCoreClient, createIamClient: fakeIam, + createLogsClient: fakeLogs, fetch, logger, }); @@ -135,6 +141,7 @@ test("control() constructs a client once per config and caches it", () => { }, createDataClient: fakeData, createIamClient: fakeIam, + createLogsClient: fakeLogs, logger: createSilentLogger(), }); @@ -154,6 +161,7 @@ test("control() builds a distinct client per distinct config", () => { }, createDataClient: fakeData, createIamClient: fakeIam, + createLogsClient: fakeLogs, logger: createSilentLogger(), }); @@ -177,6 +185,7 @@ test("data() caches independently of control()", () => { return fakeData(config); }, createIamClient: fakeIam, + createLogsClient: fakeLogs, logger: createSilentLogger(), }); @@ -194,6 +203,7 @@ test("exposes feature sub-clients", () => { createControlClient: fakeControl, createDataClient: fakeData, createIamClient: fakeIam, + createLogsClient: fakeLogs, logger: createSilentLogger(), }); expect(core.harness).toBeDefined(); @@ -346,6 +356,7 @@ test("getRuntime sends the abort signal to the control client", async () => { }) as unknown as BedrockAgentCoreControlClient, createDataClient: fakeData, createIamClient: fakeIam, + createLogsClient: fakeLogs, logger: createSilentLogger(), }); const controller = new AbortController(); @@ -380,6 +391,7 @@ test("invokeHarness sends an InvokeHarnessCommand on the data client with the ab } as unknown as BedrockAgentCoreClient; }, createIamClient: fakeIam, + createLogsClient: fakeLogs, logger: createSilentLogger(), }); diff --git a/src/core/datasetDownload.test.ts b/src/core/datasetDownload.test.ts index b48200991..790d9a22c 100644 --- a/src/core/datasetDownload.test.ts +++ b/src/core/datasetDownload.test.ts @@ -32,7 +32,7 @@ function stubClients(dataset: Record): AwsClients { throw new Error(`unexpected command: ${(command as object).constructor.name}`); }; const client = { send } as never; - return { control: () => client, data: () => client, iam: () => client }; + return { control: () => client, data: () => client, iam: () => client, logs: () => client }; } describe("EvalClient.downloadDataset", () => { diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 964c11f48..2cb86871c 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -37,19 +37,28 @@ import { type UpdateOnlineEvaluationConfigResponse, type BedrockAgentCoreControlClient, } from "@aws-sdk/client-bedrock-agentcore-control"; +import { + GetBatchEvaluationCommand, + ListBatchEvaluationsCommand, + type ListBatchEvaluationsResponse, +} from "@aws-sdk/client-bedrock-agentcore"; import { Transform } from "node:stream"; import { FileWriteError, InputValidationError, NetworkingError } from "../errors"; import type { + BatchEvaluationDetail, CodeBasedUpdate, RoleScopeWarning, CoreEvalClient, CreateDatasetInput, CreateOnlineEvalInput, + GetBatchEvaluationResult, LlmAsAJudgeUpdate, UpdateOnlineEvalInput, } from "../handlers/eval/types"; import { atomicWriteStream } from "../io"; +import { isTerminalStatus, readEvaluationResults } from "./batchEvaluationResults"; import type { AwsClients, CoreFetch, CoreOptions } from "./types"; +import type { Logger } from "../logging"; import { toClientConfig } from "./utils"; import { accountIdFromRoleArn, @@ -62,11 +71,24 @@ import { const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; +// noopLogger is the default for the optional logger arg so callers that don't +// need batch-evaluation result-log diagnostics (e.g. dataset-only tests) can +// omit it. Production (src/core/index.tsx) injects a real child logger. +const noopLogger: Logger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + child: () => noopLogger, +}; + export class EvalClient implements CoreEvalClient { constructor( private readonly clients: AwsClients, // HTTP client for datasets presigned S3 URL private readonly fetch: CoreFetch = globalThis.fetch, + // logger for batch-evaluation result-log diagnostics + private readonly logger: Logger = noopLogger, ) {} async createEvaluator( @@ -203,6 +225,61 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteEvaluatorCommand({ evaluatorId: id })); } + // getBatchEvaluation returns the service-side job (status + evaluator summaries + // + CloudWatch output config) and, by default, the per-session results read from + // the job's CloudWatch stream once it is terminal. Batch evaluation lives on the + // data plane, not control. + // + // Returns `{ detail, resultsError? }` rather than merging silently: a CloudWatch + // read failure must never hide the job status, and Core has no stderr to warn on, + // so it surfaces the error to the caller (the handler warns; the TUI ignores it). + // `includeResults: false` (the CLI's --disable-cw-results) skips the CloudWatch + // read entirely and returns metadata only. + async getBatchEvaluation( + id: string, + options: CoreOptions, + { includeResults = true }: { includeResults?: boolean } = {}, + ): Promise { + const job = await this.clients + .data(toClientConfig(options)) + .send(new GetBatchEvaluationCommand({ batchEvaluationId: id })); + + const detail: BatchEvaluationDetail = { ...job }; + const cw = job.outputConfig?.cloudWatchConfig; + if ( + !includeResults || + !isTerminalStatus(job.status) || + !cw?.logGroupName || + !cw.logStreamName + ) { + return { detail }; + } + + try { + detail.results = await readEvaluationResults( + this.clients.logs({ region: options.region }), + cw.logGroupName, + cw.logStreamName, + this.logger, + ); + return { detail }; + } catch (resultsError) { + // Return the metadata regardless — the caller decides how to surface the + // CloudWatch failure (stderr warning in the CLI, ignored in the TUI). + return { detail, resultsError }; + } + } + + async listBatchEvaluations( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new ListBatchEvaluationsCommand({ nextToken, maxResults })); + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/core/factories.tsx b/src/core/factories.tsx index 1cbc8eb11..1f2364dcf 100644 --- a/src/core/factories.tsx +++ b/src/core/factories.tsx @@ -1,7 +1,13 @@ import { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; import { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import { IAMClient } from "@aws-sdk/client-iam"; -import type { CreateControlClient, CreateDataClient, CreateIamClient } from "./types"; +import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import type { + CreateControlClient, + CreateDataClient, + CreateIamClient, + CreateLogsClient, +} from "./types"; // createControlClient / createDataClient are the production factories injected // into CoreClient at the app edge (src/index.ts). They live here — rather than @@ -15,3 +21,6 @@ export const createDataClient: CreateDataClient = (config) => new BedrockAgentCoreClient({ ...config }); export const createIamClient: CreateIamClient = (config) => new IAMClient({ ...config }); + +export const createLogsClient: CreateLogsClient = (config) => + new CloudWatchLogsClient({ ...config }); diff --git a/src/core/index.tsx b/src/core/index.tsx index f99f9347e..0928dc247 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -1,6 +1,7 @@ import { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; import { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import { IAMClient } from "@aws-sdk/client-iam"; +import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import { EvalClient } from "./eval"; import { GatewayClient } from "./gateway"; import { HarnessClient } from "./harness"; @@ -14,6 +15,7 @@ import type { CreateControlClient, CreateDataClient, CreateIamClient, + CreateLogsClient, } from "./types"; import type { Logger } from "../logging"; import type { ProjectManager } from "../handlers/project/types"; @@ -26,12 +28,14 @@ export type { CreateControlClient, CreateDataClient, CreateIamClient, + CreateLogsClient, } from "./types"; type CoreClientConfig = { createControlClient: CreateControlClient; createDataClient: CreateDataClient; createIamClient: CreateIamClient; + createLogsClient: CreateLogsClient; logger: Logger; fetch?: CoreFetch; }; @@ -44,10 +48,12 @@ export class CoreClient implements AwsClients { private controlClients = new Map(); private dataClients = new Map(); private iamClients = new Map(); + private logsClients = new Map(); private readonly createControlClient: CreateControlClient; private readonly createDataClient: CreateDataClient; private readonly createIamClient: CreateIamClient; + private readonly createLogsClient: CreateLogsClient; private logger: Logger; // Feature-scoped sub-clients. Access as e.g. `coreClient.harness.getHarness(...)`. @@ -64,12 +70,14 @@ export class CoreClient implements AwsClients { this.createControlClient = config.createControlClient; this.createDataClient = config.createDataClient; this.createIamClient = config.createIamClient; + this.createLogsClient = config.createLogsClient; this.logger = config.logger; const fetch = config.fetch ?? globalThis.fetch; this.runtime = new RuntimeClient(this, fetch, this.logger.child({ module: "runtime" })); // EvalClient shares the injected fetch: dataset content is served from a - // presigned S3 URL, outside the SDK seam the other operations use. - this.eval = new EvalClient(this, fetch); + // presigned S3 URL, outside the SDK seam the other operations use. The logger + // is used for batch-evaluation result-log diagnostics. + this.eval = new EvalClient(this, fetch, this.logger.child({ module: "eval" })); this.projectManager = new FsProjectManager({ logger: this.logger.child({ module: "projectManager" }), @@ -111,6 +119,18 @@ export class CoreClient implements AwsClients { } return client; } + + // logs returns the CloudWatch Logs client for `config`, creating and caching it + // on first use (used to read batch-evaluation result log streams). + logs(config: ClientConfig): CloudWatchLogsClient { + const key = cacheKey(config); + let client = this.logsClients.get(key); + if (!client) { + client = this.createLogsClient(config); + this.logsClients.set(key, client); + } + return client; + } } // cacheKey derives a stable cache key from a ClientConfig so that distinct diff --git a/src/core/types.tsx b/src/core/types.tsx index 41eaf2c4b..98a2f338d 100644 --- a/src/core/types.tsx +++ b/src/core/types.tsx @@ -1,6 +1,7 @@ import type { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; import type { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import type { IAMClient } from "@aws-sdk/client-iam"; +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; // CoreOptions is the standard trailing argument for Core operations. It carries // the per-call settings a handler resolves from context (the AWS region and an @@ -25,6 +26,7 @@ export interface ClientConfig { export type CreateControlClient = (config: ClientConfig) => BedrockAgentCoreControlClient; export type CreateDataClient = (config: ClientConfig) => BedrockAgentCoreClient; export type CreateIamClient = (config: ClientConfig) => IAMClient; +export type CreateLogsClient = (config: ClientConfig) => CloudWatchLogsClient; export type CoreFetch = ( ...args: Parameters ) => ReturnType; @@ -38,4 +40,8 @@ export interface AwsClients { control(config: ClientConfig): BedrockAgentCoreControlClient; data(config: ClientConfig): BedrockAgentCoreClient; iam(config: ClientConfig): IAMClient; + // logs reads the CloudWatch Logs streams AgentCore writes batch-evaluation + // results to. CloudWatch is a distinct service from the AgentCore data plane, + // so it gets its own client/factory rather than reusing `data`. + logs(config: ClientConfig): CloudWatchLogsClient; } diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.1e79922919dcb237.json b/src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.1e79922919dcb237.json new file mode 100644 index 000000000..95c8d402c --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.1e79922919dcb237.json @@ -0,0 +1,72 @@ +{ + "batchEvaluationId": "GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationName": "GTProbe2_1786034545579", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-06T16:42:26.468Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + }, + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "repro1427_ReproAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/repro1427_ReproAgent-zqNEUl8I0f-DEFAULT" + ], + "filterConfig": { + "timeRange": { + "startTime": { + "$date": "2026-07-29T00:00:00.000Z" + }, + "endTime": { + "$date": "2026-07-30T00:00:00.000Z" + } + } + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-GTProbe2_1786034545579-8ffefc851e" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 9, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 9, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 9, + "totalFailed": 0 + }, + { + "evaluatorId": "Builtin.Correctness", + "statistics": { + "averageScore": 1 + }, + "totalEvaluated": 9, + "totalFailed": 0 + } + ] + }, + "description": "GT probe #2: repro1427 recent sessions; confirm GT/assertion/trajectory attributes in result logs", + "updatedAt": { + "$date": "2026-08-06T16:43:30.696Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.94c2cede863d8fbd.json b/src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.94c2cede863d8fbd.json new file mode 100644 index 000000000..b279e53df --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/GetBatchEvaluationCommand.94c2cede863d8fbd.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ResourceNotFoundException", + "message": "Failed to retrieve the resource because it doesn't exist. Retry the request with a different resource identifier." + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.1bfa6f1de138b98c.json b/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.1bfa6f1de138b98c.json new file mode 100644 index 000000000..2b5b6f0e0 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.1bfa6f1de138b98c.json @@ -0,0 +1,96 @@ +{ + "events": [ + { + "timestamp": 1786034558088, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4b30737b856660b6a2c37e1c4b93\",\"timeUnixNano\":1785350969610484753,\"observedTimeUnixNano\":1786034558088131554,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"baseline-session-3-1785350958-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4b30737b856660b6a2c37e1c4b93\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The user asked to add 17 and 25 using the add_numbers tool and report the result. The correct mathematical answer is 17 + 25 = 42. The agent responded that the result is 42, which is mathematically correct. The expected response indicates the assistant should provide a correct, helpful, well-grounded answer, which the agent did. The agent correctly identified the sum as 42.\",\"gen_ai.evaluation.score.label\":\"Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Correct\",\"_aws\":{\"Timestamp\":1785350969610,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034558106 + }, + { + "timestamp": 1786034558088, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4b30737b856660b6a2c37e1c4b93\",\"timeUnixNano\":1785350969610484753,\"observedTimeUnixNano\":1786034558088295936,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"baseline-session-3-1785350958-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4b30737b856660b6a2c37e1c4b93\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user. The response is concise, accurate (based on tool output), and directly addresses what the user asked for. The user's goal has been fully achieved with no unnecessary detours or missing information.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785350969610,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034558106 + }, + { + "timestamp": 1786034558536, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4b1806f680cd2ed2308c1062be9b\",\"timeUnixNano\":1785350947350383874,\"observedTimeUnixNano\":1786034558536781180,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"baseline-session-1-1785350935-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4b1806f680cd2ed2308c1062be9b\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The user asked to use the add_numbers tool to add 17 and 25 and report the result. The correct mathematical answer is 17 + 25 = 42. The agent responded that the result is 42, which is mathematically correct. The expected response indicates the assistant should provide a correct, helpful, well-grounded answer, which the agent did. The agent correctly identified the sum as 42.\",\"gen_ai.evaluation.score.label\":\"Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Correct\",\"_aws\":{\"Timestamp\":1785350947350,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034558552 + }, + { + "timestamp": 1786034558536, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4b1806f680cd2ed2308c1062be9b\",\"timeUnixNano\":1785350947350383874,\"observedTimeUnixNano\":1786034558536926460,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"baseline-session-1-1785350935-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4b1806f680cd2ed2308c1062be9b\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant accomplished exactly this - it called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user in a concise, well-formatted response. The response directly addresses what the user asked for, provides the correct result from the tool, and does so efficiently without unnecessary information. This fully satisfies the user's request.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785350947350,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034558552 + }, + { + "timestamp": 1786034558732, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4b24480148ae03510a1f2dee038b\",\"timeUnixNano\":1785350958709315499,\"observedTimeUnixNano\":1786034558732530458,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"baseline-session-2-1785350947-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4b24480148ae03510a1f2dee038b\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The user asked to use the add_numbers tool to add 17 and 25 and report the result. The correct mathematical answer is 17 + 25 = 42. The agent responded that the result is 42, which is mathematically correct. The expected response indicates the assistant should provide a correct, helpful, well-grounded answer, which the agent did. The agent correctly identified the sum as 42.\",\"gen_ai.evaluation.score.label\":\"Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Correct\",\"_aws\":{\"Timestamp\":1785350958709,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034558746 + }, + { + "timestamp": 1786034558732, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4b24480148ae03510a1f2dee038b\",\"timeUnixNano\":1785350958709315499,\"observedTimeUnixNano\":1786034558732663684,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"baseline-session-2-1785350947-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4b24480148ae03510a1f2dee038b\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant did exactly this - it called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user. The response is concise, accurate (based on tool output), and directly addresses what the user asked for. The user's goal has been fully achieved with no unnecessary detours or missing information. This is a complete and satisfying response to the user's request.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785350958709,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034558746 + }, + { + "timestamp": 1786034558856, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4f7157db99706f811b26729cf203\",\"timeUnixNano\":1785352062222230500,\"observedTimeUnixNano\":1786034558856871209,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"mutated-session-1-1785352047-aaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4f7157db99706f811b26729cf203\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1785352062222,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034558870 + }, + { + "timestamp": 1786034558857, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4f7157db99706f811b26729cf203\",\"timeUnixNano\":1785352062222230500,\"observedTimeUnixNano\":1786034558857037578,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"mutated-session-1-1785352047-aaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4f7157db99706f811b26729cf203\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, direct manner. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly. This fully satisfies the user's request. There's nothing more needed here - the response is complete and accurate based on the tool output.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785352062222,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034558870 + }, + { + "timestamp": 1786034558889, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4f9c162d822072bf7b9c024376e0\",\"timeUnixNano\":1785352104104347385,\"observedTimeUnixNano\":1786034558889019111,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"mutated-session-3-1785352090-aaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4f9c162d822072bf7b9c024376e0\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which correctly reflects the tool output. This is factually accurate and directly addresses the task.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1785352104104,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034558902 + }, + { + "timestamp": 1786034558889, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4f9c162d822072bf7b9c024376e0\",\"timeUnixNano\":1785352104104347385,\"observedTimeUnixNano\":1786034558889190490,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"mutated-session-3-1785352090-aaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4f9c162d822072bf7b9c024376e0\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, well-formatted response. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly with bold formatting for emphasis. This fully satisfies the user's request without any unnecessary detours or missing information.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785352104104,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034558902 + }, + { + "timestamp": 1786034558922, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4d117acbcba224bffcdb1451334f\",\"timeUnixNano\":1785351450524248389,\"observedTimeUnixNano\":1786034558922514461,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"baseline-session-5-1785351439-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4d117acbcba224bffcdb1451334f\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1785351450524,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034558934 + }, + { + "timestamp": 1786034558922, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4d117acbcba224bffcdb1451334f\",\"timeUnixNano\":1785351450524248389,\"observedTimeUnixNano\":1786034558922666486,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"baseline-session-5-1785351439-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4d117acbcba224bffcdb1451334f\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant did exactly this - it called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user. The response is concise, accurate (based on tool output), and directly addresses what the user asked for. The user's goal has been fully achieved with no unnecessary detours or missing information. This is a complete and satisfying response to the user's request.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785351450524,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034558934 + }, + { + "timestamp": 1786034558945, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4d066e5a0bd8430a5b5937d51a4a\",\"timeUnixNano\":1785351439575170065,\"observedTimeUnixNano\":1786034558945178340,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"baseline-session-4-1785351428-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4d066e5a0bd8430a5b5937d51a4a\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1785351439575,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034558958 + }, + { + "timestamp": 1786034558945, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4d066e5a0bd8430a5b5937d51a4a\",\"timeUnixNano\":1785351439575170065,\"observedTimeUnixNano\":1786034558945323156,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"baseline-session-4-1785351428-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4d066e5a0bd8430a5b5937d51a4a\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, well-formatted response. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly. This fully satisfies the user's request without any unnecessary detours or missing information.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785351439575,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034558958 + }, + { + "timestamp": 1786034559026, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4f8708321dbf4e1235643d19eb3b\",\"timeUnixNano\":1785352082652219644,\"observedTimeUnixNano\":1786034559026221816,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"mutated-session-2-1785352070-aaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4f8708321dbf4e1235643d19eb3b\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which correctly reflects the tool output. This is factually accurate and directly addresses the task.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1785352082652,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034559035 + }, + { + "timestamp": 1786034559026, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4f8708321dbf4e1235643d19eb3b\",\"timeUnixNano\":1785352082652219644,\"observedTimeUnixNano\":1786034559026342342,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"mutated-session-2-1785352070-aaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4f8708321dbf4e1235643d19eb3b\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, well-formatted response. The answer directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the result was accurately reported. This fully satisfies the user's request with no unnecessary detours or missing information.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785352082652,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034559035 + }, + { + "timestamp": 1786034559062, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4d1c17cb3b204b884a0647affe2b\",\"timeUnixNano\":1785351462078069385,\"observedTimeUnixNano\":1786034559062951711,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"baseline-session-6-1785351450-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4d1c17cb3b204b884a0647affe2b\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1785351462078,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1786034559078 + }, + { + "timestamp": 1786034559063, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"repro1427_ReproAgent.DEFAULT\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\"}},\"traceId\":\"6a6a4d1c17cb3b204b884a0647affe2b\",\"timeUnixNano\":1785351462078069385,\"observedTimeUnixNano\":1786034559063088544,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"baseline-session-6-1785351450-aaaaaaaaaaaaaaaaaaaaaaa\",\"gen_ai.response.id\":\"6a6a4d1c17cb3b204b884a0647affe2b\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, direct manner. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly. This fully satisfies the user's request without any unnecessary detours or missing information. The response is complete and accurate based on the tool output.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluation_job.name\":\"GTProbe2_1786034545579\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"GTProbe2_1786034545579-8ffefc851e\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e\",\"runId\":\"GTProbe2_1786034545579-8ffefc851e\",\"service.name\":\"repro1427_ReproAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1785351462078,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1786034559078 + } + ], + "nextForwardToken": "f/39829901617433955900254553718948066971636896393693822977/s", + "nextBackwardToken": "b/39829901595690729331687196154775929760075568623843803136/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.5bd3c931772927ac.json b/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.5bd3c931772927ac.json new file mode 100644 index 000000000..4e2b049b6 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.5bd3c931772927ac.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39829901617445106270694635737324671751809671794238914559/s", + "nextBackwardToken": "b/39829901617433955900254553718948066971636896393693822977/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.f3f79313bdec0f23.json b/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.f3f79313bdec0f23.json new file mode 100644 index 000000000..6b2865417 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/GetLogEventsCommand.f3f79313bdec0f23.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39829901617445106270694635737324671751809671794238914559/s", + "nextBackwardToken": "b/39829901617445106270694635737324671751809671794238914559/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/batch-evaluation/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json new file mode 100644 index 000000000..d803ed804 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json @@ -0,0 +1,747 @@ +{ + "batchEvaluations": [ + { + "batchEvaluationId": "ABBatchEvalLatest-a7b8db0f3d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/ABBatchEvalLatest-a7b8db0f3d", + "batchEvaluationName": "ABBatchEvalLatest", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-17T22:15:50.980Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-06-17T22:16:55.626Z" + } + }, + { + "batchEvaluationId": "ABBatchEvalVfy-f39e3e215c", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/ABBatchEvalVfy-f39e3e215c", + "batchEvaluationName": "ABBatchEvalVfy", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-17T20:41:07.938Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-06-17T20:42:11.819Z" + } + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282542120-bc78b8200c", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282542120-bc78b8200c", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282542120", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-12T16:42:23.077Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": { + "$date": "2026-06-12T16:42:27.934Z" + } + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282564749-2116d5dcff", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282564749-2116d5dcff", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282564749", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-12T16:42:45.552Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": { + "$date": "2026-06-12T16:42:48.388Z" + } + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282587195-6953163407", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282587195-6953163407", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282587195", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-12T16:43:08.112Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": { + "$date": "2026-06-12T16:43:11.487Z" + } + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282609720-466418f536", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282609720-466418f536", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282609720", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-12T16:43:30.992Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": { + "$date": "2026-06-12T16:43:33.973Z" + } + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282632632-6e14e4d8e0", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282632632-6e14e4d8e0", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282632632", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-12T16:43:53.618Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": { + "$date": "2026-06-12T16:43:56.416Z" + } + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282655261-68c5d69e9b", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282655261-68c5d69e9b", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282655261", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-12T16:44:16.201Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": { + "$date": "2026-06-12T16:45:20.791Z" + } + }, + { + "batchEvaluationId": "GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationName": "GTProbe2_1786034545579", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-06T16:42:26.468Z" + }, + "description": "GT probe #2: repro1427 recent sessions; confirm GT/assertion/trajectory attributes in result logs", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + }, + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-08-06T16:43:30.696Z" + } + }, + { + "batchEvaluationId": "GTProbe_1786033993684-52f5045e6a", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe_1786033993684-52f5045e6a", + "batchEvaluationName": "GTProbe_1786033993684", + "status": "FAILED", + "createdAt": { + "$date": "2026-08-06T16:33:14.495Z" + }, + "description": "GT probe: confirm whether GT/assertion/trajectory outcomes surface as result-log attributes", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + }, + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "errorDetails": [ + "All 4 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-08-06T16:33:18.035Z" + } + }, + { + "batchEvaluationId": "HarnessBatchEvalCodex_20260723_1824-a6e5e74a3b", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/HarnessBatchEvalCodex_20260723_1824-a6e5e74a3b", + "batchEvaluationName": "HarnessBatchEvalCodex_20260723_1824", + "status": "FAILED", + "createdAt": { + "$date": "2026-07-23T18:23:42.907Z" + }, + "description": "Real test: batch-evaluate one session produced by InvokeHarness", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "errorDetails": [ + "All 1 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-07-23T18:24:45.539Z" + } + }, + { + "batchEvaluationId": "HarnessBatchEvalRetry_20260723_1827-72f41d37b1", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/HarnessBatchEvalRetry_20260723_1827-72f41d37b1", + "batchEvaluationName": "HarnessBatchEvalRetry_20260723_1827", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-23T18:26:50.667Z" + }, + "description": "Retry after 180s transaction-search ingestion window", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-07-23T18:27:53.396Z" + } + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781546276665-5ce2806e26", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781546276665-5ce2806e26", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781546276665", + "status": "STOPPED", + "createdAt": { + "$date": "2026-06-15T17:57:57.655Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": { + "$date": "2026-06-15T17:59:02.065Z" + } + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781546320304-280eaabe92", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781546320304-280eaabe92", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781546320304", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-15T17:58:41.095Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": { + "$date": "2026-06-15T17:59:45.093Z" + } + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781546632740-c43173cffd", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781546632740-c43173cffd", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781546632740", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-15T18:03:53.641Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": { + "$date": "2026-06-15T18:04:56.478Z" + } + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781557193089-fb5b514cfe", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781557193089-fb5b514cfe", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781557193089", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-06-15T20:59:54.052Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": { + "$date": "2026-06-15T21:00:56.284Z" + } + }, + { + "batchEvaluationId": "demoEvalEval2_20260716_180855-d6c24f32e1", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEval2_20260716_180855-d6c24f32e1", + "batchEvaluationName": "demoEvalEval2_20260716_180855", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T18:08:56.885Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": { + "$date": "2026-07-16T18:10:00.711Z" + } + }, + { + "batchEvaluationId": "demoEvalEvaluator_20260716_173405-073d204a91", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEvaluator_20260716_173405-073d204a91", + "batchEvaluationName": "demoEvalEvaluator_20260716_173405", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T17:34:06.242Z" + }, + "description": "goal-success-rate", + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": { + "$date": "2026-07-16T17:34:09.129Z" + } + }, + { + "batchEvaluationId": "demoEvalEvaluator_20260716_174250-17776eb7d0", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEvaluator_20260716_174250-17776eb7d0", + "batchEvaluationName": "demoEvalEvaluator_20260716_174250", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T17:42:51.683Z" + }, + "description": "goal-success-rate", + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": { + "$date": "2026-07-16T17:42:54.282Z" + } + }, + { + "batchEvaluationId": "demoEvalEvaluator_20260716_174359-4eb49ebe4e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEvaluator_20260716_174359-4eb49ebe4e", + "batchEvaluationName": "demoEvalEvaluator_20260716_174359", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T17:44:00.222Z" + }, + "description": "goal-success-rate", + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": { + "$date": "2026-07-16T17:45:03.585Z" + } + }, + { + "batchEvaluationId": "demoEvalInsight2_20260716_180854-42e0199ba5", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight2_20260716_180854-42e0199ba5", + "batchEvaluationName": "demoEvalInsight2_20260716_180854", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T18:08:55.918Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "updatedAt": { + "$date": "2026-07-16T18:10:02.597Z" + } + }, + { + "batchEvaluationId": "demoEvalInsight_20260716_173405-b0b4cea93d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight_20260716_173405-b0b4cea93d", + "batchEvaluationName": "demoEvalInsight_20260716_173405", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T17:34:07.167Z" + }, + "description": "failure-analysis", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": { + "$date": "2026-07-16T17:34:09.798Z" + } + }, + { + "batchEvaluationId": "demoEvalInsight_20260716_174250-6eda2d4c70", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight_20260716_174250-6eda2d4c70", + "batchEvaluationName": "demoEvalInsight_20260716_174250", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T17:42:56.078Z" + }, + "description": "failure-analysis", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": { + "$date": "2026-07-16T17:42:58.975Z" + } + }, + { + "batchEvaluationId": "demoEvalInsight_20260716_174359-a486e71be6", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight_20260716_174359-a486e71be6", + "batchEvaluationName": "demoEvalInsight_20260716_174359", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-16T17:44:01.167Z" + }, + "description": "failure-analysis", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": { + "$date": "2026-07-16T17:45:05.973Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260717T000000-81e740f320", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260717T000000-81e740f320", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260717T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-17T00:00:18.961Z" + }, + "updatedAt": { + "$date": "2026-07-17T00:00:21.457Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260718T000000-b08c7273ef", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260718T000000-b08c7273ef", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260718T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-18T00:00:15.272Z" + }, + "updatedAt": { + "$date": "2026-07-18T00:00:16.466Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260719T000000-6725a187ac", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260719T000000-6725a187ac", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260719T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-19T00:00:27.744Z" + }, + "updatedAt": { + "$date": "2026-07-19T00:00:29.241Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260720T000000-753d17f872", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260720T000000-753d17f872", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260720T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-20T00:00:15.234Z" + }, + "updatedAt": { + "$date": "2026-07-20T00:00:16.417Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260721T000000-7e9b12c2d5", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260721T000000-7e9b12c2d5", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260721T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-21T00:00:18.477Z" + }, + "updatedAt": { + "$date": "2026-07-21T00:00:19.822Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260722T000000-0eeb7036f2", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260722T000000-0eeb7036f2", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260722T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-22T00:00:15.292Z" + }, + "updatedAt": { + "$date": "2026-07-22T00:00:16.485Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260723T000000-62e70e3c39", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260723T000000-62e70e3c39", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260723T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-23T00:00:25.556Z" + }, + "updatedAt": { + "$date": "2026-07-23T00:00:48.515Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260724T000000-da1c165b6e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260724T000000-da1c165b6e", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260724T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-24T00:00:29.078Z" + }, + "updatedAt": { + "$date": "2026-07-24T00:00:30.329Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260725T000000-f34f3fe301", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260725T000000-f34f3fe301", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260725T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-25T00:00:15.442Z" + }, + "updatedAt": { + "$date": "2026-07-25T00:00:17.095Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260726T000000-eab34f736d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260726T000000-eab34f736d", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260726T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-26T00:00:26.368Z" + }, + "updatedAt": { + "$date": "2026-07-26T00:00:27.852Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260727T000000-b1d27d55aa", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260727T000000-b1d27d55aa", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260727T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-27T00:00:15.341Z" + }, + "updatedAt": { + "$date": "2026-07-27T00:00:16.908Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260728T000000-6208ac119c", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260728T000000-6208ac119c", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260728T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-28T00:00:15.384Z" + }, + "updatedAt": { + "$date": "2026-07-28T00:00:16.846Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260729T000000-d74d2e8d84", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260729T000000-d74d2e8d84", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260729T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-29T00:00:15.412Z" + }, + "updatedAt": { + "$date": "2026-07-29T00:00:16.691Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260730T000000-b1707fe880", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260730T000000-b1707fe880", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260730T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-30T00:00:15.225Z" + }, + "updatedAt": { + "$date": "2026-07-30T00:00:16.547Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260731T000000-0af248e59b", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260731T000000-0af248e59b", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260731T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-31T00:00:24.837Z" + }, + "updatedAt": { + "$date": "2026-07-31T00:00:47.847Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260801T000000-2c49e7bf38", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260801T000000-2c49e7bf38", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260801T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-01T00:00:15.438Z" + }, + "updatedAt": { + "$date": "2026-08-01T00:00:16.758Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260802T000000-d5a5c37eca", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260802T000000-d5a5c37eca", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260802T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-02T00:00:15.357Z" + }, + "updatedAt": { + "$date": "2026-08-02T00:00:16.631Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260803T000000-8585ad10f1", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260803T000000-8585ad10f1", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260803T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-03T00:00:15.392Z" + }, + "updatedAt": { + "$date": "2026-08-03T00:00:16.713Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260804T000000-ee69e14201", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260804T000000-ee69e14201", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260804T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-04T00:00:15.438Z" + }, + "updatedAt": { + "$date": "2026-08-04T00:00:17.020Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260805T000000-421bd7cc5d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260805T000000-421bd7cc5d", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260805T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-05T00:00:15.350Z" + }, + "updatedAt": { + "$date": "2026-08-05T00:00:16.889Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260806T000000-efedca8bcc", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260806T000000-efedca8bcc", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260806T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-06T00:00:15.225Z" + }, + "updatedAt": { + "$date": "2026-08-06T00:00:16.536Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260807T000000-34a8f4df8a", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260807T000000-34a8f4df8a", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260807T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-07T00:00:15.585Z" + }, + "updatedAt": { + "$date": "2026-08-07T00:00:17.284Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260717T000000-071c070564", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260717T000000-071c070564", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260717T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-17T00:00:31.359Z" + }, + "updatedAt": { + "$date": "2026-07-17T00:00:32.915Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260718T000000-4550946fea", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260718T000000-4550946fea", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260718T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-18T00:00:31.216Z" + }, + "updatedAt": { + "$date": "2026-07-18T00:00:32.766Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260719T000000-7b3cc3c5b8", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260719T000000-7b3cc3c5b8", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260719T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-19T00:00:31.272Z" + }, + "updatedAt": { + "$date": "2026-07-19T00:00:33.196Z" + } + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260720T000000-722fc2ebbd", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260720T000000-722fc2ebbd", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260720T000000", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-07-20T00:00:31.373Z" + }, + "updatedAt": { + "$date": "2026-07-20T00:00:32.917Z" + } + } + ], + "nextToken": "eyJhd3NBY2NvdW50SWQiOiI3MjU0NzY5NjQ5MTciLCJiYXRjaEV2YWx1YXRlSWQiOiJkZW1vRXZhbG9ubGluZUluc2lnaHRzQm90b19kYWlseV8yMDI2MDcyMFQwMDAwMDAtNzIyZmMyZWJiZCJ9" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/get-disable-cw.golden.json b/src/handlers/eval/batch-evaluation/__fixtures__/get-disable-cw.golden.json new file mode 100644 index 000000000..a96259ea4 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/get-disable-cw.golden.json @@ -0,0 +1,64 @@ +{ + "batchEvaluationId": "GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationName": "GTProbe2_1786034545579", + "status": "COMPLETED", + "createdAt": "2026-08-06T16:42:26.468Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + }, + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "repro1427_ReproAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/repro1427_ReproAgent-zqNEUl8I0f-DEFAULT" + ], + "filterConfig": { + "timeRange": { + "startTime": "2026-07-29T00:00:00.000Z", + "endTime": "2026-07-30T00:00:00.000Z" + } + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-GTProbe2_1786034545579-8ffefc851e" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 9, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 9, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 9, + "totalFailed": 0 + }, + { + "evaluatorId": "Builtin.Correctness", + "statistics": { + "averageScore": 1 + }, + "totalEvaluated": 9, + "totalFailed": 0 + } + ] + }, + "description": "GT probe #2: repro1427 recent sessions; confirm GT/assertion/trajectory attributes in result logs", + "updatedAt": "2026-08-06T16:43:30.696Z" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/get.golden.json b/src/handlers/eval/batch-evaluation/__fixtures__/get.golden.json new file mode 100644 index 000000000..0255b71df --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/get.golden.json @@ -0,0 +1,228 @@ +{ + "batchEvaluationId": "GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationName": "GTProbe2_1786034545579", + "status": "COMPLETED", + "createdAt": "2026-08-06T16:42:26.468Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + }, + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "repro1427_ReproAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/repro1427_ReproAgent-zqNEUl8I0f-DEFAULT" + ], + "filterConfig": { + "timeRange": { + "startTime": "2026-07-29T00:00:00.000Z", + "endTime": "2026-07-30T00:00:00.000Z" + } + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-GTProbe2_1786034545579-8ffefc851e" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 9, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 9, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 9, + "totalFailed": 0 + }, + { + "evaluatorId": "Builtin.Correctness", + "statistics": { + "averageScore": 1 + }, + "totalEvaluated": 9, + "totalFailed": 0 + } + ] + }, + "description": "GT probe #2: repro1427 recent sessions; confirm GT/assertion/trajectory attributes in result logs", + "updatedAt": "2026-08-06T16:43:30.696Z", + "results": [ + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "baseline-session-3-1785350958-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4b30737b856660b6a2c37e1c4b93", + "score": 1, + "label": "Correct", + "explanation": "The user asked to add 17 and 25 using the add_numbers tool and report the result. The correct mathematical answer is 17 + 25 = 42. The agent responded that the result is 42, which is mathematically correct. The expected response indicates the assistant should provide a correct, helpful, well-grounded answer, which the agent did. The agent correctly identified the sum as 42." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "baseline-session-3-1785350958-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4b30737b856660b6a2c37e1c4b93", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user. The response is concise, accurate (based on tool output), and directly addresses what the user asked for. The user's goal has been fully achieved with no unnecessary detours or missing information." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "baseline-session-1-1785350935-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4b1806f680cd2ed2308c1062be9b", + "score": 1, + "label": "Correct", + "explanation": "The user asked to use the add_numbers tool to add 17 and 25 and report the result. The correct mathematical answer is 17 + 25 = 42. The agent responded that the result is 42, which is mathematically correct. The expected response indicates the assistant should provide a correct, helpful, well-grounded answer, which the agent did. The agent correctly identified the sum as 42." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "baseline-session-1-1785350935-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4b1806f680cd2ed2308c1062be9b", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant accomplished exactly this - it called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user in a concise, well-formatted response. The response directly addresses what the user asked for, provides the correct result from the tool, and does so efficiently without unnecessary information. This fully satisfies the user's request." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "baseline-session-2-1785350947-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4b24480148ae03510a1f2dee038b", + "score": 1, + "label": "Correct", + "explanation": "The user asked to use the add_numbers tool to add 17 and 25 and report the result. The correct mathematical answer is 17 + 25 = 42. The agent responded that the result is 42, which is mathematically correct. The expected response indicates the assistant should provide a correct, helpful, well-grounded answer, which the agent did. The agent correctly identified the sum as 42." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "baseline-session-2-1785350947-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4b24480148ae03510a1f2dee038b", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant did exactly this - it called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user. The response is concise, accurate (based on tool output), and directly addresses what the user asked for. The user's goal has been fully achieved with no unnecessary detours or missing information. This is a complete and satisfying response to the user's request." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "mutated-session-1-1785352047-aaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4f7157db99706f811b26729cf203", + "score": 1, + "label": "Perfectly Correct", + "explanation": "The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "mutated-session-1-1785352047-aaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4f7157db99706f811b26729cf203", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, direct manner. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly. This fully satisfies the user's request. There's nothing more needed here - the response is complete and accurate based on the tool output." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "mutated-session-3-1785352090-aaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4f9c162d822072bf7b9c024376e0", + "score": 1, + "label": "Perfectly Correct", + "explanation": "The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which correctly reflects the tool output. This is factually accurate and directly addresses the task." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "mutated-session-3-1785352090-aaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4f9c162d822072bf7b9c024376e0", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, well-formatted response. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly with bold formatting for emphasis. This fully satisfies the user's request without any unnecessary detours or missing information." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "baseline-session-5-1785351439-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4d117acbcba224bffcdb1451334f", + "score": 1, + "label": "Perfectly Correct", + "explanation": "The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "baseline-session-5-1785351439-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4d117acbcba224bffcdb1451334f", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant did exactly this - it called the tool with the correct parameters (17 and 25), received the result (42), and clearly communicated the answer to the user. The response is concise, accurate (based on tool output), and directly addresses what the user asked for. The user's goal has been fully achieved with no unnecessary detours or missing information. This is a complete and satisfying response to the user's request." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "baseline-session-4-1785351428-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4d066e5a0bd8430a5b5937d51a4a", + "score": 1, + "label": "Perfectly Correct", + "explanation": "The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "baseline-session-4-1785351428-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4d066e5a0bd8430a5b5937d51a4a", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, well-formatted response. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly. This fully satisfies the user's request without any unnecessary detours or missing information." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "mutated-session-2-1785352070-aaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4f8708321dbf4e1235643d19eb3b", + "score": 1, + "label": "Perfectly Correct", + "explanation": "The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which correctly reflects the tool output. This is factually accurate and directly addresses the task." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "mutated-session-2-1785352070-aaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4f8708321dbf4e1235643d19eb3b", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, well-formatted response. The answer directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the result was accurately reported. This fully satisfies the user's request with no unnecessary detours or missing information." + }, + { + "evaluatorId": "Builtin.Correctness", + "level": "Trace", + "sessionId": "baseline-session-6-1785351450-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4d1c17cb3b204b884a0647affe2b", + "score": 1, + "label": "Perfectly Correct", + "explanation": "The tool was called with parameters a=17 and b=25, and returned the result '42'. The candidate response states 'The result of adding 17 and 25 is 42', which matches the tool output exactly. This is factually correct." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "baseline-session-6-1785351450-aaaaaaaaaaaaaaaaaaaaaaa", + "traceId": "6a6a4d1c17cb3b204b884a0647affe2b", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user's goal was simple and clear: use the add_numbers tool to add 17 and 25, then be told the result. The assistant successfully called the tool (as shown in the tool call/result), and then clearly communicated the result (42) to the user in a concise, direct manner. The response directly addresses what the user asked for - the result of the addition. The tool was used correctly, and the answer was communicated clearly. This fully satisfies the user's request without any unnecessary detours or missing information. The response is complete and accurate based on the tool output." + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/list.golden.json b/src/handlers/eval/batch-evaluation/__fixtures__/list.golden.json new file mode 100644 index 000000000..14bc3e2f2 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/list.golden.json @@ -0,0 +1,547 @@ +{ + "batchEvaluations": [ + { + "batchEvaluationId": "ABBatchEvalLatest-a7b8db0f3d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/ABBatchEvalLatest-a7b8db0f3d", + "batchEvaluationName": "ABBatchEvalLatest", + "status": "COMPLETED", + "createdAt": "2026-06-17T22:15:50.980Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": "2026-06-17T22:16:55.626Z" + }, + { + "batchEvaluationId": "ABBatchEvalVfy-f39e3e215c", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/ABBatchEvalVfy-f39e3e215c", + "batchEvaluationName": "ABBatchEvalVfy", + "status": "COMPLETED", + "createdAt": "2026-06-17T20:41:07.938Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": "2026-06-17T20:42:11.819Z" + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282542120-bc78b8200c", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282542120-bc78b8200c", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282542120", + "status": "COMPLETED", + "createdAt": "2026-06-12T16:42:23.077Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": "2026-06-12T16:42:27.934Z" + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282564749-2116d5dcff", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282564749-2116d5dcff", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282564749", + "status": "COMPLETED", + "createdAt": "2026-06-12T16:42:45.552Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": "2026-06-12T16:42:48.388Z" + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282587195-6953163407", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282587195-6953163407", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282587195", + "status": "COMPLETED", + "createdAt": "2026-06-12T16:43:08.112Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": "2026-06-12T16:43:11.487Z" + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282609720-466418f536", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282609720-466418f536", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282609720", + "status": "COMPLETED", + "createdAt": "2026-06-12T16:43:30.992Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": "2026-06-12T16:43:33.973Z" + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282632632-6e14e4d8e0", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282632632-6e14e4d8e0", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282632632", + "status": "COMPLETED", + "createdAt": "2026-06-12T16:43:53.618Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": "2026-06-12T16:43:56.416Z" + }, + { + "batchEvaluationId": "E2eArch82403176_E2eArch82403176_1781282655261-68c5d69e9b", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/E2eArch82403176_E2eArch82403176_1781282655261-68c5d69e9b", + "batchEvaluationName": "E2eArch82403176_E2eArch82403176_1781282655261", + "status": "COMPLETED", + "createdAt": "2026-06-12T16:44:16.201Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Faithfulness" + } + ], + "updatedAt": "2026-06-12T16:45:20.791Z" + }, + { + "batchEvaluationId": "GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe2_1786034545579-8ffefc851e", + "batchEvaluationName": "GTProbe2_1786034545579", + "status": "COMPLETED", + "createdAt": "2026-08-06T16:42:26.468Z", + "description": "GT probe #2: repro1427 recent sessions; confirm GT/assertion/trajectory attributes in result logs", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + }, + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": "2026-08-06T16:43:30.696Z" + }, + { + "batchEvaluationId": "GTProbe_1786033993684-52f5045e6a", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/GTProbe_1786033993684-52f5045e6a", + "batchEvaluationName": "GTProbe_1786033993684", + "status": "FAILED", + "createdAt": "2026-08-06T16:33:14.495Z", + "description": "GT probe: confirm whether GT/assertion/trajectory outcomes surface as result-log attributes", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + }, + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "errorDetails": [ + "All 4 sessions failed during batch evaluation." + ], + "updatedAt": "2026-08-06T16:33:18.035Z" + }, + { + "batchEvaluationId": "HarnessBatchEvalCodex_20260723_1824-a6e5e74a3b", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/HarnessBatchEvalCodex_20260723_1824-a6e5e74a3b", + "batchEvaluationName": "HarnessBatchEvalCodex_20260723_1824", + "status": "FAILED", + "createdAt": "2026-07-23T18:23:42.907Z", + "description": "Real test: batch-evaluate one session produced by InvokeHarness", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "errorDetails": [ + "All 1 sessions failed during batch evaluation." + ], + "updatedAt": "2026-07-23T18:24:45.539Z" + }, + { + "batchEvaluationId": "HarnessBatchEvalRetry_20260723_1827-72f41d37b1", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/HarnessBatchEvalRetry_20260723_1827-72f41d37b1", + "batchEvaluationName": "HarnessBatchEvalRetry_20260723_1827", + "status": "COMPLETED", + "createdAt": "2026-07-23T18:26:50.667Z", + "description": "Retry after 180s transaction-search ingestion window", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": "2026-07-23T18:27:53.396Z" + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781546276665-5ce2806e26", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781546276665-5ce2806e26", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781546276665", + "status": "STOPPED", + "createdAt": "2026-06-15T17:57:57.655Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": "2026-06-15T17:59:02.065Z" + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781546320304-280eaabe92", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781546320304-280eaabe92", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781546320304", + "status": "COMPLETED", + "createdAt": "2026-06-15T17:58:41.095Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": "2026-06-15T17:59:45.093Z" + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781546632740-c43173cffd", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781546632740-c43173cffd", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781546632740", + "status": "COMPLETED", + "createdAt": "2026-06-15T18:03:53.641Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": "2026-06-15T18:04:56.478Z" + }, + { + "batchEvaluationId": "bugbashagent_bugbash_agent_1781557193089-fb5b514cfe", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/bugbashagent_bugbash_agent_1781557193089-fb5b514cfe", + "batchEvaluationName": "bugbashagent_bugbash_agent_1781557193089", + "status": "COMPLETED", + "createdAt": "2026-06-15T20:59:54.052Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "updatedAt": "2026-06-15T21:00:56.284Z" + }, + { + "batchEvaluationId": "demoEvalEval2_20260716_180855-d6c24f32e1", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEval2_20260716_180855-d6c24f32e1", + "batchEvaluationName": "demoEvalEval2_20260716_180855", + "status": "COMPLETED", + "createdAt": "2026-07-16T18:08:56.885Z", + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": "2026-07-16T18:10:00.711Z" + }, + { + "batchEvaluationId": "demoEvalEvaluator_20260716_173405-073d204a91", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEvaluator_20260716_173405-073d204a91", + "batchEvaluationName": "demoEvalEvaluator_20260716_173405", + "status": "COMPLETED", + "createdAt": "2026-07-16T17:34:06.242Z", + "description": "goal-success-rate", + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": "2026-07-16T17:34:09.129Z" + }, + { + "batchEvaluationId": "demoEvalEvaluator_20260716_174250-17776eb7d0", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEvaluator_20260716_174250-17776eb7d0", + "batchEvaluationName": "demoEvalEvaluator_20260716_174250", + "status": "COMPLETED", + "createdAt": "2026-07-16T17:42:51.683Z", + "description": "goal-success-rate", + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": "2026-07-16T17:42:54.282Z" + }, + { + "batchEvaluationId": "demoEvalEvaluator_20260716_174359-4eb49ebe4e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalEvaluator_20260716_174359-4eb49ebe4e", + "batchEvaluationName": "demoEvalEvaluator_20260716_174359", + "status": "COMPLETED", + "createdAt": "2026-07-16T17:44:00.222Z", + "description": "goal-success-rate", + "evaluators": [ + { + "evaluatorId": "Builtin.GoalSuccessRate" + } + ], + "updatedAt": "2026-07-16T17:45:03.585Z" + }, + { + "batchEvaluationId": "demoEvalInsight2_20260716_180854-42e0199ba5", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight2_20260716_180854-42e0199ba5", + "batchEvaluationName": "demoEvalInsight2_20260716_180854", + "status": "COMPLETED", + "createdAt": "2026-07-16T18:08:55.918Z", + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "updatedAt": "2026-07-16T18:10:02.597Z" + }, + { + "batchEvaluationId": "demoEvalInsight_20260716_173405-b0b4cea93d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight_20260716_173405-b0b4cea93d", + "batchEvaluationName": "demoEvalInsight_20260716_173405", + "status": "COMPLETED", + "createdAt": "2026-07-16T17:34:07.167Z", + "description": "failure-analysis", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": "2026-07-16T17:34:09.798Z" + }, + { + "batchEvaluationId": "demoEvalInsight_20260716_174250-6eda2d4c70", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight_20260716_174250-6eda2d4c70", + "batchEvaluationName": "demoEvalInsight_20260716_174250", + "status": "COMPLETED", + "createdAt": "2026-07-16T17:42:56.078Z", + "description": "failure-analysis", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": "2026-07-16T17:42:58.975Z" + }, + { + "batchEvaluationId": "demoEvalInsight_20260716_174359-a486e71be6", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalInsight_20260716_174359-a486e71be6", + "batchEvaluationName": "demoEvalInsight_20260716_174359", + "status": "COMPLETED", + "createdAt": "2026-07-16T17:44:01.167Z", + "description": "failure-analysis", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": "2026-07-16T17:45:05.973Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260717T000000-81e740f320", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260717T000000-81e740f320", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260717T000000", + "status": "COMPLETED", + "createdAt": "2026-07-17T00:00:18.961Z", + "updatedAt": "2026-07-17T00:00:21.457Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260718T000000-b08c7273ef", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260718T000000-b08c7273ef", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260718T000000", + "status": "COMPLETED", + "createdAt": "2026-07-18T00:00:15.272Z", + "updatedAt": "2026-07-18T00:00:16.466Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260719T000000-6725a187ac", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260719T000000-6725a187ac", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260719T000000", + "status": "COMPLETED", + "createdAt": "2026-07-19T00:00:27.744Z", + "updatedAt": "2026-07-19T00:00:29.241Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260720T000000-753d17f872", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260720T000000-753d17f872", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260720T000000", + "status": "COMPLETED", + "createdAt": "2026-07-20T00:00:15.234Z", + "updatedAt": "2026-07-20T00:00:16.417Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260721T000000-7e9b12c2d5", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260721T000000-7e9b12c2d5", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260721T000000", + "status": "COMPLETED", + "createdAt": "2026-07-21T00:00:18.477Z", + "updatedAt": "2026-07-21T00:00:19.822Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260722T000000-0eeb7036f2", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260722T000000-0eeb7036f2", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260722T000000", + "status": "COMPLETED", + "createdAt": "2026-07-22T00:00:15.292Z", + "updatedAt": "2026-07-22T00:00:16.485Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260723T000000-62e70e3c39", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260723T000000-62e70e3c39", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260723T000000", + "status": "COMPLETED", + "createdAt": "2026-07-23T00:00:25.556Z", + "updatedAt": "2026-07-23T00:00:48.515Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260724T000000-da1c165b6e", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260724T000000-da1c165b6e", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260724T000000", + "status": "COMPLETED", + "createdAt": "2026-07-24T00:00:29.078Z", + "updatedAt": "2026-07-24T00:00:30.329Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260725T000000-f34f3fe301", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260725T000000-f34f3fe301", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260725T000000", + "status": "COMPLETED", + "createdAt": "2026-07-25T00:00:15.442Z", + "updatedAt": "2026-07-25T00:00:17.095Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260726T000000-eab34f736d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260726T000000-eab34f736d", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260726T000000", + "status": "COMPLETED", + "createdAt": "2026-07-26T00:00:26.368Z", + "updatedAt": "2026-07-26T00:00:27.852Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260727T000000-b1d27d55aa", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260727T000000-b1d27d55aa", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260727T000000", + "status": "COMPLETED", + "createdAt": "2026-07-27T00:00:15.341Z", + "updatedAt": "2026-07-27T00:00:16.908Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260728T000000-6208ac119c", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260728T000000-6208ac119c", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260728T000000", + "status": "COMPLETED", + "createdAt": "2026-07-28T00:00:15.384Z", + "updatedAt": "2026-07-28T00:00:16.846Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260729T000000-d74d2e8d84", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260729T000000-d74d2e8d84", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260729T000000", + "status": "COMPLETED", + "createdAt": "2026-07-29T00:00:15.412Z", + "updatedAt": "2026-07-29T00:00:16.691Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260730T000000-b1707fe880", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260730T000000-b1707fe880", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260730T000000", + "status": "COMPLETED", + "createdAt": "2026-07-30T00:00:15.225Z", + "updatedAt": "2026-07-30T00:00:16.547Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260731T000000-0af248e59b", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260731T000000-0af248e59b", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260731T000000", + "status": "COMPLETED", + "createdAt": "2026-07-31T00:00:24.837Z", + "updatedAt": "2026-07-31T00:00:47.847Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260801T000000-2c49e7bf38", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260801T000000-2c49e7bf38", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260801T000000", + "status": "COMPLETED", + "createdAt": "2026-08-01T00:00:15.438Z", + "updatedAt": "2026-08-01T00:00:16.758Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260802T000000-d5a5c37eca", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260802T000000-d5a5c37eca", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260802T000000", + "status": "COMPLETED", + "createdAt": "2026-08-02T00:00:15.357Z", + "updatedAt": "2026-08-02T00:00:16.631Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260803T000000-8585ad10f1", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260803T000000-8585ad10f1", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260803T000000", + "status": "COMPLETED", + "createdAt": "2026-08-03T00:00:15.392Z", + "updatedAt": "2026-08-03T00:00:16.713Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260804T000000-ee69e14201", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260804T000000-ee69e14201", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260804T000000", + "status": "COMPLETED", + "createdAt": "2026-08-04T00:00:15.438Z", + "updatedAt": "2026-08-04T00:00:17.020Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260805T000000-421bd7cc5d", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260805T000000-421bd7cc5d", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260805T000000", + "status": "COMPLETED", + "createdAt": "2026-08-05T00:00:15.350Z", + "updatedAt": "2026-08-05T00:00:16.889Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260806T000000-efedca8bcc", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260806T000000-efedca8bcc", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260806T000000", + "status": "COMPLETED", + "createdAt": "2026-08-06T00:00:15.225Z", + "updatedAt": "2026-08-06T00:00:16.536Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsights1_daily_20260807T000000-34a8f4df8a", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsights1_daily_20260807T000000-34a8f4df8a", + "batchEvaluationName": "demoEvalonlineInsights1_daily_20260807T000000", + "status": "COMPLETED", + "createdAt": "2026-08-07T00:00:15.585Z", + "updatedAt": "2026-08-07T00:00:17.284Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260717T000000-071c070564", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260717T000000-071c070564", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260717T000000", + "status": "COMPLETED", + "createdAt": "2026-07-17T00:00:31.359Z", + "updatedAt": "2026-07-17T00:00:32.915Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260718T000000-4550946fea", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260718T000000-4550946fea", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260718T000000", + "status": "COMPLETED", + "createdAt": "2026-07-18T00:00:31.216Z", + "updatedAt": "2026-07-18T00:00:32.766Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260719T000000-7b3cc3c5b8", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260719T000000-7b3cc3c5b8", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260719T000000", + "status": "COMPLETED", + "createdAt": "2026-07-19T00:00:31.272Z", + "updatedAt": "2026-07-19T00:00:33.196Z" + }, + { + "batchEvaluationId": "demoEvalonlineInsightsBoto_daily_20260720T000000-722fc2ebbd", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:batch-evaluate/demoEvalonlineInsightsBoto_daily_20260720T000000-722fc2ebbd", + "batchEvaluationName": "demoEvalonlineInsightsBoto_daily_20260720T000000", + "status": "COMPLETED", + "createdAt": "2026-07-20T00:00:31.373Z", + "updatedAt": "2026-07-20T00:00:32.917Z" + } + ], + "nextToken": "eyJhd3NBY2NvdW50SWQiOiI3MjU0NzY5NjQ5MTciLCJiYXRjaEV2YWx1YXRlSWQiOiJkZW1vRXZhbG9ubGluZUluc2lnaHRzQm90b19kYWlseV8yMDI2MDcyMFQwMDAwMDAtNzIyZmMyZWJiZCJ9" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx new file mode 100644 index 000000000..01ad371f5 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx @@ -0,0 +1,109 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with: RECORD=1 bun test src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx +// +// Batch evaluation is READ-ONLY, so — unlike the evaluator/online-eval fixture +// suites, which create-then-delete their resource during a record run — this +// pins a pre-existing COMPLETED job in the fixture account. Re-recording requires +// that job to still exist AND its CloudWatch result stream to still hold events +// (streams age out under log retention). If it has aged out, run a fresh batch +// evaluation, wait for it to complete, and repoint FIXTURE_JOB_ID at it before +// re-recording. +// +// This suite exercises the real seam end to end: parsing → handler → CoreClient → +// GetBatchEvaluation (data plane) → readEvaluationResults → GetLogEvents (the +// createLogsClient fixture seam). The TestCoreClient suite (batch-evaluation.test.tsx) +// covers the edges that can't be recorded on demand: a non-terminal job, a +// CloudWatch read failure, and list pagination. +const FIXTURE_JOB_ID = "GTProbe2_1786034545579-8ffefc851e"; + +// A well-formed but absent id, to reach the not-found path without a +// ValidationException short-circuiting the lookup. +const MISSING_JOB_ID = "missing-batch-eval-0000000000"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +// run drives the real router (parsing → middleware → handler → CoreClient) against +// the fixture-backed SDK clients and returns captured stdout. +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +describe("eval batch-evaluation (fixture-backed)", () => { + test("get returns the job with CloudWatch-backed results by default", async () => { + // Records GetBatchEvaluationCommand..json (data plane) AND + // GetLogEventsCommand..json (the CloudWatch logs seam). + const stdout = await run(["eval", "batch-evaluation", "get", "--id", FIXTURE_JOB_ID, "--json"]); + + matchGolden(FIXTURES, "get.golden.json", stdout); + const detail = JSON.parse(stdout); + expect(detail.status).toBe("COMPLETED"); + expect(Array.isArray(detail.results)).toBe(true); + expect(detail.results.length).toBeGreaterThan(0); + // Results carry the scope the old parser dropped. + expect(detail.results[0]).toHaveProperty("level"); + expect(detail.results[0]).toHaveProperty("sessionId"); + }); + + test("get --disable-cw-results returns metadata only, no CloudWatch read", async () => { + // Same job, includeResults:false → a distinct GetBatchEvaluation input hash + // (its own fixture) and no GetLogEvents call. + const stdout = await run([ + "eval", + "batch-evaluation", + "get", + "--id", + FIXTURE_JOB_ID, + "--disable-cw-results", + "--json", + ]); + + matchGolden(FIXTURES, "get-disable-cw.golden.json", stdout); + const detail = JSON.parse(stdout); + expect(detail.status).toBe("COMPLETED"); + expect(detail.results).toBeUndefined(); + }); + + test("list returns the service page", async () => { + const stdout = await run(["eval", "batch-evaluation", "list", "--json"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(Array.isArray(JSON.parse(stdout).batchEvaluations)).toBe(true); + }); + + test("get surfaces a not-found error for an absent job", async () => { + await expect( + run(["eval", "batch-evaluation", "get", "--id", MISSING_JOB_ID, "--json"]), + ).rejects.toThrow(); + }); +}); diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx new file mode 100644 index 000000000..c2513b287 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -0,0 +1,162 @@ +import { test, expect, describe } from "bun:test"; +import type { + GetBatchEvaluationResponse, + ListBatchEvaluationsResponse, +} from "@aws-sdk/client-bedrock-agentcore"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; +import type { BatchEvaluationResultEntry } from "../types"; + +// Command-flow tests for `eval batch-evaluation`, driven through the real root +// handler against a TestCoreClient (no network). These cover the edges that the +// fixture-backed suite (batch-evaluation.fixture.test.tsx) can't record on demand: +// a non-terminal job, a CloudWatch read failure, and list pagination. The +// happy-path get (with real merged results) and --disable-cw-results live there. + +const CW = { + outputConfig: { cloudWatchConfig: { logGroupName: "lg", logStreamName: "ls" } }, +} as const; + +const COMPLETED: GetBatchEvaluationResponse = { + batchEvaluationId: "batch-eval-abc123", + status: "COMPLETED", + ...CW, +} as GetBatchEvaluationResponse; + +const RUNNING: GetBatchEvaluationResponse = { + batchEvaluationId: "batch-eval-run", + status: "IN_PROGRESS", + ...CW, +} as GetBatchEvaluationResponse; + +const RESULTS: BatchEvaluationResultEntry[] = [ + { evaluatorId: "Builtin.Helpfulness", level: "Session", sessionId: "s1", score: 5 }, +]; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +describe("eval batch-evaluation command hierarchy", () => { + test("registers get + list under eval → batch-evaluation", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "batch-evaluation"); + expect(group?.children().map((c) => c.name())).toEqual(["get", "list"]); + }); + + test("prints help for `eval batch-evaluation --json` without an SDK call", async () => { + // Under --json the empty-invocation TUI middleware (inherited from the eval + // parent) prints help instead of opening the interactive UI. + const { core, stdout } = await run(["eval", "batch-evaluation", "--json"]); + expect(stdout).toContain("Usage: agentcore eval batch-evaluation"); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("eval batch-evaluation get", () => { + test("requires --id", async () => { + await expect(run(["eval", "batch-evaluation", "get", "--json"])).rejects.toThrow(/--id/); + }); + + test("includes CloudWatch results for a terminal job by default", async () => { + const { core, stdout } = await run( + ["eval", "batch-evaluation", "get", "--id", "batch-eval-abc123", "--json"], + (c) => { + c.eval.setBatchEvalGetResponse(COMPLETED); + c.eval.setBatchEvalResults(RESULTS); + }, + ); + const out = JSON.parse(stdout); + expect(out.status).toBe("COMPLETED"); + expect(out.results).toEqual(RESULTS); + // A single Core call; the CloudWatch merge happens inside it. Results were + // requested by default. + const call = core.eval.calls.find((c) => c.method === "getBatchEvaluation"); + expect(call?.args[2]).toEqual({ includeResults: true }); + }); + + test("passes includeResults:false to Core for --disable-cw-results", async () => { + // The happy-path output shape is asserted in the fixture-backed suite; here we + // only pin that the flag reaches Core as includeResults:false. + const { core } = await run( + [ + "eval", + "batch-evaluation", + "get", + "--id", + "batch-eval-abc123", + "--disable-cw-results", + "--json", + ], + (c) => c.eval.setBatchEvalGetResponse(COMPLETED), + ); + const call = core.eval.calls.find((c) => c.method === "getBatchEvaluation"); + expect(call?.args[2]).toEqual({ includeResults: false }); + }); + + test("omits results for a non-terminal job", async () => { + const { core, stdout } = await run( + ["eval", "batch-evaluation", "get", "--id", "batch-eval-run", "--json"], + (c) => c.eval.setBatchEvalGetResponse(RUNNING), + ); + expect(JSON.parse(stdout).results).toBeUndefined(); + expect(core.eval.calls.map((c) => c.method)).toEqual(["getBatchEvaluation"]); + }); + + test("a CloudWatch failure warns on stderr but never hides job status", async () => { + const { stdout, stderr } = await run( + ["eval", "batch-evaluation", "get", "--id", "batch-eval-abc123", "--json"], + (c) => { + c.eval.setBatchEvalGetResponse(COMPLETED); + // Core returns the metadata plus `resultsError` (a CloudWatch read + // failure), never throwing — so status must survive. + c.eval.setBatchEvalResultsError(new Error("AccessDenied")); + }, + ); + const out = JSON.parse(stdout); + expect(out.status).toBe("COMPLETED"); // status intact + expect(out.results).toBeUndefined(); + expect(stderr).toContain("could not retrieve CloudWatch results"); + expect(stderr).toContain("AccessDenied"); + }); +}); + +describe("eval batch-evaluation list", () => { + const PAGE: ListBatchEvaluationsResponse = { + batchEvaluations: [ + { batchEvaluationId: "b1", status: "COMPLETED" }, + { batchEvaluationId: "b2", status: "FAILED" }, + ], + nextToken: "next", + } as ListBatchEvaluationsResponse; + + test("passes pagination through and returns the page verbatim", async () => { + const { core, stdout } = await run( + ["eval", "batch-evaluation", "list", "--max-results", "10", "--json"], + (c) => c.eval.setBatchEvalListResponse(PAGE), + ); + const out = JSON.parse(stdout); + expect(out.batchEvaluations).toHaveLength(2); + expect(out.nextToken).toBe("next"); + expect(core.eval.calls[0]?.args).toEqual([undefined, 10, { region: "us-west-2" }]); + }); +}); diff --git a/src/handlers/eval/batch-evaluation/get/index.tsx b/src/handlers/eval/batch-evaluation/get/index.tsx new file mode 100644 index 000000000..8192b3a13 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/index.tsx @@ -0,0 +1,60 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import { warn, type AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { CoreOptions } from "../../../../core/types"; +import type { BatchEvaluationDetail } from "../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetBatchEvaluationHandler = (core: Core, io: AppIO) => + createHandler({ + name: "get", + description: "get a batch evaluation by id, with CloudWatch-backed results when available", + flags: [ + flag("id", "the ID of the batch evaluation", z.string().optional()), + flag( + "disable-cw-results", + "skip CloudWatch result retrieval and return only service-side job metadata", + z.boolean().optional(), + ), + ], + handle: async (ctx, flags) => { + const id = flags["id"]; + if (!id) throw new InputValidationError("required option '--id ' not specified"); + const opts = coreOptsFromCtx(ctx); + + // Core fetches the job and (unless --disable-cw-results) merges the + // per-session CloudWatch results. A CloudWatch read failure comes back as + // `resultsError` rather than throwing, so the job status is never hidden; + // surface it as a stderr warning and still print the metadata. + const { detail, resultsError } = await core.eval.getBatchEvaluation(id, opts, { + includeResults: !flags["disable-cw-results"], + }); + if (resultsError) warnCloudWatchFailure(io, detail, opts, resultsError); + + ctx.require(JsonRendererKey).renderJson(detail); + }, + }); + +// warnCloudWatchFailure emits a non-fatal advisory (to stderr, via io.warn) with a +// link to the CloudWatch results, so the machine-readable job status on stdout +// stays clean. +function warnCloudWatchFailure( + io: AppIO, + detail: BatchEvaluationDetail, + opts: CoreOptions, + error: unknown, +): void { + const cw = detail.outputConfig?.cloudWatchConfig; + const link = + cw?.logGroupName && cw.logStreamName + ? ` See CloudWatch: region ${opts.region}, log group ${cw.logGroupName}, stream ${cw.logStreamName}.` + : ""; + warn( + io, + `could not retrieve CloudWatch results (${(error as Error).message}). ` + + `Job status is unaffected.${link}`, + ); +} diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx new file mode 100644 index 000000000..2fe489a31 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -0,0 +1,16 @@ +import { Router } from "../../../router"; +import { createHelpDefault } from "../../help"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createGetBatchEvaluationHandler } from "./get"; +import { createListBatchEvaluationsHandler } from "./list"; + +// batch-evaluation is read-only for now (get + list). Unlike evaluator and +// online-eval it has no TUI screen yet, so a bare invocation prints help rather +// than launching Ink — matching the gateway/target read-only group. +export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { + return new Router("batch-evaluation", "inspect AgentCore batch evaluations") + .default(createHelpDefault(io)) + .handler(createGetBatchEvaluationHandler(core, io)) + .handler(createListBatchEvaluationsHandler(core)); +} diff --git a/src/handlers/eval/batch-evaluation/list/index.tsx b/src/handlers/eval/batch-evaluation/list/index.tsx new file mode 100644 index 000000000..8f007e26f --- /dev/null +++ b/src/handlers/eval/batch-evaluation/list/index.tsx @@ -0,0 +1,23 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListBatchEvaluationsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list batch evaluations", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listBatchEvaluations( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/dataset/dataset.fixture.test.tsx b/src/handlers/eval/dataset/dataset.fixture.test.tsx index 9201a92c6..c1bc4ad99 100644 --- a/src/handlers/eval/dataset/dataset.fixture.test.tsx +++ b/src/handlers/eval/dataset/dataset.fixture.test.tsx @@ -66,11 +66,13 @@ function writeExamples(): string { } function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index bb41c3f4f..c5c74fd68 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -44,11 +44,13 @@ const TUNED_NAME = "agentcore_cli_eval_fixture_tuned"; const TUNED_INFERENCE_CONFIG = { temperature: 0, maxTokens: 512 }; function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 9c37828d4..da3e6c686 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -6,6 +6,7 @@ import type { Core } from "../types"; import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; import { createDatasetHandler } from "./dataset"; +import { createBatchEvaluationHandler } from "./batch-evaluation"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -13,7 +14,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .default(renderTui(core, io)) .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)) - .handler(createDatasetHandler(core, io)); + .handler(createDatasetHandler(core, io)) + .handler(createBatchEvaluationHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index 1e703d61d..1188fc0b1 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -42,11 +42,13 @@ const FIXTURE_ROLE_ARN = "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-we const MISSING_CONFIG_ID = "missing-online-0000000000"; function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 9dab86006..eadbca6da 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -20,8 +20,55 @@ import type { UpdateEvaluatorResponse, UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; +import type { + GetBatchEvaluationResponse, + ListBatchEvaluationsResponse, +} from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; +// BatchEvaluationResultEntry is one per-session/-trace/-tool evaluation score, +// parsed from the CloudWatch output log stream a completed batch evaluation +// writes to. Unlike the old CLI's parser — which read only the evaluator name, +// score, label, and explanation and so flattened every level into an +// indistinguishable list — this keeps `level` and the id fields so callers can +// tell a SESSION result from a TRACE or TOOL_CALL one, and group by session. +export type BatchEvaluationResultEntry = { + evaluatorId: string; + // The scope the score applies to, read from the result log record's + // `aws.bedrock_agentcore.evaluation_level` attribute (Title-case, e.g. "Trace" + // / "Session"). The trustworthy discriminator — do not infer it from which id + // fields are set, since a trace-level result can still carry a session id. + level?: string; + sessionId?: string; + traceId?: string; + spanId?: string; + toolName?: string; + score?: number; + label?: string; + explanation?: string; + error?: string; +}; + +// BatchEvaluationDetail is a GetBatchEvaluation response augmented with the +// per-session results read from CloudWatch. `results` is present only when the +// job is terminal, the response carried a CloudWatch output config, and the +// caller did not pass --disable-cw-results. A CloudWatch read failure leaves +// `results` absent and is surfaced as a warning on stderr rather than embedded +// here, so the job status is never hidden and --json stdout stays clean. +export type BatchEvaluationDetail = GetBatchEvaluationResponse & { + results?: BatchEvaluationResultEntry[]; +}; + +// GetBatchEvaluationResult is what getBatchEvaluation returns: the detail plus an +// optional `resultsError`. Core surfaces a CloudWatch read failure here rather +// than throwing (which would hide the job status) or logging silently (Core has +// no stderr) — the handler warns on stderr, the TUI ignores it. `resultsError` is +// only ever set when results were requested and the CloudWatch read threw. +export type GetBatchEvaluationResult = { + detail: BatchEvaluationDetail; + resultsError?: unknown; +}; + // LlmAsAJudgeUpdate carries the fields a caller may change on an LLM-as-a-Judge // evaluator. Any field left undefined is preserved from the existing evaluator: // the AgentCore UpdateEvaluator API replaces the whole evaluatorConfig union, and @@ -132,6 +179,21 @@ export interface CoreEvalClient { ): Promise; deleteEvaluator(id: string, options: CoreOptions): Promise; + // getBatchEvaluation returns the service-side job and, unless `includeResults` + // is false, the per-session results read from its per-job CloudWatch stream once + // terminal. A CloudWatch read failure is returned as `resultsError` (never + // thrown) so the job status is never hidden. + getBatchEvaluation( + id: string, + options: CoreOptions, + opts?: { includeResults?: boolean }, + ): Promise; + listBatchEvaluations( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/handlers/gateway/gateway.fixture.test.tsx b/src/handlers/gateway/gateway.fixture.test.tsx index e45bdf14a..95dd197c8 100644 --- a/src/handlers/gateway/gateway.fixture.test.tsx +++ b/src/handlers/gateway/gateway.fixture.test.tsx @@ -25,11 +25,13 @@ const CONNECTOR_ID = "Z3FQ0H8JCK"; // CONNECTOR_GATEWAY_ID in CONNECTOR_REGION. Record with: // AWS_PROFILE=e2e-test RECORD=1 bun test src/handlers/gateway/gateway.fixture.test.tsx function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/handlers/harness/harness.test.tsx b/src/handlers/harness/harness.test.tsx index cc56448d2..532d6e76e 100644 --- a/src/handlers/harness/harness.test.tsx +++ b/src/handlers/harness/harness.test.tsx @@ -30,11 +30,13 @@ const REGION = "us-west-2"; // keeps tests isolated) over an in-memory io, routes `args` beneath `agentcore`, // and returns whatever the command wrote to stdout. async function run(args: string[]): Promise { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); const core = new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); const io = testIO(); diff --git a/src/handlers/identity/identity.test.tsx b/src/handlers/identity/identity.test.tsx index 5349ef5cf..0e15e888a 100644 --- a/src/handlers/identity/identity.test.tsx +++ b/src/handlers/identity/identity.test.tsx @@ -21,11 +21,13 @@ const FIXTURE_PROVIDER_NAME_2 = "agentcore-cli-identity-fixture-2"; const MISSING_PROVIDER_NAME = "missing-provider-000"; function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/handlers/identity/oauth2-credential-provider/oauth2.fixture.test.tsx b/src/handlers/identity/oauth2-credential-provider/oauth2.fixture.test.tsx index 02fe61667..970ffc3c0 100644 --- a/src/handlers/identity/oauth2-credential-provider/oauth2.fixture.test.tsx +++ b/src/handlers/identity/oauth2-credential-provider/oauth2.fixture.test.tsx @@ -21,11 +21,13 @@ const FIXTURE_PROVIDER_NAME_2 = "agentcore-cli-oauth2-fixture-2"; const MISSING_PROVIDER_NAME = "missing-oauth2-provider-000"; function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/handlers/memory/memory.test.tsx b/src/handlers/memory/memory.test.tsx index e75096282..ff945a6f2 100644 --- a/src/handlers/memory/memory.test.tsx +++ b/src/handlers/memory/memory.test.tsx @@ -58,12 +58,14 @@ const memoryRecord: MemoryRecord = { const memoryRecordSummary: MemoryRecordSummary = memoryRecord; function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/handlers/runtime/runtime.test.tsx b/src/handlers/runtime/runtime.test.tsx index 584287b07..4c2c9a8a1 100644 --- a/src/handlers/runtime/runtime.test.tsx +++ b/src/handlers/runtime/runtime.test.tsx @@ -22,12 +22,14 @@ const FIXTURE_RUNTIME_ID = "agentcore_cli_runtime_read_only_fixture-wZ7V4Q6vhx"; const MISSING_RUNTIME_ID = "missing_runtime-0000000000"; function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); return new CoreClient({ createControlClient, createDataClient, createIamClient, + createLogsClient, logger: createSilentLogger(), }); } diff --git a/src/index.ts b/src/index.ts index 90a166c04..aa2834b1b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,12 @@ import { homedir } from "os"; import { join } from "path"; import { CoreClient } from "./core"; -import { createControlClient, createDataClient, createIamClient } from "./core/factories"; +import { + createControlClient, + createDataClient, + createIamClient, + createLogsClient, +} from "./core/factories"; import { createRootHandler } from "./handlers"; import { FsReadWriteJson } from "./io"; import { createFileLogger, LOG_LEVEL } from "./logging"; @@ -64,6 +69,7 @@ process.exit( createControlClient, createDataClient, createIamClient, + createLogsClient, logger: rootLogger.child({ module: "core" }), }); diff --git a/src/io/index.ts b/src/io/index.ts index 62a65c685..a16949c7d 100644 --- a/src/io/index.ts +++ b/src/io/index.ts @@ -16,3 +16,4 @@ export { export { FsReadWriteJson } from "./json"; export { SourceResolver, type SourceResolverConfig } from "./source"; export type { AppIO, ReadWriteJson } from "./types"; +export { warn } from "./warn"; diff --git a/src/io/warn.test.ts b/src/io/warn.test.ts new file mode 100644 index 000000000..8a0d870b5 --- /dev/null +++ b/src/io/warn.test.ts @@ -0,0 +1,16 @@ +import { test, expect } from "bun:test"; +import { testIO } from "../testing"; +import { warn } from "./warn"; + +test("warn writes a prefixed advisory to stderr", () => { + const io = testIO(); + warn(io.io, "something degraded but not fatal"); + // testIO trims the trailing newline; the "warning: " prefix is what we assert. + expect(io.stderr()).toBe("warning: something degraded but not fatal"); +}); + +test("warn writes to stderr, never stdout", () => { + const io = testIO(); + warn(io.io, "keep stdout clean"); + expect(io.stdout()).toBe(""); +}); diff --git a/src/io/warn.ts b/src/io/warn.ts new file mode 100644 index 000000000..452209802 --- /dev/null +++ b/src/io/warn.ts @@ -0,0 +1,10 @@ +import type { AppIO } from "./types"; + +// warn writes a non-fatal advisory to stderr, prefixed with "warning: " and +// newline-terminated. Advisories go to stderr (not stdout) so machine-readable +// stdout — e.g. a command's --json output — stays clean and parseable. A free +// function over AppIO rather than a method, so every AppIO producer (process +// streams, testIO) gets it without implementing anything. +export function warn(io: AppIO, message: string): void { + io.stderr.write(`warning: ${message}\n`); +} diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index da72b69df..5a30b3dc0 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -57,6 +57,7 @@ import type { UpdateHarnessResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { + GetBatchEvaluationResponse, GetEventInput, GetEventOutput, GetMemoryRecordInput, @@ -69,6 +70,7 @@ import type { InvokeHarnessStreamOutput, ListActorsInput, ListActorsOutput, + ListBatchEvaluationsResponse, ListEventsInput, ListEventsOutput, ListMemoryRecordsInput, @@ -93,13 +95,17 @@ import type { RuntimeInvokeResponse, } from "../handlers/runtime/types"; import type { + BatchEvaluationDetail, + BatchEvaluationResultEntry, CodeBasedUpdate, CoreEvalClient, CreateDatasetInput, CreateOnlineEvalInput, + GetBatchEvaluationResult, LlmAsAJudgeUpdate, UpdateOnlineEvalInput, } from "../handlers/eval/types"; +import { isTerminalStatus } from "../core/batchEvaluationResults"; import { abortable } from "../core/abortable"; import type { CoreOptions } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; @@ -196,6 +202,8 @@ const DEFAULT_GET_DATASET_RESPONSE = {} as GetDatasetResponse; const DEFAULT_LIST_DATASETS_RESPONSE: ListDatasetsResponse = { datasets: [] }; const DEFAULT_DELETE_DATASET_RESPONSE = {} as DeleteDatasetResponse; const DEFAULT_PUBLISH_DATASET_RESPONSE = {} as CreateDatasetVersionResponse; +const DEFAULT_GET_BATCH_EVAL_RESPONSE = {} as GetBatchEvaluationResponse; +const DEFAULT_LIST_BATCH_EVALS_RESPONSE: ListBatchEvaluationsResponse = { batchEvaluations: [] }; // events wraps canned events as a one-shot AsyncIterable. async function* events(items: T[]): AsyncGenerator { @@ -1122,6 +1130,13 @@ export class TestEvalClient implements CoreEvalClient { private datasetListResponses = new Map(); private deleteDatasetResponse: DeleteDatasetResponse = DEFAULT_DELETE_DATASET_RESPONSE; private publishDatasetResponse: CreateDatasetVersionResponse = DEFAULT_PUBLISH_DATASET_RESPONSE; + // Batch-evaluation responses: get returns a canned job, list pages by + // nextToken, and the CloudWatch results are a canned array. batchEvalResultsError + // simulates a CloudWatch read failure surfaced as `resultsError`. + private batchEvalGetResponse: GetBatchEvaluationResponse = DEFAULT_GET_BATCH_EVAL_RESPONSE; + private batchEvalListResponses = new Map(); + private batchEvalResults: BatchEvaluationResultEntry[] = []; + private batchEvalResultsError?: unknown; private error?: Error; // setListResponse sets what listEvaluators resolves to (when not erroring). @@ -1230,6 +1245,36 @@ export class TestEvalClient implements CoreEvalClient { return this; } + // setBatchEvalGetResponse sets what getBatchEvaluation resolves to (when not + // erroring). + setBatchEvalGetResponse(response: GetBatchEvaluationResponse): this { + this.batchEvalGetResponse = response; + return this; + } + + // setBatchEvalListResponse sets what listBatchEvaluations resolves to (when + // not erroring). Pass `forNextToken` to serve a later page. + setBatchEvalListResponse(response: ListBatchEvaluationsResponse, forNextToken?: string): this { + this.batchEvalListResponses.set(forNextToken, response); + return this; + } + + // setBatchEvalResults sets the per-session results getBatchEvaluation merges in + // (when results are requested, the job is terminal, and it has a CloudWatch + // output config). + setBatchEvalResults(results: BatchEvaluationResultEntry[]): this { + this.batchEvalResults = results; + return this; + } + + // setBatchEvalResultsError makes getBatchEvaluation return `resultsError` + // instead of merged results, simulating a CloudWatch read failure. The job + // metadata is still returned. + setBatchEvalResultsError(error: unknown): this { + this.batchEvalResultsError = error; + return this; + } + // setError makes every subsequent call reject with `error`. Pass undefined to // clear it. setError(error: Error | undefined): this { @@ -1292,6 +1337,43 @@ export class TestEvalClient implements CoreEvalClient { return this.deleteResponse; } + async getBatchEvaluation( + id: string, + options: CoreOptions, + opts: { includeResults?: boolean } = {}, + ): Promise { + this.calls.push({ method: "getBatchEvaluation", args: [id, options, opts] }); + if (this.error) throw this.error; + + const detail: BatchEvaluationDetail = { ...this.batchEvalGetResponse }; + const includeResults = opts.includeResults ?? true; + const cw = detail.outputConfig?.cloudWatchConfig; + // Mirror the real client's gate: only merge results when requested, terminal, + // and a CloudWatch output config exists. + if (!includeResults || !isTerminalStatus(detail.status) || !cw?.logGroupName) { + return { detail }; + } + if (this.batchEvalResultsError !== undefined) { + return { detail, resultsError: this.batchEvalResultsError }; + } + detail.results = this.batchEvalResults; + return { detail }; + } + + async listBatchEvaluations( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listBatchEvaluations", args: [nextToken, maxResults, options] }); + if (this.error) throw this.error; + return ( + this.batchEvalListResponses.get(nextToken) ?? + this.batchEvalListResponses.get(undefined) ?? + DEFAULT_LIST_BATCH_EVALS_RESPONSE + ); + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/testing/fixtures.tsx b/src/testing/fixtures.tsx index d294ce50d..49b5dc798 100644 --- a/src/testing/fixtures.tsx +++ b/src/testing/fixtures.tsx @@ -4,13 +4,20 @@ import { expect } from "bun:test"; import type { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; import type { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import type { IAMClient } from "@aws-sdk/client-iam"; +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import type { ClientConfig, CreateControlClient, CreateDataClient, CreateIamClient, + CreateLogsClient, } from "../core/types"; -import { createControlClient, createDataClient, createIamClient } from "../core/factories"; +import { + createControlClient, + createDataClient, + createIamClient, + createLogsClient, +} from "../core/factories"; import { parse, stringify } from "./serialization"; // Golden-file record/replay for the AWS SDK seam. @@ -141,6 +148,7 @@ export function fixtureFactories(dir: string): { createControlClient: CreateControlClient; createDataClient: CreateDataClient; createIamClient: CreateIamClient; + createLogsClient: CreateLogsClient; } { return { createControlClient: (config: ClientConfig) => { @@ -163,6 +171,12 @@ export function fixtureFactories(dir: string): { send: makeRecordingSend(real, dir), } as unknown as IAMClient; }, + createLogsClient: (config: ClientConfig) => { + const real = createLogsClient(config); + return { + send: makeRecordingSend(real, dir), + } as unknown as CloudWatchLogsClient; + }, }; }