diff --git a/src/core/configBundle.test.ts b/src/core/configBundle.test.ts new file mode 100644 index 000000000..eac360b59 --- /dev/null +++ b/src/core/configBundle.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, test } from "bun:test"; +import { + CreateConfigurationBundleCommand, + DeleteConfigurationBundleCommand, + GetConfigurationBundleCommand, + GetConfigurationBundleVersionCommand, + ListConfigurationBundlesCommand, + ListConfigurationBundleVersionsCommand, + UpdateConfigurationBundleCommand, + type BedrockAgentCoreControlClient, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { NetworkingError } from "../errors"; +import { EvalClient } from "./eval"; +import type { AwsClients, ClientConfig } from "./types"; + +const OPTIONS = { region: "us-west-2", endpointUrl: "https://control.test" }; +const COMPONENTS = { + "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/orders-agent": { + configuration: { system_prompt: "Help with orders." }, + }, +}; + +function subject(respond: (command: unknown) => Promise): { + client: EvalClient; + configs: ClientConfig[]; +} { + const configs: ClientConfig[] = []; + const control = { send: respond } as unknown as BedrockAgentCoreControlClient; + const clients = { + control: (config: ClientConfig) => { + configs.push(config); + return control; + }, + } as unknown as AwsClients; + return { client: new EvalClient(clients), configs }; +} + +describe("EvalClient configuration bundles", () => { + test("create sends CreateConfigurationBundleCommand unchanged", async () => { + const sent: unknown[] = []; + const response = { + bundleArn: "arn:bundle:b-1", + bundleId: "b-1", + versionId: "v-1", + createdAt: new Date("2026-08-07T00:00:00Z"), + }; + const { client, configs } = subject(async (command) => { + sent.push(command); + return response; + }); + const input = { + bundleName: "orders-prompt", + components: COMPONENTS, + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/abc", + }; + + expect(await client.createConfigurationBundle(input, OPTIONS)).toBe(response); + expect(sent[0]).toBeInstanceOf(CreateConfigurationBundleCommand); + expect((sent[0] as CreateConfigurationBundleCommand).input).toEqual(input); + expect(configs).toEqual([{ region: "us-west-2", endpoint: "https://control.test" }]); + }); + + test("get selects the latest or immutable-version SDK operation", async () => { + const sent: unknown[] = []; + const { client } = subject(async (command) => { + sent.push(command); + return {}; + }); + + await client.getConfigurationBundle("b-1", undefined, OPTIONS); + await client.getConfigurationBundle("b-1", "v-2", OPTIONS); + + expect(sent[0]).toBeInstanceOf(GetConfigurationBundleCommand); + expect((sent[0] as GetConfigurationBundleCommand).input).toEqual({ bundleId: "b-1" }); + expect(sent[1]).toBeInstanceOf(GetConfigurationBundleVersionCommand); + expect((sent[1] as GetConfigurationBundleVersionCommand).input).toEqual({ + bundleId: "b-1", + versionId: "v-2", + }); + }); + + test("list sends only the aligned pagination fields", async () => { + const sent: unknown[] = []; + const { client } = subject(async (command) => { + sent.push(command); + return { bundles: [] }; + }); + + await client.listConfigurationBundles("token-1", 10, OPTIONS); + + expect(sent[0]).toBeInstanceOf(ListConfigurationBundlesCommand); + expect((sent[0] as ListConfigurationBundlesCommand).input).toEqual({ + nextToken: "token-1", + maxResults: 10, + }); + }); + + test("update gets the latest version and sends it as the sole parent", async () => { + const sent: unknown[] = []; + const response = { + bundleArn: "arn:bundle:b-1", + bundleId: "b-1", + versionId: "v-3", + updatedAt: new Date("2026-08-07T00:00:00Z"), + }; + const { client } = subject(async (command) => { + sent.push(command); + if (command instanceof GetConfigurationBundleCommand) { + return { + versionId: "v-2", + lineageMetadata: { branchName: "custom-branch" }, + }; + } + return response; + }); + + expect( + await client.updateConfigurationBundle( + "b-1", + { + components: COMPONENTS, + commitMessage: "Replace order support configuration", + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/new", + }, + OPTIONS, + ), + ).toBe(response); + + expect(sent).toHaveLength(2); + expect(sent[0]).toBeInstanceOf(GetConfigurationBundleCommand); + expect((sent[0] as GetConfigurationBundleCommand).input).toEqual({ bundleId: "b-1" }); + expect(sent[1]).toBeInstanceOf(UpdateConfigurationBundleCommand); + expect((sent[1] as UpdateConfigurationBundleCommand).input).toEqual({ + bundleId: "b-1", + components: COMPONENTS, + commitMessage: "Replace order support configuration", + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/new", + parentVersionIds: ["v-2"], + }); + }); + + test("update fails before sending when latest has no version id", async () => { + const sent: unknown[] = []; + const { client } = subject(async (command) => { + sent.push(command); + return {}; + }); + + const promise = client.updateConfigurationBundle( + "b-1", + { + components: COMPONENTS, + commitMessage: "Replace order support configuration", + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/new", + }, + OPTIONS, + ); + + await expect(promise).rejects.toBeInstanceOf(NetworkingError); + await expect(promise).rejects.toThrow(/returned no latest version/); + expect(sent).toHaveLength(1); + expect(sent[0]).toBeInstanceOf(GetConfigurationBundleCommand); + }); + + test("delete sends DeleteConfigurationBundleCommand", async () => { + const sent: unknown[] = []; + const response = { bundleId: "b-1", status: "DELETING" as const }; + const { client } = subject(async (command) => { + sent.push(command); + return response; + }); + + expect(await client.deleteConfigurationBundle("b-1", OPTIONS)).toBe(response); + expect(sent[0]).toBeInstanceOf(DeleteConfigurationBundleCommand); + expect((sent[0] as DeleteConfigurationBundleCommand).input).toEqual({ bundleId: "b-1" }); + }); + + test("version list sends the parent bundle and pagination fields", async () => { + const sent: unknown[] = []; + const { client } = subject(async (command) => { + sent.push(command); + return { versions: [] }; + }); + + await client.listConfigurationBundleVersions("b-1", "token-1", 5, OPTIONS); + + expect(sent[0]).toBeInstanceOf(ListConfigurationBundleVersionsCommand); + expect((sent[0] as ListConfigurationBundleVersionsCommand).input).toEqual({ + bundleId: "b-1", + nextToken: "token-1", + maxResults: 5, + }); + }); +}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 964c11f48..f037798b5 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -1,38 +1,52 @@ import { + CreateConfigurationBundleCommand, CreateDatasetCommand, CreateDatasetVersionCommand, CreateEvaluatorCommand, CreateOnlineEvaluationConfigCommand, + DeleteConfigurationBundleCommand, DeleteDatasetCommand, DeleteEvaluatorCommand, DeleteOnlineEvaluationConfigCommand, + GetConfigurationBundleCommand, + GetConfigurationBundleVersionCommand, GetAgentRuntimeCommand, GetDatasetCommand, GetEvaluatorCommand, GetHarnessCommand, GetOnlineEvaluationConfigCommand, + ListConfigurationBundlesCommand, + ListConfigurationBundleVersionsCommand, ListDatasetsCommand, ListEvaluatorsCommand, ListOnlineEvaluationConfigsCommand, + UpdateConfigurationBundleCommand, UpdateEvaluatorCommand, UpdateOnlineEvaluationConfigCommand, + type CreateConfigurationBundleResponse, type CreateDatasetResponse, type CreateDatasetVersionResponse, type CreateEvaluatorRequest, type CreateEvaluatorResponse, type CreateOnlineEvaluationConfigResponse, + type DeleteConfigurationBundleResponse, type DeleteDatasetResponse, type DeleteEvaluatorResponse, type DeleteOnlineEvaluationConfigResponse, type EvaluatorConfig, + type GetConfigurationBundleResponse, + type GetConfigurationBundleVersionResponse, type GetDatasetResponse, type GetEvaluatorResponse, type GetOnlineEvaluationConfigResponse, + type ListConfigurationBundlesResponse, + type ListConfigurationBundleVersionsResponse, type ListDatasetsResponse, type ListEvaluatorsResponse, type DataSourceConfig, type ListOnlineEvaluationConfigsResponse, type Rule, + type UpdateConfigurationBundleResponse, type UpdateEvaluatorResponse, type UpdateOnlineEvaluationConfigResponse, type BedrockAgentCoreControlClient, @@ -43,9 +57,11 @@ import type { CodeBasedUpdate, RoleScopeWarning, CoreEvalClient, + CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, LlmAsAJudgeUpdate, + UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; import { atomicWriteStream } from "../io"; @@ -465,6 +481,83 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); } + async createConfigurationBundle( + input: CreateConfigurationBundleInput, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new CreateConfigurationBundleCommand(input)); + } + + async getConfigurationBundle( + id: string, + version: string | undefined, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + return version === undefined + ? control.send(new GetConfigurationBundleCommand({ bundleId: id })) + : control.send( + new GetConfigurationBundleVersionCommand({ bundleId: id, versionId: version }), + ); + } + + async listConfigurationBundles( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListConfigurationBundlesCommand({ nextToken, maxResults })); + } + + async updateConfigurationBundle( + id: string, + update: UpdateConfigurationBundleInput, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send(new GetConfigurationBundleCommand({ bundleId: id })); + if (!current.versionId) { + throw new NetworkingError( + `Configuration bundle "${id}" returned no latest version and cannot be updated`, + { meta: { bundleId: id } }, + ); + } + + return control.send( + new UpdateConfigurationBundleCommand({ + bundleId: id, + components: update.components, + commitMessage: update.commitMessage, + kmsKeyArn: update.kmsKeyArn, + parentVersionIds: [current.versionId], + }), + ); + } + + async deleteConfigurationBundle( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new DeleteConfigurationBundleCommand({ bundleId: id })); + } + + async listConfigurationBundleVersions( + id: string, + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListConfigurationBundleVersionsCommand({ bundleId: id, nextToken, maxResults })); + } + async createDataset( input: CreateDatasetInput, options: CoreOptions, diff --git a/src/handlers/eval/config-bundle/components.ts b/src/handlers/eval/config-bundle/components.ts new file mode 100644 index 000000000..7941c1dd0 --- /dev/null +++ b/src/handlers/eval/config-bundle/components.ts @@ -0,0 +1,31 @@ +import z from "zod"; +import type { ComponentConfiguration } from "@aws-sdk/client-bedrock-agentcore-control"; +import type { SourceResolver } from "../../../io"; +import { InputValidationError } from "../../../errors"; +import { parseJsonFlagWithSchema } from "../../utils"; + +const componentConfigurationSchema = z + .object({ + configuration: z.unknown().refine((value) => value !== undefined, "configuration is required"), + }) + .strict(); + +const componentMapSchema = z + .record(z.string().min(1), componentConfigurationSchema) + .refine((components) => Object.keys(components).length > 0, { + message: "must contain at least one component", + }); + +export type ConfigurationBundleComponents = Record; + +export async function resolveConfigurationBundleComponents( + value: string, + source: SourceResolver, +): Promise { + const text = await source.resolveText("components", value); + const components = parseJsonFlagWithSchema("components", text, componentMapSchema); + if (components === undefined) { + throw new InputValidationError("required option '--components ' not specified"); + } + return components as ConfigurationBundleComponents; +} diff --git a/src/handlers/eval/config-bundle/config-bundle.test.tsx b/src/handlers/eval/config-bundle/config-bundle.test.tsx new file mode 100644 index 000000000..dda683ab7 --- /dev/null +++ b/src/handlers/eval/config-bundle/config-bundle.test.tsx @@ -0,0 +1,423 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; +import type { CreateConfigurationBundleInput, UpdateConfigurationBundleInput } from "../types"; + +const REGION = "us-west-2"; +const COMPONENT_ARN = + "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/orders-agent-abc123"; +const COMPONENTS = { + [COMPONENT_ARN]: { + configuration: { + system_prompt: "You are an order-support assistant.", + settings: { cite_sources: true }, + }, + }, +}; + +const dirs: string[] = []; +afterEach(async () => { + await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function writeTempJson(value: unknown): Promise { + const dir = mkdtempSync(join(tmpdir(), "agentcore-config-bundle-")); + dirs.push(dir); + const path = join(dir, "components.json"); + await Bun.write(path, JSON.stringify(value)); + return path; +} + +function testConfigBundleCommand(stdin?: string) { + const core = new TestCoreClient(); + const io = testIO(); + if (stdin !== undefined) { + io.io.stdin.push(stdin); + io.io.stdin.push(null); + } + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + return { + core, + stdout: io.stdout, + route: (args: string[]) => root.route(["bun", "agentcore", ...args, "--region", REGION]), + }; +} + +function callArgs(core: TestCoreClient, method: string): unknown[] { + const call = core.eval.calls.find((candidate) => candidate.method === method); + if (!call) throw new Error(`${method} was not called`); + return call.args; +} + +describe("eval config-bundle command hierarchy", () => { + test("registers CRUDL and nested version list commands", () => { + const root = createRootHandler(new TestCoreClient(), { + io: testIO().io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const configBundle = root + .children() + .find((child) => child.name() === "eval") + ?.children() + .find((child) => child.name() === "config-bundle"); + + expect(configBundle?.children().map((child) => child.name())).toEqual([ + "create", + "get", + "list", + "update", + "delete", + "version", + ]); + expect( + configBundle + ?.children() + .find((child) => child.name() === "version") + ?.children() + .map((child) => child.name()), + ).toEqual(["list"]); + expect( + configBundle + ?.children() + .find((child) => child.name() === "create") + ?.flags() + .map((candidate) => candidate.name), + ).toEqual(["name", "components", "kms-key-arn"]); + expect( + configBundle + ?.children() + .find((child) => child.name() === "update") + ?.flags() + .map((candidate) => candidate.name), + ).toEqual(["id", "components", "commit-message", "kms-key-arn"]); + }); + + test("prints help for a bare config-bundle command", async () => { + const { core, stdout, route } = testConfigBundleCommand(); + + await route(["eval", "config-bundle", "--json"]); + + expect(stdout()).toContain("Usage: agentcore eval config-bundle"); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("config-bundle create", () => { + test("accepts an inline component map and renders the SDK response directly", async () => { + const { core, stdout, route } = testConfigBundleCommand(); + core.eval.setCreateConfigurationBundleResponse({ + bundleArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:configuration-bundle/b-1", + bundleId: "b-1", + versionId: "v-1", + createdAt: new Date("2026-08-07T00:00:00Z"), + }); + + await route([ + "eval", + "config-bundle", + "create", + "--name", + "orders-prompt", + "--components", + JSON.stringify(COMPONENTS), + "--kms-key-arn", + "arn:aws:kms:us-west-2:123456789012:key/abc", + ]); + + expect(callArgs(core, "createConfigurationBundle")[0]).toEqual({ + bundleName: "orders-prompt", + components: COMPONENTS, + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/abc", + } satisfies CreateConfigurationBundleInput); + expect(JSON.parse(stdout())).toMatchObject({ + bundleArn: expect.any(String), + bundleId: "b-1", + versionId: "v-1", + }); + }); + + test("reads components from stdin", async () => { + const { core, route } = testConfigBundleCommand(JSON.stringify(COMPONENTS)); + + await route([ + "eval", + "config-bundle", + "create", + "--name", + "orders-prompt", + "--components", + "-", + ]); + + expect(callArgs(core, "createConfigurationBundle")[0]).toMatchObject({ + components: COMPONENTS, + }); + }); + + test.each([ + ["an empty map", {}], + ["a component without configuration", { [COMPONENT_ARN]: {} }], + [ + "an unexpected component field", + { [COMPONENT_ARN]: { configuration: {}, description: "not accepted" } }, + ], + ])("rejects %s", async (_name, contents) => { + const { core, route } = testConfigBundleCommand(); + + await expect( + route([ + "eval", + "config-bundle", + "create", + "--name", + "orders-prompt", + "--components", + JSON.stringify(contents), + ]), + ).rejects.toThrow(/Invalid value for option '--components'/); + expect(core.eval.calls).toHaveLength(0); + }); + + test("rejects malformed component JSON", async () => { + const { core, route } = testConfigBundleCommand(); + + await expect( + route([ + "eval", + "config-bundle", + "create", + "--name", + "orders-prompt", + "--components", + "{not-json", + ]), + ).rejects.toThrow(/Invalid JSON for option '--components'/); + expect(core.eval.calls).toHaveLength(0); + }); + + test("requires both --name and --components", async () => { + const { core, route } = testConfigBundleCommand(); + + await expect( + route(["eval", "config-bundle", "create", "--components", JSON.stringify(COMPONENTS)]), + ).rejects.toThrow(/--name/); + await expect( + route(["eval", "config-bundle", "create", "--name", "orders-prompt"]), + ).rejects.toThrow(/--components/); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("config-bundle get", () => { + test("gets the latest bundle when --version is absent", async () => { + const { core, stdout, route } = testConfigBundleCommand(); + core.eval.setGetConfigurationBundleResponse({ + bundleId: "b-1", + bundleArn: "arn:bundle:b-1", + bundleName: "orders-prompt", + versionId: "latest-v", + components: COMPONENTS, + createdAt: new Date("2026-08-06T00:00:00Z"), + updatedAt: new Date("2026-08-07T00:00:00Z"), + }); + + await route(["eval", "config-bundle", "get", "--id", "b-1"]); + + expect(callArgs(core, "getConfigurationBundle").slice(0, 2)).toEqual(["b-1", undefined]); + expect(JSON.parse(stdout()).versionId).toBe("latest-v"); + }); + + test("passes an explicit version through unchanged", async () => { + const { core, route } = testConfigBundleCommand(); + + await route(["eval", "config-bundle", "get", "--id", "b-1", "--version", "v-2"]); + + expect(callArgs(core, "getConfigurationBundle").slice(0, 2)).toEqual(["b-1", "v-2"]); + }); +}); + +describe("config-bundle list", () => { + test("passes pagination flags and renders the unmodified response", async () => { + const { core, stdout, route } = testConfigBundleCommand(); + core.eval.setListConfigurationBundlesResponse( + { + bundles: [ + { + bundleArn: "arn:bundle:b-2", + bundleId: "b-2", + bundleName: "second-page", + }, + ], + nextToken: "token-2", + }, + "token-1", + ); + + await route(["eval", "config-bundle", "list", "--max-results", "1", "--next-token", "token-1"]); + + expect(callArgs(core, "listConfigurationBundles").slice(0, 2)).toEqual(["token-1", 1]); + expect(JSON.parse(stdout())).toMatchObject({ + bundles: [{ bundleId: "b-2", bundleName: "second-page" }], + nextToken: "token-2", + }); + }); +}); + +describe("config-bundle update", () => { + test("passes a complete replacement component map and KMS key", async () => { + const path = await writeTempJson(COMPONENTS); + const { core, route } = testConfigBundleCommand(); + + await route([ + "eval", + "config-bundle", + "update", + "--id", + "b-1", + "--components", + `file://${path}`, + "--commit-message", + "Replace order support configuration", + "--kms-key-arn", + "arn:aws:kms:us-west-2:123456789012:key/replacement", + ]); + + expect(callArgs(core, "updateConfigurationBundle").slice(0, 2)).toEqual([ + "b-1", + { + components: COMPONENTS, + commitMessage: "Replace order support configuration", + kmsKeyArn: "arn:aws:kms:us-west-2:123456789012:key/replacement", + } satisfies UpdateConfigurationBundleInput, + ]); + }); + + test("requires components even when a KMS key is provided", async () => { + const { core, route } = testConfigBundleCommand(); + + await expect( + route([ + "eval", + "config-bundle", + "update", + "--id", + "b-1", + "--commit-message", + "Rotate encryption key", + "--kms-key-arn", + "arn:aws:kms:us-west-2:123456789012:key/replacement", + ]), + ).rejects.toThrow(/required option '--components ' not specified/); + expect(core.eval.calls).toHaveLength(0); + }); + + test("requires a commit message", async () => { + const path = await writeTempJson(COMPONENTS); + const { core, route } = testConfigBundleCommand(); + + await expect( + route(["eval", "config-bundle", "update", "--id", "b-1", "--components", `file://${path}`]), + ).rejects.toThrow(/required option '--commit-message ' not specified/); + expect(core.eval.calls).toHaveLength(0); + }); + + test("requires an id", async () => { + const path = await writeTempJson(COMPONENTS); + const { core, route } = testConfigBundleCommand(); + + await expect( + route([ + "eval", + "config-bundle", + "update", + "--components", + `file://${path}`, + "--commit-message", + "Replace order support configuration", + ]), + ).rejects.toThrow(/required option '--id ' not specified/); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("config-bundle delete", () => { + test("takes only --id and renders the SDK response", async () => { + const { core, stdout, route } = testConfigBundleCommand(); + core.eval.setDeleteConfigurationBundleResponse({ bundleId: "b-1", status: "DELETING" }); + + await route(["eval", "config-bundle", "delete", "--id", "b-1"]); + + expect(callArgs(core, "deleteConfigurationBundle")[0]).toBe("b-1"); + expect(JSON.parse(stdout())).toEqual({ bundleId: "b-1", status: "DELETING" }); + + const root = createRootHandler(new TestCoreClient(), { + io: testIO().io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const deleteCommand = root + .children() + .find((child) => child.name() === "eval") + ?.children() + .find((child) => child.name() === "config-bundle") + ?.children() + .find((child) => child.name() === "delete"); + expect(deleteCommand?.flags().map((candidate) => candidate.name)).toEqual(["id"]); + }); +}); + +describe("config-bundle version list", () => { + test("passes the bundle id and pagination flags", async () => { + const { core, stdout, route } = testConfigBundleCommand(); + core.eval.setListConfigurationBundleVersionsResponse( + { + versions: [ + { + bundleArn: "arn:bundle:b-1", + bundleId: "b-1", + versionId: "v-2", + versionCreatedAt: new Date("2026-08-07T00:00:00Z"), + }, + ], + }, + "token-1", + ); + + await route([ + "eval", + "config-bundle", + "version", + "list", + "--id", + "b-1", + "--max-results", + "5", + "--next-token", + "token-1", + ]); + + expect(callArgs(core, "listConfigurationBundleVersions").slice(0, 3)).toEqual([ + "b-1", + "token-1", + 5, + ]); + expect(JSON.parse(stdout()).versions).toEqual([ + expect.objectContaining({ bundleId: "b-1", versionId: "v-2" }), + ]); + }); +}); diff --git a/src/handlers/eval/config-bundle/create/index.tsx b/src/handlers/eval/config-bundle/create/index.tsx new file mode 100644 index 000000000..b5224bc9e --- /dev/null +++ b/src/handlers/eval/config-bundle/create/index.tsx @@ -0,0 +1,50 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { SourceResolver, type AppIO } from "../../../../io"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { resolveConfigurationBundleComponents } from "../components"; + +export const createCreateConfigBundleHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create a configuration bundle and its initial immutable version", + flags: [ + flag("name", "the name of the configuration bundle", z.string().optional()), + flag( + "components", + "complete component configuration map (JSON inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "kms-key-arn", + "customer managed KMS key ARN for component configurations", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) { + throw new InputValidationError("required option '--name ' not specified"); + } + if (!flags["components"]) { + throw new InputValidationError("required option '--components ' not specified"); + } + + const components = await resolveConfigurationBundleComponents( + flags["components"], + new SourceResolver({ stdin: io.stdin }), + ); + ctx.require(JsonRendererKey).renderJson( + await core.eval.createConfigurationBundle( + { + bundleName: flags["name"], + components, + kmsKeyArn: flags["kms-key-arn"], + }, + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/config-bundle/delete/index.tsx b/src/handlers/eval/config-bundle/delete/index.tsx new file mode 100644 index 000000000..4e7066647 --- /dev/null +++ b/src/handlers/eval/config-bundle/delete/index.tsx @@ -0,0 +1,22 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteConfigBundleHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete a configuration bundle and all of its versions", + flags: [flag("id", "the ID of the configuration bundle", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) { + throw new InputValidationError("required option '--id ' not specified"); + } + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteConfigurationBundle(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/config-bundle/get/index.tsx b/src/handlers/eval/config-bundle/get/index.tsx new file mode 100644 index 000000000..153a43e2f --- /dev/null +++ b/src/handlers/eval/config-bundle/get/index.tsx @@ -0,0 +1,31 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetConfigBundleHandler = (core: Core) => + createHandler({ + name: "get", + description: "get the latest or a specific configuration bundle version", + flags: [ + flag("id", "the ID of the configuration bundle", z.string().optional()), + flag("version", "the immutable version ID to retrieve", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) { + throw new InputValidationError("required option '--id ' not specified"); + } + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.getConfigurationBundle( + flags["id"], + flags["version"], + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/config-bundle/index.tsx b/src/handlers/eval/config-bundle/index.tsx new file mode 100644 index 000000000..6accd1f89 --- /dev/null +++ b/src/handlers/eval/config-bundle/index.tsx @@ -0,0 +1,21 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createCreateConfigBundleHandler } from "./create"; +import { createDeleteConfigBundleHandler } from "./delete"; +import { createGetConfigBundleHandler } from "./get"; +import { createListConfigBundlesHandler } from "./list"; +import { createUpdateConfigBundleHandler } from "./update"; +import { createConfigBundleVersionHandler } from "./version"; + +export function createConfigBundleHandler(core: Core, io: AppIO): Router { + return new Router("config-bundle", "manage AgentCore configuration bundles") + .default(createHelpDefault(io)) + .handler(createCreateConfigBundleHandler(core, io)) + .handler(createGetConfigBundleHandler(core)) + .handler(createListConfigBundlesHandler(core)) + .handler(createUpdateConfigBundleHandler(core, io)) + .handler(createDeleteConfigBundleHandler(core)) + .handler(createConfigBundleVersionHandler(core, io)); +} diff --git a/src/handlers/eval/config-bundle/list/index.tsx b/src/handlers/eval/config-bundle/list/index.tsx new file mode 100644 index 000000000..6c064883e --- /dev/null +++ b/src/handlers/eval/config-bundle/list/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListConfigBundlesHandler = (core: Core) => + createHandler({ + name: "list", + description: "list configuration bundles", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.listConfigurationBundles( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/config-bundle/update/index.tsx b/src/handlers/eval/config-bundle/update/index.tsx new file mode 100644 index 000000000..d06e99965 --- /dev/null +++ b/src/handlers/eval/config-bundle/update/index.tsx @@ -0,0 +1,61 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { SourceResolver, type AppIO } from "../../../../io"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { resolveConfigurationBundleComponents } from "../components"; + +export const createUpdateConfigBundleHandler = (core: Core, io: AppIO) => + createHandler({ + name: "update", + description: "create a new immutable configuration bundle version", + flags: [ + flag("id", "the ID of the configuration bundle", z.string().optional()), + flag( + "components", + "replacement component configuration map (JSON inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "commit-message", + "message describing the configuration bundle update", + z.string().max(500).optional(), + ), + flag( + "kms-key-arn", + "customer managed KMS key ARN to rotate component encryption to", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) { + throw new InputValidationError("required option '--id ' not specified"); + } + if (!flags["components"]) { + throw new InputValidationError("required option '--components ' not specified"); + } + if (!flags["commit-message"]) { + throw new InputValidationError( + "required option '--commit-message ' not specified", + ); + } + + const components = await resolveConfigurationBundleComponents( + flags["components"], + new SourceResolver({ stdin: io.stdin }), + ); + ctx.require(JsonRendererKey).renderJson( + await core.eval.updateConfigurationBundle( + flags["id"], + { + components, + commitMessage: flags["commit-message"], + kmsKeyArn: flags["kms-key-arn"], + }, + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/config-bundle/version/index.tsx b/src/handlers/eval/config-bundle/version/index.tsx new file mode 100644 index 000000000..ba848ed41 --- /dev/null +++ b/src/handlers/eval/config-bundle/version/index.tsx @@ -0,0 +1,11 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createHelpDefault } from "../../../help"; +import { createListConfigBundleVersionsHandler } from "./list"; + +export function createConfigBundleVersionHandler(core: Core, io: AppIO): Router { + return new Router("version", "inspect immutable configuration bundle versions") + .default(createHelpDefault(io)) + .handler(createListConfigBundleVersionsHandler(core)); +} diff --git a/src/handlers/eval/config-bundle/version/list/index.tsx b/src/handlers/eval/config-bundle/version/list/index.tsx new file mode 100644 index 000000000..1a5e9f04f --- /dev/null +++ b/src/handlers/eval/config-bundle/version/list/index.tsx @@ -0,0 +1,33 @@ +import z from "zod"; +import { InputValidationError } from "../../../../../errors"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { Core } from "../../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; + +export const createListConfigBundleVersionsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list immutable versions of a configuration bundle", + flags: [ + flag("id", "the ID of the configuration bundle", z.string().optional()), + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) { + throw new InputValidationError("required option '--id ' not specified"); + } + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.listConfigurationBundleVersions( + flags["id"], + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 9c37828d4..ca34f0a27 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -6,6 +6,7 @@ import type { Core } from "../types"; import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; import { createDatasetHandler } from "./dataset"; +import { createConfigBundleHandler } from "./config-bundle"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -13,7 +14,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .default(renderTui(core, io)) .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)) - .handler(createDatasetHandler(core, io)); + .handler(createDatasetHandler(core, io)) + .handler(createConfigBundleHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 9dab86006..65deb6474 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -1,22 +1,31 @@ import type { + CreateConfigurationBundleRequest, + CreateConfigurationBundleResponse, CreateDatasetRequest, CreateDatasetResponse, CreateDatasetVersionResponse, CreateEvaluatorRequest, CreateEvaluatorResponse, CreateOnlineEvaluationConfigResponse, + DeleteConfigurationBundleResponse, DeleteDatasetResponse, DeleteEvaluatorResponse, DeleteOnlineEvaluationConfigResponse, + GetConfigurationBundleResponse, + GetConfigurationBundleVersionResponse, GetDatasetResponse, GetEvaluatorResponse, GetOnlineEvaluationConfigResponse, + ListConfigurationBundlesResponse, + ListConfigurationBundleVersionsResponse, ListDatasetsResponse, ListEvaluatorsResponse, ListOnlineEvaluationConfigsResponse, DataSourceConfig, RatingScale, Rule, + UpdateConfigurationBundleRequest, + UpdateConfigurationBundleResponse, UpdateEvaluatorResponse, UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; @@ -102,6 +111,14 @@ export type RoleScopeWarning = { }; export type CreateDatasetInput = CreateDatasetRequest; +export type CreateConfigurationBundleInput = Pick< + CreateConfigurationBundleRequest, + "bundleName" | "components" | "kmsKeyArn" +>; +export type UpdateConfigurationBundleInput = Required< + Pick +> & + Pick; // CoreEvalClient is the evaluator, online evaluation, and dataset surface the eval // handlers depend on. It is declared here, next to the handlers that consume it, @@ -165,6 +182,39 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; + createConfigurationBundle( + input: CreateConfigurationBundleInput, + options: CoreOptions, + ): Promise; + // Omitting version returns the latest mainline version; an explicit version + // selects the immutable version API. + getConfigurationBundle( + id: string, + version: string | undefined, + options: CoreOptions, + ): Promise; + listConfigurationBundles( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + // Updates are appended to the latest mainline version by the Core client. + updateConfigurationBundle( + id: string, + update: UpdateConfigurationBundleInput, + options: CoreOptions, + ): Promise; + deleteConfigurationBundle( + id: string, + options: CoreOptions, + ): Promise; + listConfigurationBundleVersions( + id: string, + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + // createDataset seeds a new dataset's DRAFT from `source`, which is required. // `schemaType` governs the structure of every example and is immutable after creation. // The response reports status CREATING — ingestion is asynchronous, and the dataset is not diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index da72b69df..023ed05c4 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -32,21 +32,28 @@ import type { ListGatewayRulesResponse, ListGatewaysResponse, ListGatewayTargetsResponse, + CreateConfigurationBundleResponse, CreateDatasetResponse, CreateDatasetVersionResponse, CreateEvaluatorRequest, CreateEvaluatorResponse, CreateOnlineEvaluationConfigResponse, + DeleteConfigurationBundleResponse, DeleteDatasetResponse, DeleteEvaluatorResponse, DeleteOnlineEvaluationConfigResponse, + GetConfigurationBundleResponse, + GetConfigurationBundleVersionResponse, GetDatasetResponse, + ListConfigurationBundlesResponse, + ListConfigurationBundleVersionsResponse, ListDatasetsResponse, GetEvaluatorResponse, GetOnlineEvaluationConfigResponse, ListEvaluatorsResponse, ListOnlineEvaluationConfigsResponse, MemoryView, + UpdateConfigurationBundleResponse, UpdateEvaluatorResponse, UpdateOnlineEvaluationConfigResponse, UpdateApiKeyCredentialProviderResponse, @@ -95,9 +102,11 @@ import type { import type { CodeBasedUpdate, CoreEvalClient, + CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, LlmAsAJudgeUpdate, + UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; import { abortable } from "../core/abortable"; @@ -191,6 +200,15 @@ const DEFAULT_CREATE_ONLINE_EVAL_RESPONSE = {} as CreateOnlineEvaluationConfigRe const DEFAULT_UPDATE_ONLINE_EVAL_RESPONSE = {} as UpdateOnlineEvaluationConfigResponse; const DEFAULT_GET_ONLINE_EVAL_RESPONSE = {} as GetOnlineEvaluationConfigResponse; const DEFAULT_DELETE_ONLINE_EVAL_RESPONSE = {} as DeleteOnlineEvaluationConfigResponse; +const DEFAULT_CREATE_CONFIG_BUNDLE_RESPONSE = {} as CreateConfigurationBundleResponse; +const DEFAULT_GET_CONFIG_BUNDLE_RESPONSE = {} as GetConfigurationBundleResponse; +const DEFAULT_GET_CONFIG_BUNDLE_VERSION_RESPONSE = {} as GetConfigurationBundleVersionResponse; +const DEFAULT_LIST_CONFIG_BUNDLES_RESPONSE: ListConfigurationBundlesResponse = { bundles: [] }; +const DEFAULT_UPDATE_CONFIG_BUNDLE_RESPONSE = {} as UpdateConfigurationBundleResponse; +const DEFAULT_DELETE_CONFIG_BUNDLE_RESPONSE = {} as DeleteConfigurationBundleResponse; +const DEFAULT_LIST_CONFIG_BUNDLE_VERSIONS_RESPONSE: ListConfigurationBundleVersionsResponse = { + versions: [], +}; const DEFAULT_CREATE_DATASET_RESPONSE = {} as CreateDatasetResponse; const DEFAULT_GET_DATASET_RESPONSE = {} as GetDatasetResponse; const DEFAULT_LIST_DATASETS_RESPONSE: ListDatasetsResponse = { datasets: [] }; @@ -1117,6 +1135,24 @@ export class TestEvalClient implements CoreEvalClient { DEFAULT_GET_ONLINE_EVAL_RESPONSE; private onlineEvalDeleteResponse: DeleteOnlineEvaluationConfigResponse = DEFAULT_DELETE_ONLINE_EVAL_RESPONSE; + private createConfigBundleResponse: CreateConfigurationBundleResponse = + DEFAULT_CREATE_CONFIG_BUNDLE_RESPONSE; + private getConfigBundleResponse: GetConfigurationBundleResponse = + DEFAULT_GET_CONFIG_BUNDLE_RESPONSE; + private getConfigBundleVersionResponse: GetConfigurationBundleVersionResponse = + DEFAULT_GET_CONFIG_BUNDLE_VERSION_RESPONSE; + private configBundleListResponses = new Map< + string | undefined, + ListConfigurationBundlesResponse + >(); + private updateConfigBundleResponse: UpdateConfigurationBundleResponse = + DEFAULT_UPDATE_CONFIG_BUNDLE_RESPONSE; + private deleteConfigBundleResponse: DeleteConfigurationBundleResponse = + DEFAULT_DELETE_CONFIG_BUNDLE_RESPONSE; + private configBundleVersionListResponses = new Map< + string | undefined, + ListConfigurationBundleVersionsResponse + >(); private createDatasetResponse: CreateDatasetResponse = DEFAULT_CREATE_DATASET_RESPONSE; private getDatasetResponse: GetDatasetResponse = DEFAULT_GET_DATASET_RESPONSE; private datasetListResponses = new Map(); @@ -1195,6 +1231,47 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setCreateConfigurationBundleResponse(response: CreateConfigurationBundleResponse): this { + this.createConfigBundleResponse = response; + return this; + } + + setGetConfigurationBundleResponse(response: GetConfigurationBundleResponse): this { + this.getConfigBundleResponse = response; + return this; + } + + setGetConfigurationBundleVersionResponse(response: GetConfigurationBundleVersionResponse): this { + this.getConfigBundleVersionResponse = response; + return this; + } + + setListConfigurationBundlesResponse( + response: ListConfigurationBundlesResponse, + forNextToken?: string, + ): this { + this.configBundleListResponses.set(forNextToken, response); + return this; + } + + setUpdateConfigurationBundleResponse(response: UpdateConfigurationBundleResponse): this { + this.updateConfigBundleResponse = response; + return this; + } + + setDeleteConfigurationBundleResponse(response: DeleteConfigurationBundleResponse): this { + this.deleteConfigBundleResponse = response; + return this; + } + + setListConfigurationBundleVersionsResponse( + response: ListConfigurationBundleVersionsResponse, + forNextToken?: string, + ): this { + this.configBundleVersionListResponses.set(forNextToken, response); + return this; + } + // setCreateDatasetResponse sets what createDataset resolves to (when not // erroring). setCreateDatasetResponse(response: CreateDatasetResponse): this { @@ -1355,6 +1432,78 @@ export class TestEvalClient implements CoreEvalClient { return this.onlineEvalDeleteResponse; } + async createConfigurationBundle( + input: CreateConfigurationBundleInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createConfigurationBundle", args: [input, options] }); + if (this.error) throw this.error; + return this.createConfigBundleResponse; + } + + async getConfigurationBundle( + id: string, + version: string | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "getConfigurationBundle", args: [id, version, options] }); + if (this.error) throw this.error; + return version === undefined + ? this.getConfigBundleResponse + : this.getConfigBundleVersionResponse; + } + + async listConfigurationBundles( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listConfigurationBundles", args: [nextToken, maxResults, options] }); + if (this.error) throw this.error; + return ( + this.configBundleListResponses.get(nextToken) ?? + this.configBundleListResponses.get(undefined) ?? + DEFAULT_LIST_CONFIG_BUNDLES_RESPONSE + ); + } + + async updateConfigurationBundle( + id: string, + update: UpdateConfigurationBundleInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "updateConfigurationBundle", args: [id, update, options] }); + if (this.error) throw this.error; + return this.updateConfigBundleResponse; + } + + async deleteConfigurationBundle( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "deleteConfigurationBundle", args: [id, options] }); + if (this.error) throw this.error; + return this.deleteConfigBundleResponse; + } + + async listConfigurationBundleVersions( + id: string, + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ + method: "listConfigurationBundleVersions", + args: [id, nextToken, maxResults, options], + }); + if (this.error) throw this.error; + return ( + this.configBundleVersionListResponses.get(nextToken) ?? + this.configBundleVersionListResponses.get(undefined) ?? + DEFAULT_LIST_CONFIG_BUNDLE_VERSIONS_RESPONSE + ); + } + async createDataset( input: CreateDatasetInput, options: CoreOptions,