feat(migrate): retain unsupported v0 resources automatically

scripts/generate-v0-cdk-apps.ts

@@ -1,9 +1,11 @@
 #!/usr/bin/env bun
 /**
- * Regenerates src/core/project/migrate/v0CdkApps.ts: what every CLI release that
- * creates v0 projects (v0.4.0 to v0.31.1, 1.0.0-preview.1 to 1.0.0-preview.33)
- * vended in agentcore/cdk, read from its git tag's src/assets/cdk. Releases that
- * vended identical apps share an entry. Run from a clone with the release tags:
+ * Migration must recognize the CDK app every v0 release generated, so it can tell
+ * an unmodified app (safe to replace with v1's) from one with customer changes
+ * (which replacing would lose). This regenerates that list,
+ * src/core/project/migrate/v0CdkApps.ts, from each release's git tag
+ * (v0.4.0 to v0.31.1, 1.0.0-preview.1 to 1.0.0-preview.33), rather than keeping
+ * it by hand, so a release can't be missed. Run it from a clone with the tags:
  *
  *   bun scripts/generate-v0-cdk-apps.ts
  */

src/core/project/migrate/__fixtures__/physical-names-contract.agentcore.json

@@ -291,24 +291,7 @@
       "strategies": []
     }
   ],
-  "knowledgeBases": [
-    {
-      "type": "AgentCoreKnowledgeBase",
-      "name": "kb-docs",
-      "description": "kb",
-      "dataSources": [
-        {
-          "type": "S3",
-          "uri": "s3://my-bucket/docs"
-        },
-        {
-          "type": "CONFLUENCE",
-          "connectorConfigFile": "kb/confluence.json"
-        }
-      ],
-      "gateway": "gw1"
-    }
-  ],
+  "knowledgeBases": [],
   "credentials": [
     {
       "authorizerType": "ApiKeyCredentialProvider",
@@ -592,7 +575,7 @@
               "name": "Retrieve",
               "description": "d",
               "parameterValues": {
-                "knowledgeBaseId": "kb-docs"
+                "knowledgeBaseId": "KBDOCS001A"
               },
               "parameterOverrides": [
                 {

src/core/project/migrate/apply.test.ts

@@ -507,3 +507,182 @@ describe("physicalIdChanges", () => {
     ]);
   });
 });
+
+describe("applyMigration retaining unsupported resources", () => {
+  const DATASET = "ApplicationDatasetGolden8AF7116D";
+  const retained = {
+    Resources: {
+      OldMemory: { Type: "AWS::BedrockAgentCore::Memory" },
+      [DATASET]: {
+        Type: "AWS::BedrockAgentCore::Dataset",
+        DeletionPolicy: "Retain",
+        UpdateReplacePolicy: "Retain",
+      },
+    },
+  };
+  const stack = (status: string) => ({
+    Stacks: [{ StackName: "AgentCore-Proj-default", StackStatus: status, Parameters: [] }],
+  });
+  const policyChange = {
+    Type: "Resource",
+    ResourceChange: {
+      Action: "Modify",
+      LogicalResourceId: DATASET,
+      Replacement: "False",
+      Details: [
+        { Target: { Attribute: "DeletionPolicy" } },
+        { Target: { Attribute: "UpdateReplacePolicy" } },
+      ],
+    },
+  };
+
+  function retainingTarget(execute: unknown = {}) {
+    const { client, sent } = fakeCloudFormation({
+      DescribeStacks: [
+        stack("UPDATE_COMPLETE"),
+        stack("UPDATE_IN_PROGRESS"),
+        stack("UPDATE_COMPLETE"),
+      ],
+      CreateChangeSet: [{ Id: "cs" }],
+      DescribeChangeSet: [{ Status: "CREATE_COMPLETE", Changes: [policyChange] }],
+      ExecuteChangeSet: [execute],
+      GetTemplate: [{ TemplateBody: JSON.stringify(retained) }],
+      ListStackResources: [
+        stackResources([
+          ["OldMemory", "mem"],
+          [DATASET, "ds"],
+        ]),
+        stackResources([
+          ["NewMemory", "mem"],
+          [DATASET, "ds"],
+        ]),
+      ],
+      CreateStackRefactor: [{ StackRefactorId: "rf-default" }],
+      DescribeStackRefactor: [
+        { Status: "CREATE_COMPLETE" },
+        { ExecutionStatus: "EXECUTE_COMPLETE" },
+      ],
+      ListStackRefactorActions: [
+        moveActions("AgentCore-Proj-default", [["OldMemory", "NewMemory"]]),
+      ],
+      ExecuteStackRefactor: [{}],
+    });
+    const entry: ApplyTarget = {
+      target: { name: "default", account: "111111111111", region: "us-east-1" },
+      stackName: "AgentCore-Proj-default",
+      renames: new Map([["OldMemory", "NewMemory"]]),
+      bridge: {
+        Resources: {
+          NewMemory: retained.Resources.OldMemory,
+          [DATASET]: retained.Resources[DATASET],
+        },
+      },
+      retain: { template: retained, logicalIds: new Set([DATASET]) },
+      client,
+      templates: async () => ({ put: async () => "", remove: async () => {} }),
+    };
+    return { entry, sent };
+  }
+
+  test("sets the retain policies before the refactor, recording the change set while it runs", async () => {
+    const root = tempRoot();
+    const { entry, sent } = retainingTarget();
+    const { applier, saved } = config(root, freshState("default"), entry);
+
+    const result = await applyMigration(applier);
+
+    expect(result.migrated).toBe(true);
+    expect(result.stacks[0]).toMatchObject({ outcome: "refactored", refactorId: "rf-default" });
+    expect(result.stacks[0]?.retainChangeSet).toStartWith("agentcore-migrate-retain-");
+    const names = sent.map((command) => command.name);
+    expect(names.indexOf("ExecuteChangeSet")).toBeLessThan(names.indexOf("CreateStackRefactor"));
+    const statuses = saved.flatMap((state) => state.targets.map((t) => t.status));
+    expect(statuses).toContain("retaining");
+    expect(statuses.indexOf("retaining")).toBeLessThan(statuses.indexOf("refactoring"));
+  });
+
+  test("an unknown retain outcome stops before the refactor, with the change set's check commands", async () => {
+    const root = tempRoot();
+    const { entry, sent } = retainingTarget(new Error("socket hang up"));
+    const { applier, events } = config(root, freshState("default"), entry);
+
+    const result = await applyMigration(applier);
+
+    expect(result.stacks[0]?.outcome).toBe("needs-repair");
+    expect(sent.some((command) => command.name === "CreateStackRefactor")).toBe(false);
+    // An update whose outcome isn't known may have run, so the files stay as written.
+    expect(result.restored).toBeUndefined();
+    expect(events).toEqual(["write"]);
+    const recovery = result.recovery.join("\n");
+    expect(recovery).toContain(
+      "aws cloudformation describe-change-set --stack-name AgentCore-Proj-default",
+    );
+    expect(recovery).toContain("agentcore migrate --target default");
+    expect(recovery).not.toContain("describe-stack-refactor");
+    const state = await readMigrationState(root);
+    expect(state?.targets[0]).toMatchObject({ status: "retaining" });
+  });
+
+  test("a retain update CloudFormation refused puts the v0 files back", async () => {
+    const root = tempRoot();
+    const { entry } = retainingTarget(clientError("ValidationError"));
+    const { applier, events } = config(root, freshState("default"), entry);
+
+    const result = await applyMigration(applier);
+
+    expect(result.stacks[0]?.outcome).toBe("failed");
+    expect(result.restored).toBe(true);
+    expect(events).toEqual(["write", "restore"]);
+  });
+
+  test("a successful rerun cleans up the retain change set recorded by the unknown run", async () => {
+    const root = tempRoot();
+    const { entry, sent } = target("default");
+    const state = freshState("default");
+    state.files!.status = "installed";
+    state.targets[0] = {
+      ...state.targets[0]!,
+      status: "retaining",
+      changeSetName: "retain-from-unknown-run",
+    };
+    const { applier } = config(root, state, entry);
+
+    const result = await applyMigration(applier);
+
+    expect(result.migrated).toBe(true);
+    expect(sent).toContainEqual({
+      name: "DeleteChangeSet",
+      input: {
+        StackName: "AgentCore-Proj-default",
+        ChangeSetName: "retain-from-unknown-run",
+      },
+    });
+  });
+
+  test("a successful no-refactor rerun also cleans up the recorded retain change set", async () => {
+    const root = tempRoot();
+    const { entry, sent } = target("default");
+    entry.renames = new Map();
+    entry.retain = undefined;
+    const state = freshState("default");
+    state.files!.status = "installed";
+    state.targets[0] = {
+      ...state.targets[0]!,
+      status: "retaining",
+      changeSetName: "retain-from-unknown-run",
+    };
+    const { applier } = config(root, state, entry);
+
+    const result = await applyMigration(applier);
+
+    expect(result.migrated).toBe(true);
+    expect(result.stacks[0]?.outcome).toBe("unchanged");
+    expect(sent).toContainEqual({
+      name: "DeleteChangeSet",
+      input: {
+        StackName: "AgentCore-Proj-default",
+        ChangeSetName: "retain-from-unknown-run",
+      },
+    });
+  });
+});

src/core/project/migrate/apply.ts

@@ -34,9 +34,15 @@ import {
   type PollOptions,
   type RefactorAction,
 } from "./refactor";
+import { deleteRetainChangeSet, retainCheckCommands, retainResources } from "./retain";
 import type { CfnTemplate } from "./templates";
 import type { TemplateStore } from "./templateStore";
 
+/**
+ * Applying records each step in the journal before taking it, so a migration
+ * stopped at any point (a crash, Ctrl-C, a failed refactor) can be finished or
+ * undone, and deploying can refuse a target that's only half migrated.
+ */
 /** The one deployment target whose stack this run refactors. */
 export type ApplyTarget = {
   target: AwsDeploymentTarget;
@@ -45,6 +51,11 @@ export type ApplyTarget = {
   renames: ReadonlyMap<string, string>;
   /** The deployed template with the renames applied; absent when nothing moves. */
   bridge?: CfnTemplate;
+  /**
+   * The update made before the refactor: the deployed template with retain
+   * policies on the unsupported resources the migration keeps.
+   */
+  retain?: { template: CfnTemplate; logicalIds: ReadonlySet<string> };
   /** A CloudFormation client for the target's account and region. */
   client: CloudFormationClient;
   /** Where a template too big to pass inline is uploaded. */
@@ -112,6 +123,9 @@ export async function applyMigration(
   };
   const state: MigrationState = structuredClone(config.state);
   const entry = config.target;
+  const previousRetainChangeSet = entry
+    ? state.targets.find((target) => target.name === entry.target.name)?.changeSetName
+    : undefined;
   const results: StackRefactorResult[] = entry
     ? [
         {
@@ -184,29 +198,67 @@ export async function applyMigration(
 
     // 3. The selected target's stack.
     let outcome: StackRefactorResult["outcome"] | undefined;
-    if (entry && entry.renames.size === 0) {
+    if (entry && entry.renames.size === 0 && !entry.retain) {
       dropTarget(state, entry.target.name);
       outcome = "unchanged";
     } else if (entry) {
       if (guard.stop()) {
         result({ outcome: "not-started", message: INTERRUPTED });
       } else {
+        const record = (status: JournalTarget["status"], extra: Partial<JournalTarget> = {}) =>
+          setTarget(state, {
+            name: entry.target.name,
+            account: entry.target.account,
+            region: entry.target.region,
+            stackName: entry.stackName,
+            status,
+            ...extra,
+          });
         try {
-          await refactorTarget(entry, poll, async (refactorId) => {
-            result({ refactorId });
-            setTarget(state, {
-              name: entry.target.name,
-              account: entry.target.account,
-              region: entry.target.region,
-              stackName: entry.stackName,
-              status: "refactoring",
-              refactorId,
-              renames: [...entry.renames],
-            });
+          if (entry.retain) {
+            // Before the refactor, so the stack it moves (and the v1 deploy after it)
+            // already keeps the resources v1 has no counterpart for.
+            await retainResources(
+              {
+                stackName: entry.stackName,
+                region: entry.target.region,
+                template: entry.retain.template,
+                logicalIds: entry.retain.logicalIds,
+                client: entry.client,
+                templates: entry.templates,
+              },
+              {
+                ...poll,
+                onCreated: async (changeSetName) => {
+                  result({ retainChangeSet: changeSetName });
+                  record("retaining", { changeSetName });
+                  await save(rootPath, state);
+                },
+              },
+            );
+            record("pending");
             await save(rootPath, state);
-          });
-          result({ outcome: "refactored" });
-          markRefactored(state, entry.target.name);
+          }
+          if (entry.renames.size === 0) {
+            dropTarget(state, entry.target.name);
+            result({ outcome: "unchanged" });
+          } else {
+            await refactorTarget(entry, poll, async (refactorId) => {
+              result({ refactorId });
+              setTarget(state, {
+                name: entry.target.name,
+                account: entry.target.account,
+                region: entry.target.region,
+                stackName: entry.stackName,
+                status: "refactoring",
+                refactorId,
+                renames: [...entry.renames],
+              });
+              await save(rootPath, state);
+            });
+            result({ outcome: "refactored" });
+            markRefactored(state, entry.target.name);
+          }
         } catch (error) {
           if (error instanceof ActionsMismatchError) {
             result({ outcome: "mismatch", message: error.message });
@@ -230,6 +282,14 @@ export async function applyMigration(
       }
       outcome = results[0]!.outcome;
     }
+    if (
+      entry &&
+      previousRetainChangeSet &&
+      outcome !== undefined &&
+      ["refactored", "unchanged"].includes(outcome)
+    ) {
+      await deleteRetainChangeSet(entry.client, entry.stackName, previousRetainChangeSet);
+    }
 
     // 4. Done, undone, or recorded for later.
     const stackError = results
@@ -513,6 +573,20 @@ async function discard(rootPath: string): Promise<string[]> {
 }
 
 function repairSteps(entry: StackRefactorResult): string[] {
+  if (entry.retainChangeSet && !entry.refactorId) {
+    const [changeSet, stack] = retainCheckCommands(
+      entry.stackName,
+      entry.region,
+      entry.retainChangeSet,
+    );
+    return [
+      `Stack ${entry.stackName} (target '${entry.target}') needs a look before anything else is ` +
+        `done to it: ${entry.message ?? "the outcome of the update that retains its unsupported resources isn't known"}. ` +
+        "That update only sets retain policies; it doesn't change or replace any resource.",
+      `Check it with \`${changeSet}\` and \`${stack}\`. Once the stack is UPDATE_COMPLETE (or ` +
+        `UPDATE_ROLLBACK_COMPLETE), run \`agentcore migrate --target ${entry.target}\` to carry on.`,
+    ];
+  }
   const id = entry.refactorId ?? "<refactor ID>";
   const describe = `aws cloudformation describe-stack-refactor --stack-refactor-id ${id} --region ${entry.region}`;
   const list = `aws cloudformation list-stack-resources --stack-name ${entry.stackName} --region ${entry.region}`;

src/core/project/migrate/contract.test.ts

@@ -40,7 +40,7 @@ describe("migratedFrom contract with @aws/agentcore-cdk", () => {
       "2dac5fc18e6893f04276cddab70ad50cd739968f6af25a314b34e8e5b7d5168c",
     );
     expect(sha256("physical-names-contract.agentcore.json")).toBe(
-      "25e3f3a77da888068fba1709a8507419c33826b429e04b8d600e9e2b7f61db5d",
+      "1bd381c22d9919ecb7e7ee7be6f59dbb4d3013a6c0ce8e8282b2e12d46e3f761",
     );
   });