diff --git a/docusaurus/docs/how-to-guides/updating-a-plugin.md b/docusaurus/docs/how-to-guides/updating-a-plugin.md index 63592f2602..9b105a148c 100644 --- a/docusaurus/docs/how-to-guides/updating-a-plugin.md +++ b/docusaurus/docs/how-to-guides/updating-a-plugin.md @@ -11,6 +11,7 @@ keywords: --- import UpdateNPM from '@shared/createplugin-update.md'; +import UpdateNPMAgent from '@shared/createplugin-update-agent.md'; import UpdateNPMCommit from '@shared/createplugin-update-commit.md'; import UpdateNPMForce from '@shared/createplugin-update-force.md'; @@ -50,6 +51,18 @@ The force flag can be used to bypass all safety checks related to uncommitted ch +#### `--agent` + +Some migrations include instructions for an AI coding agent instead of, or in addition to, an automated code transform. The agent flag controls whether an agent CLI installed on your machine runs those instructions. Pass `--agent=` to use a specific agent, `--agent` to use whichever supported agent is installed, or `--no-agent` to skip agent steps entirely. Supported agents: `claude-code`, `codex`. + + + +:::note + +Always pass the flag with an equals sign (`--agent=claude-code`) and after the `update` command. `create-plugin --agent update` is parsed as a value assignment and runs the wrong command. + +::: + ### What happens when you update The update command applies a series of changes, known as migrations, to your plugin to align it with the latest `create-plugin` standards. @@ -64,10 +77,26 @@ As each migration runs, its name and description are output to the terminal, alo If you pass the `--commit` flag, after each migration finishes it adds a Git commit to the current branch with the name of the migration. +### AI-assisted migrations + +Some changes can't be applied reliably by an automated code transform, for example porting user-authored configuration or refactoring plugin source code. Migrations for such changes ship natural-language instructions that an AI coding agent can apply, either on their own (prompt migrations) or as a follow-up to an automated transform (hybrid migrations). + +When the update includes such a migration and a supported agent CLI is installed, `create-plugin` asks whether to run it (or use the [`--agent`](#--agent) flag to decide up front). If you opt in: + +- The agent runs in an interactive session: you can watch it work and redirect it at any point. +- Each agent step ends automatically when the agent reports its result. If the update runs without `--commit`, per-migration commits are enabled so each agent step sees an isolated diff; pass `--no-commit` to prevent this. +- If an agent reports failure or its session ends unexpectedly, the update stops (or asks you how to proceed) without recording the update as complete. + +If you decline, pass `--no-agent`, or have no supported agent installed, the automated portions of the update still run. The instructions of any skipped agent steps are printed at the end of the update as manual next steps — the update is not blocked by them. + +If you run `create-plugin update` from inside an AI agent session (for example, asking Claude Code to update your plugin), `create-plugin` never starts a second agent. Instead it prints the deferred instructions in a machine-readable block for the agent driving your session to complete. + ## Automate updates via CI To make it even easier to keep your plugin up to date, use the provided GitHub workflow that runs the update command and automatically opens a PR if there are any changes. Follow [these steps](/set-up/set-up-github#the-create-plugin-update-workflow) to enable it in your repository. +In CI and other non-interactive environments, AI-assisted migrations are always skipped: automated transforms run as normal, and the instructions of any skipped agent steps are printed to the job log as manual next steps. Passing `--agent` in CI prints a warning and has no other effect. + ## Automate dependency updates `create-plugin` will only update dependencies if they are required for other changes to function correctly. Besides running the update command regularly, use [dependabot](https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide) or [renovatebot](https://docs.renovatebot.com/) to keep all dependencies up to date. diff --git a/docusaurus/docs/reference/cli-commands.mdx b/docusaurus/docs/reference/cli-commands.mdx index bae3924871..9316b69e08 100644 --- a/docusaurus/docs/reference/cli-commands.mdx +++ b/docusaurus/docs/reference/cli-commands.mdx @@ -14,6 +14,7 @@ sidebar_position: 40 import ScaffoldNPM from '@shared/createplugin-scaffold.md'; import UpdateNPM from '@shared/createplugin-update.md'; +import UpdateAgentNPM from '@shared/createplugin-update-agent.md'; import UpdateForceNPM from '@shared/createplugin-update-force.md'; # CLI commands @@ -107,3 +108,9 @@ By default, the `update` command will stop if there are any uncommitted changes +### update --agent + +Some migrations ship AI-agent instructions for changes an automated code transform can't apply. Use `--agent=` to run them with a specific installed agent CLI (`claude-code` or `codex`), bare `--agent` to use whichever supported agent is installed, or `--no-agent` to always skip them. Without the flag, `update` asks when such a migration is queued and a supported agent is installed. Skipped agent steps are printed as manual next steps. See [AI-assisted migrations](/how-to-guides/updating-a-plugin#ai-assisted-migrations) for details. + + + diff --git a/docusaurus/docs/shared/createplugin-update-agent.md b/docusaurus/docs/shared/createplugin-update-agent.md new file mode 100644 index 0000000000..2e9d962634 --- /dev/null +++ b/docusaurus/docs/shared/createplugin-update-agent.md @@ -0,0 +1,3 @@ +```shell npm2yarn +npx @grafana/create-plugin@latest update --agent=claude-code +``` diff --git a/packages/create-plugin/src/codemods/AGENTS.md b/packages/create-plugin/src/codemods/AGENTS.md index a36dafb934..91e79d1c3b 100644 --- a/packages/create-plugin/src/codemods/AGENTS.md +++ b/packages/create-plugin/src/codemods/AGENTS.md @@ -2,6 +2,16 @@ This guide provides specific instructions for working with migrations and additions in the create-plugin package. +## Migration shapes + +A migration registry entry in @./migrations/migrations.ts takes one of three shapes: + +- **Script migration** — `scriptPath` only. A codemod applies the change automatically. +- **Prompt migration** — `prompt` only. A Markdown instructions file that an installed AI agent (or the user, manually) applies. Use when the change touches user-authored code or configuration that a codemod cannot transform reliably. +- **Hybrid migration** — `scriptPath` and `prompt`. The codemod applies the deterministic part first; the agent finishes the parts that depend on user code. + +Prefer a script migration whenever the change is mechanically expressible. Reach for a prompt only when judgment over user-authored code is unavoidable. + ## Agent Behavior - Refer to current migrations and additions typescript files found in @./additions/scripts and @./migrations/scripts @@ -23,3 +33,13 @@ This guide provides specific instructions for working with migrations and additi - `NNN-migration-title.ts` - main migration logic - `NNN-migration-title.test.ts` - migration logic tests - Each migration should export a default function named "migrate" + +## Prompt authoring rules + +Prompt files live under @./migrations/prompts as `NNN-migration-title.md` (same `NNN` as the script for hybrid migrations) and are registered with `prompt: import.meta.resolve('./prompts/NNN-migration-title.md')`. See @./migrations/prompts/000-example.md for a template. Rules: + +- **Standalone**: when no agent is available the file is surfaced to the user as a manual next step, so it must read as complete manual instructions with no agent-specific context. +- **Idempotency check first**: deferred prompt migrations are version-stamped past, so the instructions may be followed on a project where the work is already done. Start with how to detect that nothing needs doing. +- **No handoff or completion mechanics**: the agent handoff contract is injected by the system prompt at run time and does not exist on the manual path. +- **Bounded scope**: include a "Verify" section (what command proves it worked) and an "Out of scope" section (at minimum: never modify `.config/`). +- The registry test in @./migrations/migrations.test.ts asserts every registered `prompt` file exists; prompt files ship verbatim to dist via the root rollup config. diff --git a/packages/create-plugin/src/codemods/agentic/definitions.test.ts b/packages/create-plugin/src/codemods/agentic/definitions.test.ts new file mode 100644 index 0000000000..ef966cda55 --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/definitions.test.ts @@ -0,0 +1,50 @@ +import { AGENT_DEFINITIONS } from './definitions.js'; +import { AgentInvocationContext } from './types.js'; + +const invocationContext: AgentInvocationContext = { + systemPrompt: 'system prompt contents', + userPrompt: 'user prompt contents', + workspaceRoot: '/virtual/workspace', + handoffDir: '/virtual/workspace/node_modules/.cache/grafana-create-plugin/migrate-runs/run-1', +}; + +function getDefinition(id: string) { + const definition = AGENT_DEFINITIONS.find((agentDefinition) => agentDefinition.id === id); + if (!definition) { + throw new Error(`No agent definition found for ${id}`); + } + return definition; +} + +describe('AGENT_DEFINITIONS', () => { + it('should define claude-code and codex in preference order', () => { + expect(AGENT_DEFINITIONS.map((agentDefinition) => agentDefinition.id)).toEqual(['claude-code', 'codex']); + }); + + AGENT_DEFINITIONS.forEach((definition) => { + describe(definition.id, () => { + it('should have at least one binary name to probe', () => { + expect(definition.binaryNames.length).toBeGreaterThan(0); + }); + + it('should embed both prompts in the interactive invocation', () => { + const invocation = definition.buildInteractive(invocationContext); + const serialisedArgs = invocation.args.join(' '); + expect(serialisedArgs).toContain(invocationContext.systemPrompt); + expect(serialisedArgs).toContain(invocationContext.userPrompt); + }); + }); + }); + + it('should pre-authorize the handoff write for claude-code', () => { + const invocation = getDefinition('claude-code').buildInteractive(invocationContext); + const allowedToolsIndex = invocation.args.indexOf('--allowedTools'); + expect(allowedToolsIndex).toBeGreaterThan(-1); + expect(invocation.args[allowedToolsIndex + 1]).toBe(`Write(${invocationContext.handoffDir}/**)`); + }); + + it('should pass the system prompt as developer instructions for codex', () => { + const invocation = getDefinition('codex').buildInteractive(invocationContext); + expect(invocation.args).toContain(`developer_instructions=${invocationContext.systemPrompt}`); + }); +}); diff --git a/packages/create-plugin/src/codemods/agentic/definitions.ts b/packages/create-plugin/src/codemods/agentic/definitions.ts new file mode 100644 index 0000000000..38108563ad --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/definitions.ts @@ -0,0 +1,29 @@ +import { AgentDefinition } from './types.js'; + +// Pure data. Invocation args mirror the Nx agentic migrate implementation (nrwl/nx#35718, #35889) +// and must be verified against the currently shipping agent CLIs when changed. +export const AGENT_DEFINITIONS: AgentDefinition[] = [ + { + id: 'claude-code', + displayName: 'Claude Code', + binaryNames: ['claude'], + buildInteractive: (invocationContext) => ({ + args: [ + '--system-prompt', + invocationContext.systemPrompt, + // pre-authorize the handoff write so the session cannot stall on a permission prompt + '--allowedTools', + `Write(${invocationContext.handoffDir}/**)`, + invocationContext.userPrompt, + ], + }), + }, + { + id: 'codex', + displayName: 'OpenAI Codex', + binaryNames: ['codex'], + buildInteractive: (invocationContext) => ({ + args: ['-c', `developer_instructions=${invocationContext.systemPrompt}`, invocationContext.userPrompt], + }), + }, +]; diff --git a/packages/create-plugin/src/codemods/agentic/detect.test.ts b/packages/create-plugin/src/codemods/agentic/detect.test.ts new file mode 100644 index 0000000000..a5cca4000e --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/detect.test.ts @@ -0,0 +1,77 @@ +import which from 'which'; +import { detectInstalledAgents, isInsideAgent } from './detect.js'; +import { AgentDefinition } from './types.js'; + +vi.mock('which'); + +const whichMock = vi.mocked(which); + +function createDefinition(overrides: Partial): AgentDefinition { + return { + id: 'claude-code', + displayName: 'Claude Code', + binaryNames: ['claude'], + buildInteractive: () => ({ args: [] }), + ...overrides, + }; +} + +describe('detectInstalledAgents', () => { + beforeEach(() => { + vi.resetAllMocks(); + }); + + it('should detect an agent found on the PATH', async () => { + whichMock.mockResolvedValue('/usr/local/bin/claude'); + + const installed = await detectInstalledAgents([createDefinition({})]); + + expect(installed).toHaveLength(1); + expect(installed[0].binaryPath).toBe('/usr/local/bin/claude'); + expect(whichMock).toHaveBeenCalledWith('claude', { nothrow: true }); + }); + + it('should return an empty list when nothing is installed', async () => { + // @ts-expect-error - the nothrow overload returns string | null which the mocked union cannot infer + whichMock.mockResolvedValue(null); + + const installed = await detectInstalledAgents([createDefinition({})]); + + expect(installed).toEqual([]); + }); + + it('should preserve definition order even when probes resolve out of order', async () => { + whichMock.mockImplementation((binaryName) => { + if (binaryName === 'claude') { + return new Promise((resolve) => { + setTimeout(() => resolve('/usr/local/bin/claude'), 20); + }); + } + return Promise.resolve('/usr/local/bin/codex'); + }); + + const installed = await detectInstalledAgents([ + createDefinition({}), + createDefinition({ id: 'codex', displayName: 'OpenAI Codex', binaryNames: ['codex'] }), + ]); + + expect(installed.map((agent) => agent.definition.id)).toEqual(['claude-code', 'codex']); + }); +}); + +describe('isInsideAgent', () => { + it.each(['CLAUDECODE', 'CURSOR_TRACE_ID', 'OPENCODE', 'CODEX_THREAD_ID', 'GEMINI_CLI', 'VSCODE_AGENT', 'REPL_ID'])( + 'should return true when %s is set', + (envVar) => { + expect(isInsideAgent({ [envVar]: '1' })).toBe(true); + } + ); + + it('should return false when no agent environment variables are set', () => { + expect(isInsideAgent({ PATH: '/usr/bin', TERM: 'xterm-256color' })).toBe(false); + }); + + it('should return false when an agent environment variable is set but empty', () => { + expect(isInsideAgent({ CLAUDECODE: '' })).toBe(false); + }); +}); diff --git a/packages/create-plugin/src/codemods/agentic/detect.ts b/packages/create-plugin/src/codemods/agentic/detect.ts new file mode 100644 index 0000000000..9b14ba47f7 --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/detect.ts @@ -0,0 +1,37 @@ +import which from 'which'; +import { AGENT_DEFINITIONS } from './definitions.js'; +import { AgentDefinition, InstalledAgent } from './types.js'; + +// Environment variables set by AI agent CLIs/IDEs when they run a shell command. +// Used to avoid spawning an agent from inside another agent. +const AGENT_ENV_VARS = [ + 'CLAUDECODE', + 'CURSOR_TRACE_ID', + 'OPENCODE', + 'CODEX_THREAD_ID', + 'GEMINI_CLI', + 'VSCODE_AGENT', + 'REPL_ID', +]; + +export async function detectInstalledAgents( + definitions: AgentDefinition[] = AGENT_DEFINITIONS +): Promise { + const probes = await Promise.all(definitions.map(detectAgent)); + return probes.filter((agent): agent is InstalledAgent => agent !== null); +} + +export function isInsideAgent(env: NodeJS.ProcessEnv = process.env): boolean { + return AGENT_ENV_VARS.some((envVar) => Boolean(env[envVar])); +} + +async function detectAgent(definition: AgentDefinition): Promise { + for (const binaryName of definition.binaryNames) { + const binaryPath = await which(binaryName, { nothrow: true }); + if (binaryPath) { + return { definition, binaryPath }; + } + } + + return null; +} diff --git a/packages/create-plugin/src/codemods/agentic/handoff.test.ts b/packages/create-plugin/src/codemods/agentic/handoff.test.ts new file mode 100644 index 0000000000..9a0fe20ef3 --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/handoff.test.ts @@ -0,0 +1,102 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + createRunId, + ensureRunDir, + getHandoffPath, + getRunDir, + MIGRATE_RUNS_DIR, + parseHandoff, + wipeMigrateRuns, +} from './handoff.js'; + +describe('run dir lifecycle', () => { + let basePath: string; + + beforeEach(() => { + basePath = mkdtempSync(join(tmpdir(), 'cp-handoff-test-')); + }); + + afterEach(() => { + rmSync(basePath, { recursive: true, force: true }); + }); + + it('should build run dirs under the migrate-runs cache dir', () => { + expect(getRunDir(basePath, 'run-1')).toBe(join(basePath, MIGRATE_RUNS_DIR, 'run-1')); + }); + + it('should build handoff paths from the migration name', () => { + const runDir = getRunDir(basePath, 'run-1'); + expect(getHandoffPath(runDir, '013-example')).toBe(join(runDir, '013-example.json')); + }); + + it('should create run dirs and tolerate them already existing', () => { + const runDir = getRunDir(basePath, 'run-1'); + ensureRunDir(runDir); + expect(existsSync(runDir)).toBe(true); + expect(() => ensureRunDir(runDir)).not.toThrow(); + }); + + it('should wipe all previous runs', () => { + const runDir = getRunDir(basePath, 'run-1'); + ensureRunDir(runDir); + writeFileSync(getHandoffPath(runDir, '013-example'), '{}'); + + wipeMigrateRuns(basePath); + + expect(existsSync(join(basePath, MIGRATE_RUNS_DIR))).toBe(false); + }); + + it('should not throw when wiping a base path with no runs', () => { + expect(() => wipeMigrateRuns(basePath)).not.toThrow(); + }); +}); + +describe('createRunId', () => { + it('should create ids that are safe to use as directory names', () => { + expect(createRunId()).toMatch(/^[a-z0-9-]+$/); + }); + + it('should create unique ids', () => { + expect(createRunId()).not.toBe(createRunId()); + }); +}); + +describe('parseHandoff', () => { + it('should parse a success handoff', () => { + expect(parseHandoff('{"status":"success","summary":"done"}')).toEqual({ status: 'success', summary: 'done' }); + }); + + it('should parse a failed handoff', () => { + expect(parseHandoff('{"status":"failed","summary":"could not migrate"}')).toEqual({ + status: 'failed', + summary: 'could not migrate', + }); + }); + + it('should drop unknown keys', () => { + expect(parseHandoff('{"status":"success","summary":"done","extras":{"foo":1}}')).toEqual({ + status: 'success', + summary: 'done', + }); + }); + + it('should return undefined for malformed JSON', () => { + expect(parseHandoff('{"status": "succ')).toBeUndefined(); + }); + + it('should return undefined for unexpected status values', () => { + expect(parseHandoff('{"status":"maybe","summary":"done"}')).toBeUndefined(); + }); + + it('should return undefined when summary is missing', () => { + expect(parseHandoff('{"status":"success"}')).toBeUndefined(); + }); + + it('should not be vulnerable to prototype pollution', () => { + const handoff = parseHandoff('{"status":"success","summary":"done","__proto__":{"polluted":true}}'); + expect(handoff).toEqual({ status: 'success', summary: 'done' }); + expect('polluted' in {}).toBe(false); + }); +}); diff --git a/packages/create-plugin/src/codemods/agentic/handoff.ts b/packages/create-plugin/src/codemods/agentic/handoff.ts new file mode 100644 index 0000000000..058ab9e6a9 --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/handoff.ts @@ -0,0 +1,45 @@ +import { randomBytes } from 'node:crypto'; +import { mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import * as v from 'valibot'; +import { Handoff } from './types.js'; + +// The handoff file is ephemeral single-run IPC between the spawned agent and create-plugin: +// the agent writes it as its final action, the runner polls and consumes it immediately. +// It lives under node_modules so it is gitignored in every scaffold and never needs to persist. +export const MIGRATE_RUNS_DIR = join('node_modules', '.cache', 'grafana-create-plugin', 'migrate-runs'); + +const handoffSchema = v.object({ + status: v.picklist(['success', 'failed']), + summary: v.string(), +}); + +export function createRunId(): string { + return `${Date.now().toString(36)}-${randomBytes(4).toString('hex')}`; +} + +export function getRunDir(basePath: string, runId: string): string { + return join(basePath, MIGRATE_RUNS_DIR, runId); +} + +export function getHandoffPath(runDir: string, migrationName: string): string { + return join(runDir, `${migrationName}.json`); +} + +// re-run before every agent session; a package manager install may have pruned the cache dir +export function ensureRunDir(runDir: string): void { + mkdirSync(runDir, { recursive: true }); +} + +export function wipeMigrateRuns(basePath: string): void { + rmSync(join(basePath, MIGRATE_RUNS_DIR), { recursive: true, force: true }); +} + +export function parseHandoff(raw: string): Handoff | undefined { + try { + const parsed = v.parse(handoffSchema, JSON.parse(raw)); + return { status: parsed.status, summary: parsed.summary }; + } catch { + return undefined; + } +} diff --git a/packages/create-plugin/src/codemods/agentic/index.test.ts b/packages/create-plugin/src/codemods/agentic/index.test.ts new file mode 100644 index 0000000000..f7e5372755 --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/index.test.ts @@ -0,0 +1,238 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { output, selectPrompt } from '../../utils/utils.console.js'; +import { Context } from '../context.js'; +import { PromptOnlyMigration, ScriptMigration } from '../migrations/migrations.js'; +import { MIGRATE_RUNS_DIR } from './handoff.js'; +import { ABORT_CHOICE, CONTINUE_CHOICE, createAgenticRuntime } from './index.js'; +import { resolveAgenticMode } from './resolve.js'; +import { runAgentSession } from './runner.js'; +import { AgentSessionResult, InstalledAgent } from './types.js'; + +vi.mock('./resolve.js'); +vi.mock('./runner.js'); +vi.mock('../../utils/utils.console.js', () => ({ + output: { + log: vi.fn(), + warning: vi.fn(), + logSingleLine: vi.fn(), + statusList: vi.fn(), + }, + confirmPrompt: vi.fn(), + selectPrompt: vi.fn(), +})); +vi.mock('../../utils/utils.packageManager.js', () => ({ + getPackageManagerWithFallback: () => ({ packageManagerName: 'npm', packageManagerVersion: '11.0.0' }), + getPackageManagerSilentInstallCmd: () => 'npm install --silent', + getPackageManagerExecCmd: () => 'npx -y', +})); + +const resolveMock = vi.mocked(resolveAgenticMode); +const runAgentSessionMock = vi.mocked(runAgentSession); +const selectPromptMock = vi.mocked(selectPrompt); +const outputMock = vi.mocked(output); + +const fakeAgent: InstalledAgent = { + definition: { + id: 'claude-code', + displayName: 'Claude Code', + binaryNames: ['claude'], + buildInteractive: (invocationContext) => ({ + args: ['--sys', invocationContext.systemPrompt, invocationContext.userPrompt], + env: { FAKE_AGENT: '1' }, + }), + }, + binaryPath: '/usr/local/bin/claude', +}; + +function handoffResult(status: 'success' | 'failed', summary: string): AgentSessionResult { + return { outcome: 'handoff', handoff: { status, summary } }; +} + +describe('createAgenticRuntime', () => { + let basePath: string; + let migration: PromptOnlyMigration; + + beforeEach(() => { + vi.clearAllMocks(); + basePath = mkdtempSync(join(tmpdir(), 'cp-agentic-test-')); + const promptPath = join(basePath, '013-example.md'); + writeFileSync(promptPath, '# Port things'); + migration = { + name: '013-example', + version: '8.0.0', + description: 'Port custom things.', + prompt: pathToFileURL(promptPath).href, + }; + }); + + afterEach(() => { + rmSync(basePath, { recursive: true, force: true }); + }); + + function createEnabledRuntime(commitEachMigration?: boolean) { + resolveMock.mockResolvedValue({ mode: 'enabled', agent: fakeAgent }); + return createAgenticRuntime({ agentFlag: undefined, commitEachMigration, basePath }); + } + + describe('disabled mode', () => { + it('should defer prompt steps and print them as manual next steps', async () => { + resolveMock.mockResolvedValue({ mode: 'disabled', reason: 'no-agents' }); + const runtime = await createAgenticRuntime({ agentFlag: undefined, commitEachMigration: undefined, basePath }); + + const step = await runtime.runPromptStep(migration); + runtime.printSummary(); + + expect(step).toEqual({ kind: 'deferred' }); + expect(runtime.softForceCommits).toBe(false); + expect(runAgentSessionMock).not.toHaveBeenCalled(); + expect(outputMock.warning).toHaveBeenCalledWith( + expect.objectContaining({ + body: [expect.stringContaining('013-example')], + }) + ); + }); + }); + + describe('inside-agent mode', () => { + it('should defer prompt steps and print a directive block for the outer agent', async () => { + resolveMock.mockResolvedValue({ mode: 'inside-agent' }); + const runtime = await createAgenticRuntime({ agentFlag: undefined, commitEachMigration: undefined, basePath }); + + const step = await runtime.runPromptStep(migration); + runtime.printSummary(); + + expect(step).toEqual({ kind: 'deferred' }); + expect(outputMock.logSingleLine).toHaveBeenCalledWith(expect.stringContaining('')); + expect(outputMock.logSingleLine).toHaveBeenCalledWith(expect.stringContaining('013-example')); + }); + }); + + describe('enabled mode', () => { + it('should wipe previous runs when the runtime is created', async () => { + const staleHandoff = join(basePath, MIGRATE_RUNS_DIR, 'old-run'); + mkdirSync(staleHandoff, { recursive: true }); + writeFileSync(join(staleHandoff, 'stale.json'), '{}'); + + await createEnabledRuntime(); + + expect(existsSync(staleHandoff)).toBe(false); + }); + + it('should run the agent session and report an applied step on a success handoff', async () => { + runAgentSessionMock.mockResolvedValue(handoffResult('success', 'ported everything')); + const runtime = await createEnabledRuntime(); + + const step = await runtime.runPromptStep(migration); + runtime.printSummary(); + + expect(step).toEqual({ kind: 'applied', summary: 'ported everything' }); + expect(runAgentSessionMock).toHaveBeenCalledWith( + expect.objectContaining({ + binaryPath: '/usr/local/bin/claude', + cwd: basePath, + env: { FAKE_AGENT: '1' }, + handoffPath: expect.stringContaining(join(MIGRATE_RUNS_DIR, '')), + }) + ); + const sessionOptions = runAgentSessionMock.mock.calls[0][0]; + expect(sessionOptions.handoffPath).toContain('013-example.json'); + // the fake agent puts the system prompt at args[1] and the user prompt at args[2] + expect(sessionOptions.args[1]).toContain(sessionOptions.handoffPath); + expect(sessionOptions.args[2]).toContain('# Port things'); + expect(outputMock.statusList).toHaveBeenCalledWith('success', [expect.stringContaining('ported everything')]); + }); + + it('should throw when the agent hands off a failure', async () => { + runAgentSessionMock.mockResolvedValue(handoffResult('failed', 'could not port')); + const runtime = await createEnabledRuntime(); + + await expect(runtime.runPromptStep(migration)).rejects.toThrow(/could not port/); + }); + + it('should throw when the user aborts the agent session', async () => { + runAgentSessionMock.mockResolvedValue({ outcome: 'user-aborted' }); + const runtime = await createEnabledRuntime(); + + await expect(runtime.runPromptStep(migration)).rejects.toThrow(/aborted/i); + }); + + it('should ask the user on an ambiguous exit and continue when told to', async () => { + runAgentSessionMock.mockResolvedValue({ outcome: 'ambiguous-exit', exitCode: 1, signal: null }); + selectPromptMock.mockResolvedValue(CONTINUE_CHOICE); + const runtime = await createEnabledRuntime(); + + const step = await runtime.runPromptStep(migration); + + expect(selectPromptMock).toHaveBeenCalledWith(expect.any(String), [ABORT_CHOICE, CONTINUE_CHOICE]); + expect(step).toEqual({ kind: 'assumed-applied' }); + }); + + it('should abort on an ambiguous exit when told to', async () => { + runAgentSessionMock.mockResolvedValue({ outcome: 'ambiguous-exit', exitCode: 1, signal: null }); + selectPromptMock.mockResolvedValue(ABORT_CHOICE); + const runtime = await createEnabledRuntime(); + + await expect(runtime.runPromptStep(migration)).rejects.toThrow(/aborted/i); + }); + + it('should treat a cancelled ambiguous-exit prompt as an abort', async () => { + runAgentSessionMock.mockResolvedValue({ outcome: 'ambiguous-exit', exitCode: null, signal: null }); + selectPromptMock.mockRejectedValue(new Error('')); + const runtime = await createEnabledRuntime(); + + await expect(runtime.runPromptStep(migration)).rejects.toThrow(/aborted/i); + }); + }); + + describe('hybrid steps', () => { + let hybridMigration: ScriptMigration & { prompt: string }; + let context: Context; + + beforeEach(() => { + hybridMigration = { ...migration, scriptPath: 'file:///virtual/scripts/013-example.js' }; + context = new Context(basePath); + context.addFile('src/new-file.ts', 'content'); + runAgentSessionMock.mockResolvedValue(handoffResult('success', 'done')); + }); + + it('should point the agent at the git diff when per-migration commits are on', async () => { + const runtime = await createEnabledRuntime(true); + + await runtime.runPromptStep(hybridMigration, context); + + const userPrompt = runAgentSessionMock.mock.calls[0][0].args[2]; + expect(userPrompt).toContain('git diff'); + expect(userPrompt).not.toContain(''); + }); + + it('should embed the changed file list when commits are off', async () => { + const runtime = await createEnabledRuntime(false); + + await runtime.runPromptStep(hybridMigration, context); + + const userPrompt = runAgentSessionMock.mock.calls[0][0].args[2]; + expect(userPrompt).toContain(''); + expect(userPrompt).toContain('[ADD] src/new-file.ts'); + }); + + it('should warn that the agent loses diff context when commits are explicitly off', async () => { + await createEnabledRuntime(false); + expect(outputMock.warning).toHaveBeenCalled(); + }); + }); + + describe('softForceCommits', () => { + it('should soft-force commits when enabled and the commit flag was not given', async () => { + const runtime = await createEnabledRuntime(undefined); + expect(runtime.softForceCommits).toBe(true); + }); + + it('should not soft-force commits when the user set the flag explicitly', async () => { + expect((await createEnabledRuntime(true)).softForceCommits).toBe(false); + expect((await createEnabledRuntime(false)).softForceCommits).toBe(false); + }); + }); +}); diff --git a/packages/create-plugin/src/codemods/agentic/index.ts b/packages/create-plugin/src/codemods/agentic/index.ts new file mode 100644 index 0000000000..aabd5aef62 --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/index.ts @@ -0,0 +1,199 @@ +import { output, selectPrompt } from '../../utils/utils.console.js'; +import { + getPackageManagerExecCmd, + getPackageManagerSilentInstallCmd, + getPackageManagerWithFallback, +} from '../../utils/utils.packageManager.js'; +import { Context } from '../context.js'; +import { Migration } from '../migrations/migrations.js'; +import { createRunId, ensureRunDir, getHandoffPath, getRunDir, wipeMigrateRuns } from './handoff.js'; +import { + buildDirectiveBlock, + buildHybridUserPrompt, + buildNextStepsList, + buildPromptOnlyUserPrompt, + buildSystemPrompt, + CodemodChanges, + DeferredMigration, + getPromptPath, + loadPromptInstructions, +} from './prompts.js'; +import { resolveAgenticMode } from './resolve.js'; +import { runAgentSession } from './runner.js'; +import { AgenticResolution, AgentSessionResult, PromptStepResult } from './types.js'; + +export const ABORT_CHOICE = 'Abort the update'; +export const CONTINUE_CHOICE = 'Treat the migration step as completed'; + +export interface AgenticRuntimeOptions { + // raw minimist --agent value: undefined | '' | '' | false + agentFlag: string | boolean | undefined; + // tri-state: undefined = flag absent, which allows agentic to soft-force commits + commitEachMigration: boolean | undefined; + basePath: string; +} + +export interface AgenticRuntime { + resolution: AgenticResolution; + softForceCommits: boolean; + runPromptStep: (migration: Migration & { prompt: string }, context?: Context) => Promise; + printSummary: () => void; +} + +export async function createAgenticRuntime(options: AgenticRuntimeOptions): Promise { + const resolution = await resolveAgenticMode({ agentFlag: options.agentFlag }); + const softForceCommits = resolution.mode === 'enabled' && options.commitEachMigration === undefined; + const commitsEnabled = options.commitEachMigration ?? softForceCommits; + const runDir = getRunDir(options.basePath, createRunId()); + const deferredMigrations: DeferredMigration[] = []; + const appliedMigrations: string[] = []; + + if (resolution.mode === 'enabled') { + wipeMigrateRuns(options.basePath); + if (softForceCommits) { + output.log({ title: 'Enabling per-migration commits so each agent step sees an isolated diff.' }); + } + if (options.commitEachMigration === false) { + output.warning({ + title: 'Running agent migrations without per-migration commits.', + body: [ + 'The agent cannot inspect an isolated git diff; the changed file list is embedded in its prompt instead.', + ], + }); + } + } + + async function runPromptStep( + migration: Migration & { prompt: string }, + context?: Context + ): Promise { + const instructionsPath = getPromptPath(migration.prompt); + + if (resolution.mode !== 'enabled') { + deferredMigrations.push({ name: migration.name, description: migration.description, instructionsPath }); + return { kind: 'deferred' }; + } + + const agent = resolution.agent; + const handoffPath = getHandoffPath(runDir, migration.name); + ensureRunDir(runDir); + + const { packageManagerName, packageManagerVersion } = getPackageManagerWithFallback(); + const systemPrompt = buildSystemPrompt({ + workspaceRoot: options.basePath, + packageManagerName, + packageManagerVersion, + installCmd: getPackageManagerSilentInstallCmd(packageManagerName, packageManagerVersion), + execCmd: getPackageManagerExecCmd(packageManagerName, packageManagerVersion), + handoffPath, + }); + + const userPromptOptions = { + migration: { name: migration.name, version: migration.version, description: migration.description }, + instructions: loadPromptInstructions(migration.prompt), + instructionsPath, + handoffPath, + }; + const userPrompt = context + ? buildHybridUserPrompt({ ...userPromptOptions, codemodChanges: getCodemodChanges(context) }) + : buildPromptOnlyUserPrompt(userPromptOptions); + + output.log({ + title: `Handing ${migration.name} to ${agent.definition.displayName}.`, + body: [ + 'The session is interactive: you can watch and redirect the agent.', + 'It ends automatically once the agent reports completion.', + ], + }); + + const invocation = agent.definition.buildInteractive({ + systemPrompt, + userPrompt, + workspaceRoot: options.basePath, + handoffDir: runDir, + }); + + const sessionResult = await runAgentSession({ + binaryPath: agent.binaryPath, + args: invocation.args, + env: invocation.env, + cwd: options.basePath, + handoffPath, + }); + + if (sessionResult.outcome === 'handoff') { + if (sessionResult.handoff.status === 'failed') { + throw new Error(`The agent reported failure for ${migration.name}: ${sessionResult.handoff.summary}`); + } + appliedMigrations.push(`${migration.name} — ${sessionResult.handoff.summary}`); + return { kind: 'applied', summary: sessionResult.handoff.summary }; + } + + if (sessionResult.outcome === 'user-aborted') { + throw new Error(`The agent session for ${migration.name} was aborted.`); + } + + return handleAmbiguousExit(migration.name, sessionResult); + } + + async function handleAmbiguousExit( + migrationName: string, + sessionResult: Extract + ): Promise { + output.warning({ + title: `The agent session for ${migrationName} ended without reporting a result.`, + body: [ + `Exit code: ${sessionResult.exitCode ?? 'none'}, signal: ${sessionResult.signal ?? 'none'}.`, + 'Inspect the working tree before deciding how to proceed.', + ], + }); + + const choice = await safeSelect('How should the update proceed?', [ABORT_CHOICE, CONTINUE_CHOICE]); + if (choice === CONTINUE_CHOICE) { + appliedMigrations.push(`${migrationName} — assumed applied (the agent did not hand off a result).`); + return { kind: 'assumed-applied' }; + } + + throw new Error(`Update aborted after the agent session for ${migrationName} ended unexpectedly.`); + } + + function getCodemodChanges(context: Context): CodemodChanges { + if (commitsEnabled) { + return { kind: 'git-diff' }; + } + const files = Object.entries(context.listChanges()).map(([path, change]) => ({ + path, + changeType: change.changeType, + })); + return { kind: 'file-list', files }; + } + + function printSummary(): void { + if (deferredMigrations.length > 0) { + if (resolution.mode === 'inside-agent') { + output.logSingleLine(buildDirectiveBlock(deferredMigrations)); + return; + } + output.warning({ + title: 'Some migrations include AI-agent instructions that were not applied automatically.', + body: buildNextStepsList(deferredMigrations), + }); + return; + } + + if (appliedMigrations.length > 0) { + output.statusList('success', appliedMigrations); + } + } + + return { resolution, softForceCommits, runPromptStep, printSummary }; +} + +async function safeSelect(message: string, choices: string[]): Promise { + try { + return await selectPrompt(message, choices); + } catch { + // enquirer rejects when the prompt is cancelled (ctrl+c) — treat as an abort + return ABORT_CHOICE; + } +} diff --git a/packages/create-plugin/src/codemods/agentic/prompts.test.ts b/packages/create-plugin/src/codemods/agentic/prompts.test.ts new file mode 100644 index 0000000000..7071fdb314 --- /dev/null +++ b/packages/create-plugin/src/codemods/agentic/prompts.test.ts @@ -0,0 +1,187 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + buildDirectiveBlock, + buildHybridUserPrompt, + buildNextStepsList, + buildPromptOnlyUserPrompt, + buildSystemPrompt, + escapeXmlBody, + getPromptPath, + loadPromptInstructions, +} from './prompts.js'; + +const migrationMeta = { + name: '013-example', + version: '8.0.0', + description: 'Port custom webpack overrides to rspack.', +}; + +const systemPromptOptions = { + workspaceRoot: '/virtual/workspace', + packageManagerName: 'npm', + packageManagerVersion: '11.0.0', + installCmd: 'npm install --silent', + execCmd: 'npx -y', + handoffPath: '/virtual/workspace/node_modules/.cache/grafana-create-plugin/migrate-runs/run-1/013-example.json', +}; + +describe('escapeXmlBody', () => { + it('should escape ampersands and angle brackets', () => { + expect(escapeXmlBody('use