diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74a88e6..6cdf3c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,11 @@ on: jobs: test: strategy: + # Let every matrix cell report: a canceled job hides whether its + # platform actually passes, which has already cost a debugging round. + fail-fast: false matrix: - node: [20, 22] + node: ['20.18.1', 22] os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: diff --git a/.gitignore b/.gitignore index 8612a42..be494af 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ Thumbs.db # Test coverage coverage/ +test-results/ # Cache .cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f41ca11..155a668 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,78 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Concurrent `ChatEngine.sendMessage` calls now queue and run in call order instead of interleaving shared history and preview state; the interactive REPL already serialized calls, so this protects programmatic callers +- Chat responses that wrap a JSON tool call in prose (text before or after the object, braces inside string values) now parse correctly: the greedy first-`{`-to-last-`}` fallback was replaced with a brace-depth scanner that respects string literals and escapes; fenced-block parsing and pure-JSON responses are unchanged +- Destructive execution now fails closed: if the automatic `dry_run` preview errors or returns an unsuccessful result, the command exits 4 without sending a confirm request. The successful preview is shown before the confirmation prompt (and included in the `--json` envelope); `--force` skips only the prompt, never the preview +- Chat tool calling now works against the real OpenAI, Anthropic, and Gemini APIs: ability names are aliased to provider-safe tool names (all three reject `/`), assistant tool-call blocks are preserved across turns so continuations pair correctly with their results, and the destructive-approval flow keeps the original tool-call id +- Malformed chat tool calls are rejected instead of executing with empty input: unparseable argument JSON, non-object input, multiple tool calls in one response, responses containing both an answer and a tool call, and responses truncated by `length` or `content_filter` all return a protocol error to the model +- Chat now validates tool input against the ability's JSON schema before execution, matching the CLI path; validation failures go back to the model as tool errors +- Replaced retired LLM model defaults: Anthropic `claude-sonnet-4-20250514` (retired June 2026) with `claude-sonnet-4-6`, Gemini `gemini-1.5-flash` (shut down 2025) with `gemini-3.5-flash` +- Preview summaries no longer claim "No items would be affected" when the dry-run response is in an unrecognized format; the raw response is shown with a warning instead +- `--max-context-messages` rejects negative values; `0` remains the only way to disable truncation +- Chat context truncation no longer orphans tool results mid tool-calling loop or at the destructive-action approval step, which could cause provider API errors on the next message +- Caller-cancelled requests now report "Request cancelled" instead of "Request timed out" +- Gemini 3 thought signatures on function calls are preserved through parsing, chat history, and streaming, and echoed back on the continuation request; previously they were discarded, which Gemini 3 models (including the default `gemini-3.5-flash`) reject with HTTP 400 +- Context truncation no longer treats the synthetic approval echo after a destructive action as a turn boundary, which with a small `--max-context-messages` could erase the executed action and its result from history right after approval +- HTTP method selection now resolves destructiveness the same way the safety classifier does, so a destructive-named ability is never sent as a read-only GET even if the server mislabels it; non-boolean annotation values (e.g. `readonly: "true"` as a string) are likewise ignored. When the destructive classification comes from the name override rather than the annotations, the request uses POST instead of trusting the annotations' `idempotent` flag for DELETE +- Keychain credential-removal failures now warn in non-interactive (CI) runs instead of only when attached to a terminal +- Warning shown when the active profile no longer exists and the CLI falls back to another profile +- Chat error responses name the failing ability (`[mainwp/update-site-v1] Error: ...`) so a failed preview or tool call is attributable; engine-level errors stay bare +- `login --url` with embedded credentials fails at intake with the friendly configuration error (exit 2) instead of an opaque network error from the connection test +- Malformed Dashboard ability schemas (PHP artifacts such as `properties: []` or `"inputSchema": []`) are repaired centrally for both `abilities run` and chat tool execution; schemas that remain invalid exit 4 (Dashboard error), not 5 +- The Dashboard's queued-job envelope (`job_id`) is recognized, so `abilities run --wait` polls to completion instead of returning immediately +- One-shot chat failures exit non-zero through the documented JSON error envelope instead of printing a raw response object and exiting 0; an empty provider stream is an error, not a blank successful answer +- Malformed `--input` JSON reports the parse position instead of echoing the raw payload, and JSON input that is not an object (array, string, number) is rejected locally instead of being sent to the Dashboard +- Parse-time flag errors under `--json` emit a single JSON error envelope on stdout (exit code 1 unchanged) +- `jobs watch` prints the `cancelled` batch status, mapped to `BATCH_CANCELLED` +- Batch polling rejects unknown statuses, job-ID mismatches, invalid numeric fields, oversized arrays, and terminal-state regressions instead of trusting them +- A keychain read error during login aborts before overwriting, instead of being treated as "nothing stored" and later rolling back a credential that still existed; keychain delete distinguishes not-found from failure, and profile delete reports not-found as the goal state +- Cyclic or over-deep error details no longer crash `--json` output: both sanitizers bound depth and truncate cycles while legitimately shared references survive +- Atomic config writes sync file contents to disk before the rename, so a crash at the wrong moment cannot leave a truncated file behind +- Failures to repair config and audit file permissions now warn instead of being silently ignored + +### Changed + +- **Breaking:** keychain credentials are now stored bound to the profile's canonical Dashboard identity, and unbound credentials stored by earlier versions are refused for authenticated requests. Each existing profile needs a one-time `mainwpcontrol login` to re-bind its credential; the error message says so. Repointing a profile at a different host by editing `profiles.json` now gets an authentication error instead of the stored password +- A transport failure after a destructive confirm was dispatched exits 3 (`OUTCOME_UNKNOWN`) with audit entries for both the dispatch and the unknown outcome, instead of a generic network error that implied nothing ran; chat reports the same stable code and keeps the session alive +- 2xx Dashboard responses must be parseable JSON with a JSON content type; empty or HTML responses are `INVALID_RESPONSE` errors, never treated as success +- Ability discovery validates entries, caps pagination, warns and keeps the first entry on duplicate names, and no longer generates ambiguous short aliases +- `--json` now emits exactly one JSON document when a batch job times out, fails, or completes partially: an error envelope with the job status in `error.details` (previously a success envelope was printed before the error envelope) +- `jobs watch` and `abilities run --wait` exit 4 when the job ends `failed` or `partial`; `jobs watch` exits 130/143 with an error envelope when interrupted by SIGINT/SIGTERM (previously all of these exited 0 with a success envelope) +- Flag and argument parse errors (for example passing `--dry-run` with `--confirm`) exit 1 (user input error) instead of 2 +- Minimum Node.js version is 20.18.1 (required by the bundled undici) +- `npm test` no longer requires a reachable MainWP Dashboard; live integration tests run only via `npm run test:live` with `MAINWP_LIVE_TEST=1` +- Unified sensitive-key redaction into one shared utility covering compound keys (`apiToken`, `appPassword`) across error output, debug logging, and input sanitization +- Broader destructive-ability name patterns (`reset-`, `restore-`, `rollback-`, `wipe-`, `purge-`, `uninstall-`) in the defense-in-depth safety classification +- Exit code 130 on Ctrl-C at prompts documented as the intentional SIGINT convention + +### Security + +- Dashboard URLs with embedded credentials (`https://user:pass@host`) are rejected at login with a hint to use `--username` and the password prompt; profiles stored before this fix have the userinfo masked as `***:***@` in `login`, `config show`, and `doctor` output (human, JSON, and echoed error messages) +- The audit log directory and file permissions now self-heal to `0700`/`0600` on every write, and the log is opened atomically in append mode, removing a check-then-act window that could truncate the log +- `config show` sanitizes every untrusted value in human-readable output to a single safe line: environment-derived provider names and paths (`MAINWP_LLM_PROVIDER`, `XDG_CONFIG_HOME`) and profile-derived fields (profile name, dashboard URL, username), closing line-injection via a crafted `profiles.json` or hostile environment +- `doctor` and `config show` human-readable output now strips terminal escape sequences from error- and config-derived text, matching the sanitization the `--json` path already applied +- HTTP responses are size-checked after buffering even when the server sends a parseable `Content-Length`, so an inaccurate header can no longer bypass the response size limit +- All remaining single-row terminal output (login summary, profile fallback and keychain warnings, ability names, table cells, list items, preview labels) collapses untrusted values to a single line instead of only stripping non-CR/LF control characters +- Input keys containing `[` or `]` are now rejected; they could canonicalize server-side (PHP query parsing) to alias a control flag like `confirm` past the executor's flag-stripping guard +- Mutual exclusion of `dry_run` and `confirm` is now also asserted at the executor boundary, not only at the flag layer +- Updated `undici` to 7.28.0, resolving TLS certificate validation bypass and response queue poisoning advisories +- Updated `fast-uri` (transitive, via `ajv`) to 3.1.4, resolving a high-severity host-confusion advisory (GHSA-v2hh-gcrm-f6hx) +- Updated `@oclif/core`, `@oclif/plugin-help`, `@oclif/plugin-autocomplete`, and transitive dependencies; `npm audit --omit=dev` reports zero production vulnerabilities, dev-chain advisories are tracked separately +- Abilities with missing or malformed safety annotations are classified destructive (fail closed) instead of defaulting to read-only; every real Dashboard ability declares all three annotation keys +- Case-variant ability names (`Mainwp/Delete-Site-V1`) are refused at discovery, so a case variant can never evade destructive-name classification or alias a cache key +- `pattern` and `patternProperties` from Dashboard schemas are stripped from every node of the tree and never compiled, so a hostile Dashboard cannot stall the CLI with a catastrophic regex; the whole tree, including arrays and literal data values, counts against a 32-level depth budget +- Tool results are key-redacted before entering provider-bound chat history (local display stays raw); provider requests refuse redirects; hosted providers refuse `http://` base URLs; the local provider allows HTTP only to loopback and private-range hosts +- Profile `skipSSLVerification` must be strictly boolean; the string `"false"` no longer disables TLS verification +- SSE streams are bounded (line and buffer caps, idle and absolute timeouts), and provider error bodies are read bounded and key-redacted before they can reach an error message +- Dashboard response bodies stream against a byte cap and the request timeout covers the body read, so an unbounded or stalled body cannot hang the process +- Config files write via random-suffix `O_EXCL` temporary files; the audit log opens with `O_NOFOLLOW`; audit-log input is bounded at 8 KiB and free text at 2 KiB with visible truncation markers +- URLs with username-only credentials now redact, and userinfo masking is greedy through the last `@` so passwords containing `@` mask fully; malformed URLs are no longer echoed in profile-store error messages +- Sensitive query-string parameters (`access_token=...`) are redacted in error output, and keychain store errors pass through the same sanitizer as the rest of the keychain surface +- Redaction and terminal sanitization build results on null-prototype objects, so a crafted `__proto__` key cannot pollute prototypes; terminal output also strips Unicode bidirectional and isolate controls to prevent right-to-left display spoofing + ## [1.1.0-beta.1] - 2026-03-26 ### Added diff --git a/README.md b/README.md index bb0c227..7b9532d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A CLI for managing your MainWP Dashboard from the terminal. List sites, push upd > **On Windows?** Use [Git Bash](https://gitforwindows.org/) and every example below works without changes. For scheduled workflows (cron), see [WSL](https://learn.microsoft.com/en-us/windows/wsl/install). -You need Node.js 20+ and a MainWP Dashboard (v6+) with an [Application Password](https://make.wordpress.org/core/2020/11/05/application-passwords-integration-guide/). +You need Node.js 20.18.1+ and a MainWP Dashboard (v6+) with an [Application Password](https://make.wordpress.org/core/2020/11/05/application-passwords-integration-guide/). ```bash npm install -g @mainwp/control @@ -70,7 +70,7 @@ mainwpcontrol abilities run list-updates-v1 --json mainwpcontrol abilities run get-site-v1 --input '{"site_id_or_domain": 1}' --json ``` -> **Windows?** This works as-is in [Git Bash](https://gitforwindows.org/). In PowerShell, escape the inner quotes: `'{\"site_id_or_domain\": 1}'`. Or skip quoting entirely with `--input-file` ([details](docs/workflows/input-from-file.md)). +> **Windows?** This works as-is in [Git Bash](https://gitforwindows.org/). In PowerShell, use `--input-file` instead of inline JSON: how PowerShell passes quoted arguments to native commands varies by version ([details](docs/workflows/input-from-file.md)). **Preview a destructive action before running it:** @@ -136,7 +136,7 @@ A terminal is where you type commands instead of clicking buttons. You'll see it **How to open it:** - **macOS**: Open **Terminal** (search in Spotlight, or look in Applications > Utilities) -- **Windows**: Open **Git Bash** (installed with [Git for Windows](https://gitforwindows.org/)). If you don't have it, PowerShell works too — see the [quoting notes](#json-quoting-on-the-command-line) below. +- **Windows**: Open **Git Bash** (installed with [Git for Windows](https://gitforwindows.org/)). If you don't have it, PowerShell works too; see the [quoting notes](#json-quoting-on-the-command-line) below. - **Linux**: Open your distribution's **Terminal** app (usually in the applications menu) ### What does `npm install -g` do? @@ -178,14 +178,11 @@ When you pass JSON with `--input`, quoting depends on your shell: ```bash # macOS / Linux / Git Bash on Windows mainwpcontrol abilities run get-site-v1 --input '{"site_id_or_domain": 1}' --json - -# Windows PowerShell -mainwpcontrol abilities run get-site-v1 --input '{\"site_id_or_domain\": 1}' --json ``` **Git Bash on Windows** (comes with [Git for Windows](https://gitforwindows.org/)) handles quoting the same way macOS and Linux do. If you use Git Bash, all the examples in this documentation work without changes. -PowerShell strips the inner double quotes unless you escape them with backslashes. If this gets annoying, put your parameters in a file and use `--input-file`: +**Windows PowerShell** quoting of inline JSON is unreliable: whether backslash-escaped quotes inside a single-quoted string reach the command intact depends on your PowerShell version. Don't fight it, put your parameters in a file and use `--input-file`: ```bash mainwpcontrol abilities run get-site-v1 --input-file params.json --json @@ -219,10 +216,7 @@ mainwpcontrol abilities run list-sites-v1 --json # Run with input parameters mainwpcontrol abilities run get-site-v1 --input '{"site_id_or_domain": 1}' --json -# Windows PowerShell: escape inner quotes (Git Bash doesn't need this) -mainwpcontrol abilities run get-site-v1 --input '{\"site_id_or_domain\": 1}' --json - -# Or use a file (works everywhere) +# Or use a file (works everywhere, and is the reliable option on Windows PowerShell) mainwpcontrol abilities run get-site-v1 --input-file params.json --json ``` @@ -277,7 +271,7 @@ See [Chat Mode Configuration](#chat-mode-configuration) for all supported provid ### Global Flags -These flags work on every command. +These flags work on every `mainwpcontrol` command except the built-in `help` and `autocomplete` commands. | Flag | Description | |------|-------------| @@ -477,6 +471,7 @@ Step-by-step guides for common automation patterns: | 3 | Network error | Retry or check connectivity | | 4 | API error | Check ability parameters | | 5 | Internal error | Report bug | +| 130 | Interrupted (SIGINT) | Ctrl-C during a prompt or `jobs watch`; standard Unix 128+SIGINT convention, outside the 0-5 contract | ### Environment Variables @@ -606,7 +601,7 @@ GPL-3.0-or-later ## Requirements -- Node.js 20 LTS or later +- Node.js 20.18.1 or later - MainWP Dashboard 6+ with Abilities API - WordPress Application Password diff --git a/docs/acceptance-testing.md b/docs/acceptance-testing.md new file mode 100644 index 0000000..432672d --- /dev/null +++ b/docs/acceptance-testing.md @@ -0,0 +1,190 @@ +# Acceptance testing + +The acceptance harness exercises the CLI as a process and verifies results with direct Abilities API reads. Packed mode is the default. It creates the npm tarball, checks its contents, installs it in a temporary consumer, and runs the installed `mainwpcontrol` binary. Source mode uses the repository's `bin/run.js` and existing build output. + +The deterministic layer chooses every CLI command itself. The agent layer gives Claude a natural-language task and grades the ordered `mainwpcontrol` invocations and final answer against independent state. Agent runs supplement the deterministic suite; they do not replace it. + +## Modes and targets + +The runner supports two modes: + +- `packed` builds and installs the current working tree as a package in a temporary consumer. This is the default. +- `source` runs the repository binary and current `dist/` output. Run `npm run build` first. + +The deterministic runner supports two targets: + +- `live` uses a configured MainWP Dashboard. This is the default. +- `fixture` starts the local deterministic Dashboard fixture on `127.0.0.1`. + +The agent runner has no `--target` flag. Its three read scenarios are live: each reads ground truth directly from the live Abilities API before starting Claude. The delete scenario runs against a per-scenario local fixture Dashboard instead. A live delete victim is not repeatable — MainWP Child locks to a dashboard key on first connect, so every add-then-delete cycle would leave the child rejecting the next add — and pre-existing testbed sites are never valid targets. The mcp reference harness scopes its agent delete scenario the same way. + +## Prerequisites + +- Node.js 20.18.1 or newer +- Dependencies installed in this repository +- `npm`, `git`, and `tar` on `PATH` +- A completed build for source mode +- Live Dashboard credentials for live and agent runs +- The `claude` CLI with working model access for agent runs + +The full harness must run from an unsandboxed local shell. Packed installs and fixtures bind temporary listeners on `127.0.0.1`, `tsx` creates a temporary IPC socket, live verification contacts the Dashboard, and agent scenarios start `claude`. Restricted sandboxes can reject these operations with `listen EPERM`. + +## Credentials + +Live credentials are resolved in this order: + +1. `MAINWP_URL`, `MAINWP_USER`, and `MAINWP_APP_PASSWORD` in the process environment +2. The file named by `MAINWP_CONTROL_ACCEPTANCE_ENV` +3. `~/github/dev-tools/network-testbed/.env` + +The environment file maps `LLM_DASH_URL` to the Dashboard URL and reads `MAINWP_USER` and `MAINWP_APP_PASSWORD`. + +The agent runner creates one temporary XDG configuration directory per scenario. Its profile contains the Dashboard URL and username. The Application Password exists only in the Claude child environment as `MAINWP_APP_PASSWORD`; it is not written to the profile, consumer, transcript, command record, or result files. `MAINWPCONTROL_NO_KEYTAR=1` keeps the run independent of the OS keychain. + +`MAINWP_CONTROL_ACCEPTANCE_TOGGLE_PLUGIN` can select the plugin slug preferred by the `agent-plugin-active` scenario and the reversible deterministic plugin scenario. + +## Commands + +The package defines six acceptance entry points: + +```bash +npm run test:acceptance +npm run test:acceptance:fast +npm run test:acceptance:fixture +npm run test:acceptance:writes +npm run test:acceptance:agent +npm run test:acceptance:human +``` + +Their behavior is: + +- `test:acceptance`: packed mode against the live target. +- `test:acceptance:fast`: builds, then runs source mode against the live target. +- `test:acceptance:fixture`: packed mode against the local fixture. +- `test:acceptance:writes`: packed mode against live with guarded writes enabled. +- `test:acceptance:agent`: packed agent scenarios (live reads plus the fixture delete scenario). +- `test:acceptance:human`: fixture, guarded writes, and agent layers in that order. The `&&` chain stops on the first non-zero exit. + +List or select deterministic scenarios: + +```bash +npx tsx tests/acceptance/run.ts --list +npx tsx tests/acceptance/run.ts --target fixture --scenario count-sites-consistency +npx tsx tests/acceptance/run.ts --mode source --scenario startup-doctor +``` + +List or select agent scenarios: + +```bash +npx tsx tests/acceptance/agent-run.ts --list +npx tsx tests/acceptance/agent-run.ts --scenario agent-count-sites +npx tsx tests/acceptance/agent-run.ts --scenario agent-confirm-delete-site +npx tsx tests/acceptance/agent-run.ts --mode source --scenario agent-updates +``` + +Both runners accept repeatable `--scenario `, `--mode packed|source`, `--list`, `--keep-consumer`, and `--help`. The deterministic runner also accepts `--target live|fixture`. `--writes` enables guarded live mutation scenarios in both runners; the agent delete scenario uses the fixture Dashboard and does not need it. Unknown flags fail the run. + +## Write guard + +Live writes require both conditions: + +1. The runner received `--writes`. +2. The Dashboard hostname is `localhost`, `127.0.0.1`, or ends in `.local`. + +The shared guard in `tests/acceptance/lib/guards.ts` checks these conditions before a live mutation. Missing authorization reports the scenario as skipped. Fixture writes remain local and do not require `--writes`. + +The agent delete scenario targets a synthetic site on its own in-process fixture Dashboard, so no real site record is ever at risk and the scenario is repeatable. The transcript must contain a target-matching `mainwpcontrol abilities run mainwp/delete-site-v1 ... --dry-run` invocation before a target-matching invocation with `--confirm --force`. Independent reads of the fixture must then show exactly one fewer site. The fixture server and its state are discarded when the scenario ends. + +## Agent layer + +`tests/acceptance/agent-run.ts` runs four scenarios: + +- `agent-count-sites` +- `agent-updates` +- `agent-plugin-active` +- `agent-confirm-delete-site` + +For packed mode, the child's `PATH` begins with the temporary consumer's `node_modules/.bin`, so `mainwpcontrol` resolves to the package installed from the current tarball. Source mode adds a temporary `mainwpcontrol` link to the repository binary. + +Claude receives only: + +```text +claude -p + --allowedTools "Bash(mainwpcontrol *)" + --disallowedTools "mcp__*" + --strict-mcp-config + --settings '{"sandbox": {"enabled": false}}' + --append-system-prompt + --output-format stream-json + --verbose + --max-turns 20 +``` + +`--strict-mcp-config` and the MCP disallow pattern isolate the run from any MCP servers configured on the host machine — without them, a host-configured MainWP MCP server can answer the task and the CLI is never exercised. The child's Bash sandbox is disabled because it would block CLI network access to the Dashboard host and force error-and-retry churn on every command; tool access stays restricted through `--allowedTools`. The appended system prompt tells the model that the Dashboard is managed exclusively through the `mainwpcontrol` CLI with `--json`, `--dry-run`, and `--confirm --force`, mirroring what the CLI's own help teaches. Grading tolerates failed intermediate CLI attempts (a CLI agent discovers input schemas by trying) and requires correct non-error results, a non-error final attempt, and a final answer consistent with ground truth. The runner parses stream JSON line by line and collects Bash `tool_use` blocks only when their command starts with `mainwpcontrol`. Grading requires explicit `abilities run` subcommands, expected ability names, valid inline JSON arguments where needed, successful correlated Bash results, and a final answer consistent with verifier ground truth. + +Ground truth is computed before Claude starts. If the verifier cannot establish it, the scenario is unverified and Claude is not invoked. A failed scenario makes the runner exit non-zero; skipped and unverified scenarios remain visible but are not counted as passed. + +Each agent result records: + +- Claude process wall-clock milliseconds +- `duration_ms` from the terminal stream result +- `duration_api_ms` from the terminal stream result +- TTFT from process spawn to the first `assistant` stream event +- Parsed `mainwpcontrol` invocation count + +`summary.md` shows the timing for every scenario and identifies the slowest completed Claude process. + +## Evidence order + +Deterministic and agent checks use this evidence order: + +1. The independent verifier reads current state through the Abilities API. +2. The installed or source CLI runs as a child process. +3. Structured output or parsed Bash tool results are compared with the direct read. +4. A second independent read proves state preservation or the requested mutation. +5. Final prose is checked only after structured and state evidence. + +The model never grades itself. If the initial verifier read throws, no model call is made. + +## Artifacts + +Every run writes to: + +```text +test-results/acceptance/-[-dirty][-agent]/ +``` + +The directory can contain: + +- `manifest.json`: branch, commit, dirty state, diff hash, mode, target, runtime versions, flags, and packed tarball metadata +- `results.json`: statuses, assertions or agent evaluation fields, timing, parsed invocations, and artifact-audit result +- `summary.md`: totals, scenario status, timing, and slowest-run information +- `commands.jsonl`: package, install, CLI, and Claude command records with durations and redacted output tails +- `events.jsonl`: ordered deterministic runner events +- `scenario-.stderr.log`: deterministic CLI diagnostics +- `agent-transcript.jsonl`: every raw Claude stream line, including lines that did not parse + +Use `--keep-consumer` to preserve the temporary packed consumer for inspection. + +## Redaction audit + +The redactor registers the live credentials and the fixture credentials, including usernames, spaced and compact Application Passwords, Dashboard origins, and Basic Authorization values. Every artifact write goes through the `Artifacts` class, which applies the redactor before writing. + +At the end of a run, the harness scans the artifact directory for every registered raw value. A finding is recorded as an artifact-audit failure and makes the runner exit non-zero. The transcript retains full model stream lines only after this redaction path. + +## Reproducing a failure + +1. Read `summary.md` for the failing or slow scenario. +2. Open `results.json` for exact evaluation evidence, parsed argv, timing, and the failure reason. +3. Inspect the scenario's records in `commands.jsonl` and `agent-transcript.jsonl` or `events.jsonl`. +4. Confirm the mode, commit, dirty state, and tarball identity in `manifest.json`. +5. Re-run only that scenario with the same mode and target. + +Example: + +```bash +npx tsx tests/acceptance/agent-run.ts \ + --mode packed \ + --scenario agent-plugin-active \ + --keep-consumer +``` diff --git a/docs/workflows/daily-health-check.md b/docs/workflows/daily-health-check.md index b5a8d8c..0e8390f 100644 --- a/docs/workflows/daily-health-check.md +++ b/docs/workflows/daily-health-check.md @@ -524,7 +524,7 @@ An asterisk (`*`) means "every," so `* * * * *` means every minute of every hour Add this line to schedule the health check to run every day at 7:00 AM: -``` +```cron 0 7 * * * /full/path/to/mainwp-health-check.sh ``` @@ -536,13 +536,13 @@ realpath mainwp-health-check.sh For example, if the script is in your home directory, the line might be: -``` +```cron 0 7 * * * /Users/yourname/mainwp-health-check.sh ``` **Note on PATH:** Cron runs in a minimal environment. It does not load your shell profile, so commands like `mainwpcontrol` or `jq` may not be found by their short names. If you encounter issues, add a `PATH` line at the top of your crontab: -``` +```cron PATH=/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin 0 7 * * * /full/path/to/mainwp-health-check.sh ``` @@ -649,11 +649,35 @@ If the doctor command reports authentication issues, run `mainwpcontrol login` a ### Authentication errors in cron -Cron runs in a minimal environment and may not have access to your system keychain where MainWP Control stores credentials. If the health check works when you run it manually but fails from cron, you can set the credentials as environment variables directly in your crontab: +Cron runs in a minimal environment and may not have access to your system keychain where MainWP Control stores credentials. If the health check works when you run it manually but fails from cron, store the Application Password in a restricted-permission env file and have cron source it before running the script. + +Don't put the password directly in the crontab. Crontab contents are easy to expose: `crontab -l` output ends up in shared logs, and system backups often capture the crontab file itself. + +Be aware of the trade-off: this file stores the Application Password in plain text on disk, protected only by its file permissions. That is inherent to unattended runs — cron cannot unlock your OS keychain. `MAINWP_APP_PASSWORD` is MainWP Control's supported environment fallback for exactly this situation; for interactive use, keep credentials in the keychain via `mainwpcontrol login`. If the password may have been exposed, revoke it in WordPress and issue a new one. +Create the env file: + +```bash +mkdir -p ~/.config/mainwpcontrol +nano ~/.config/mainwpcontrol/cron.env ``` -MAINWP_APP_PASSWORD='your-app-password' -0 7 * * * /full/path/to/mainwp-health-check.sh + +Add this line, using the Application Password from Step 1 (spaces removed): + +```bash +export MAINWP_APP_PASSWORD='your-app-password' +``` + +Save the file, then restrict its permissions so only you can read it: + +```bash +chmod 600 ~/.config/mainwpcontrol/cron.env +``` + +Update the crontab entry to source the file before running the script: + +```cron +0 7 * * * . "$HOME/.config/mainwpcontrol/cron.env" && /full/path/to/mainwp-health-check.sh ``` -Replace `your-app-password` with the Application Password from Step 1 (spaces removed). Environment variables set at the top of the crontab apply to all jobs below them. +The `. "$HOME/.config/mainwpcontrol/cron.env"` part loads the environment variable from the file, and `&&` runs the script only if that succeeds. diff --git a/docs/workflows/input-from-file.md b/docs/workflows/input-from-file.md index 68b3a54..cf831e9 100644 --- a/docs/workflows/input-from-file.md +++ b/docs/workflows/input-from-file.md @@ -28,11 +28,10 @@ Some MainWP Control abilities need more than a flag. For example, updating speci ```bash # macOS / Linux / Git Bash mainwpcontrol abilities run get-site-v1 --input '{"site_id_or_domain": 5}' --json - -# Windows PowerShell (escape inner quotes with backslashes) -mainwpcontrol abilities run get-site-v1 --input '{\"site_id_or_domain\": 5}' --json ``` +On Windows PowerShell, quoting of inline JSON is unreliable and varies by version -- use `--input-file` instead, which is what the rest of this guide covers. + But when parameters get complex (nested objects, arrays, multiple fields), inline JSON becomes hard to read and easy to get wrong. A misplaced quote or missing comma can cause confusing errors. File input solves this. You write your parameters in a file (or pipe them from another command), and MainWP Control reads them cleanly. diff --git a/docs/workflows/monthly-batch-updates.md b/docs/workflows/monthly-batch-updates.md index 3d64360..9a01ba7 100644 --- a/docs/workflows/monthly-batch-updates.md +++ b/docs/workflows/monthly-batch-updates.md @@ -148,7 +148,7 @@ Before writing any scripts, it is important to understand how MainWP Control pre MainWP Control enforces this through four flags: -- **`--dry-run`:** Asks MainWP to show you what *would* happen, without making any changes. Think of it as a preview. Your sites are not touched. You can run `--dry-run` as many times as you want with zero risk. +- **`--dry-run`:** Asks MainWP to show you what *would* happen, without making any changes. Think of it as a preview. `--dry-run` makes no changes on any site, so you can repeat it freely while you refine your filters. - **`--confirm`:** Tells MainWP to go ahead and execute the operation for real. This is required for any operation that changes something (applying updates, deleting plugins, etc.). Without `--confirm`, the command will only show you what it would do. @@ -169,6 +169,19 @@ The typical flow in any script is: --- +## Before You Automate + +Once you schedule `--confirm --force`, updates apply without anyone watching. Before you turn on either option below, make sure: + +- **Backups are current for every site in scope.** Use the MainWP Backups extension or your host's backup tool, and confirm a recent, restorable backup exists before the first automated run. +- **You've run a canary first.** Run the workflow against one or two low-risk sites before widening it to your full network. `run-updates-v1` accepts a `site_ids_or_domains` input that limits its scope, so a canary run looks like `mainwpcontrol abilities run run-updates-v1 --input '{"site_ids_or_domains": [12, 34]}' --confirm --force --wait --json` with your low-risk site IDs (or domains). Only expand once a full cycle has run clean. +- **You know your rollback path.** If an update breaks a site, you need a way back: restoring from backup, or rolling back the specific plugin or theme version. Confirm this actually works before you rely on it. +- **The confirmed run lands inside a maintenance window you can monitor.** Even with `--wait`, something can go wrong. Schedule the `--confirm` run for a time when you, or someone, can check the result and react. + +Both Option A and Option B below assume these are in place. + +--- + > **Windows users:** Option A builds a bash script with cron, which needs macOS or Linux. If you're on Windows, skip to **Option B: GitHub Actions**. It runs on Linux in the cloud and works regardless of your local OS. ## Option A: Scripted Updates @@ -290,7 +303,7 @@ Expected output: ### Step 6: Apply Updates -When you are satisfied with the preview, apply the updates for real: +When you are satisfied with the preview, apply the updates for real. The prerequisites above apply here: confirm backups are current and run against your canary sites before pointing this at your full network. ```bash mainwpcontrol abilities run run-updates-v1 --confirm --force --wait --json @@ -353,7 +366,7 @@ If the number is not zero, some updates may have failed, or new updates appeared ### Complete Script -Here is everything combined into a single script with error handling. Create a file called `monthly-updates.sh`: +Here is everything combined into a single script with error handling. This is the script you'll schedule with cron, so make sure the [prerequisites above](#before-you-automate) are in place before you rely on it. Create a file called `monthly-updates.sh`: ```bash #!/bin/bash @@ -555,6 +568,8 @@ jobs: #### Apply Step (Conditional) +The prerequisites above apply here too: confirm backups are current and run this workflow against your canary sites before scheduling it against your full network. + ```yaml - name: Apply updates if: steps.preview.outputs.count != '0' diff --git a/docs/workflows/plugin-deployment-verification.md b/docs/workflows/plugin-deployment-verification.md index d2de059..22e6e3b 100644 --- a/docs/workflows/plugin-deployment-verification.md +++ b/docs/workflows/plugin-deployment-verification.md @@ -93,13 +93,7 @@ Expected output: You should see `@mainwp/control/` followed by a version number. If you see `command not found`, make sure Node.js 20+ is installed and try opening a new terminal window. -> **Windows PowerShell note:** When running `mainwpcontrol` locally with `--input`, you need to escape the inner double quotes: -> -> ```powershell -> mainwpcontrol abilities run get-site-plugins-v1 --input '{\"site_id_or_domain\": 1}' --json -> ``` -> -> The GitHub Actions workflow runs on Linux, so this quoting issue only affects local testing. You can also use `--input-file params.json` to avoid it entirely. +> **Windows PowerShell note:** PowerShell's quoting of inline JSON is unreliable and varies by version. When running `mainwpcontrol` locally with `--input`, use `--input-file params.json` instead. The GitHub Actions workflow runs on Linux, so this only affects local testing. --- diff --git a/package-lock.json b/package-lock.json index 7be7c58..b0fdd3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,11 @@ "version": "1.1.0-beta.1", "license": "GPL-3.0-or-later", "dependencies": { - "@oclif/core": "~4.0.0", - "@oclif/plugin-autocomplete": "~3.2.39", - "@oclif/plugin-help": "~6.0.0", + "@oclif/core": "^4.11.14", + "@oclif/plugin-autocomplete": "^3.2.53", + "@oclif/plugin-help": "^6.2.53", "ajv": "^8.18.0", - "undici": "^7.24.0" + "undici": "^7.28.0" }, "bin": { "mainwpcontrol": "bin/run.js" @@ -25,11 +25,12 @@ "@typescript-eslint/parser": "^7.0.0", "eslint": "^8.57.0", "oclif": "^4.0.0", + "tsx": "^4.21.0", "typescript": "^5.4.0", "vitest": "^1.6.0" }, "engines": { - "node": ">=20.0.0" + "node": ">=20.18.1" }, "optionalDependencies": { "keytar": "~7.9.0" @@ -1450,9 +1451,9 @@ } }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", + "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", "dev": true, "license": "MIT", "dependencies": { @@ -1507,9 +1508,9 @@ } }, "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", + "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", "dev": true, "license": "MIT", "dependencies": { @@ -2568,6 +2569,7 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -2581,6 +2583,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -2590,6 +2593,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -2600,26 +2604,26 @@ } }, "node_modules/@oclif/core": { - "version": "4.0.37", - "resolved": "https://registry.npmjs.org/@oclif/core/-/core-4.0.37.tgz", - "integrity": "sha512-D69KH08/08kAPaDUPzDALYfqC2BXi4LRDQ/+59P3zZoVKYMoFO1kOouNILpI4bQKA+PpI2REctXMebB8U/bYHQ==", + "version": "4.11.14", + "resolved": "https://registry.npmjs.org/@oclif/core/-/core-4.11.14.tgz", + "integrity": "sha512-cZ5Ktd+rT0PO+o7KBH4vRFTgg+xMLf8F41WK39p8MkXEViZA/Qqe+4lzZT6102zgUxMORET1HtF9t5w8CB3tnQ==", "license": "MIT", "dependencies": { "ansi-escapes": "^4.3.2", - "ansis": "^3.3.2", + "ansis": "^3.17.0", "clean-stack": "^3.0.1", "cli-spinners": "^2.9.2", - "debug": "^4.4.0", + "debug": "^4.4.3", "ejs": "^3.1.10", "get-package-type": "^0.1.0", - "globby": "^11.1.0", "indent-string": "^4.0.0", "is-wsl": "^2.2.0", "lilconfig": "^3.1.3", - "minimatch": "^9.0.5", - "semver": "^7.6.3", + "minimatch": "^10.2.5", + "semver": "^7.8.1", "string-width": "^4.2.3", "supports-color": "^8", + "tinyglobby": "^0.2.17", "widest-line": "^3.1.0", "wordwrap": "^1.0.0", "wrap-ansi": "^7.0.0" @@ -2628,92 +2632,67 @@ "node": ">=18.0.0" } }, - "node_modules/@oclif/plugin-autocomplete": { - "version": "3.2.39", - "resolved": "https://registry.npmjs.org/@oclif/plugin-autocomplete/-/plugin-autocomplete-3.2.39.tgz", - "integrity": "sha512-OwAZNnSpuDjKyhAwoOJkFWxGswPFKBB4hpNIMsj6PUtbKwGBPmD+2wGGPgTsDioVwLmUELSb2bZ+1dxHfvXmvg==", + "node_modules/@oclif/core/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "license": "MIT", - "dependencies": { - "@oclif/core": "^4", - "ansis": "^3.16.0", - "debug": "^4.4.1", - "ejs": "^3.1.10" - }, "engines": { - "node": ">=18.0.0" + "node": "18 || 20 || >=22" } }, - "node_modules/@oclif/plugin-help": { - "version": "6.0.22", - "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-6.0.22.tgz", - "integrity": "sha512-IPgUvPSdZMCHzCwCRVDUMWtFkWZSoU6Z7igNclugLIpF3Ac3vKkZGguWZ+SLK3e7012etDzgAHjXFELYOqqbsw==", + "node_modules/@oclif/core/node_modules/brace-expansion": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", + "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", "license": "MIT", "dependencies": { - "@oclif/core": "^3.26.6" + "balanced-match": "^4.0.2" }, "engines": { - "node": ">=18.0.0" + "node": "18 || 20 || >=22" } }, - "node_modules/@oclif/plugin-help/node_modules/@oclif/core": { - "version": "3.27.0", - "resolved": "https://registry.npmjs.org/@oclif/core/-/core-3.27.0.tgz", - "integrity": "sha512-Fg93aNFvXzBq5L7ztVHFP2nYwWU1oTCq48G0TjF/qC1UN36KWa2H5Hsm72kERd5x/sjy2M2Tn4kDEorUlpXOlw==", - "license": "MIT", + "node_modules/@oclif/core/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", "dependencies": { - "@types/cli-progress": "^3.11.5", - "ansi-escapes": "^4.3.2", - "ansi-styles": "^4.3.0", - "cardinal": "^2.1.1", - "chalk": "^4.1.2", - "clean-stack": "^3.0.1", - "cli-progress": "^3.12.0", - "color": "^4.2.3", - "debug": "^4.3.5", - "ejs": "^3.1.10", - "get-package-type": "^0.1.0", - "globby": "^11.1.0", - "hyperlinker": "^1.0.0", - "indent-string": "^4.0.0", - "is-wsl": "^2.2.0", - "js-yaml": "^3.14.1", - "minimatch": "^9.0.4", - "natural-orderby": "^2.0.3", - "object-treeify": "^1.1.33", - "password-prompt": "^1.1.3", - "slice-ansi": "^4.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1", - "supports-color": "^8.1.1", - "supports-hyperlinks": "^2.2.0", - "widest-line": "^3.1.0", - "wordwrap": "^1.0.0", - "wrap-ansi": "^7.0.0" + "brace-expansion": "^5.0.5" }, "engines": { - "node": ">=18.0.0" + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@oclif/plugin-help/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "node_modules/@oclif/plugin-autocomplete": { + "version": "3.2.53", + "resolved": "https://registry.npmjs.org/@oclif/plugin-autocomplete/-/plugin-autocomplete-3.2.53.tgz", + "integrity": "sha512-cGAgN9ujTDxa6C84d6XNVsmoFnDFfHwdiykAnAfym3oBLxoXBNYMZhmfnZ8KBWDy/r/DYf11BrGJwqKl2P0iSA==", "license": "MIT", "dependencies": { - "sprintf-js": "~1.0.2" + "@oclif/core": "^4", + "ansis": "^3.16.0", + "debug": "^4.4.1", + "ejs": "^3.1.10" + }, + "engines": { + "node": ">=18.0.0" } }, - "node_modules/@oclif/plugin-help/node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "node_modules/@oclif/plugin-help": { + "version": "6.2.53", + "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-6.2.53.tgz", + "integrity": "sha512-njx2nTH87EQEEuz4ShNtL0gzzN981MRkDPqScbu+Tkd7NpIv30OHdTjQK1GzGtkf+V2RvSYIrX+LrLsUVh9DJw==", "license": "MIT", "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "@oclif/core": "^4" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "engines": { + "node": ">=18.0.0" } }, "node_modules/@oclif/plugin-not-found": { @@ -2732,75 +2711,6 @@ "node": ">=18.0.0" } }, - "node_modules/@oclif/plugin-not-found/node_modules/@oclif/core": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/@oclif/core/-/core-4.9.0.tgz", - "integrity": "sha512-k/ntRgDcUprTT+aaNoF+whk3cY3f9fRD2lkF6ul7JeCUg2MaMXVXZXfbRhJCfsiX51X8/5Pqo0LGdO9SLYXNHg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-escapes": "^4.3.2", - "ansis": "^3.17.0", - "clean-stack": "^3.0.1", - "cli-spinners": "^2.9.2", - "debug": "^4.4.3", - "ejs": "^3.1.10", - "get-package-type": "^0.1.0", - "indent-string": "^4.0.0", - "is-wsl": "^2.2.0", - "lilconfig": "^3.1.3", - "minimatch": "^10.2.4", - "semver": "^7.7.3", - "string-width": "^4.2.3", - "supports-color": "^8", - "tinyglobby": "^0.2.14", - "widest-line": "^3.1.0", - "wordwrap": "^1.0.0", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@oclif/plugin-not-found/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@oclif/plugin-not-found/node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@oclif/plugin-not-found/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@oclif/plugin-warn-if-update-available": { "version": "3.1.53", "resolved": "https://registry.npmjs.org/@oclif/plugin-warn-if-update-available/-/plugin-warn-if-update-available-3.1.53.tgz", @@ -4048,15 +3958,6 @@ "node": ">=14.16" } }, - "node_modules/@types/cli-progress": { - "version": "3.11.6", - "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.6.tgz", - "integrity": "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -4085,6 +3986,7 @@ "version": "20.19.27", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.27.tgz", "integrity": "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -4491,12 +4393,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/ansicolors": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.3.2.tgz", - "integrity": "sha512-QXu7BPrP29VllRxH8GwB7x5iX5qWKAAMLqKQGWTeLWVlNHNOpVMJ91dsxQAIWXpjuW5wqvxu3Jd/nRjrJ+0pqg==", - "license": "MIT" - }, "node_modules/ansis": { "version": "3.17.0", "resolved": "https://registry.npmjs.org/ansis/-/ansis-3.17.0.tgz", @@ -4517,6 +4413,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4532,15 +4429,6 @@ "node": "*" } }, - "node_modules/astral-regex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -4604,9 +4492,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", + "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -4616,6 +4504,7 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -4721,19 +4610,6 @@ "upper-case-first": "^2.0.2" } }, - "node_modules/cardinal": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/cardinal/-/cardinal-2.1.1.tgz", - "integrity": "sha512-JSr5eOgoEymtYHBjNWyjrMqet9Am2miJhlfKNdqLp6zoeAh0KN5dRAcxlecj5mAJrmQomgiOBj35xHLrFjqBpw==", - "license": "MIT", - "dependencies": { - "ansicolors": "~0.3.2", - "redeyed": "~2.1.0" - }, - "bin": { - "cdl": "bin/cdl.js" - } - }, "node_modules/chai": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", @@ -4757,6 +4633,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -4773,6 +4650,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -4844,18 +4722,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/cli-progress": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", - "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", - "license": "MIT", - "dependencies": { - "string-width": "^4.2.3" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/cli-spinners": { "version": "2.9.2", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", @@ -4878,19 +4744,6 @@ "node": ">= 12" } }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -4909,16 +4762,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "license": "MIT", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -4970,6 +4813,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -5116,6 +4960,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, "license": "MIT", "dependencies": { "path-type": "^4.0.0" @@ -5345,9 +5190,9 @@ } }, "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", + "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", "dev": true, "license": "MIT", "dependencies": { @@ -5393,19 +5238,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", @@ -5532,6 +5364,7 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -5548,6 +5381,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -5574,9 +5408,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", "funding": [ { "type": "github", @@ -5640,6 +5474,7 @@ "version": "1.19.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz", "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", + "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -5659,9 +5494,9 @@ } }, "node_modules/filelist": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", - "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.6.tgz", + "integrity": "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==", "license": "Apache-2.0", "dependencies": { "minimatch": "^5.0.1" @@ -5683,6 +5518,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -5839,6 +5675,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-tsconfig": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", + "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/git-hooks-list": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/git-hooks-list/-/git-hooks-list-3.2.0.tgz", @@ -5899,9 +5748,9 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", + "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", "dev": true, "license": "MIT", "dependencies": { @@ -5955,6 +5804,7 @@ "version": "11.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, "license": "MIT", "dependencies": { "array-union": "^2.1.0", @@ -6093,15 +5943,6 @@ "node": ">=16.17.0" } }, - "node_modules/hyperlinker": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/hyperlinker/-/hyperlinker-1.0.0.tgz", - "integrity": "sha512-Ty8UblRWFEcfSuIaajM34LdPXIhbs1ajEX/BBPv24J+enSVaEVY63xQ6lTO9VRYS5LAoghIG0IDJ+p+IPzKUQQ==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/iconv-lite": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", @@ -6144,6 +5985,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -6237,6 +6079,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -6255,6 +6098,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -6267,6 +6111,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -6334,6 +6179,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, "license": "ISC" }, "node_modules/jake": { @@ -6361,10 +6207,20 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", + "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -6566,6 +6422,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -6575,6 +6432,7 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -6614,6 +6472,7 @@ "version": "9.0.9", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.2" @@ -6711,15 +6570,6 @@ "dev": true, "license": "MIT" }, - "node_modules/natural-orderby": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/natural-orderby/-/natural-orderby-2.0.3.tgz", - "integrity": "sha512-p7KTHxU0CUrcOXe62Zfrb5Z13nLvPhSWR/so3kFulUQU0sgUll2Z0LwpsLN351eOOD+hRGu/F1g+6xDfPeD++Q==", - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/no-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", @@ -6808,15 +6658,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/object-treeify": { - "version": "1.1.33", - "resolved": "https://registry.npmjs.org/object-treeify/-/object-treeify-1.1.33.tgz", - "integrity": "sha512-EFVjAYfzWqWsBMRHPMAXLCDIJnpMhdWAqR7xG6M6a2cs6PMFpl/+Z20w9zDW4vkxOFfddegBKq9Rehd0bxWE7A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, "node_modules/oclif": { "version": "4.22.57", "resolved": "https://registry.npmjs.org/oclif/-/oclif-4.22.57.tgz", @@ -6856,112 +6697,30 @@ "node": ">=18.0.0" } }, - "node_modules/oclif/node_modules/@oclif/core": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/@oclif/core/-/core-4.9.0.tgz", - "integrity": "sha512-k/ntRgDcUprTT+aaNoF+whk3cY3f9fRD2lkF6ul7JeCUg2MaMXVXZXfbRhJCfsiX51X8/5Pqo0LGdO9SLYXNHg==", - "dev": true, - "license": "MIT", + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "devOptional": true, + "license": "ISC", "dependencies": { - "ansi-escapes": "^4.3.2", - "ansis": "^3.17.0", - "clean-stack": "^3.0.1", - "cli-spinners": "^2.9.2", - "debug": "^4.4.3", - "ejs": "^3.1.10", - "get-package-type": "^0.1.0", - "indent-string": "^4.0.0", - "is-wsl": "^2.2.0", - "lilconfig": "^3.1.3", - "minimatch": "^10.2.4", - "semver": "^7.7.3", - "string-width": "^4.2.3", - "supports-color": "^8", - "tinyglobby": "^0.2.14", - "widest-line": "^3.1.0", - "wordwrap": "^1.0.0", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=18.0.0" + "wrappy": "1" } }, - "node_modules/oclif/node_modules/@oclif/plugin-help": { - "version": "6.2.38", - "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-6.2.38.tgz", - "integrity": "sha512-aTVQ8qPy5kD/Neq2B4OEo2joukHWdEabTMHfQyXtsagW1O2MvhM58+JUWADvieX67OSjSXseD6f6O/e5SA2N/Q==", + "node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", "dev": true, "license": "MIT", "dependencies": { - "@oclif/core": "^4" + "mimic-fn": "^4.0.0" }, "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/oclif/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/oclif/node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/oclif/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "devOptional": true, - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/onetime": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", - "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "mimic-fn": "^4.0.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/optionator": { @@ -7080,16 +6839,6 @@ "tslib": "^2.0.3" } }, - "node_modules/password-prompt": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/password-prompt/-/password-prompt-1.1.3.tgz", - "integrity": "sha512-HkrjG2aJlvF0t2BMH0e2LB/EHf3Lcq3fNMzy4GYHcQblAvOl+QQji1Lx7WRBMqpVK8p+KR7bCg7oqAMXtdgqyw==", - "license": "0BSD", - "dependencies": { - "ansi-escapes": "^4.3.2", - "cross-spawn": "^7.0.3" - } - }, "node_modules/path-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/path-case/-/path-case-3.0.4.tgz", @@ -7141,6 +6890,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -7150,6 +6900,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -7182,6 +6933,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -7335,6 +7087,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, "funding": [ { "type": "github", @@ -7412,15 +7165,6 @@ "node": ">= 6" } }, - "node_modules/redeyed": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/redeyed/-/redeyed-2.1.1.tgz", - "integrity": "sha512-FNpGGo1DycYAdnrKFxCMmKYgo/mILAqtRYbkdQD8Ep/Hk2PQ5+aEAEx+IU713RTDmuBaH0c8P5ZozurNu5ObRQ==", - "license": "MIT", - "dependencies": { - "esprima": "~4.0.0" - } - }, "node_modules/registry-auth-token": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-5.1.0.tgz", @@ -7460,6 +7204,16 @@ "node": ">=4" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/responselike": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz", @@ -7490,6 +7244,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -7562,6 +7317,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, "funding": [ { "type": "github", @@ -7610,9 +7366,9 @@ "license": "MIT" }, "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -7637,6 +7393,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -7649,6 +7406,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -7721,47 +7479,16 @@ "simple-concat": "^1.0.0" } }, - "node_modules/simple-swizzle": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", - "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", - "license": "MIT", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/simple-swizzle/node_modules/is-arrayish": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", - "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==", - "license": "MIT" - }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, - "node_modules/slice-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", - "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, "node_modules/snake-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.4.tgz", @@ -7846,12 +7573,6 @@ "dev": true, "license": "CC0-1.0" }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "license": "BSD-3-Clause" - }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -7969,31 +7690,6 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/supports-hyperlinks": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", - "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0", - "supports-color": "^7.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-hyperlinks/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tar-fs": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", @@ -8046,14 +7742,13 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "dev": true, + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -8066,7 +7761,6 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -8084,7 +7778,6 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -8117,6 +7810,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -8145,6 +7839,510 @@ "dev": true, "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.0.tgz", + "integrity": "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.0.tgz", + "integrity": "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.0.tgz", + "integrity": "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.0.tgz", + "integrity": "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.0.tgz", + "integrity": "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.0.tgz", + "integrity": "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.0.tgz", + "integrity": "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.0.tgz", + "integrity": "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.0.tgz", + "integrity": "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.0.tgz", + "integrity": "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.0.tgz", + "integrity": "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.0.tgz", + "integrity": "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.0.tgz", + "integrity": "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.0.tgz", + "integrity": "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.0.tgz", + "integrity": "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.0.tgz", + "integrity": "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.0.tgz", + "integrity": "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.0.tgz", + "integrity": "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.0.tgz", + "integrity": "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.0.tgz", + "integrity": "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.0.tgz", + "integrity": "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.0.tgz", + "integrity": "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.0.tgz", + "integrity": "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.0.tgz", + "integrity": "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.0.tgz", + "integrity": "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.0.tgz", + "integrity": "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", + "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.0", + "@esbuild/android-arm": "0.27.0", + "@esbuild/android-arm64": "0.27.0", + "@esbuild/android-x64": "0.27.0", + "@esbuild/darwin-arm64": "0.27.0", + "@esbuild/darwin-x64": "0.27.0", + "@esbuild/freebsd-arm64": "0.27.0", + "@esbuild/freebsd-x64": "0.27.0", + "@esbuild/linux-arm": "0.27.0", + "@esbuild/linux-arm64": "0.27.0", + "@esbuild/linux-ia32": "0.27.0", + "@esbuild/linux-loong64": "0.27.0", + "@esbuild/linux-mips64el": "0.27.0", + "@esbuild/linux-ppc64": "0.27.0", + "@esbuild/linux-riscv64": "0.27.0", + "@esbuild/linux-s390x": "0.27.0", + "@esbuild/linux-x64": "0.27.0", + "@esbuild/netbsd-arm64": "0.27.0", + "@esbuild/netbsd-x64": "0.27.0", + "@esbuild/openbsd-arm64": "0.27.0", + "@esbuild/openbsd-x64": "0.27.0", + "@esbuild/openharmony-arm64": "0.27.0", + "@esbuild/sunos-x64": "0.27.0", + "@esbuild/win32-arm64": "0.27.0", + "@esbuild/win32-ia32": "0.27.0", + "@esbuild/win32-x64": "0.27.0" + } + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -8215,9 +8413,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "7.24.4", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.4.tgz", - "integrity": "sha512-BM/JzwwaRXxrLdElV2Uo6cTLEjhSb3WXboncJamZ15NgUURmvlXvxa6xkwIOILIjPNo9i8ku136ZvWV0Uly8+w==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", "license": "MIT", "engines": { "node": ">=20.18.1" @@ -8227,6 +8425,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, "license": "MIT" }, "node_modules/universalify": { @@ -8450,6 +8649,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" diff --git a/package.json b/package.json index de253d8..6a65f0d 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,14 @@ "lint": "eslint src --ext .ts", "postpack": "rm -f oclif.manifest.json", "prepack": "npm run clean && npm run build && oclif manifest", + "pretest": "npm run build", "test": "vitest run", + "test:acceptance": "tsx tests/acceptance/run.ts", + "test:acceptance:fast": "npm run build && tsx tests/acceptance/run.ts --mode source", + "test:acceptance:fixture": "tsx tests/acceptance/run.ts --target fixture", + "test:acceptance:writes": "tsx tests/acceptance/run.ts --writes", + "test:acceptance:agent": "tsx tests/acceptance/agent-run.ts", + "test:acceptance:human": "npm run test:acceptance:fixture && npm run test:acceptance:writes && npm run test:acceptance:agent", "test:process": "npm run build && vitest run --config vitest.process.config.ts", "test:live": "npm run build && MAINWP_LIVE_TEST=1 vitest run --config vitest.live.config.ts", "test:all": "npm run test && npm run test:process", @@ -54,7 +61,7 @@ } }, "engines": { - "node": ">=20.0.0" + "node": ">=20.18.1" }, "keywords": [ "mainwp", @@ -79,11 +86,11 @@ "url": "https://github.com/mainwp/mainwp-control/issues" }, "dependencies": { - "@oclif/core": "~4.0.0", - "@oclif/plugin-autocomplete": "~3.2.39", - "@oclif/plugin-help": "~6.0.0", + "@oclif/core": "^4.11.14", + "@oclif/plugin-autocomplete": "^3.2.53", + "@oclif/plugin-help": "^6.2.53", "ajv": "^8.18.0", - "undici": "^7.24.0" + "undici": "^7.28.0" }, "optionalDependencies": { "keytar": "~7.9.0" @@ -95,6 +102,7 @@ "@typescript-eslint/parser": "^7.0.0", "eslint": "^8.57.0", "oclif": "^4.0.0", + "tsx": "^4.21.0", "typescript": "^5.4.0", "vitest": "^1.6.0" } diff --git a/src/__tests__/e2e/batch-polling-flow.test.ts b/src/__tests__/e2e/batch-polling-flow.test.ts index 6c50761..2be10a8 100644 --- a/src/__tests__/e2e/batch-polling-flow.test.ts +++ b/src/__tests__/e2e/batch-polling-flow.test.ts @@ -511,7 +511,7 @@ describe('E2E: Batch Operation → Polling Flow', () => { ['completed', 'completed'], ['failed', 'failed'], ['partial', 'partial'], - ['unknown', 'pending'], // Defaults to pending + ['cancelled', 'cancelled'], ]; for (const [input, expected] of statusMappings) { @@ -524,6 +524,16 @@ describe('E2E: Batch Operation → Polling Flow', () => { expect(status.status).toBe(expected); }); } + + it('rejects an unknown status', async () => { + mockHttpGet.mockResolvedValueOnce({ + data: { job_id: 'job_test', status: 'unknown' }, + }); + + await expect(manager.getJobStatus('job_test')).rejects.toMatchObject({ + code: 'INVALID_RESPONSE', + }); + }); }); // ========================================================================== @@ -644,14 +654,15 @@ describe('E2E: Batch Operation → Polling Flow', () => { describe('Edge Cases', () => { it('handles job with zero total items', async () => { - mockHttpGet.mockResolvedValue( - createJobStatusResponse({ + mockHttpGet.mockResolvedValue({ + data: { + job_id: 'job_test', status: 'completed', total: 0, processed: 0, results: [], - }) - ); + }, + }); const result = await manager.resumeJob('job_test', { initialDelay: 1 }); diff --git a/src/__tests__/e2e/chat-destructive-flow.test.ts b/src/__tests__/e2e/chat-destructive-flow.test.ts index 896371b..45aa7de 100644 --- a/src/__tests__/e2e/chat-destructive-flow.test.ts +++ b/src/__tests__/e2e/chat-destructive-flow.test.ts @@ -87,7 +87,7 @@ function createTestEngine(options: { */ function createInvalidJsonResponse(): LLMResponse { return { - content: 'This is not valid JSON { broken', + content: '{"type": "tool_call", "tool": broken', finishReason: 'stop', model: 'test-model', }; @@ -896,7 +896,7 @@ describe('E2E: Chat → Destructive Action Flow', () => { expect(engine.hasPendingPreview()).toBe(false); }); - it('handles cancelPendingPreview programmatically', async () => { + it('pairs the pending tool call with a USER_DECLINED result', async () => { const mockProvider = createMockProvider([ createMockLLMToolCallResponse('delete-site-v1', { site_id: 123 }), ]); @@ -910,10 +910,26 @@ describe('E2E: Chat → Destructive Action Flow', () => { await engine.sendMessage('Delete site'); expect(engine.hasPendingPreview()).toBe(true); - engine.cancelPendingPreview(); + const pendingToolCallId = engine + .getHistory() + .find((message) => message.role === 'assistant' && message.toolCalls?.length) + ?.toolCalls?.[0]?.id; + expect(pendingToolCallId).toBeDefined(); + + await engine.sendMessage('cancel'); expect(engine.hasPendingPreview()).toBe(false); expect(engine.getPendingPreview()).toBeNull(); + const declineResult = engine + .getHistory() + .find( + (message) => message.role === 'tool' && message.toolCallId === pendingToolCallId + ); + expect(declineResult).toBeDefined(); + expect(JSON.parse(declineResult!.content)).toMatchObject({ + success: false, + error: { code: 'USER_DECLINED' }, + }); }); it('mixed readonly and destructive in sequence', async () => { diff --git a/src/__tests__/e2e/command-workflows.test.ts b/src/__tests__/e2e/command-workflows.test.ts index 9aaf295..120646c 100644 --- a/src/__tests__/e2e/command-workflows.test.ts +++ b/src/__tests__/e2e/command-workflows.test.ts @@ -23,6 +23,8 @@ import { clearEnvVar, restoreEnvVars, STANDARD_ABILITIES, + createCommandHarness, + type CapturedOutput, } from './test-helpers.js'; // ============================================================================ @@ -37,7 +39,11 @@ const mockProfileStoreList = vi.fn(); const mockProfileStoreDelete = vi.fn(); const mockProfileStoreSetActive = vi.fn(); -vi.mock('../../config/profile-store.js', () => ({ +vi.mock('../../config/profile-store.js', async (importOriginal) => ({ + // Keep the real validateDashboardUrl: login calls it at intake, and these + // workflows should exercise the genuine validation behavior. + validateDashboardUrl: (await importOriginal()) + .validateDashboardUrl, getProfileStore: vi.fn(() => ({ get: mockProfileStoreGet, getActive: mockProfileStoreGetActive, @@ -51,6 +57,7 @@ vi.mock('../../config/profile-store.js', () => ({ // Mock keychain singleton const mockKeychainGet = vi.fn(); +const mockKeychainGetStored = vi.fn(); const mockKeychainGetOrThrow = vi.fn(); const mockKeychainSet = vi.fn(); const mockKeychainDelete = vi.fn(); @@ -58,6 +65,7 @@ const mockKeychainDelete = vi.fn(); vi.mock('../../config/keychain.js', () => ({ getKeychain: vi.fn(() => ({ get: mockKeychainGet, + getStored: mockKeychainGetStored, getOrThrow: mockKeychainGetOrThrow, set: mockKeychainSet, delete: mockKeychainDelete, @@ -149,15 +157,6 @@ import ChatCommand from '../../commands/chat.js'; // Test Utilities // ============================================================================ -/** - * Captured output from command execution - */ -interface CapturedOutput { - stdout: string[]; - stderr: string[]; - exitCode?: number; -} - /** * Parse argv into flags and args * @@ -226,53 +225,12 @@ function createCommandWithCapture argv: string[] = [], flagDefs: Record = {} ): { command: T; output: CapturedOutput } { - const output: CapturedOutput = { - stdout: [], - stderr: [], - }; - - const mockConfig = { - root: '/mock/root', - bin: 'mainwpcontrol', - name: 'mainwpcontrol', - version: '1.0.0', - pjson: { name: 'mainwpcontrol', version: '1.0.0' }, - dataDir: '/mock/data', - cacheDir: '/mock/cache', - configDir: '/mock/config', - findCommand: vi.fn(), - runCommand: vi.fn(), - runHook: vi.fn(), - }; - - const command = new CommandClass(argv, mockConfig as never); + const { command, output } = createCommandHarness(CommandClass, argv); // Mock parse to return our parsed argv const parsed = parseArgv(argv, flagDefs); command.parse = vi.fn().mockResolvedValue(parsed) as never; - // Capture log output - command.log = vi.fn((...args: unknown[]) => { - output.stdout.push(args.map(String).join(' ')); - }); - - command.logToStderr = vi.fn((...args: unknown[]) => { - output.stderr.push(args.map(String).join(' ')); - }); - - // Capture exit - command.exit = vi.fn((code?: number) => { - output.exitCode = code ?? 0; - throw new Error(`EXIT:${code ?? 0}`); - }) as never; - - // Mock error to capture exit codes - command.error = vi.fn((message: string | Error, options?: { exit?: number }) => { - output.stderr.push(message instanceof Error ? message.message : message); - output.exitCode = options?.exit ?? 1; - throw new Error(`EXIT:${output.exitCode}`); - }) as never; - return { command, output }; } @@ -349,9 +307,10 @@ describe('E2E: Command-Level Workflows', () => { mockProfileStoreSetActive.mockReset().mockResolvedValue(undefined); mockKeychainGet.mockReset(); + mockKeychainGetStored.mockReset().mockResolvedValue({ status: 'not-found' }); mockKeychainGetOrThrow.mockReset(); mockKeychainSet.mockReset().mockResolvedValue({ stored: true, location: 'keychain' }); - mockKeychainDelete.mockReset().mockResolvedValue(undefined); + mockKeychainDelete.mockReset().mockResolvedValue({ deleted: true }); mockHttpGet.mockReset(); mockHttpPost.mockReset(); @@ -399,7 +358,7 @@ describe('E2E: Command-Level Workflows', () => { // Verify profile was saved expect(mockProfileStoreSave).toHaveBeenCalled(); - expect(mockKeychainSet).toHaveBeenCalledWith('test-profile', 'secret123'); + expect(mockKeychainSet).toHaveBeenCalledWith('test-profile', 'secret123', 'https://dashboard.test'); }); it('outputs JSON envelope with --json flag', async () => { @@ -454,7 +413,50 @@ describe('E2E: Command-Level Workflows', () => { '--name', 'keychain-test', ], LOGIN_FLAGS); - expect(mockKeychainSet).toHaveBeenCalledWith('keychain-test', 'mypassword'); + expect(mockKeychainSet).toHaveBeenCalledWith('keychain-test', 'mypassword', 'https://dashboard.test'); + }); + + it('restores an existing credential when profile persistence fails', async () => { + mockHttpGet.mockResolvedValueOnce( + createMockHttpResponse(200, { abilities: [] }) + ); + mockProfileStoreGet.mockResolvedValueOnce(createMockProfile({ name: 'existing' })); + mockKeychainGetStored.mockResolvedValueOnce({ status: 'found', password: 'old-password' }); + mockProfileStoreSave.mockRejectedValueOnce(new Error('disk full')); + + await runCommand(Login, [ + '--url', 'https://dashboard.test', + '--username', 'admin', + '--password', 'new-password', + '--name', 'existing', + ], LOGIN_FLAGS); + + expect(mockKeychainSet).toHaveBeenNthCalledWith(1, 'existing', 'new-password', 'https://dashboard.test'); + expect(mockKeychainSet).toHaveBeenNthCalledWith(2, 'existing', 'old-password'); + expect(mockProfileStoreSetActive).not.toHaveBeenCalled(); + }); + + it('aborts before overwriting when the existing credential cannot be read', async () => { + mockHttpGet.mockResolvedValueOnce( + createMockHttpResponse(200, { abilities: [] }) + ); + mockProfileStoreGet.mockResolvedValueOnce(createMockProfile({ name: 'existing' })); + // A failed read is NOT "nothing stored": overwriting here and later + // rolling back would delete a credential that still exists. + mockKeychainGetStored.mockResolvedValueOnce({ status: 'error', error: 'keychain locked' }); + + const output = await runCommand(Login, [ + '--url', 'https://dashboard.test', + '--username', 'admin', + '--password', 'new-password', + '--name', 'existing', + ], LOGIN_FLAGS); + + expect(output.exitCode).toBe(1); + expect(mockKeychainSet).not.toHaveBeenCalled(); + expect(mockKeychainDelete).not.toHaveBeenCalled(); + expect(mockProfileStoreSave).not.toHaveBeenCalled(); + expect(mockProfileStoreSetActive).not.toHaveBeenCalled(); }); it('handles authentication failure with error exit', async () => { @@ -602,7 +604,7 @@ describe('E2E: Command-Level Workflows', () => { it('handles single message mode with readonly tool call', async () => { // LLM returns a tool call followed by answer mockProviderChat - .mockResolvedValueOnce(createMockLLMToolCallResponse('mainwp/list-sites-v1', {})) + .mockResolvedValueOnce(createMockLLMToolCallResponse('mainwp__list-sites-v1', {})) .mockResolvedValueOnce(createMockLLMAnswerResponse('Found 3 sites')); // Mock tool execution @@ -621,7 +623,7 @@ describe('E2E: Command-Level Workflows', () => { it('shows destructive preview and requires approval in non-interactive mode', async () => { // LLM returns a destructive tool call mockProviderChat.mockResolvedValueOnce( - createMockLLMToolCallResponse('mainwp/delete-site-v1', { site_id: 123 }) + createMockLLMToolCallResponse('mainwp__delete-site-v1', { site_id: 123 }) ); // Mock preview execution @@ -639,7 +641,7 @@ describe('E2E: Command-Level Workflows', () => { it('outputs JSON for tool results with --json flag', { timeout: 10000 }, async () => { mockProviderChat - .mockResolvedValueOnce(createMockLLMToolCallResponse('mainwp/list-sites-v1', {})) + .mockResolvedValueOnce(createMockLLMToolCallResponse('mainwp__list-sites-v1', {})) .mockResolvedValueOnce(createMockLLMAnswerResponse('Done')); mockExecutorExecute.mockResolvedValueOnce({ diff --git a/src/__tests__/e2e/exit-codes.test.ts b/src/__tests__/e2e/exit-codes.test.ts index 284580e..bffd78b 100644 --- a/src/__tests__/e2e/exit-codes.test.ts +++ b/src/__tests__/e2e/exit-codes.test.ts @@ -18,6 +18,8 @@ import { createMockHttpResponse, restoreEnvVars, STANDARD_ABILITIES, + createCommandHarness, + type CapturedOutput, } from './test-helpers.js'; // ============================================================================ @@ -112,53 +114,10 @@ import AbilitiesRun from '../../commands/abilities/run.js'; // Test Utilities // ============================================================================ -interface CapturedOutput { - stdout: string[]; - stderr: string[]; - exitCode?: number; -} - function createRunCommand( argv: string[] = [] ): { command: AbilitiesRun; output: CapturedOutput } { - const output: CapturedOutput = { stdout: [], stderr: [] }; - - const mockConfig = { - root: '/mock/root', - bin: 'mainwpcontrol', - name: 'mainwpcontrol', - version: '1.0.0', - pjson: { name: 'mainwpcontrol', version: '1.0.0' }, - dataDir: '/mock/data', - cacheDir: '/mock/cache', - configDir: '/mock/config', - findCommand: vi.fn(), - runCommand: vi.fn(), - runHook: vi.fn(), - }; - - const command = new AbilitiesRun(argv, mockConfig as never); - - command.log = vi.fn((...args: unknown[]) => { - output.stdout.push(args.map(String).join(' ')); - }); - - command.logToStderr = vi.fn((...args: unknown[]) => { - output.stderr.push(args.map(String).join(' ')); - }); - - command.exit = vi.fn((code?: number) => { - output.exitCode = code ?? 0; - throw new Error(`EXIT:${code ?? 0}`); - }) as never; - - command.error = vi.fn((message: string | Error, options?: { exit?: number }) => { - output.stderr.push(message instanceof Error ? message.message : message); - output.exitCode = options?.exit ?? 1; - throw new Error(`EXIT:${output.exitCode}`); - }) as never; - - return { command, output }; + return createCommandHarness(AbilitiesRun, argv); } async function runAbilitiesRun( @@ -309,11 +268,7 @@ describe('E2E: Exit Code Contract', () => { { name: 'list-sites-v1' } ); - // ConfigError maps to exit code 2 (AUTH_ERROR) - expect(output.exitCode).toBeDefined(); - // The exit code comes from the catch handler; in test setup it may be 1 - // because command.catch is not fully wired. The important thing is it fails. - expect(output.exitCode).toBeGreaterThanOrEqual(1); + expect(output.exitCode).toBe(2); }); // API error (exit code 4): API returns error response diff --git a/src/__tests__/e2e/json-contract.test.ts b/src/__tests__/e2e/json-contract.test.ts index 45496d7..bc95b86 100644 --- a/src/__tests__/e2e/json-contract.test.ts +++ b/src/__tests__/e2e/json-contract.test.ts @@ -15,6 +15,8 @@ import { createMockProfile, createMockAbility, restoreEnvVars, + createCommandHarness, + type CapturedOutput, } from './test-helpers.js'; // ============================================================================ @@ -75,6 +77,7 @@ vi.mock('../../utils/audit-logger.js', () => ({ getAuditLogger: vi.fn(() => ({ logDestructiveAction: vi.fn().mockResolvedValue(undefined), })), + logDestructiveActionSafe: vi.fn().mockResolvedValue(undefined), })); vi.mock('node:readline', () => ({ @@ -104,51 +107,11 @@ import AbilitiesList from '../../commands/abilities/list.js'; // Test Utilities // ============================================================================ -interface CapturedOutput { - stdout: string[]; - stderr: string[]; - exitCode?: number; -} - function createCommand( CommandClass: new (argv: string[], config: unknown) => T, argv: string[] = [] ): { command: T; output: CapturedOutput } { - const output: CapturedOutput = { stdout: [], stderr: [] }; - - const mockConfig = { - root: '/mock/root', - bin: 'mainwpcontrol', - name: 'mainwpcontrol', - version: '1.0.0', - pjson: { name: 'mainwpcontrol', version: '1.0.0' }, - dataDir: '/mock/data', - cacheDir: '/mock/cache', - configDir: '/mock/config', - findCommand: vi.fn(), - runCommand: vi.fn(), - runHook: vi.fn(), - }; - - const command = new CommandClass(argv, mockConfig as never); - - command.log = vi.fn((...args: unknown[]) => { - output.stdout.push(args.map(String).join(' ')); - }); - command.logToStderr = vi.fn((...args: unknown[]) => { - output.stderr.push(args.map(String).join(' ')); - }); - command.exit = vi.fn((code?: number) => { - output.exitCode = code ?? 0; - throw new Error(`EXIT:${code ?? 0}`); - }) as never; - command.error = vi.fn((message: string | Error, options?: { exit?: number }) => { - output.stderr.push(message instanceof Error ? message.message : message); - output.exitCode = options?.exit ?? 1; - throw new Error(`EXIT:${output.exitCode}`); - }) as never; - - return { command, output }; + return createCommandHarness(CommandClass, argv); } function findJsonOutput(lines: string[]): unknown | undefined { @@ -333,6 +296,123 @@ describe('E2E: JSON Output Contract', () => { }); }); + describe('abilities run --json envelope shapes (contract pins)', () => { + const runFlags = { + json: true, + quiet: false, + debug: false, + input: '{}', + 'dry-run': false, + confirm: false, + force: false, + wait: false, + 'wait-timeout': 300, + }; + + const expectedPreview = { + affected: [{ id: 5, name: 'Site Five' }], + summary: expect.any(String), + requiresApproval: true, + abilityName: 'mainwp/delete-site-v1', + input: {}, + }; + + it('pins the --dry-run preview envelope: mode, preview block, data.data nesting', async () => { + mockExecutorGetAbility.mockResolvedValue( + createMockAbility('delete-site-v1', { destructive: true }) + ); + mockExecutorExecute.mockResolvedValue({ + success: true, + data: { affected: [{ id: 5, name: 'Site Five' }] }, + }); + + const { command, output } = createCommand(AbilitiesRun); + command.parse = vi.fn().mockResolvedValue({ + flags: { ...runFlags, 'dry-run': true }, + args: { name: 'delete-site-v1' }, + }) as never; + + try { await command.run(); } catch { /* exit */ } + + const json = findJsonOutput(output.stdout); + expect(json).toEqual({ + success: true, + data: { + mode: 'preview', + ability: 'mainwp/delete-site-v1', + success: true, + data: { affected: [{ id: 5, name: 'Site Five' }] }, + preview: expectedPreview, + }, + }); + }); + + it('pins the destructive execute envelope: preview from the approved dry_run is present', async () => { + mockExecutorGetAbility.mockResolvedValue( + createMockAbility('delete-site-v1', { destructive: true }) + ); + mockExecutorExecute.mockImplementation( + (_name: string, _input: Record, options?: { dryRun?: boolean }) => + Promise.resolve( + options?.dryRun + ? { success: true, data: { affected: [{ id: 5, name: 'Site Five' }] } } + : { success: true, data: { deleted: true } } + ) + ); + + const { command, output } = createCommand(AbilitiesRun); + command.parse = vi.fn().mockResolvedValue({ + flags: { ...runFlags, confirm: true, force: true }, + args: { name: 'delete-site-v1' }, + }) as never; + + try { await command.run(); } catch { /* exit */ } + + const json = findJsonOutput(output.stdout); + expect(json).toEqual({ + success: true, + data: { + mode: 'execute', + ability: 'mainwp/delete-site-v1', + success: true, + data: { deleted: true }, + preview: expectedPreview, + }, + }); + }); + + it('pins the direct execute envelope: no preview key (no preview ran)', async () => { + mockExecutorGetAbility.mockResolvedValue( + createMockAbility('list-sites-v1', { readonly: true }) + ); + mockExecutorExecute.mockResolvedValue({ + success: true, + data: { sites: [{ id: 1 }] }, + }); + + const { command, output } = createCommand(AbilitiesRun); + command.parse = vi.fn().mockResolvedValue({ + flags: { ...runFlags }, + args: { name: 'list-sites-v1' }, + }) as never; + + try { await command.run(); } catch { /* exit */ } + + const json = findJsonOutput(output.stdout); + expect(json).toEqual({ + success: true, + data: { + mode: 'execute', + ability: 'mainwp/list-sites-v1', + success: true, + data: { sites: [{ id: 1 }] }, + }, + }); + expect(json as Record).toBeDefined(); + expect((json as { data: Record }).data).not.toHaveProperty('preview'); + }); + }); + describe('JSON parsability', () => { it('all --json output lines parse cleanly with JSON.parse', async () => { const abilities = [ diff --git a/src/__tests__/e2e/login-abilities-flow.test.ts b/src/__tests__/e2e/login-abilities-flow.test.ts index 709aad0..f5fcb4a 100644 --- a/src/__tests__/e2e/login-abilities-flow.test.ts +++ b/src/__tests__/e2e/login-abilities-flow.test.ts @@ -37,12 +37,22 @@ const mockFsWriteFile = vi.fn(); const mockFsMkdir = vi.fn(); const mockFsRename = vi.fn(); +// atomicWriteFile writes through an exclusive fs.open handle; delegate the +// handle's writeFile back to mockFsWriteFile as (path, content) so existing +// assertions keep their shape. +const mockFsOpen = vi.fn((path: unknown) => Promise.resolve({ + writeFile: (content: unknown) => mockFsWriteFile(path, content) as Promise, + sync: () => Promise.resolve(), + close: () => Promise.resolve(), +})); + vi.mock('node:fs', () => ({ promises: { readFile: (...args: unknown[]) => mockFsReadFile(...args), writeFile: (...args: unknown[]) => mockFsWriteFile(...args), mkdir: (...args: unknown[]) => mockFsMkdir(...args), rename: (...args: unknown[]) => mockFsRename(...args), + open: (...args: unknown[]) => mockFsOpen(args[0]), }, })); @@ -443,7 +453,7 @@ describe('E2E: Login → Abilities Flow', () => { expect(result).toBe('env-password'); }); - it('delete() completes silently when keytar hangs', async () => { + it('delete() reports failure when keytar hangs', async () => { mockKeytarDeletePassword.mockReturnValue(new Promise(() => {})); const testKeychain = new Keychain(); @@ -451,7 +461,10 @@ describe('E2E: Login → Abilities Flow', () => { await vi.advanceTimersByTimeAsync(5_000); - await expect(resultPromise).resolves.toBeUndefined(); + await expect(resultPromise).resolves.toMatchObject({ + deleted: false, + error: expect.stringContaining('timed out'), + }); }); }); diff --git a/src/__tests__/e2e/non-tty-behavior.test.ts b/src/__tests__/e2e/non-tty-behavior.test.ts index 0f94825..e81018c 100644 --- a/src/__tests__/e2e/non-tty-behavior.test.ts +++ b/src/__tests__/e2e/non-tty-behavior.test.ts @@ -21,6 +21,8 @@ import { createMockLLMToolCallResponse, restoreEnvVars, setEnvVar, + createCommandHarness, + type CapturedOutput, } from './test-helpers.js'; // ============================================================================ @@ -165,51 +167,11 @@ import AbilitiesRun from '../../commands/abilities/run.js'; // Test Utilities // ============================================================================ -interface CapturedOutput { - stdout: string[]; - stderr: string[]; - exitCode?: number; -} - function createCommandInstance( CommandClass: new (argv: string[], config: unknown) => T, argv: string[] = [] ): { command: T; output: CapturedOutput } { - const output: CapturedOutput = { stdout: [], stderr: [] }; - - const mockConfig = { - root: '/mock/root', - bin: 'mainwpcontrol', - name: 'mainwpcontrol', - version: '1.0.0', - pjson: { name: 'mainwpcontrol', version: '1.0.0' }, - dataDir: '/mock/data', - cacheDir: '/mock/cache', - configDir: '/mock/config', - findCommand: vi.fn(), - runCommand: vi.fn(), - runHook: vi.fn(), - }; - - const command = new CommandClass(argv, mockConfig as never); - - command.log = vi.fn((...args: unknown[]) => { - output.stdout.push(args.map(String).join(' ')); - }); - command.logToStderr = vi.fn((...args: unknown[]) => { - output.stderr.push(args.map(String).join(' ')); - }); - command.exit = vi.fn((code?: number) => { - output.exitCode = code ?? 0; - throw new Error(`EXIT:${code ?? 0}`); - }) as never; - command.error = vi.fn((message: string | Error, options?: { exit?: number }) => { - output.stderr.push(message instanceof Error ? message.message : message); - output.exitCode = options?.exit ?? 1; - throw new Error(`EXIT:${output.exitCode}`); - }) as never; - - return { command, output }; + return createCommandHarness(CommandClass, argv); } // ============================================================================ @@ -390,30 +352,31 @@ describe('E2E: Non-TTY Behavior', () => { try { await command.run(); } catch { /* exit */ } - // Contract: exactly one JSON object emitted, no preamble text + // Contract: exactly one JSON envelope emitted, no preamble text expect(output.stdout).toHaveLength(1); const allOutput = output.stdout[0]!; const parsed = JSON.parse(allOutput); - expect(parsed.type).toBe('tool_result'); - expect(parsed.tool).toBe('mainwp/list-sites-v1'); - expect(parsed.result.success).toBe(true); + expect(parsed.success).toBe(true); + expect(parsed.data.type).toBe('tool_result'); + expect(parsed.data.tool).toBe('mainwp/list-sites-v1'); + expect(parsed.data.result.success).toBe(true); expect(mockExecutorExecute).toHaveBeenCalledTimes(1); expect(mockCreateInterface).not.toHaveBeenCalled(); }); it('emits final tool result (not intermediate) for multi-tool-call --json', async () => { const listSitesAbility = createMockAbility('list-sites-v1', { readonly: true }); - const updatePluginsAbility = createMockAbility('update-site-plugins-v1', { readonly: false }); + const syncSitesAbility = createMockAbility('sync-sites-v1', { readonly: false }); - // LLM does two tool calls: list-sites (intermediate) then update-plugins (final), then answers + // LLM does two non-destructive tool calls: list-sites (intermediate) then sync-sites (final) mockProviderChat .mockResolvedValueOnce(createMockLLMToolCallResponse('list-sites-v1', {})) - .mockResolvedValueOnce(createMockLLMToolCallResponse('update-site-plugins-v1', { site_id: 1 })) - .mockResolvedValueOnce(createMockLLMAnswerResponse('Plugins updated')); - mockExecutorListAbilities.mockResolvedValue([listSitesAbility, updatePluginsAbility]); + .mockResolvedValueOnce(createMockLLMToolCallResponse('sync-sites-v1', {})) + .mockResolvedValueOnce(createMockLLMAnswerResponse('Sites synced')); + mockExecutorListAbilities.mockResolvedValue([listSitesAbility, syncSitesAbility]); mockExecutorGetAbility .mockResolvedValueOnce(listSitesAbility) - .mockResolvedValueOnce(updatePluginsAbility); + .mockResolvedValueOnce(syncSitesAbility); mockExecutorExecute .mockResolvedValueOnce({ success: true, @@ -421,7 +384,7 @@ describe('E2E: Non-TTY Behavior', () => { }) .mockResolvedValueOnce({ success: true, - data: { updated: ['akismet/akismet.php'], site_id: 1 }, + data: { synced: [1] }, }); const { command, output } = createCommandInstance(ChatCommand); @@ -438,22 +401,66 @@ describe('E2E: Non-TTY Behavior', () => { 'max-context-messages': undefined, stream: false, }, - args: { message: 'update plugins on site 1' }, + args: { message: 'sync all sites' }, }) as never; try { await command.run(); } catch { /* exit */ } const allOutput = output.stdout.join('\n'); const parsed = JSON.parse(allOutput); - // Contract: exactly one JSON object, selecting the final tool result + // Contract: exactly one JSON envelope, selecting the final tool result expect(output.stdout).toHaveLength(1); - expect(parsed.type).toBe('tool_result'); - expect(parsed.tool).toBe('mainwp/update-site-plugins-v1'); - expect(parsed.result.data.updated).toContain('akismet/akismet.php'); + expect(parsed.success).toBe(true); + expect(parsed.data.type).toBe('tool_result'); + expect(parsed.data.tool).toBe('mainwp/sync-sites-v1'); + expect(parsed.data.result.data.synced).toContain(1); expect(mockExecutorExecute).toHaveBeenCalledTimes(2); expect(mockCreateInterface).not.toHaveBeenCalled(); }); + it('rejects with a non-zero-exit APIError when the turn ends in a failed tool result', async () => { + const listSitesAbility = createMockAbility('list-sites-v1', { readonly: true }); + + mockProviderChat.mockResolvedValueOnce( + createMockLLMToolCallResponse('list-sites-v1', {}) + ); + mockExecutorListAbilities.mockResolvedValue([listSitesAbility]); + mockExecutorGetAbility.mockResolvedValue(listSitesAbility); + mockExecutorExecute.mockResolvedValue({ + success: false, + error: { code: 'SITE_NOT_FOUND', message: 'No such site' }, + }); + // Turn ends on the failed tool result (no recovery answer) + mockProviderChat.mockResolvedValueOnce( + createMockLLMAnswerResponse('') + ); + + const { command } = createCommandInstance(ChatCommand); + command.parse = vi.fn().mockResolvedValue({ + flags: { + json: true, + quiet: false, + debug: false, + provider: undefined, + model: undefined, + 'api-key': undefined, + 'base-url': undefined, + 'max-turns': 1, + 'max-context-messages': undefined, + stream: false, + }, + args: { message: 'list all sites' }, + }) as never; + + // CI honesty: a failed MainWP operation must not exit 0. The thrown + // APIError carries the ability's error code and a non-zero exit code; + // BaseCommand.catch() turns it into the error envelope in real runs. + await expect(command.run()).rejects.toMatchObject({ + code: 'SITE_NOT_FOUND', + exitCode: 4, + }); + }); + it('exits with code 2 when no provider is configured for single-message mode', async () => { mockResolveProviderSelection.mockReturnValueOnce({ source: 'none', diff --git a/src/__tests__/e2e/test-helpers.ts b/src/__tests__/e2e/test-helpers.ts index 2c0cf92..fa10bce 100644 --- a/src/__tests__/e2e/test-helpers.ts +++ b/src/__tests__/e2e/test-helpers.ts @@ -6,6 +6,7 @@ */ import { vi } from 'vitest'; +import type { Command } from '@oclif/core'; import type { Ability, ExecutionResult } from '../../core/abilities-executor.js'; import type { JobStatus } from '../../core/batch-manager.js'; import type { LLMProvider, LLMResponse, Message, ChatOptions } from '../../chat/providers/provider.js'; @@ -135,7 +136,7 @@ export function createMockLLMToolCallResponse( ): LLMResponse { return { content: '', - toolCalls: [{ id, name: toolName, arguments: input }], + toolCalls: [{ id, name: toolName.replaceAll('/', '__'), arguments: input }], finishReason: 'tool_calls', model: 'test-model', }; @@ -232,6 +233,65 @@ export function createMockReadlineInterface(responses: string[] = []): MockReadl }; } +// ============================================================================ +// Command Harness Factory +// ============================================================================ + +/** + * Output captured from a mocked command run + */ +export interface CapturedOutput { + stdout: string[]; + stderr: string[]; + exitCode?: number; +} + +/** + * Create a command instance with a mocked oclif Config and captured + * log/logToStderr/exit/error output. Shared by the e2e command-harness + * factories, which layer command-specific `parse` wiring on top. + */ +export function createCommandHarness( + CommandClass: new (argv: string[], config: unknown) => T, + argv: string[] = [] +): { command: T; output: CapturedOutput } { + const output: CapturedOutput = { stdout: [], stderr: [] }; + + const mockConfig = { + root: '/mock/root', + bin: 'mainwpcontrol', + name: 'mainwpcontrol', + version: '1.0.0', + pjson: { name: 'mainwpcontrol', version: '1.0.0' }, + dataDir: '/mock/data', + cacheDir: '/mock/cache', + configDir: '/mock/config', + findCommand: vi.fn(), + runCommand: vi.fn(), + runHook: vi.fn(), + }; + + const command = new CommandClass(argv, mockConfig as never); + + command.log = vi.fn((...args: unknown[]) => { + output.stdout.push(args.map(String).join(' ')); + }); + command.logToStderr = vi.fn((...args: unknown[]) => { + output.stderr.push(args.map(String).join(' ')); + }); + command.exit = vi.fn((code?: number) => { + output.exitCode = code ?? 0; + throw new Error(`EXIT:${code ?? 0}`); + }) as never; + command.error = vi.fn((message: string | Error, options?: { exit?: number }) => { + output.stderr.push(message instanceof Error ? message.message : message); + output.exitCode = options?.exit ?? 1; + throw new Error(`EXIT:${output.exitCode}`); + }) as never; + + return { command, output }; +} + // ============================================================================ // Mock Executor Factory // ============================================================================ diff --git a/src/__tests__/process/abilities-run.test.ts b/src/__tests__/process/abilities-run.test.ts index 433349d..b9fd73e 100644 --- a/src/__tests__/process/abilities-run.test.ts +++ b/src/__tests__/process/abilities-run.test.ts @@ -14,7 +14,7 @@ import { tmpdir } from 'node:os'; import { MockServer } from './fixtures/mock-server.js'; import { runCLI, type CLIResult } from './fixtures/cli-runner.js'; import { ConfigDir } from './fixtures/config-dir.js'; -import { abilityRunSuccess } from './fixtures/api-responses.js'; +import { abilityRunSuccess, mockAbility } from './fixtures/api-responses.js'; describe('abilities run', () => { const server = new MockServer(); @@ -107,6 +107,41 @@ describe('abilities run', () => { }); }); + describe('PHP-artifact input schema --json', () => { + it('exits 0 with one success envelope when properties is an empty array', async () => { + server.reset(); + server.setAbilities([ + mockAbility({ + name: 'mainwp/php-empty-properties-v1', + readonly: true, + input_schema: { + type: 'object', + properties: [] as unknown as Record, + }, + }), + ]); + server.setRunResponse( + 'php-empty-properties-v1', + abilityRunSuccess({ status: 'ok' }), + ); + + const result = await run([ + 'abilities', 'run', 'php-empty-properties-v1', '--json', + ]); + + expect(result.exitCode).toBe(0); + expect(result.json).toEqual({ + success: true, + data: { + mode: 'execute', + ability: 'mainwp/php-empty-properties-v1', + success: true, + data: { status: 'ok' }, + }, + }); + }); + }); + // ------------------------------------------------------------------------- // 2. get-site-v1 --input '{"site_id_or_domain": 5}' --json → GET with query params // ------------------------------------------------------------------------- diff --git a/src/__tests__/process/auth.test.ts b/src/__tests__/process/auth.test.ts index 643e73e..4539d57 100644 --- a/src/__tests__/process/auth.test.ts +++ b/src/__tests__/process/auth.test.ts @@ -239,6 +239,42 @@ describe('login command', () => { expect(result.stderr).toMatch(/process list|MAINWP_APP_PASSWORD/); }); + // ------------------------------------------------------------------------- + // 4b. URL with embedded userinfo → rejected before any connection attempt + // ------------------------------------------------------------------------- + + it.each([ + ['user and password', (base: string) => base.replace('://', '://user:pass@')], + ['user only', (base: string) => base.replace('://', '://user@')], + ])( + 'login with embedded credentials in URL (%s) exits 2 without contacting the server', + async (_label, embed) => { + configDir = await ConfigDir.create({ profiles: [] }); + + const result = await runCLI( + [ + 'login', + '--url', embed(server.baseUrl), + '--username', 'admin', + '--password', 'test-pass', + ], + { + xdgConfigHome: configDir.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + expect(result.exitCode).toBe(2); + expect(result.stderr).toContain('Embedded credentials'); + + // Rejected at intake: nothing may reach the server + expect(server.getRecordedRequests()).toHaveLength(0); + + const profiles = await configDir.readProfiles(); + expect(profiles.profiles).toHaveLength(0); + }, + ); + // ------------------------------------------------------------------------- // 5. URL normalization: protocol-less URL gets https:// prefix // ------------------------------------------------------------------------- diff --git a/src/__tests__/process/batch-wait.test.ts b/src/__tests__/process/batch-wait.test.ts index f8aa889..7ad0c86 100644 --- a/src/__tests__/process/batch-wait.test.ts +++ b/src/__tests__/process/batch-wait.test.ts @@ -8,9 +8,9 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; import { MockServer } from './fixtures/mock-server.js'; -import { runCLI } from './fixtures/cli-runner.js'; +import { runCLI, runCLIWithSignal } from './fixtures/cli-runner.js'; import { ConfigDir } from './fixtures/config-dir.js'; -import { abilityRunBatch, jobStatus } from './fixtures/api-responses.js'; +import { dashboardQueuedResponse, jobStatus } from './fixtures/api-responses.js'; describe('batch job waiting', () => { const server = new MockServer(); @@ -60,7 +60,7 @@ describe('batch job waiting', () => { const cfg = await createConfig(); // Register the ability run endpoint to return a batch job - server.setRunResponse('sync-sites-v1', abilityRunBatch('sync_123')); + server.setRunResponse('sync-sites-v1', dashboardQueuedResponse('sync_123')); // Register the batch status progression: running → completed server.setJobProgression('sync_123', [ @@ -109,7 +109,7 @@ describe('batch job waiting', () => { it('abilities run --wait with timeout exits 4 (BATCH_TIMEOUT)', async () => { const cfg = await createConfig(); - server.setRunResponse('sync-sites-v1', abilityRunBatch('sync_123')); + server.setRunResponse('sync-sites-v1', dashboardQueuedResponse('sync_123')); // Job never completes: stays running forever (last status repeated) server.setJobProgression('sync_123', [ @@ -128,9 +128,40 @@ describe('batch job waiting', () => { // APIError with code BATCH_TIMEOUT maps to exit code 4 expect(result.exitCode).toBe(4); - // stdout contains partial results + error envelope (two JSON objects) - // Verify that BATCH_TIMEOUT appears in the output - expect(result.stdout).toContain('BATCH_TIMEOUT'); + // JSON mode emits exactly ONE document: an error envelope whose details + // carry the partial status (no preceding success envelope). + const envelope = JSON.parse(result.stdout) as Record; + expect(envelope['success']).toBe(false); + const error = envelope['error'] as Record; + expect(error['code']).toBe('BATCH_TIMEOUT'); + const details = error['details'] as Record; + expect(details).toHaveProperty('partialStatus'); + }); + + it('abilities run --wait exits 4 when the batch is cancelled', async () => { + const cfg = await createConfig(); + + server.setRunResponse('sync-sites-v1', dashboardQueuedResponse('sync_123')); + server.setJobProgression('sync_123', [ + jobStatus({ job_id: 'sync_123', status: 'cancelled', progress: 25, processed: 2, total: 10 }), + ]); + + const result = await runCLI( + ['abilities', 'run', 'sync-sites-v1', '--wait', '--json'], + { + xdgConfigHome: cfg.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + timeout: 15_000, + }, + ); + + expect(result.exitCode).toBe(4); + const envelope = JSON.parse(result.stdout) as { + success: boolean; + error?: { code?: string }; + }; + expect(envelope.success).toBe(false); + expect(envelope.error?.code).toBe('BATCH_CANCELLED'); }); // --------------------------------------------------------------------------- @@ -177,6 +208,99 @@ describe('batch job waiting', () => { expect(envelope.data.results).toBeDefined(); }); + it('jobs watch honors settings-derived JSON without progress output', async () => { + configDir = await ConfigDir.create({ + profiles: [{ name: 'test', dashboardUrl: server.baseUrl, username: 'admin' }], + activeProfile: 'test', + settings: { defaultJsonOutput: true }, + }); + server.setJobProgression('sync_123', [ + jobStatus({ job_id: 'sync_123', status: 'running', progress: 50 }), + jobStatus({ job_id: 'sync_123', status: 'completed', progress: 100 }), + ]); + + const result = await runCLI( + ['jobs', 'watch', 'sync_123', '--initial-delay', '100'], + { + xdgConfigHome: configDir.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + expect(result.exitCode).toBe(0); + expect(() => JSON.parse(result.stdout)).not.toThrow(); + expect((JSON.parse(result.stdout) as { success: boolean }).success).toBe(true); + }); + + it('jobs watch exits 4 when the job fails', async () => { + const cfg = await createConfig(); + server.setJobProgression('sync_123', [ + jobStatus({ job_id: 'sync_123', status: 'failed', errors: [{ message: 'failed' }] }), + ]); + + const result = await runCLI( + ['jobs', 'watch', 'sync_123', '--json', '--initial-delay', '100'], + { + xdgConfigHome: cfg.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + expect(result.exitCode).toBe(4); + expect(result.stdout).toContain('BATCH_FAILED'); + }); + + // On Windows, child.kill('SIGINT') terminates the process without running + // signal handlers (no POSIX signals), so the cancellation contract these + // two tests pin cannot execute there. The contract itself is POSIX-only: + // exit 130 is the SIGINT convention. + it.skipIf(process.platform === 'win32')('jobs watch --json emits one envelope and exits 130 on SIGINT', async () => { + const cfg = await createConfig(); + server.setJobProgression('sync_123', [ + jobStatus({ job_id: 'sync_123', status: 'running', progress: 10 }), + ]); + + const result = await runCLIWithSignal( + ['jobs', 'watch', 'sync_123', '--json', '--initial-delay', '100'], + { + xdgConfigHome: cfg.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + 'SIGINT', + // Deliver SIGINT only after the first status poll: the watch command + // has installed its handler by then, on any speed of runner. + server.waitForRequest('get-batch-job-status'), + ); + + expect(result.exitCode).toBe(130); + const envelope = JSON.parse(result.stdout) as { + success: boolean; + error?: { code?: string }; + }; + expect(envelope.success).toBe(false); + expect(envelope.error?.code).toBe('CANCELLED'); + }); + + it.skipIf(process.platform === 'win32')('jobs watch reports SIGINT cancellation on stderr in human mode', async () => { + const cfg = await createConfig(); + server.setJobProgression('sync_123', [ + jobStatus({ job_id: 'sync_123', status: 'running', progress: 10 }), + ]); + + const result = await runCLIWithSignal( + ['jobs', 'watch', 'sync_123', '--no-progress', '--initial-delay', '100'], + { + xdgConfigHome: cfg.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + 'SIGINT', + server.waitForRequest('get-batch-job-status'), + ); + + expect(result.exitCode).toBe(130); + expect(result.stderr).toMatch(/cancelled by signal/i); + }); + // --------------------------------------------------------------------------- // 4. jobs watch sync_123 --timeout 5 → exit 0 (custom timeout, job completes) // --------------------------------------------------------------------------- diff --git a/src/__tests__/process/config-show.test.ts b/src/__tests__/process/config-show.test.ts index 66c49b9..92f0a92 100644 --- a/src/__tests__/process/config-show.test.ts +++ b/src/__tests__/process/config-show.test.ts @@ -28,6 +28,50 @@ describe('config show command', () => { } }); + it('keeps provider warnings and configuration paths on one safe line', async () => { + configDir = await ConfigDir.create(); + const unsafeConfigHome = `${configDir.xdgHome}/\x1b[31mconfig\r\ninjected-path`; + + const result = await runCLI(['config', 'show'], { + xdgConfigHome: unsafeConfigHome, + env: { + MAINWP_LLM_PROVIDER: '\x1b[31minvalid\r\ninjected-provider', + }, + }); + + expect(result.exitCode).toBe(0); + expect(result.stdout).not.toContain('\x1b'); + expect(result.stdout).not.toContain('\ninjected-provider'); + expect(result.stdout).not.toContain('\ninjected-path'); + expect(result.stdout).toContain('invalid injected-provider'); + expect(result.stdout).toContain('config injected-path'); + }); + + it('keeps profile-derived fields on one safe line (hostile profiles.json)', async () => { + configDir = await ConfigDir.create({ + profiles: [ + { + name: 'evil\r\nInjected Profile: fake', + dashboardUrl: `${server.baseUrl}/\x1b[2Jclear`, + username: 'admin\r\nPassword: hunter2', + }, + ], + activeProfile: 'evil\r\nInjected Profile: fake', + }); + + const result = await runCLI(['config', 'show'], { + xdgConfigHome: configDir.xdgHome, + }); + + expect(result.exitCode).toBe(0); + expect(result.stdout).not.toContain('\x1b'); + expect(result.stdout).not.toContain('\nInjected Profile'); + expect(result.stdout).not.toContain('\nPassword: hunter2'); + // Values survive, flattened to one line + expect(result.stdout).toContain('evil Injected Profile: fake'); + expect(result.stdout).toContain('admin Password: hunter2'); + }); + it('reports effective settings and provider resolution in JSON mode', async () => { configDir = await ConfigDir.create({ profiles: [ @@ -95,4 +139,30 @@ describe('config show command', () => { expect(envelope.data.effectiveSettings.skipSSLVerification).toBe(true); expect(envelope.data.effectiveSettings.allowInsecureHttp).toBe(true); }); + + it('masks userinfo from a legacy profile in JSON mode', async () => { + configDir = await ConfigDir.create({ + profiles: [ + { + name: 'legacy', + dashboardUrl: 'https://legacy:secret@dashboard.example.com', + username: 'admin', + }, + ], + activeProfile: 'legacy', + }); + + const result = await runCLI(['config', 'show', '--json'], { + xdgConfigHome: configDir.xdgHome, + }); + + expect(result.exitCode).toBe(0); + const envelope = result.json as { + data: { profile: { dashboardUrl: string } }; + }; + expect(envelope.data.profile.dashboardUrl).toBe( + 'https://***:***@dashboard.example.com' + ); + expect(result.stdout).not.toContain('legacy:secret'); + }); }); diff --git a/src/__tests__/process/doctor.test.ts b/src/__tests__/process/doctor.test.ts index 50bfb9e..10bde0f 100644 --- a/src/__tests__/process/doctor.test.ts +++ b/src/__tests__/process/doctor.test.ts @@ -298,6 +298,45 @@ describe('doctor command', () => { expect(result.stderr).not.toContain('WARNING: SSL verification'); }); + // --------------------------------------------------------------------------- + // 4b. Human output strips escape sequences from check messages/details + // --------------------------------------------------------------------------- + + it('doctor human output strips terminal escape sequences from config-derived text', async () => { + // A profile name with an embedded CSI clear-screen sequence, as a stand-in + // for any hostile/corrupted text reaching a check's message or details + // (profiles.json is user-editable on disk, so this needs no login-path bypass). + configDir = await ConfigDir.create({ + profiles: [ + { + name: 'evil\u001b[2Jpwn', + dashboardUrl: server.baseUrl, + username: 'admin', + }, + ], + activeProfile: 'evil\u001b[2Jpwn', + }); + + const result = await runCLI(['doctor', '-v'], { + xdgConfigHome: configDir.xdgHome, + env: { + MAINWP_APP_PASSWORD: 'test-pass', + ANTHROPIC_API_KEY: '', + OPENAI_API_KEY: '', + GOOGLE_API_KEY: '', + OPENROUTER_API_KEY: '', + LOCAL_LLM_URL: '', + MAINWP_LLM_PROVIDER: '', + }, + }); + + const combined = result.stdout + result.stderr; + // The escape sequence must not survive to the terminal... + expect(combined).not.toContain('\u001b[2J'); + // ...but the surrounding profile-name text still renders (Active Profile check). + expect(combined).toContain('evilpwn'); + }); + // --------------------------------------------------------------------------- // 5. Doctor --json stability with env var fallback // --------------------------------------------------------------------------- @@ -359,4 +398,39 @@ describe('doctor command', () => { ]) ); }); + + it('masks userinfo from a legacy profile in JSON mode', async () => { + configDir = await ConfigDir.create({ + profiles: [ + { + name: 'legacy', + dashboardUrl: 'https://legacy:secret@dashboard.example.com', + username: 'admin', + }, + ], + activeProfile: 'legacy', + }); + + const result = await runCLI(['doctor', '--json'], { + xdgConfigHome: configDir.xdgHome, + env: { + MAINWP_APP_PASSWORD: 'test-pass', + ANTHROPIC_API_KEY: '', + OPENAI_API_KEY: '', + GOOGLE_API_KEY: '', + OPENROUTER_API_KEY: '', + LOCAL_LLM_URL: '', + MAINWP_LLM_PROVIDER: '', + }, + }); + + const envelope = result.json as { + data: { checks: Array<{ name: string; details?: string }> }; + }; + const activeProfile = envelope.data.checks.find( + (check) => check.name === 'Active Profile' + ); + expect(activeProfile?.details).toBe('https://***:***@dashboard.example.com'); + expect(result.stdout).not.toContain('legacy:secret'); + }); }); diff --git a/src/__tests__/process/exit-codes.test.ts b/src/__tests__/process/exit-codes.test.ts index c25e7d6..ccf5da3 100644 --- a/src/__tests__/process/exit-codes.test.ts +++ b/src/__tests__/process/exit-codes.test.ts @@ -14,6 +14,8 @@ */ import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; import { MockServer } from './fixtures/mock-server.js'; import { runCLI, type CLIResult } from './fixtures/cli-runner.js'; import { ConfigDir } from './fixtures/config-dir.js'; @@ -87,7 +89,7 @@ describe('exit code contract', () => { // --------------------------------------------------------------------------- describe('exit 1: mutually exclusive flags', () => { - it('abilities run delete-site-v1 --dry-run --confirm exits non-zero with exclusive flag error', async () => { + it('abilities run delete-site-v1 --dry-run --confirm exits 1 with prose on stderr', async () => { config = await ConfigDir.create({ profiles: [ { name: 'test', dashboardUrl: server.baseUrl, username: 'admin' }, @@ -100,23 +102,41 @@ describe('exit code contract', () => { const result = await run([ 'abilities', 'run', 'delete-site-v1', '--dry-run', '--confirm', - '--json', ]); - // oclif throws a CLIError for exclusive flag violations. - // The exact exit code depends on oclif's internal handling - // (typically 2 for arg validation), so we assert non-zero and - // verify the error message references the flag conflict. - expect(result.exitCode).not.toBe(0); + expect(result.exitCode).toBe(1); + expect(result.stdout).toBe(''); - const combined = result.stdout + result.stderr; const mentionsExclusion = - /exclusive/i.test(combined) || - /cannot also be provided/i.test(combined) || - /mutually exclusive/i.test(combined) || - /dry-run.*confirm/i.test(combined); + /exclusive/i.test(result.stderr) || + /cannot also be provided/i.test(result.stderr) || + /mutually exclusive/i.test(result.stderr) || + /dry-run.*confirm/i.test(result.stderr); expect(mentionsExclusion).toBe(true); }); + + it('emits exactly one JSON error envelope on stdout for a parse-time error', async () => { + config = await ConfigDir.create({ + profiles: [ + { name: 'test', dashboardUrl: server.baseUrl, username: 'admin' }, + ], + activeProfile: 'test', + }); + + const result = await run([ + 'abilities', 'run', 'delete-site-v1', + '--dry-run', '--confirm', '--json', + ]); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toBe(''); + const envelope = JSON.parse(result.stdout) as { + success: boolean; + error?: { message?: string }; + }; + expect(envelope.success).toBe(false); + expect(envelope.error?.message).toMatch(/confirm|exclusive|provided/i); + }); }); // --------------------------------------------------------------------------- @@ -241,3 +261,69 @@ describe('exit code contract', () => { }); }); }); + +// ----------------------------------------------------------------------------- +// Exit 5 — Internal error: untyped settings I/O failure +// ----------------------------------------------------------------------------- + +describe('exit 5: unexpected settings read failure', () => { + let config: ConfigDir; + + afterEach(async () => { + if (config) await config.cleanup(); + }); + + it('abilities list exits 5 when settings.json cannot be read as a file', async () => { + config = await ConfigDir.create({ profiles: [] }); + await mkdir(join(config.configPath, 'settings.json')); + + const result = await runCLI(['abilities', 'list', '--json'], { + xdgConfigHome: config.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }); + + expect(result.exitCode).toBe(5); + // stderr carries the human-readable error line; the --json contract + // guarantees stdout purity, not stderr silence. + expect(JSON.parse(result.stdout)).toMatchObject({ + success: false, + error: expect.any(Object), + }); + }); +}); + +// ----------------------------------------------------------------------------- +// Exit 130 — Ctrl-C at an interactive password prompt +// ----------------------------------------------------------------------------- + +describe('exit 130: password prompt interrupted', () => { + let config: ConfigDir; + + afterEach(async () => { + if (config) await config.cleanup(); + }); + + it('login exits 130 when Ctrl-C interrupts the password prompt', async () => { + config = await ConfigDir.create({ profiles: [] }); + + const result = await runCLI( + [ + 'login', + '--url', 'https://dashboard.example.com', + '--username', 'admin', + ], + { + xdgConfigHome: config.xdgHome, + env: {}, + // Keep stdin pipe-based for deterministic CI input while emulating the + // TTY flags checked by login. In raw mode, Ctrl-C arrives as ETX. + emulateTTY: true, + stdin: '\u0003', + stdinWaitFor: 'Application password', + }, + ); + + expect(result.stdout).toContain('Application password'); + expect(result.exitCode).toBe(130); + }); +}); diff --git a/src/__tests__/process/fixtures/api-responses.ts b/src/__tests__/process/fixtures/api-responses.ts index 1e1e7d3..e9ad567 100644 --- a/src/__tests__/process/fixtures/api-responses.ts +++ b/src/__tests__/process/fixtures/api-responses.ts @@ -64,8 +64,13 @@ export function abilityDryRunResponse( /** * A batch-job-started result. */ -export function abilityRunBatch(jobId: string): Record { - return { success: true, jobId }; +export function dashboardQueuedResponse(jobId: string): Record { + return { + queued: true, + job_id: jobId, + status_url: `https://dashboard.test/wp-json/mainwp/v2/jobs/${jobId}`, + sites_queued: 10, + }; } /** diff --git a/src/__tests__/process/fixtures/cli-runner.ts b/src/__tests__/process/fixtures/cli-runner.ts index 5164508..de614a8 100644 --- a/src/__tests__/process/fixtures/cli-runner.ts +++ b/src/__tests__/process/fixtures/cli-runner.ts @@ -7,6 +7,7 @@ import { execFile, spawn } from 'node:child_process'; import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; const PROJECT_ROOT = resolve(import.meta.dirname, '..', '..', '..', '..'); const BIN_PATH = resolve(PROJECT_ROOT, 'bin', 'run.js'); @@ -20,6 +21,10 @@ export interface CLIRunnerOptions { timeout?: number; /** Data to pipe to stdin */ stdin?: string; + /** Wait for this stdout text before piping stdin */ + stdinWaitFor?: string; + /** Emulate TTY flags while retaining pipe-based stdin/stdout */ + emulateTTY?: boolean; } export interface CLIResult { @@ -32,40 +37,65 @@ export interface CLIResult { duration: number; } -/** - * Run the CLI with the given arguments and return the result. - */ -export async function runCLI( - args: string[], - options: CLIRunnerOptions, -): Promise { - const timeout = options.timeout ?? 15_000; - const start = Date.now(); - - const env: Record = { - // Minimal PATH for node +function buildEnv(options: CLIRunnerOptions): Record { + // On Windows, children must inherit the OS plumbing (SystemRoot, TEMP, + // PATHEXT, APPDATA, ...): a hand-built minimal env sends node into + // multi-second fallback paths on every boot (measured 20-40s per child + // on CI runners vs ~800ms with a full env). Hermeticity comes from + // stripping MAINWP* and overriding every variable the CLI reads, not + // from starting empty. POSIX keeps the fully minimal env. + const base: Record = {}; + if (process.platform === 'win32') { + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined && !key.toUpperCase().startsWith('MAINWP')) { + base[key] = value; + } + } + } + return { + ...base, PATH: process.env['PATH'] ?? '', - // Isolated config XDG_CONFIG_HOME: options.xdgConfigHome, - // Prevent reads from real home HOME: options.xdgConfigHome, - // Suppress Node.js warnings in output NODE_NO_WARNINGS: '1', - // Test environment NODE_ENV: 'test', - // Process tests use a local mock HTTP server; opt in explicitly so - // runtime defaults can remain HTTPS-first. MAINWP_ALLOW_HTTP: '1', - // Skip native keytar — process tests run with isolated HOME where - // macOS Keychain access is slow/unavailable. MAINWPCONTROL_NO_KEYTAR: '1', - // Spread any extra env ...options.env, }; +} + +/** + * Run the CLI with the given arguments and return the result. + */ +/** + * Default child timeout. Windows children run ~1-3s with the inherited env + * (see buildEnv), but CI runner speed varies; the extra headroom there is + * free when tests are healthy and prevents a slow runner from SIGKILLing a + * working child mid-boot, which reports as empty output. + */ +const DEFAULT_TIMEOUT = process.platform === 'win32' ? 60_000 : 15_000; + +export async function runCLI( + args: string[], + options: CLIRunnerOptions, +): Promise { + const timeout = options.timeout ?? DEFAULT_TIMEOUT; + const start = Date.now(); + + const env = buildEnv(options); // If stdin is provided, we need to use spawn to pipe data if (options.stdin !== undefined) { - return runWithStdin(args, env, timeout, options.stdin, start); + return runWithStdin( + args, + env, + timeout, + options.stdin, + start, + options.stdinWaitFor, + options.emulateTTY ?? false, + ); } return new Promise((resolve) => { @@ -92,15 +122,94 @@ export async function runCLI( }); } +/** Run the CLI, deliver a real process signal, and collect its final output. */ +export function runCLIWithSignal( + args: string[], + options: CLIRunnerOptions, + signal: NodeJS.Signals = 'SIGINT', + signalAfter: number | Promise = 750, +): Promise { + const start = Date.now(); + return new Promise((resolve) => { + const child = spawn(process.execPath, [BIN_PATH, ...args], { + env: buildEnv(options), + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + // A numeric delay races slow CI runners: SIGINT can land before the CLI + // has booted and installed its handler, killing it with no output. Pass a + // readiness promise (e.g. MockServer.waitForRequest) to deliver the + // signal only once the CLI is demonstrably up. + let signalTimer: NodeJS.Timeout | undefined; + if (typeof signalAfter === 'number') { + signalTimer = setTimeout(() => child.kill(signal), signalAfter); + } else { + void signalAfter.then(() => child.kill(signal)); + } + const timeoutTimer = setTimeout(() => child.kill('SIGKILL'), options.timeout ?? DEFAULT_TIMEOUT); + + child.stdout.on('data', (chunk: Buffer) => stdoutChunks.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderrChunks.push(chunk)); + let spawned = false; + child.on('spawn', () => { + spawned = true; + }); + child.on('error', (error) => { + // 'error' also fires when a later kill() fails. In that case the child + // is still running: leave the timers armed (the SIGKILL fallback must + // stay live) and let 'close' remain the terminal resolution path. Only + // a spawn failure, where 'close' is not guaranteed, resolves here. + if (spawned) { + return; + } + clearTimeout(signalTimer); + clearTimeout(timeoutTimer); + resolve({ stdout: '', stderr: String(error), exitCode: 1, json: undefined, duration: Date.now() - start }); + }); + child.on('close', (code, closeSignal) => { + clearTimeout(signalTimer); + clearTimeout(timeoutTimer); + const stdout = Buffer.concat(stdoutChunks).toString('utf8'); + const stderr = Buffer.concat(stderrChunks).toString('utf8'); + const exitCode = code ?? (closeSignal === 'SIGINT' ? 130 : closeSignal === 'SIGTERM' ? 143 : 1); + let json: unknown; + try { + json = JSON.parse(stdout); + } catch { + // not JSON + } + resolve({ stdout, stderr, exitCode, json, duration: Date.now() - start }); + }); + }); +} + function runWithStdin( args: string[], env: Record, timeout: number, stdinData: string, start: number, + stdinWaitFor: string | undefined, + emulateTTY: boolean, ): Promise { return new Promise((resolve) => { - const child = spawn(process.execPath, [BIN_PATH, ...args], { + const childArgs = emulateTTY + ? [ + '--input-type=module', + '--eval', + [ + "Object.defineProperty(process.stdin, 'isTTY', { value: true });", + "Object.defineProperty(process.stdout, 'isTTY', { value: true });", + "Object.defineProperty(process.stdout, 'getWindowSize', { value: () => [80, 24] });", + `process.argv = [process.execPath, ${JSON.stringify(BIN_PATH)}, ...process.argv.slice(1)];`, + `await import(${JSON.stringify(pathToFileURL(BIN_PATH).href)});`, + ].join('\n'), + ...args, + ] + : [BIN_PATH, ...args]; + + const child = spawn(process.execPath, childArgs, { env, stdio: ['pipe', 'pipe', 'pipe'], timeout, @@ -108,8 +217,23 @@ function runWithStdin( const stdoutChunks: Buffer[] = []; const stderrChunks: Buffer[] = []; + let stdinSent = false; - child.stdout.on('data', (chunk: Buffer) => stdoutChunks.push(chunk)); + const sendStdin = (): void => { + if (stdinSent) return; + stdinSent = true; + child.stdin.end(stdinData); + }; + + child.stdout.on('data', (chunk: Buffer) => { + stdoutChunks.push(chunk); + if ( + stdinWaitFor !== undefined && + Buffer.concat(stdoutChunks).toString('utf-8').includes(stdinWaitFor) + ) { + sendStdin(); + } + }); child.stderr.on('data', (chunk: Buffer) => stderrChunks.push(chunk)); child.on('close', (code) => { @@ -138,8 +262,8 @@ function runWithStdin( }); }); - // Write stdin and close - child.stdin.write(stdinData); - child.stdin.end(); + if (stdinWaitFor === undefined) { + sendStdin(); + } }); } diff --git a/src/__tests__/process/fixtures/mock-server.ts b/src/__tests__/process/fixtures/mock-server.ts index 7670b24..ca95f07 100644 --- a/src/__tests__/process/fixtures/mock-server.ts +++ b/src/__tests__/process/fixtures/mock-server.ts @@ -41,6 +41,7 @@ export class MockServer { private server: Server | null = null; private routes: Route[] = []; private recorded: RecordedRequest[] = []; + private requestWaiters: Array<{ match: (r: RecordedRequest) => boolean; resolve: () => void }> = []; private credentials = { username: 'admin', password: 'test-pass' }; /** The port the server is listening on (available after start()). */ @@ -88,6 +89,7 @@ export class MockServer { reset(): void { this.routes = []; this.recorded = []; + this.requestWaiters = []; this.credentials = { username: 'admin', password: 'test-pass' }; } @@ -170,6 +172,21 @@ export class MockServer { return [...this.recorded]; } + /** + * Resolves once a request whose path contains the given substring arrives + * (immediately if one is already recorded). Lets a test key an action, like + * delivering a signal, off evidence the CLI is booted and talking to the + * server instead of a wall-clock delay that races slow CI runners. + */ + waitForRequest(pathSubstring: string): Promise { + if (this.recorded.some((r) => r.path.includes(pathSubstring))) { + return Promise.resolve(); + } + return new Promise((resolve) => { + this.requestWaiters.push({ match: (r) => r.path.includes(pathSubstring), resolve }); + }); + } + /** Return the last recorded request matching the given path substring. */ getLastRequest(pathSubstring: string): RecordedRequest | undefined { return [...this.recorded].reverse().find((r) => r.path.includes(pathSubstring)); @@ -198,6 +215,13 @@ export class MockServer { body, }; this.recorded.push(recorded); + this.requestWaiters = this.requestWaiters.filter((waiter) => { + if (waiter.match(recorded)) { + waiter.resolve(); + return false; + } + return true; + }); // Auth check if (!this.checkAuth(req)) { diff --git a/src/__tests__/process/live-api.test.ts b/src/__tests__/process/live-api.test.ts index b65b08d..5db4840 100644 --- a/src/__tests__/process/live-api.test.ts +++ b/src/__tests__/process/live-api.test.ts @@ -34,9 +34,8 @@ function loadTestbedEnv(path: string): Record { } } -const testbedEnv = loadTestbedEnv( - '/Users/denni1/github/dev-tools/network-testbed/.env', -); +const testbedEnvPath = process.env['MAINWP_TESTBED_ENV']; +const testbedEnv = testbedEnvPath ? loadTestbedEnv(testbedEnvPath) : {}; const DASH_URL = process.env['MAINWP_API_URL'] ?? testbedEnv['MAINWP_API_URL'] ?? ''; @@ -71,10 +70,24 @@ async function checkDashboard( } } -// Set for the connectivity check (self-signed cert) -process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'; +// Only explicit opt-in values enable live tests; "false"/"0"/anything else +// stays disabled — this gate also controls the TLS-verification override. +const rawLiveFlag = process.env['MAINWP_LIVE_TEST']; +const liveTestsEnabled = rawLiveFlag === '1' || rawLiveFlag === 'true'; +if (liveTestsEnabled && (!DASH_URL || !DASH_USER || !DASH_PASS)) { + throw new Error( + 'MAINWP_LIVE_TEST is enabled but live credentials are incomplete. ' + + 'Set MAINWP_TESTBED_ENV to your testbed .env file, or export ' + + 'MAINWP_API_URL, MAINWP_USER, and MAINWP_APP_PASSWORD.' + ); +} +if (liveTestsEnabled) { + // Set only for explicitly enabled live tests using the self-signed testbed. + process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'; +} -const dashboardOnline = await checkDashboard(DASH_URL, DASH_USER, DASH_PASS); +const dashboardOnline = liveTestsEnabled + && await checkDashboard(DASH_URL, DASH_USER, DASH_PASS); // --------------------------------------------------------------------------- // Helpers — typed access to CLI JSON output @@ -144,7 +157,6 @@ describe.skipIf(!dashboardOnline)('live integration tests', () => { 'login', '--url', DASH_URL, '--username', DASH_USER, - '--password', DASH_PASS, '--skip-ssl-verify', '--json', ]); diff --git a/src/__tests__/process/live-workflow-docs.test.ts b/src/__tests__/process/live-workflow-docs.test.ts index 23c5997..688c024 100644 --- a/src/__tests__/process/live-workflow-docs.test.ts +++ b/src/__tests__/process/live-workflow-docs.test.ts @@ -40,9 +40,8 @@ function loadTestbedEnv(path: string): Record { } } -const testbedEnv = loadTestbedEnv( - '/Users/denni1/github/dev-tools/network-testbed/.env', -); +const testbedEnvPath = process.env['MAINWP_TESTBED_ENV']; +const testbedEnv = testbedEnvPath ? loadTestbedEnv(testbedEnvPath) : {}; const DASH_URL = process.env['MAINWP_API_URL'] ?? testbedEnv['MAINWP_API_URL'] ?? ''; @@ -55,7 +54,21 @@ const DASH_PASS = // Connectivity gate // --------------------------------------------------------------------------- -process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'; +// Only explicit opt-in values enable live tests; the TLS-verification +// override must never apply to normal (non-live) runs of this suite. +const rawLiveFlag = process.env['MAINWP_LIVE_TEST']; +const liveTestsEnabled = rawLiveFlag === '1' || rawLiveFlag === 'true'; +if (liveTestsEnabled && (!DASH_URL || !DASH_USER || !DASH_PASS)) { + throw new Error( + 'MAINWP_LIVE_TEST is enabled but live credentials are incomplete. ' + + 'Set MAINWP_TESTBED_ENV to your testbed .env file, or export ' + + 'MAINWP_API_URL, MAINWP_USER, and MAINWP_APP_PASSWORD.' + ); +} +if (liveTestsEnabled) { + // Set only for explicitly enabled live tests using the self-signed testbed. + process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'; +} async function checkDashboard(): Promise { if (!DASH_URL || !DASH_USER || !DASH_PASS) return false; @@ -75,7 +88,7 @@ async function checkDashboard(): Promise { } } -const dashboardOnline = await checkDashboard(); +const dashboardOnline = liveTestsEnabled && await checkDashboard(); // --------------------------------------------------------------------------- // Helpers diff --git a/src/__tests__/process/safety.test.ts b/src/__tests__/process/safety.test.ts index d835636..b1b9bdb 100644 --- a/src/__tests__/process/safety.test.ts +++ b/src/__tests__/process/safety.test.ts @@ -1,4 +1,6 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; import { MockServer } from './fixtures/mock-server.js'; import { runCLI } from './fixtures/cli-runner.js'; import { ConfigDir } from './fixtures/config-dir.js'; @@ -171,8 +173,7 @@ describe('safety / destructive action handling', () => { // ------------------------------------------------------------------------- // 3. --dry-run --confirm together → oclif rejects at flag parsing time // oclif's exclusive flag validation fires before the command runs. - // The FailedFlagValidationError lacks an exitCode property, so - // BaseCommand.catch defaults to ExitCode.INTERNAL_ERROR (5). + // Parse failures are user input errors → exit 1 (INPUT_ERROR). // ------------------------------------------------------------------------- it('--dry-run and --confirm together are rejected as mutually exclusive', async () => { await createConfig(); @@ -189,8 +190,8 @@ describe('safety / destructive action handling', () => { }, ); - // Exit code is non-zero (oclif flag validation error) - expect(result.exitCode).not.toBe(0); + // User input error (oclif flag validation) maps to exit code 1 + expect(result.exitCode).toBe(1); // Error output should mention the mutual exclusion const combined = result.stdout + result.stderr; @@ -309,4 +310,251 @@ describe('safety / destructive action handling', () => { expect(runReq.query).not.toHaveProperty('input[dry_run]'); expect(runReq.query).not.toHaveProperty('input[confirm]'); }); + + // ------------------------------------------------------------------------- + // 7. Preview failure blocks destructive execution (fail closed). + // A dry_run that errors at the HTTP layer must abort the flow with + // exit 4 and must never send a confirm request — even with --force. + // ------------------------------------------------------------------------- + it('--confirm --force with HTTP-failing preview exits 4 and sends no confirm request', async () => { + await createConfig(); + + const runPath = '/wp-json/wp-abilities/v1/abilities/mainwp/delete-site-v1/run'; + server.addRoute('POST', runPath, (_req, res) => { + const body = _req.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + if (input?.['dry_run'] === true) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ code: 'internal_error', message: 'preview exploded' })); + } else { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(abilityRunSuccess({ deleted: true }))); + } + }); + + const result = await runCLI( + [ + 'abilities', 'run', 'delete-site-v1', + '--input', '{"site_id_or_domain":1}', + '--confirm', '--force', '--json', + ], + { + xdgConfigHome: config.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + expect(result.exitCode).toBe(4); + + // No confirm request may have reached the server + const confirmReq = server + .getRecordedRequests() + .filter((r) => r.path.includes('delete-site-v1/run')) + .find((r) => { + const body = r.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + return input?.['confirm'] === true; + }); + expect(confirmReq).toBeUndefined(); + + const envelope = result.json as Record; + expect(envelope).toHaveProperty('success', false); + expect(String((envelope['error'] as Record)?.['message'] ?? '')).toMatch(/preview/i); + }); + + // ------------------------------------------------------------------------- + // 8. Preview returning success:false also fails closed (no confirm request) + // ------------------------------------------------------------------------- + it('--confirm --force with unsuccessful preview result exits 4 and sends no confirm request', async () => { + await createConfig(); + + const runPath = '/wp-json/wp-abilities/v1/abilities/mainwp/delete-site-v1/run'; + server.addRoute('POST', runPath, (_req, res) => { + const body = _req.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + res.writeHead(200, { 'Content-Type': 'application/json' }); + if (input?.['dry_run'] === true) { + res.end(JSON.stringify({ success: false, error: { code: 'preview_unavailable', message: 'cannot preview' } })); + } else { + res.end(JSON.stringify(abilityRunSuccess({ deleted: true }))); + } + }); + + const result = await runCLI( + [ + 'abilities', 'run', 'delete-site-v1', + '--input', '{"site_id_or_domain":1}', + '--confirm', '--force', '--json', + ], + { + xdgConfigHome: config.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + expect(result.exitCode).toBe(4); + + const confirmReq = server + .getRecordedRequests() + .filter((r) => r.path.includes('delete-site-v1/run')) + .find((r) => { + const body = r.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + return input?.['confirm'] === true; + }); + expect(confirmReq).toBeUndefined(); + }); + + // ------------------------------------------------------------------------- + // 9. Successful preview is rendered to the operator before execution, + // even with --force (which skips only the prompt, never the preview). + // ------------------------------------------------------------------------- + it('--confirm --force renders the preview before the execution result (human mode)', async () => { + await createConfig(); + + const runPath = '/wp-json/wp-abilities/v1/abilities/mainwp/delete-site-v1/run'; + server.addRoute('POST', runPath, (_req, res) => { + const body = _req.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + res.writeHead(200, { 'Content-Type': 'application/json' }); + if (input?.['dry_run'] === true) { + res.end(JSON.stringify(abilityDryRunResponse([{ site_id: 1, name: 'Test Site' }]))); + } else { + res.end(JSON.stringify(abilityRunSuccess({ deleted: true }))); + } + }); + + const result = await runCLI( + [ + 'abilities', 'run', 'delete-site-v1', + '--input', '{"site_id_or_domain":1}', + '--confirm', '--force', + ], + { + xdgConfigHome: config.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + expect(result.exitCode).toBe(0); + + const previewIdx = result.stdout.indexOf('Preview:'); + const executedIdx = result.stdout.indexOf('Executed:'); + expect(previewIdx).toBeGreaterThanOrEqual(0); + expect(executedIdx).toBeGreaterThan(previewIdx); + }); + + // ------------------------------------------------------------------------- + // 10. In JSON mode the preview is included in the success envelope + // ------------------------------------------------------------------------- + it('--confirm --force --json includes preview data in the success envelope', async () => { + await createConfig(); + + const runPath = '/wp-json/wp-abilities/v1/abilities/mainwp/delete-site-v1/run'; + server.addRoute('POST', runPath, (_req, res) => { + const body = _req.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + res.writeHead(200, { 'Content-Type': 'application/json' }); + if (input?.['dry_run'] === true) { + res.end(JSON.stringify(abilityDryRunResponse([{ site_id: 1, name: 'Test Site' }]))); + } else { + res.end(JSON.stringify(abilityRunSuccess({ deleted: true }))); + } + }); + + const result = await runCLI( + [ + 'abilities', 'run', 'delete-site-v1', + '--input', '{"site_id_or_domain":1}', + '--confirm', '--force', '--json', + ], + { + xdgConfigHome: config.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + expect(result.exitCode).toBe(0); + + const envelope = result.json as Record; + expect(envelope).toHaveProperty('success', true); + const data = envelope['data'] as Record; + expect(data).toHaveProperty('preview'); + const preview = data['preview'] as Record; + expect(preview).toHaveProperty('summary'); + expect(preview).toHaveProperty('affected'); + }); + + // ------------------------------------------------------------------------- + // 11. Transport failure AFTER the confirm call is dispatched is an unknown + // outcome: OUTCOME_UNKNOWN in the envelope, exit 3, and BOTH audit + // entries (dispatch-stage before the call, outcomeUnknown after) are + // on disk even though no response ever arrived. + // ------------------------------------------------------------------------- + it('--confirm --force with connection dropped mid-confirm exits 3 with OUTCOME_UNKNOWN and audits the dispatch', async () => { + await createConfig(); + + const runPath = '/wp-json/wp-abilities/v1/abilities/mainwp/delete-site-v1/run'; + server.addRoute('POST', runPath, (_req, res) => { + const body = _req.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + if (input?.['dry_run'] === true) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(abilityDryRunResponse([{ site_id: 1, name: 'Test Site' }]))); + } else { + // Confirm call: the server got the request, then the connection dies + // before any response — the Dashboard may have executed the action. + res.socket?.destroy(); + } + }); + + const result = await runCLI( + [ + 'abilities', 'run', 'delete-site-v1', + '--input', '{"site_id_or_domain":1}', + '--confirm', '--force', '--json', + ], + { + xdgConfigHome: config.xdgHome, + env: { MAINWP_APP_PASSWORD: 'test-pass' }, + }, + ); + + // NETWORK_ERROR exit class, but labeled as an unknown outcome + expect(result.exitCode).toBe(3); + const envelope = result.json as Record; + expect(envelope).toHaveProperty('success', false); + const error = envelope['error'] as Record; + expect(error['code']).toBe('OUTCOME_UNKNOWN'); + expect(String(error['message'])).toContain('may or may not have executed'); + + // The confirm request really was dispatched + const confirmRequests = server + .getRecordedRequests() + .filter((r) => r.path.includes('delete-site-v1')) + .filter((r) => { + const body = r.body as Record | undefined; + const input = body?.['input'] as Record | undefined; + return input?.['dry_run'] !== true; + }); + expect(confirmRequests.length).toBe(1); + + // Audit trail: a dispatch-stage entry written before the confirm call, + // then an outcomeUnknown entry after the transport failure. + const auditRaw = await readFile(join(config.configPath, 'audit.log'), 'utf-8'); + const entries = auditRaw + .trim() + .split('\n') + .map((line) => JSON.parse(line) as Record); + const dispatchEntry = entries.find((e) => e['stage'] === 'dispatch'); + expect(dispatchEntry).toBeDefined(); + expect(dispatchEntry!['userDecision']).toBe('approved'); + expect(dispatchEntry!['abilityName']).toContain('delete-site-v1'); + const unknownEntry = entries.find( + (e) => (e['execution'] as Record | undefined)?.['outcomeUnknown'] === true + ); + expect(unknownEntry).toBeDefined(); + expect(unknownEntry!['userDecision']).toBe('approved'); + expect((unknownEntry!['execution'] as Record)['success']).toBe(false); + }); }); diff --git a/src/__tests__/process/scenarios.test.ts b/src/__tests__/process/scenarios.test.ts index f1d80c8..ca01e9c 100644 --- a/src/__tests__/process/scenarios.test.ts +++ b/src/__tests__/process/scenarios.test.ts @@ -25,7 +25,7 @@ import { STANDARD_ABILITIES, abilityRunSuccess, abilityDryRunResponse, - abilityRunBatch, + dashboardQueuedResponse, jobStatus, } from './fixtures/api-responses.js'; @@ -252,7 +252,7 @@ describe('Scenario: Batch Update', () => { it('step 2: --confirm --force --wait executes and waits for job completion', async () => { // The run-updates-v1 --confirm returns a batch job - server.setRunResponse('run-updates-v1', abilityRunBatch(BATCH_JOB_ID)); + server.setRunResponse('run-updates-v1', dashboardQueuedResponse(BATCH_JOB_ID)); // Set up job status progression: pending → running → completed server.setJobProgression(BATCH_JOB_ID, [ @@ -332,7 +332,7 @@ describe('Scenario: Batch Update', () => { // Step 2: reset routes and set up for confirm+wait server.reset(); server.setAbilities(STANDARD_ABILITIES); - server.setRunResponse('run-updates-v1', abilityRunBatch(BATCH_JOB_ID)); + server.setRunResponse('run-updates-v1', dashboardQueuedResponse(BATCH_JOB_ID)); server.setJobProgression(BATCH_JOB_ID, [ jobStatus({ job_id: BATCH_JOB_ID, status: 'pending', progress: 0, processed: 0, total: 2 }), jobStatus({ job_id: BATCH_JOB_ID, status: 'completed', progress: 100, processed: 2, total: 2, results: [{ ok: true }] }), diff --git a/src/chat/chat-engine.test.ts b/src/chat/chat-engine.test.ts index f0249f1..1bb17c3 100644 --- a/src/chat/chat-engine.test.ts +++ b/src/chat/chat-engine.test.ts @@ -16,6 +16,11 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { ChatEngine, createChatEngine, type ChatResponse } from './chat-engine.js'; import type { LLMProvider, LLMResponse, Message, ToolDefinition, ChatOptions } from './providers/provider.js'; import type { Ability, ExecutionResult, ExecutionOptions } from '../core/abilities-executor.js'; +import { logDestructiveActionSafe } from '../utils/audit-logger.js'; + +vi.mock('../utils/audit-logger.js', () => ({ + logDestructiveActionSafe: vi.fn().mockResolvedValue(undefined), +})); // ============================================================================ // Test Fixtures @@ -76,6 +81,18 @@ const READONLY_DESTRUCTIVE_ABILITY = createTestAbility('special-v1', { }); const ABILITY_WITHOUT_ANNOTATIONS = createAbilityWithoutAnnotations('legacy-ability-v1'); const UPDATE_ABILITY = createTestAbility('update-site-v1', { destructive: false }); +const NAMESPACED_ABILITY = createTestAbility( + 'mainwp/list-sites-v1', + { readonly: true }, + { + type: 'object', + properties: { + page: { type: 'integer' }, + }, + required: ['page'], + additionalProperties: false, + } +); // Standard LLM responses function createToolCallResponse(toolName: string, input: Record): LLMResponse { @@ -105,7 +122,7 @@ function createAnswerResponse(answer: string): LLMResponse { function createInvalidJsonResponse(): LLMResponse { return { - content: 'This is not valid JSON { broken', + content: '{"type": "tool_call", "tool": broken', finishReason: 'stop', model: 'test-model', }; @@ -325,6 +342,266 @@ describe('ChatEngine', () => { expect(mockExecutor.listAbilities).toHaveBeenCalledTimes(1); }); + + it('declares namespaced abilities with protocol-safe aliases', async () => { + const mockProvider = createMockProvider([createAnswerResponse('Done')]); + const { engine } = createTestEngine({ + provider: mockProvider, + abilities: [NAMESPACED_ABILITY], + }); + + await engine.sendMessage('List sites'); + + const options = vi.mocked(mockProvider.chat).mock.calls[0]?.[1]; + expect(options?.tools).toEqual([ + expect.objectContaining({ name: 'mainwp__list-sites-v1' }), + ]); + expect(options?.tools?.[0]?.name).not.toContain('/'); + }); + + it('rejects colliding protocol-safe aliases', async () => { + const { engine } = createTestEngine({ + abilities: [ + createTestAbility('mainwp/list-sites-v1', { readonly: true }), + createTestAbility('mainwp__list-sites-v1', { readonly: true }), + ], + }); + + await expect(engine.initialize()).rejects.toThrow('Tool alias collision'); + }); + }); + + describe('Protocol-strict tool calls', () => { + it('resolves a native wire alias before ability lookup and execution', async () => { + const mockProvider = createMockProvider([ + createNativeToolCallResponse('mainwp__list-sites-v1', { page: 1 }, 'call_alias'), + createAnswerResponse('Done'), + ]); + const { engine, mockExecutor } = createTestEngine({ + provider: mockProvider, + abilities: [NAMESPACED_ABILITY], + }); + + await engine.sendMessage('List sites'); + + expect(mockExecutor.getAbility).toHaveBeenCalledWith('mainwp/list-sites-v1'); + expect(mockExecutor.execute).toHaveBeenCalledWith('mainwp/list-sites-v1', { page: 1 }); + }); + + it.each([ + { + name: 'invalid native argument JSON', + response: { + content: '', + toolCalls: [{ + id: 'call_bad_json', + name: 'mainwp__list-sites-v1', + arguments: '{bad json' as unknown as Record, + }], + finishReason: 'tool_calls' as const, + model: 'test-model', + }, + }, + { + name: 'non-object envelope input', + response: { + content: JSON.stringify({ tool: 'mainwp/list-sites-v1', input: 'not-an-object' }), + finishReason: 'stop' as const, + model: 'test-model', + }, + }, + { + name: 'multiple native tool calls', + response: { + content: '', + toolCalls: [ + { id: 'call_1', name: 'mainwp__list-sites-v1', arguments: { page: 1 } }, + { id: 'call_2', name: 'mainwp__list-sites-v1', arguments: { page: 2 } }, + ], + finishReason: 'tool_calls' as const, + model: 'test-model', + }, + }, + { + name: 'answer and tool in one envelope', + response: { + content: JSON.stringify({ + answer: 'Done', + tool: 'mainwp/list-sites-v1', + input: { page: 1 }, + }), + finishReason: 'stop' as const, + model: 'test-model', + }, + }, + { + name: 'length finish reason with a tool call', + response: { + content: '', + toolCalls: [ + { id: 'call_length', name: 'mainwp__list-sites-v1', arguments: { page: 1 } }, + ], + finishReason: 'length' as const, + model: 'test-model', + }, + }, + { + name: 'content-filter finish reason with a tool call', + response: { + content: '', + toolCalls: [ + { id: 'call_filter', name: 'mainwp__list-sites-v1', arguments: { page: 1 } }, + ], + finishReason: 'content_filter' as const, + model: 'test-model', + }, + }, + { + name: 'length finish reason with a content tool envelope', + response: { + content: JSON.stringify({ + tool: 'mainwp/list-sites-v1', + input: { page: 1 }, + }), + finishReason: 'length' as const, + model: 'test-model', + }, + }, + { + name: 'content-filter finish reason with a content tool envelope', + response: { + content: JSON.stringify({ + tool: 'mainwp/list-sites-v1', + input: { page: 1 }, + }), + finishReason: 'content_filter' as const, + model: 'test-model', + }, + }, + ])('never executes $name', async ({ response }) => { + const { engine, mockExecutor } = createTestEngine({ + provider: createMockProvider([response]), + abilities: [NAMESPACED_ABILITY], + maxParseRetries: 0, + }); + + const responses = await engine.sendMessage('List sites'); + + expect(responses[0]?.type).toBe('error'); + expect(mockExecutor.execute).not.toHaveBeenCalled(); + }); + + it('returns schema-invalid input to the model as a tool error without executing', async () => { + const mockProvider = createMockProvider([ + createNativeToolCallResponse( + 'mainwp__list-sites-v1', + { page: 'not-an-integer' }, + 'call_schema' + ), + createAnswerResponse('Please provide a numeric page.'), + ]); + const { engine, mockExecutor } = createTestEngine({ + provider: mockProvider, + abilities: [NAMESPACED_ABILITY], + }); + + const responses = await engine.sendMessage('List page nope'); + + expect(mockExecutor.execute).not.toHaveBeenCalled(); + expect(responses.at(-1)).toEqual({ + type: 'message', + content: 'Please provide a numeric page.', + }); + const secondMessages = vi.mocked(mockProvider.chat).mock.calls[1]?.[0]; + const validationResult = secondMessages?.find( + (message) => message.role === 'tool' && message.toolCallId === 'call_schema' + ); + expect(validationResult).toMatchObject({ + role: 'tool', + toolCallId: 'call_schema', + toolName: 'mainwp__list-sites-v1', + }); + expect(validationResult?.content).toContain('SCHEMA_VALIDATION_ERROR'); + }); + + it('executes with the coerced AJV input', async () => { + const mockProvider = createMockProvider([ + createNativeToolCallResponse( + 'mainwp__list-sites-v1', + { page: '2' }, + 'call_coerced' + ), + createAnswerResponse('Done'), + ]); + const { engine, mockExecutor } = createTestEngine({ + provider: mockProvider, + abilities: [NAMESPACED_ABILITY], + }); + + await engine.sendMessage('List page 2'); + + expect(mockExecutor.execute).toHaveBeenCalledWith( + 'mainwp/list-sites-v1', + { page: 2 } + ); + }); + + it('preserves native assistant tool calls in history', async () => { + const mockProvider = createMockProvider([ + createNativeToolCallResponse('mainwp__list-sites-v1', { page: 1 }, 'call_original'), + createAnswerResponse('Done'), + ]); + const { engine } = createTestEngine({ + provider: mockProvider, + abilities: [NAMESPACED_ABILITY], + }); + + await engine.sendMessage('List sites'); + + expect(engine.getHistory().find((message) => message.role === 'assistant')).toMatchObject({ + toolCalls: [ + { + id: 'call_original', + name: 'mainwp__list-sites-v1', + arguments: { page: 1 }, + }, + ], + }); + }); + + it('preserves the original native call id through destructive approval', async () => { + const namespacedDelete = createTestAbility( + 'mainwp/delete-site-v1', + { destructive: true }, + DESTRUCTIVE_ABILITY.input_schema + ); + const mockProvider = createMockProvider([ + createNativeToolCallResponse( + 'mainwp__delete-site-v1', + { site_id: 123 }, + 'call_delete_original' + ), + ]); + const { engine } = createTestEngine({ + provider: mockProvider, + abilities: [namespacedDelete], + executeHandler: (_name, _input, options) => + options?.dryRun + ? createPreviewResult([{ id: 123 }]) + : createSuccessResult({ deleted: true }), + }); + + await engine.sendMessage('Delete site 123'); + await engine.sendMessage('yes'); + + const history = engine.getHistory(); + expect(history.find((message) => message.role === 'assistant')?.toolCalls?.[0]?.id) + .toBe('call_delete_original'); + expect(history.find((message) => message.role === 'tool')).toMatchObject({ + toolCallId: 'call_delete_original', + toolName: 'mainwp__delete-site-v1', + }); + }); }); // ========================================================================== @@ -484,6 +761,33 @@ describe('ChatEngine', () => { ); }); + it('names the ability when the mandatory preview fails', async () => { + const mockProvider = createMockProvider([ + createToolCallResponse('delete-site-v1', { site_id: 123 }), + createAnswerResponse('Could not preview'), + ]); + + const { engine } = createTestEngine({ + provider: mockProvider, + abilities: [DESTRUCTIVE_ABILITY], + executeHandler: (_name, _input, options) => { + if (options?.dryRun) { + return createErrorResult('NOT_FOUND', 'Resource not found'); + } + return createSuccessResult({ deleted: true }); + }, + }); + + const responses = await engine.sendMessage('Delete site 123'); + + const errorResponse = responses.find((response) => response.type === 'error'); + expect(errorResponse).toMatchObject({ + type: 'error', + tool: 'delete-site-v1', + error: 'Resource not found', + }); + }); + it('returns preview response type for destructive action', async () => { const mockProvider = createMockProvider([ createToolCallResponse('delete-site-v1', { site_id: 123 }), @@ -663,6 +967,22 @@ describe('ChatEngine', () => { { confirm: true } ); expect(responses[0]!.type).toBe('tool_result'); + + // Audit is written twice: once at dispatch (before the confirm call, so + // a mid-confirm failure still leaves durable evidence) and once with + // the execution outcome. + expect(logDestructiveActionSafe).toHaveBeenCalledTimes(2); + expect(logDestructiveActionSafe).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ stage: 'dispatch', userDecision: 'approved' }) + ); + expect(logDestructiveActionSafe).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + userDecision: 'approved', + execution: expect.objectContaining({ success: true }), + }) + ); }); it('approves with "y"', async () => { @@ -810,6 +1130,113 @@ describe('ChatEngine', () => { }); }); + // ========================================================================== + // Golden Test: Confirm Failure Leaves Unknown Outcome + // ========================================================================== + + describe('Golden Test: Confirm Failure Leaves Unknown Outcome', () => { + it('reports an unknown-outcome error and preserves history when the confirm call rejects after dispatch', async () => { + const mockProvider = createMockProvider([ + createNativeToolCallResponse('delete-site-v1', { site_id: 123 }, 'call_confirm_fail'), + ]); + + const { engine, mockExecutor } = createTestEngine({ + provider: mockProvider, + abilities: [DESTRUCTIVE_ABILITY], + executeHandler: (_name, _input, options) => { + if (options?.dryRun) { + return createPreviewResult([{ id: 123 }]); + } + return createErrorResult('UNEXPECTED', 'Unexpected call'); + }, + }); + + await engine.sendMessage('Delete site 123'); + expect(engine.hasPendingPreview()).toBe(true); + + mockExecutor.execute.mockRejectedValueOnce(new Error('Request timed out')); + + const responses = await engine.sendMessage('yes'); + + // A single error response, never a rejected sendMessage — the caller + // must not have to catch this. + expect(responses).toHaveLength(1); + expect(responses[0]!.type).toBe('error'); + if (responses[0]!.type === 'error') { + expect(responses[0]!.error).toContain('delete-site-v1'); + expect(responses[0]!.error).toContain('verify'); + // Callers map this stable code to the OUTCOME_UNKNOWN process exit — + // it must never be downgraded to a generic chat error. + expect(responses[0]!.code).toBe('OUTCOME_UNKNOWN'); + } + + // Dispatch is audited before the failure is known, then the failure + // itself is audited as an unknown outcome. + const auditCalls = vi.mocked(logDestructiveActionSafe).mock.calls; + expect(auditCalls).toHaveLength(2); + expect(auditCalls[0]![0]).toEqual( + expect.objectContaining({ stage: 'dispatch', userDecision: 'approved' }) + ); + expect(auditCalls[1]![0]).toEqual( + expect.objectContaining({ + userDecision: 'approved', + execution: expect.objectContaining({ success: false, outcomeUnknown: true }), + }) + ); + + // History stays coherent: the pending tool call got a matching tool + // message rather than being left dangling. + const history = engine.getHistory(); + const toolMessage = history.find( + (m) => m.role === 'tool' && m.toolCallId === 'call_confirm_fail' + ); + expect(toolMessage).toBeDefined(); + expect(toolMessage!.content).toContain('OUTCOME_UNKNOWN'); + }); + }); + + // ========================================================================== + // Golden Test: Provider-Bound Redaction + // ========================================================================== + + describe('Golden Test: Provider-Bound Redaction', () => { + it('redacts sensitive keys in provider-bound history but returns raw values to the local caller', async () => { + const mockProvider = createMockProvider([ + createToolCallResponse('list-sites-v1', {}), + createAnswerResponse('Done'), + ]); + + const sensitiveData = { + site: 'a.com', + appPassword: 'hunter2', + nested: { api_key: 'k' }, + }; + + const { engine } = createTestEngine({ + provider: mockProvider, + abilities: [READONLY_ABILITY], + executeHandler: () => createSuccessResult(sensitiveData), + }); + + const responses = await engine.sendMessage('List sites'); + + const toolResult = responses.find((r) => r.type === 'tool_result'); + expect(toolResult).toBeDefined(); + if (toolResult?.type === 'tool_result') { + expect(toolResult.result.data).toEqual(sensitiveData); + } + + const history = engine.getHistory(); + const toolMessage = history.find((m) => m.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(JSON.parse(toolMessage!.content)).toEqual({ + site: 'a.com', + appPassword: '[REDACTED]', + nested: { api_key: '[REDACTED]' }, + }); + }); + }); + // ========================================================================== // Golden Test: User Decline Handling // ========================================================================== @@ -1339,21 +1766,54 @@ describe('ChatEngine', () => { expect(engine.hasPendingPreview()).toBe(false); }); - it('hasPendingPreview returns false after cancelPendingPreview', async () => { + it.each(['no', 'cancel'])('records a complete decline for "%s"', async (reply) => { const mockProvider = createMockProvider([ - createToolCallResponse('delete-site-v1', { site_id: 1 }), + createNativeToolCallResponse('delete-site-v1', { site_id: 1 }, 'call_decline'), ]); - const { engine } = createTestEngine({ + const { engine, mockExecutor } = createTestEngine({ provider: mockProvider, abilities: [DESTRUCTIVE_ABILITY], executeHandler: () => createPreviewResult([{ id: 1 }]), }); await engine.sendMessage('Delete site'); - engine.cancelPendingPreview(); + mockExecutor.execute.mockClear(); + await engine.sendMessage(reply); expect(engine.hasPendingPreview()).toBe(false); + expect(mockExecutor.execute).not.toHaveBeenCalled(); + expect(logDestructiveActionSafe).toHaveBeenCalledWith( + expect.objectContaining({ + abilityName: 'delete-site-v1', + userDecision: 'declined', + }) + ); + + const history = engine.getHistory(); + const declineResult = history.find( + (message) => message.role === 'tool' && message.toolCallId === 'call_decline' + ); + expect(declineResult).toMatchObject({ + role: 'tool', + toolCallId: 'call_decline', + toolName: 'delete-site-v1', + }); + expect(JSON.parse(declineResult!.content)).toMatchObject({ + success: false, + error: { code: 'USER_DECLINED' }, + }); + + const unansweredToolCalls = history + .filter((message) => message.role === 'assistant') + .flatMap((message) => message.toolCalls ?? []) + .filter( + (toolCall) => + !history.some( + (message) => message.role === 'tool' && message.toolCallId === toolCall.id + ) + ); + expect(unansweredToolCalls).toEqual([]); }); it('getPendingPreview returns null initially', async () => { @@ -1385,26 +1845,6 @@ describe('ChatEngine', () => { expect(preview!.summary).toBeDefined(); }); - it('cancelPendingPreview clears state', async () => { - const mockProvider = createMockProvider([ - createToolCallResponse('delete-site-v1', { site_id: 1 }), - createAnswerResponse('OK, cancelled'), - ]); - - const { engine } = createTestEngine({ - provider: mockProvider, - abilities: [DESTRUCTIVE_ABILITY], - executeHandler: () => createPreviewResult([{ id: 1 }]), - }); - - await engine.sendMessage('Delete site'); - expect(engine.hasPendingPreview()).toBe(true); - - engine.cancelPendingPreview(); - expect(engine.hasPendingPreview()).toBe(false); - expect(engine.getPendingPreview()).toBeNull(); - }); - it('clearHistory also clears pending preview', async () => { const mockProvider = createMockProvider([ createToolCallResponse('delete-site-v1', { site_id: 1 }), @@ -1808,7 +2248,7 @@ describe('ChatEngine', () => { // ========================================================================== describe('Edge Cases and Boundary Conditions', () => { - it('handles ability without annotations (defaults to safe)', async () => { + it('handles ability without annotations as destructive', async () => { const mockProvider = createMockProvider([ createToolCallResponse('legacy-ability-v1', {}), createAnswerResponse('Done'), @@ -1822,9 +2262,12 @@ describe('ChatEngine', () => { await engine.sendMessage('Run legacy'); - // Should execute directly (no preview) - expect(mockExecutor.execute).toHaveBeenCalledWith('legacy-ability-v1', {}); - expect(engine.hasPendingPreview()).toBe(false); + expect(mockExecutor.execute).toHaveBeenCalledWith( + 'legacy-ability-v1', + {}, + { dryRun: true }, + ); + expect(engine.hasPendingPreview()).toBe(true); }); it('handles LLM returning answer immediately', async () => { @@ -2296,16 +2739,52 @@ describe('ChatEngine', () => { // History should be managed const history = engine.getHistory(); expect(history[0]!.role).toBe('system'); + + // Pairing integrity: truncation must never orphan a tool result from + // its preceding assistant tool call — providers reject orphaned + // tool results on the next call (regression test for the unsafe + // mid-tool-loop truncation fallback) + for (let i = 1; i < history.length; i++) { + if (history[i]!.role === 'tool') { + expect(history[i - 1]!.role).toBe('assistant'); + } + } + // A truncation cut is only safe immediately before a user message, + // so the first non-system message is never a dangling tool result + expect(history[1]!.role).not.toBe('tool'); }); - it('should handle empty messages array gracefully', async () => { + it('should keep pairing integrity across the next turn after a mid-tool-loop overflow', async () => { + const mockProvider = createMockProvider([ + createToolCallResponse('list-sites-v1', { page: 1 }), + createToolCallResponse('list-sites-v1', { page: 2 }), + createAnswerResponse('Found all sites'), + createAnswerResponse('Done'), + ]); + const { engine } = createEngineWithContext({ - maxContextMessages: 5, + provider: mockProvider, + maxContextMessages: 4, + executeHandler: () => createSuccessResult({ sites: [] }), }); - // Before initialization, getContextStats should work - const stats = engine.getContextStats(); - expect(stats.messageCount).toBe(-1); // No messages yet + // First turn overflows the window mid tool-loop (truncation is + // deferred until a safe boundary exists) + await engine.sendMessage('List all sites'); + // Next user turn provides the safe boundary and the window catches up + await engine.sendMessage('Thanks'); + + const history = engine.getHistory(); + expect(history[0]!.role).toBe('system'); + // After a cut, the window is bounded again (system + max + current exchange) + expect(history.length).toBeLessThanOrEqual(6); + // And the cut landed on a user boundary, not inside a tool exchange + expect(history[1]!.role).toBe('user'); + for (let i = 1; i < history.length; i++) { + if (history[i]!.role === 'tool') { + expect(history[i - 1]!.role).toBe('assistant'); + } + } }); }); @@ -2389,6 +2868,9 @@ describe('ChatEngine', () => { }); describe('Configuration Tests', () => { + // The resolved maxContextMessages is observable through the system + // prompt's context-window constraint line (the stats accessor it was + // previously asserted through was speculative plumbing and is gone) it('should use provided maxContextMessages option', async () => { const { engine } = createEngineWithContext({ maxContextMessages: 10, @@ -2396,8 +2878,8 @@ describe('ChatEngine', () => { await engine.initialize(); - const stats = engine.getContextStats(); - expect(stats.maxMessages).toBe(10); + const systemPrompt = engine.getHistory()[0]?.content as string; + expect(systemPrompt).toContain('Context window: 10 messages'); }); it('should apply default limit (20) when not specified', async () => { @@ -2413,8 +2895,8 @@ describe('ChatEngine', () => { await engine.initialize(); - const stats = engine.getContextStats(); - expect(stats.maxMessages).toBe(20); + const systemPrompt = engine.getHistory()[0]?.content as string; + expect(systemPrompt).toContain('Context window: 20 messages'); }); it('should apply default limit when undefined is passed (use 0 to disable)', async () => { @@ -2426,19 +2908,8 @@ describe('ChatEngine', () => { await engine.initialize(); - const stats = engine.getContextStats(); - expect(stats.maxMessages).toBe(20); - }); - - it('should disable truncation when 0 is passed', async () => { - const { engine } = createEngineWithContext({ - maxContextMessages: 0, - }); - - await engine.initialize(); - - const stats = engine.getContextStats(); - expect(stats.maxMessages).toBe(0); + const systemPrompt = engine.getHistory()[0]?.content as string; + expect(systemPrompt).toContain('Context window: 20 messages'); }); it('should not include context constraint in system prompt when 0 is passed', async () => { @@ -2467,54 +2938,6 @@ describe('ChatEngine', () => { }); }); - describe('Context Stats', () => { - it('should return correct message count', async () => { - const mockProvider = createMockProvider([ - createAnswerResponse('Response 1'), - createAnswerResponse('Response 2'), - ]); - - const { engine } = createEngineWithContext({ - provider: mockProvider, - maxContextMessages: 20, - }); - - await engine.sendMessage('Message 1'); - await engine.sendMessage('Message 2'); - - const stats = engine.getContextStats(); - // 2 user + 2 assistant = 4 messages (excluding system prompt) - expect(stats.messageCount).toBe(4); - }); - - it('should return correct max messages value', async () => { - const { engine } = createEngineWithContext({ - maxContextMessages: 15, - }); - - await engine.initialize(); - - const stats = engine.getContextStats(); - expect(stats.maxMessages).toBe(15); - }); - - it('should estimate tokens based on character count', async () => { - const mockProvider = createMockProvider([ - createAnswerResponse('This is a response with some text content'), - ]); - - const { engine } = createEngineWithContext({ - provider: mockProvider, - maxContextMessages: 20, - }); - - await engine.sendMessage('Hello world'); - - const stats = engine.getContextStats(); - // Should have some estimated tokens - expect(stats.estimatedTokens).toBeGreaterThan(0); - }); - }); }); // ========================================================================== @@ -2544,7 +2967,7 @@ describe('ChatEngine', () => { arguments: { truncated: true }, }, }; - throw new Error('Stream interrupted'); + throw new Error('\x1b]0;Injected\x07Stream interrupted'); }), isConfigured: () => true, getModels: () => ['test-model'], @@ -2562,6 +2985,7 @@ describe('ChatEngine', () => { await engine.initialize(); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); const responses = await engine.sendMessage('list sites'); // Should return an error response, not execute the partial tool call @@ -2573,6 +2997,181 @@ describe('ChatEngine', () => { // Executor should NOT have been called with partial tool call expect(mockExecutor.execute).not.toHaveBeenCalled(); + expect(consoleError).toHaveBeenCalledWith(expect.not.stringContaining('\x1b')); + }); + + it('treats a stream that yields nothing as an error, not an empty success', async () => { + const emptyStreamProvider: LLMProvider = { + name: 'mock-streaming-provider', + capabilities: { + functionCalling: true, + streaming: true, + systemMessages: true, + vision: false, + maxContextLength: 4096, + }, + chat: vi.fn(), + chatStream: vi.fn(async function* () { + // Ends cleanly without yielding any content or tool calls + }), + isConfigured: () => true, + getModels: () => ['test-model'], + getDefaultModel: () => 'test-model', + }; + + const abilities = [READONLY_ABILITY]; + const mockExecutor = createMockExecutor(abilities); + + const engine = createChatEngine({ + provider: emptyStreamProvider, + executor: mockExecutor as never, + stream: true, + }); + + await engine.initialize(); + + const responses = await engine.sendMessage('list sites'); + + expect(responses).toHaveLength(1); + expect(responses[0]!.type).toBe('error'); + if (responses[0]!.type === 'error') { + expect(responses[0]!.error).toContain('interrupted'); + } + expect(mockExecutor.execute).not.toHaveBeenCalled(); + }); + + it('still succeeds when the stream yields content before done', async () => { + const contentStreamProvider: LLMProvider = { + name: 'mock-streaming-provider', + capabilities: { + functionCalling: true, + streaming: true, + systemMessages: true, + vision: false, + maxContextLength: 4096, + }, + chat: vi.fn(), + chatStream: vi.fn(async function* () { + yield { content: JSON.stringify({ answer: 'Hi there' }) }; + yield { done: true }; + }), + isConfigured: () => true, + getModels: () => ['test-model'], + getDefaultModel: () => 'test-model', + }; + + const abilities = [READONLY_ABILITY]; + const mockExecutor = createMockExecutor(abilities); + + const engine = createChatEngine({ + provider: contentStreamProvider, + executor: mockExecutor as never, + stream: true, + }); + + await engine.initialize(); + + const responses = await engine.sendMessage('hello'); + + expect(responses).toEqual([{ type: 'message', content: 'Hi there' }]); + }); + }); + + // ========================================================================== + // sendMessage Re-entrancy + // ========================================================================== + + describe('sendMessage Re-entrancy', () => { + it('serializes concurrent sendMessage calls in call order', async () => { + let releaseFirst!: () => void; + const firstGate = new Promise((resolve) => (releaseFirst = resolve)); + + const responses = [createAnswerResponse('first answer'), createAnswerResponse('second answer')]; + let callIndex = 0; + const gatedProvider: LLMProvider = { + name: 'mock-provider', + capabilities: { + functionCalling: true, + streaming: false, + systemMessages: true, + vision: false, + maxContextLength: 4096, + }, + chat: vi.fn(async (): Promise => { + const index = callIndex++; + if (index === 0) { + await firstGate; + } + return responses[index]!; + }), + isConfigured: () => true, + getModels: () => ['test-model'], + getDefaultModel: () => 'test-model', + }; + + const { engine } = createTestEngine({ provider: gatedProvider }); + await engine.initialize(); + + // Fire both without awaiting the first + const first = engine.sendMessage('first question'); + const second = engine.sendMessage('second question'); + + // While the first call is blocked in the provider, the second must be + // queued: no second provider call, no second user message in history + await new Promise((resolve) => setImmediate(resolve)); + expect(gatedProvider.chat).toHaveBeenCalledTimes(1); + expect( + engine.getHistory().filter((m) => m.role === 'user') + ).toHaveLength(1); + + releaseFirst(); + const [firstResponses, secondResponses] = await Promise.all([first, second]); + + expect(firstResponses[0]).toEqual({ type: 'message', content: 'first answer' }); + expect(secondResponses[0]).toEqual({ type: 'message', content: 'second answer' }); + + // History interleaves strictly: user1, assistant1, user2, assistant2 + const conversation = engine + .getHistory() + .filter((m) => m.role === 'user' || m.role === 'assistant') + .map((m) => m.role); + expect(conversation).toEqual(['user', 'assistant', 'user', 'assistant']); + }); + + it('runs a queued call even when the previous call rejects', async () => { + let callIndex = 0; + const flakyProvider: LLMProvider = { + name: 'mock-provider', + capabilities: { + functionCalling: true, + streaming: false, + systemMessages: true, + vision: false, + maxContextLength: 4096, + }, + chat: vi.fn(async (): Promise => { + if (callIndex++ === 0) { + throw new Error('provider exploded'); + } + return createAnswerResponse('recovered'); + }), + isConfigured: () => true, + getModels: () => ['test-model'], + getDefaultModel: () => 'test-model', + }; + + const { engine } = createTestEngine({ provider: flakyProvider }); + await engine.initialize(); + + const first = engine.sendMessage('first question'); + const second = engine.sendMessage('second question'); + + // The first call rejects (provider errors propagate to the caller); + // the rejection must not poison the queue for the second call + await expect(first).rejects.toThrow('provider exploded'); + const secondResponses = await second; + + expect(secondResponses[0]).toEqual({ type: 'message', content: 'recovered' }); }); }); }); diff --git a/src/chat/chat-engine.ts b/src/chat/chat-engine.ts index 5e56181..ea3b57d 100644 --- a/src/chat/chat-engine.ts +++ b/src/chat/chat-engine.ts @@ -18,6 +18,7 @@ import type { ChatOptions, ToolDefinition, StreamChunk, + ToolCall, } from './providers/provider.js'; import { buildConfiguredPrompt, @@ -39,8 +40,14 @@ import { type PreviewResult, } from '../core/safety-controller.js'; import { abilityToTool } from './providers/provider.js'; +import { ContextWindow } from './context-window.js'; import { logDestructiveActionSafe } from '../utils/audit-logger.js'; import { getInputSanitizer } from '../validation/input-sanitizer.js'; +import { getSchemaValidator } from '../validation/schema-validator.js'; +import { SchemaValidationError } from '../utils/errors.js'; +import { stripControlChars } from '../utils/terminal-sanitizer.js'; +import { redactSensitiveKeys } from '../utils/redaction.js'; +import { executeAbilityWithPolicy } from '../core/execute-ability-with-policy.js'; /** * Chat response types @@ -54,7 +61,18 @@ export type ChatResponse = preview?: PreviewResult; } | { type: 'preview'; preview: PreviewResult; requiresApproval: boolean } - | { type: 'error'; error: string }; + | { + type: 'error'; + error: string; + /** Ability name, when the error occurred while handling a specific tool */ + tool?: string; + /** + * Stable error code for callers that map chat errors to process + * outcomes (e.g. 'OUTCOME_UNKNOWN' for a confirm call that failed + * after dispatch). + */ + code?: string; + }; /** * Chat engine options @@ -76,8 +94,6 @@ export interface ChatEngineOptions { promptConfig?: Partial; /** Maximum messages to keep in context (excluding system prompt). undefined = no limit */ maxContextMessages?: number; - /** Maximum estimated tokens in context. Reserved for future use. */ - maxContextTokens?: number; /** Whether to use streaming responses (default: false) */ stream?: boolean; /** Callback for streaming content chunks (called as content arrives) */ @@ -91,6 +107,8 @@ interface PendingPreview { ability: Ability; input: Record; preview: PreviewResult; + toolCallId: string; + toolAlias: string; } /** @@ -112,15 +130,21 @@ export class ChatEngine { private readonly model: string | undefined; private readonly temperature: number | undefined; private readonly promptConfig: SystemPromptConfig; - private readonly maxContextMessages: number | undefined; + private readonly contextWindow: ContextWindow; private readonly stream: boolean; private readonly onStreamChunk?: (content: string) => void; private messages: Message[] = []; private abilities: Ability[] = []; private tools: ToolDefinition[] = []; + private readonly toolAliases = new Map(); + private readonly abilityAliases = new Map(); private pendingPreview: PendingPreview | null = null; private initialized = false; + // Engine-lifetime counter for fallback tool-call IDs. A per-turn counter + // would repeat call_1, call_2, ... across turns while history is retained, + // producing duplicate IDs in the conversation sent to providers. + private fallbackToolCallId = 0; constructor(options: ChatEngineOptions) { this.provider = options.provider; @@ -151,19 +175,13 @@ export class ChatEngine { options.promptConfig?.maxContextMessages ?? defaultPromptConfig.maxContextMessages; - this.maxContextMessages = resolvedContextMessages; + this.contextWindow = new ContextWindow(resolvedContextMessages); // Sync the resolved value into promptConfig for system prompt generation if (resolvedContextMessages !== undefined) { mergedPromptConfig.maxContextMessages = resolvedContextMessages; } - // Handle optional token limit (reserved for future use) - const contextTokens = options.maxContextTokens ?? options.promptConfig?.maxContextTokens; - if (contextTokens !== undefined) { - mergedPromptConfig.maxContextTokens = contextTokens; - } - this.promptConfig = mergedPromptConfig; } @@ -176,10 +194,20 @@ export class ChatEngine { // Load abilities this.abilities = await this.executor.listAbilities(); - // Convert to tool definitions - this.tools = this.abilities.map((a) => - abilityToTool(a.name, a.description, a.input_schema) - ); + // Convert to protocol-safe tool definitions and keep a collision-checked + // reverse map so execution always uses the real ability name. + this.tools = this.abilities.map((ability) => { + const alias = ability.name.replaceAll('/', '__'); + const existing = this.toolAliases.get(alias); + if (existing && existing !== ability.name) { + throw new Error( + `Tool alias collision: "${existing}" and "${ability.name}" both map to "${alias}"` + ); + } + this.toolAliases.set(alias, ability.name); + this.abilityAliases.set(ability.name, alias); + return abilityToTool(alias, ability.description, ability.input_schema); + }); // Build system prompt const systemPrompt = buildConfiguredPrompt(this.abilities, this.promptConfig); @@ -190,10 +218,31 @@ export class ChatEngine { this.initialized = true; } + // Serializes sendMessage calls. History and pendingPreview are shared + // mutable state with no other concurrency protection; the readline REPL + // happens to serialize calls today, but programmatic callers may not. + private inFlight: Promise = Promise.resolve(); + /** * Send a user message and process the response + * + * Concurrent calls are queued and run in call order; a rejected call does + * not block the calls queued behind it. */ async sendMessage(userMessage: string): Promise { + const previous = this.inFlight; + let release!: () => void; + this.inFlight = new Promise((resolve) => (release = resolve)); + + await previous; + try { + return await this.sendMessageSerialized(userMessage); + } finally { + release(); + } + } + + private async sendMessageSerialized(userMessage: string): Promise { if (!this.initialized) { await this.initialize(); } @@ -219,6 +268,19 @@ export class ChatEngine { return responses; } + /** + * Build the audit-log preview payload for a pending preview + */ + private static previewAuditPayload(preview: PendingPreview): { + summary: string; + affectedCount: number; + } { + return { + summary: preview.preview.summary, + affectedCount: preview.preview.affected.length, + }; + } + /** * Handle response to a pending preview */ @@ -237,6 +299,18 @@ export class ChatEngine { if (!approved) { // User declined + this.messages.push({ + role: 'tool', + content: JSON.stringify({ + success: false, + error: { + code: 'USER_DECLINED', + message: 'The user declined the destructive action.', + }, + }), + toolCallId: preview.toolCallId, + toolName: preview.toolAlias, + }); this.messages.push({ role: 'user', content: userMessage, @@ -252,10 +326,7 @@ export class ChatEngine { // Log audit entry for declined action (fire-and-forget) await logDestructiveActionSafe({ abilityName: preview.ability.name, - preview: { - summary: preview.preview.summary, - affectedCount: preview.preview.affected.length, - }, + preview: ChatEngine.previewAuditPayload(preview), userDecision: 'declined', input: preview.input, }); @@ -271,17 +342,66 @@ export class ChatEngine { ]; } - // User approved - execute with confirm - this.messages.push({ - role: 'user', - content: 'User approved: yes', + // Record the approval BEFORE dispatching the confirm call, so a transport + // failure (or process death) mid-confirm still leaves durable evidence + // that an approved destructive action may have reached the Dashboard. + await logDestructiveActionSafe({ + abilityName: preview.ability.name, + preview: ChatEngine.previewAuditPayload(preview), + userDecision: 'approved', + stage: 'dispatch', + input: preview.input, }); - const result = await this.executor.execute( - preview.ability.name, - preview.input, - { confirm: true } - ); + // User approved - execute with confirm. A throw here arrives after + // dispatch was initiated: the outcome is unknown. Fail closed — audit the + // uncertainty, keep history coherent, and tell the user to verify before + // retrying. Never auto-retry the confirm. + let result: ExecutionResult; + try { + result = await executeAbilityWithPolicy( + this.executor, + preview.ability, + preview.input, + { confirm: true } + ); + } catch (error) { + const reason = getInputSanitizer().sanitizeErrorMessage( + error instanceof Error ? error.message : String(error) + ); + await logDestructiveActionSafe({ + abilityName: preview.ability.name, + preview: ChatEngine.previewAuditPayload(preview), + userDecision: 'approved', + execution: { success: false, error: reason, outcomeUnknown: true }, + input: preview.input, + }); + this.messages.push({ + role: 'tool', + content: JSON.stringify({ + success: false, + error: { + code: 'OUTCOME_UNKNOWN', + message: + 'The confirm call failed after dispatch; the action may or may not have executed. Do not retry without verifying Dashboard state.', + }, + }), + toolCallId: preview.toolCallId, + toolName: preview.toolAlias, + }); + this.messages.push({ role: 'user', content: userMessage }); + this.truncateHistory(); + return [ + { + type: 'error', + error: + `Confirm call for "${preview.ability.name}" failed after dispatch: ${reason}. ` + + 'The Dashboard may or may not have executed the action — verify its state before retrying.', + tool: preview.ability.name, + code: 'OUTCOME_UNKNOWN', + }, + ]; + } // Log audit entry for approved and executed action (fire-and-forget) const executionAudit: { success: boolean; error?: string } = { @@ -292,23 +412,26 @@ export class ChatEngine { } await logDestructiveActionSafe({ abilityName: preview.ability.name, - preview: { - summary: preview.preview.summary, - affectedCount: preview.preview.affected.length, - }, + preview: ChatEngine.previewAuditPayload(preview), userDecision: 'approved', execution: executionAudit, input: preview.input, }); - // Add result to context + // Add result to context. PROVIDER BOUNDARY: only a key-redacted copy of + // the result enters the history sent to the LLM provider; the unredacted + // result still goes back to the local user below. const toolResultMsg = { role: 'tool' as const, - content: JSON.stringify(result), - toolCallId: `execute_${preview.ability.name}`, - toolName: preview.ability.name, + content: JSON.stringify(redactSensitiveKeys(result)), + toolCallId: preview.toolCallId, + toolName: preview.toolAlias, }; this.messages.push(toolResultMsg); + this.messages.push({ + role: 'user', + content: 'User approved: yes', + }); // Truncate after preview resolution (safe boundary) this.truncateHistory(); @@ -361,6 +484,7 @@ export class ChatEngine { const parseResult = parseResponse(llmResponse, { abilities: this.abilities, validateToolExists: true, + toolAliases: this.toolAliases, }); // Handle parse errors with retry @@ -370,7 +494,9 @@ export class ChatEngine { // Add retry prompt this.messages.push({ role: 'assistant', - content: llmResponse.content, + content: + llmResponse.content || + 'Invalid tool call omitted due to a protocol error.', }); this.messages.push({ role: 'user', @@ -406,16 +532,32 @@ export class ChatEngine { const toolResponse = parseResult.response; toolCallCount++; + const toolCallId = toolResponse.id ?? `call_${++this.fallbackToolCallId}`; + const toolAlias = + this.abilityAliases.get(toolResponse.tool) ?? toolResponse.tool; + // Add assistant message with tool call this.messages.push({ role: 'assistant', content: llmResponse.content, + toolCalls: [ + { + id: toolCallId, + name: toolAlias, + arguments: toolResponse.input, + ...(toolResponse.thoughtSignature !== undefined + ? { thoughtSignature: toolResponse.thoughtSignature } + : {}), + }, + ], }); // Execute tool const toolResult = await this.executeTool( toolResponse.tool, - toolResponse.input + toolResponse.input, + toolCallId, + toolAlias ); if (toolResult.type === 'preview') { @@ -426,18 +568,22 @@ export class ChatEngine { responses.push(toolResult); - // Add tool result to context + // Add tool result to context. PROVIDER BOUNDARY: ability output is + // Dashboard-controlled data leaving the machine for a third-party LLM + // provider — redact sensitive-looking keys before it enters history + // (AGENTS.md: "minimize and redact secrets/customer data"). The + // unredacted result was already returned to the local user above. if (toolResult.type === 'tool_result') { const resultContent = toolResult.result.success - ? JSON.stringify(toolResult.result.data) - : JSON.stringify(toolResult.result.error); + ? JSON.stringify(redactSensitiveKeys(toolResult.result.data)) + : JSON.stringify(redactSensitiveKeys(toolResult.result.error)); this.messages.push({ role: 'tool', content: resultContent, - toolCallId: toolResponse.id ?? `call_${toolCallCount}`, - toolName: toolResponse.tool, + toolCallId, + toolName: toolAlias, }); // Truncate between tool-call iterations to enforce context limit @@ -448,8 +594,8 @@ export class ChatEngine { this.messages.push({ role: 'tool', content: JSON.stringify({ error: toolResult.error }), - toolCallId: toolResponse.id ?? `call_${toolCallCount}`, - toolName: toolResponse.tool, + toolCallId, + toolName: toolAlias, }); break; // Stop on error } @@ -473,7 +619,9 @@ export class ChatEngine { */ private async executeTool( toolName: string, - input: Record + input: Record, + toolCallId: string, + toolAlias: string ): Promise { // Find ability const ability = await this.executor.getAbility(toolName); @@ -484,18 +632,62 @@ export class ChatEngine { }; } + input = getInputSanitizer().sanitize(input); + if (ability.input_schema) { + try { + const validated = getSchemaValidator().validateOrThrow( + input, + ability.input_schema, + ability.name + ); + input = validated.coerced ?? input; + } catch (error) { + if (error instanceof SchemaValidationError) { + const validationError: NonNullable = { + code: error.code, + message: error.message, + details: error.details, + }; + if (error.hint) { + validationError.hint = error.hint; + } + return { + type: 'tool_result', + tool: ability.name, + result: { + success: false, + error: validationError, + }, + }; + } + // A throw here would leave the already-pushed assistant tool call + // dangling in history (no matching tool message), corrupting the next + // provider request. Surface it like the execution catch-all below. + return { + type: 'error', + error: error instanceof Error ? error.message : String(error), + tool: ability.name, + }; + } + } + // Check if destructive const classification = this.safetyController.classify(ability); if (classification.requiresSafetyFlow) { // SAFETY: Destructive actions always preview first // AI cannot skip this step - return this.executeWithPreview(ability, input); + return this.executeWithPreview( + ability, + input, + toolCallId, + toolAlias + ); } // Safe to execute directly try { - const result = await this.executor.execute(ability.name, input); + const result = await executeAbilityWithPolicy(this.executor, ability, input); return { type: 'tool_result', tool: ability.name, @@ -505,6 +697,7 @@ export class ChatEngine { return { type: 'error', error: error instanceof Error ? error.message : String(error), + tool: ability.name, }; } } @@ -514,12 +707,15 @@ export class ChatEngine { */ private async executeWithPreview( ability: Ability, - input: Record + input: Record, + toolCallId: string, + toolAlias: string ): Promise { try { // Execute with dry_run - const previewResult = await this.executor.execute( - ability.name, + const previewResult = await executeAbilityWithPolicy( + this.executor, + ability, input, { dryRun: true } ); @@ -528,6 +724,7 @@ export class ChatEngine { return { type: 'error', error: previewResult.error?.message ?? 'Preview failed', + tool: ability.name, }; } @@ -539,7 +736,7 @@ export class ChatEngine { ); // Store pending preview for approval - this.pendingPreview = { ability, input, preview }; + this.pendingPreview = { ability, input, preview, toolCallId, toolAlias }; return { type: 'preview', @@ -550,6 +747,7 @@ export class ChatEngine { return { type: 'error', error: error instanceof Error ? error.message : String(error), + tool: ability.name, }; } } @@ -567,7 +765,7 @@ export class ChatEngine { ): Promise { let content = ''; // Providers yield complete tool calls (not deltas), so we collect them directly - const toolCalls: Array<{ id: string; name: string; arguments: Record }> = []; + const toolCalls: ToolCall[] = []; try { for await (const chunk of stream) { @@ -586,7 +784,10 @@ export class ChatEngine { toolCalls.push({ id: chunk.toolCall.id, name: chunk.toolCall.name, - arguments: chunk.toolCall.arguments ?? {}, + arguments: chunk.toolCall.arguments, + ...(chunk.toolCall.thoughtSignature !== undefined + ? { thoughtSignature: chunk.toolCall.thoughtSignature } + : {}), }); } @@ -599,7 +800,9 @@ export class ChatEngine { // If streaming fails mid-response, return what we have so far if (content || toolCalls.length > 0) { console.error( - `[ChatEngine] Stream interrupted: ${error instanceof Error ? error.message : String(error)}` + `[ChatEngine] Stream interrupted: ${stripControlChars( + error instanceof Error ? error.message : String(error) + )}` ); return { content, @@ -612,6 +815,19 @@ export class ChatEngine { throw error; } + // A stream that ended cleanly but delivered nothing is a failed response, + // not an empty answer: reporting finishReason 'stop' here would flow into + // the envelope parser's plain-text fallback and surface as a successful + // blank reply (exit 0 in one-shot mode). + if (!content && toolCalls.length === 0) { + return { + content: '', + toolCalls: undefined, + finishReason: 'error', + model: this.provider.getDefaultModel(), + }; + } + // Return accumulated LLMResponse const parsedToolCalls = toolCalls; @@ -637,13 +853,6 @@ export class ChatEngine { return this.pendingPreview?.preview ?? null; } - /** - * Cancel pending preview - */ - cancelPendingPreview(): void { - this.pendingPreview = null; - } - /** * Get conversation history (for debugging) */ @@ -665,151 +874,16 @@ export class ChatEngine { } /** - * Get message count (excluding system prompt) - */ - private getMessageCount(): number { - return this.messages.length - 1; - } - - /** - * Estimate tokens for a set of messages using character count as a rough proxy. - * Uses character_count / 4 as a heuristic (common approximation for English text). - * - * NOTE: This is a rough estimate. Actual token counts from LLMResponse.usage - * are more accurate when available. + * Truncate message history via the context window. * - * @param messages - Messages to estimate tokens for - * @returns Estimated token count - */ - private estimateTokens(messages: Message[]): number { - let totalChars = 0; - for (const msg of messages) { - if (typeof msg.content === 'string') { - totalChars += msg.content.length; - } - } - // Character count / 4 is a common heuristic for English text tokenization - return Math.ceil(totalChars / 4); - } - - /** - * Check if context truncation should occur based on message count limits. - * Token-based limits are reserved for future implementation. - * - * @returns true if truncation should occur, false if: - * - maxContextMessages is undefined (no limit) - * - maxContextMessages is 0 (explicit unlimited) - * - message count is within limit - */ - private shouldTruncate(): boolean { - if (this.maxContextMessages === undefined || this.maxContextMessages === 0) { - return false; // No limit configured or explicitly unlimited - } - return this.getMessageCount() > this.maxContextMessages; - } - - /** - * Find the index where truncation should start, respecting message boundaries. - * This ensures we keep complete user-assistant exchanges and tool call-result pairs. - * - * @returns Index in the messages array where truncation should start (exclusive of system prompt) - */ - private findTruncationPoint(): number { - if (this.maxContextMessages === undefined || this.maxContextMessages <= 0) { - return 1; // Keep only system prompt - } - - // Calculate how many messages to keep (plus 1 for system prompt) - const targetLength = this.maxContextMessages + 1; - - if (this.messages.length <= targetLength) { - return this.messages.length; // No truncation needed - } - - // Start from where we'd ideally cut - const idealTruncationIndex = this.messages.length - this.maxContextMessages; - - // Ensure we don't cut the system prompt - let truncationIndex = Math.max(1, idealTruncationIndex); - - // Walk forward to find a safe boundary (start of a user message) - // This ensures we don't split: - // - user message + assistant response - // - tool call + tool result - // - retry prompts from their original failed attempt - const maxSearchIndex = this.messages.length; - while (truncationIndex < maxSearchIndex) { - const msg = this.messages[truncationIndex]; - // Safe to cut at the start of a user message - if (msg && msg.role === 'user') { - break; - } - truncationIndex++; - } - - // Fallback: if no user boundary found, keep at least maxContextMessages - // This ensures we don't drop all messages when the limit is very small - if (truncationIndex >= this.messages.length) { - truncationIndex = Math.max(1, idealTruncationIndex); - } - - return truncationIndex; - } - - /** - * Truncate message history using a sliding window approach. - * Preserves: - * - System prompt (always first message) - * - Messages since pending preview (if any) - * - Most recent N messages where N = maxContextMessages - * - Complete message exchanges (user-assistant, tool call-result pairs) + * Safety: Never truncates while a preview is pending — this preserves + * context for the approval decision. */ private truncateHistory(): void { - if (!this.shouldTruncate()) { - return; - } - - // Safety: Never truncate if there's a pending preview - // This preserves context for the approval decision if (this.pendingPreview !== null) { return; } - - const systemPrompt = this.messages[0]; - if (!systemPrompt) { - return; - } - - const truncationIndex = this.findTruncationPoint(); - const messagesBefore = this.messages.length; - - // Keep system prompt + messages from truncation point onwards - this.messages = [systemPrompt, ...this.messages.slice(truncationIndex)]; - - // Debug logging (only if significant truncation occurred) - const messagesRemoved = messagesBefore - this.messages.length; - if (messagesRemoved > 0 && process.env['DEBUG']) { - console.debug( - `[ChatEngine] Truncated ${messagesRemoved} messages (${messagesBefore - 1} -> ${this.messages.length - 1})` - ); - } - } - - /** - * Get context window statistics for monitoring. - * - * @returns Object with current message count, max limit, and estimated tokens - */ - getContextStats(): { - messageCount: number; - maxMessages: number | undefined; - estimatedTokens: number; - } { - return { - messageCount: this.getMessageCount(), - maxMessages: this.maxContextMessages, - estimatedTokens: this.estimateTokens(this.messages), - }; + this.messages = this.contextWindow.truncate(this.messages); } /** diff --git a/src/chat/context-window.test.ts b/src/chat/context-window.test.ts new file mode 100644 index 0000000..5e9b639 --- /dev/null +++ b/src/chat/context-window.test.ts @@ -0,0 +1,207 @@ +/** + * Tests for ContextWindow truncation boundary logic. + * + * The critical invariant: a cut is only safe immediately before a `user` + * message. Cutting mid tool-exchange orphans a `tool` result from its + * assistant tool call, which providers reject on the next call. + */ + +import { describe, it, expect } from 'vitest'; +import { ContextWindow } from './context-window.js'; +import type { Message } from './providers/provider.js'; + +const system: Message = { role: 'system', content: 'system prompt' }; +const user = (content: string): Message => ({ role: 'user', content }); +const assistant = (content: string): Message => ({ role: 'assistant', content }); +const tool = (name: string): Message => ({ + role: 'tool', + content: '{"ok":true}', + toolCallId: `call_${name}`, + toolName: name, +}); + +describe('ContextWindow', () => { + describe('shouldTruncate', () => { + it('returns false when no limit is configured', () => { + const window = new ContextWindow(undefined); + expect(window.shouldTruncate([system, user('a'), assistant('b')])).toBe(false); + }); + + it('returns false when limit is 0 (explicit unlimited)', () => { + const window = new ContextWindow(0); + expect(window.shouldTruncate([system, user('a'), assistant('b')])).toBe(false); + }); + + it('rejects a negative limit', () => { + expect(() => new ContextWindow(-5)).toThrow( + 'maxMessages must be non-negative' + ); + }); + + it('returns false while within the limit', () => { + const window = new ContextWindow(2); + expect(window.shouldTruncate([system, user('a'), assistant('b')])).toBe(false); + }); + + it('returns true when message count (excluding system) exceeds the limit', () => { + const window = new ContextWindow(2); + expect( + window.shouldTruncate([system, user('a'), assistant('b'), user('c')]) + ).toBe(true); + }); + }); + + describe('truncate', () => { + it('returns the original array unchanged when within the limit', () => { + const window = new ContextWindow(5); + const messages = [system, user('a'), assistant('b')]; + expect(window.truncate(messages)).toBe(messages); + }); + + it('cuts at a user boundary and always keeps the system prompt', () => { + const window = new ContextWindow(2); + const messages = [system, user('m1'), assistant('r1'), user('m2'), assistant('r2')]; + + const result = window.truncate(messages); + + expect(result[0]).toBe(system); + expect(result[1]!.role).toBe('user'); + expect(result[1]!.content).toBe('m2'); + expect(result).toHaveLength(3); + }); + + it('never orphans a tool result mid tool-calling loop (skips instead)', () => { + // Mid tool-loop overflow: only assistant/tool messages after the ideal + // cut point. The old fallback cut here and orphaned tool results. + const window = new ContextWindow(4); + const messages = [ + system, + user('list sites'), + assistant('tc1'), + tool('list-sites-v1'), + assistant('tc2'), + tool('list-sites-v1'), + ]; + + const result = window.truncate(messages); + + // No safe boundary exists — truncation is deferred, not forced + expect(result).toBe(messages); + }); + + it('catches up at the next user boundary after a deferred cycle', () => { + const window = new ContextWindow(4); + const messages = [ + system, + user('list sites'), + assistant('tc1'), + tool('list-sites-v1'), + assistant('tc2'), + tool('list-sites-v1'), + assistant('answer'), + user('thanks'), + ]; + + const result = window.truncate(messages); + + expect(result[0]).toBe(system); + // The only user boundary at/after the ideal cut is the final message + expect(result).toEqual([system, user('thanks')]); + // Pairing integrity holds: no tool message without its assistant + for (let i = 1; i < result.length; i++) { + if (result[i]!.role === 'tool') { + expect(result[i - 1]!.role).toBe('assistant'); + } + } + }); + + it('keeps a complete tool exchange when the cut lands before its user turn', () => { + const window = new ContextWindow(4); + const messages = [ + system, + user('old'), + assistant('old reply'), + user('list sites'), + assistant('tc1'), + tool('list-sites-v1'), + assistant('answer'), + ]; + + const result = window.truncate(messages); + + expect(result).toEqual([ + system, + user('list sites'), + assistant('tc1'), + tool('list-sites-v1'), + assistant('answer'), + ]); + }); + + it('handles an empty history without throwing', () => { + const window = new ContextWindow(2); + const messages: Message[] = []; + expect(window.truncate(messages)).toBe(messages); + }); + + // Regression: ChatEngine injects a synthetic `User approved: yes` user + // message after an assistant tool-call and its confirm result. Treating + // that as a real turn boundary would discard the call and result. + it('does not cut at the synthetic approval message (defers instead)', () => { + const window = new ContextWindow(3); + const messages = [ + system, + user('delete site 1'), + assistant('tc-delete'), + tool('delete-site-v1'), + user('User approved: yes'), + ]; + + // Only boundary at/after the ideal cut is the synthetic approval, whose + // preceding message is a tool result — not a real turn start, so defer. + expect(window.truncate(messages)).toBe(messages); + }); + + it.each([1, 2, 3])( + 'never isolates an approval fragment with maxMessages %i', + (maxMessages) => { + const window = new ContextWindow(maxMessages); + const messages = [ + system, + user('delete site 1'), + assistant('tc-delete'), + tool('delete-site-v1'), + user('User approved: yes'), + ]; + + const result = window.truncate(messages); + + expect(result).toBe(messages); + expect(result).not.toEqual([system, user('User approved: yes')]); + } + ); + + it('catches up cleanly at the next real user turn after an approval', () => { + const window = new ContextWindow(3); + const messages = [ + system, + user('delete site 1'), + assistant('tc-delete'), + tool('delete-site-v1'), + user('User approved: yes'), + assistant('done'), + user('next question'), + ]; + + const result = window.truncate(messages); + + expect(result).toEqual([system, user('next question')]); + // No orphaned tool result + for (let i = 1; i < result.length; i++) { + if (result[i]!.role === 'tool') { + expect(result[i - 1]!.role).toBe('assistant'); + } + } + }); + }); +}); diff --git a/src/chat/context-window.ts b/src/chat/context-window.ts new file mode 100644 index 0000000..cb9ab76 --- /dev/null +++ b/src/chat/context-window.ts @@ -0,0 +1,108 @@ +/** + * Context window management for chat message history. + * + * Extracted from ChatEngine so the truncation boundary logic is unit-testable + * without constructing a full engine. + */ + +import type { Message } from './providers/provider.js'; + +/** + * Sliding-window truncation over a message history. + * + * INVARIANT: A cut is only safe immediately before a `user` message. + * Cutting anywhere else can orphan a `tool` result from its preceding + * assistant tool call, which providers reject on the next call (Anthropic + * returns 400 for a tool_result with no matching tool_use; OpenAI-compatible + * APIs reject unmatched tool_call_ids). + */ +export class ContextWindow { + /** + * @param maxMessages - Maximum messages to keep (excluding system prompt). + * undefined or 0 = unlimited. + */ + constructor(private readonly maxMessages: number | undefined) { + if (maxMessages !== undefined && maxMessages < 0) { + throw new RangeError('maxMessages must be non-negative'); + } + } + + /** + * Check if the history exceeds the configured limit. + */ + shouldTruncate(messages: Message[]): boolean { + if (this.maxMessages === undefined || this.maxMessages === 0) { + return false; // No limit configured or explicitly unlimited + } + return messages.length - 1 > this.maxMessages; + } + + /** + * Truncate history using a sliding window approach. Preserves: + * - System prompt (always first message) + * - Complete exchanges (user-assistant, tool call-result pairs), by only + * cutting immediately before a `user` message + * + * If no safe boundary exists at or after the ideal cut point (mid + * tool-calling loop, when only assistant/tool messages follow), truncation + * is skipped for this cycle rather than making an unsafe cut — the next + * user turn provides a safe boundary and the window catches up then. + * + * @returns A new truncated array, or the original array unchanged when no + * truncation occurs. + */ + truncate(messages: Message[]): Message[] { + if (!this.shouldTruncate(messages)) { + return messages; + } + + const systemPrompt = messages[0]; + if (!systemPrompt) { + return messages; + } + + const cutIndex = this.findSafeCut(messages); + if (cutIndex === null) { + return messages; + } + + const truncated = [systemPrompt, ...messages.slice(cutIndex)]; + + if (process.env['DEBUG']) { + console.debug( + `[ContextWindow] Truncated ${messages.length - truncated.length} messages (${messages.length - 1} -> ${truncated.length - 1})` + ); + } + + return truncated; + } + + /** + * Find the first safe cut index at or after the ideal cut point. + * + * A safe boundary is the start of a genuine user turn. A `user` message + * immediately followed by a `tool` message is NOT a genuine turn start — + * cutting there would orphan the result from its assistant tool call. A + * `user` message immediately preceded by a `tool` message is also synthetic: + * it is the approval/decline echo appended by ChatEngine, not a new turn. + * + * @returns Index of the first genuine user-turn boundary at or after the + * ideal cut point (never 0, the system prompt), or null when none exists. + */ + private findSafeCut(messages: Message[]): number | null { + // shouldTruncate() guarantees maxMessages is a positive number here + const idealCut = messages.length - (this.maxMessages as number); + + for (let i = Math.max(1, idealCut); i < messages.length; i++) { + if ( + messages[i]?.role === 'user' && + messages[i - 1]?.role !== 'tool' && + messages[i + 1]?.role !== 'tool' + ) { + return i; + } + } + + return null; + } +} diff --git a/src/chat/providers/anthropic.test.ts b/src/chat/providers/anthropic.test.ts new file mode 100644 index 0000000..f152a10 --- /dev/null +++ b/src/chat/providers/anthropic.test.ts @@ -0,0 +1,158 @@ +/** + * Tests for AnthropicProvider message conversion + * + * Audit-remainders item 4 (2026-07-13): the audit flagged that after a + * declined destructive preview, chat-engine history holds a `tool` result + * followed directly by a `user` message with no assistant turn between, and + * convertMessages() maps the tool result to a `user`-role entry — so the + * request body sent to /v1/messages contains two consecutive `user` entries. + * + * That is NOT a bug. The Messages API does not enforce strict role + * alternation; per the API reference (platform.claude.com/docs/en/api/messages, + * checked 2026-07-13): "Consecutive `user` or `assistant` turns in your + * request will be combined into a single turn." Merging client-side is + * therefore unnecessary, so convertMessages intentionally does not do it. + * The test below pins the current wire shape so a future change to it is a + * conscious decision rather than an accident. + */ + +import { describe, expect, it, vi, beforeEach } from 'vitest'; +import type { Message } from './provider.js'; + +const mockMakeProviderRequest = vi.fn(); + +vi.mock('./provider-fetch.js', () => ({ + makeProviderRequest: (...args: unknown[]) => mockMakeProviderRequest(...args), +})); + +const { createAnthropicProvider } = await import('./anthropic.js'); + +interface CapturedMessage { + role: string; + content: unknown; +} + +function anthropicTextResponse(text: string) { + return { + id: 'msg_test', + type: 'message', + role: 'assistant', + content: [{ type: 'text', text }], + model: 'claude-sonnet-4-6', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }; +} + +describe('AnthropicProvider.convertMessages (via chat)', () => { + beforeEach(() => { + mockMakeProviderRequest.mockReset(); + mockMakeProviderRequest.mockResolvedValue(anthropicTextResponse('ok')); + }); + + function createProvider() { + return createAnthropicProvider({ apiKey: 'test-key' }); + } + + async function captureMessages(history: Message[]): Promise { + await createProvider().chat(history); + const request = mockMakeProviderRequest.mock.calls[0]![0] as { + body: { messages: CapturedMessage[] }; + }; + return request.body.messages; + } + + it('sends consecutive user-role entries after a declined preview (API merges them)', async () => { + // Decline-path history: assistant tool call → tool result → user decline echo + const history: Message[] = [ + { role: 'system', content: 'system prompt' }, + { role: 'user', content: 'delete site 7' }, + { + role: 'assistant', + content: '', + toolCalls: [ + { id: 'call_1', name: 'delete-site-v1', arguments: { site_id: 7 } }, + ], + }, + { + role: 'tool', + content: '{"declined":true}', + toolCallId: 'call_1', + toolName: 'delete-site-v1', + }, + { role: 'user', content: 'User declined the action.' }, + ]; + + const messages = await captureMessages(history); + + expect(messages.map((m) => m.role)).toEqual([ + 'user', + 'assistant', + 'user', // tool result converted to user role + 'user', // decline echo — consecutive user entries are valid; API merges + ]); + expect(messages[1]).toEqual({ + role: 'assistant', + content: [{ + type: 'tool_use', + id: 'call_1', + name: 'delete-site-v1', + input: { site_id: 7 }, + }], + }); + expect(messages[2]).toEqual({ + role: 'user', + content: [{ + type: 'tool_result', + tool_use_id: 'call_1', + content: '{"declined":true}', + }], + }); + }); + + it('keeps a normally alternating history unchanged', async () => { + const history: Message[] = [ + { role: 'system', content: 'system prompt' }, + { role: 'user', content: 'hello' }, + { role: 'assistant', content: 'hi there' }, + { role: 'user', content: 'list my sites' }, + ]; + + const messages = await captureMessages(history); + + expect(messages.map((m) => m.role)).toEqual(['user', 'assistant', 'user']); + expect(messages[0]).toEqual({ + role: 'user', + content: [{ type: 'text', text: 'hello' }], + }); + }); + + it('appends a tool result into a directly preceding user message', async () => { + // Two tool results in a row: the second merges into the user-role entry + // created for the first instead of opening another user entry + const history: Message[] = [ + { role: 'user', content: 'check both sites' }, + { + role: 'assistant', + content: '', + toolCalls: [ + { id: 'call_1', name: 'check-site-v1', arguments: { site_id: 1 } }, + { id: 'call_2', name: 'check-site-v1', arguments: { site_id: 2 } }, + ], + }, + { role: 'tool', content: '{"ok":true}', toolCallId: 'call_1' }, + { role: 'tool', content: '{"ok":true}', toolCallId: 'call_2' }, + ]; + + const messages = await captureMessages(history); + + expect(messages.map((m) => m.role)).toEqual(['user', 'assistant', 'user']); + expect(messages[2]).toEqual({ + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: '{"ok":true}' }, + { type: 'tool_result', tool_use_id: 'call_2', content: '{"ok":true}' }, + ], + }); + }); +}); diff --git a/src/chat/providers/anthropic.ts b/src/chat/providers/anthropic.ts index f233bf2..37415a9 100644 --- a/src/chat/providers/anthropic.ts +++ b/src/chat/providers/anthropic.ts @@ -14,6 +14,7 @@ import { type StreamChunk, type ToolCall, registerProvider, + splitSystemMessage, } from './provider.js'; import { makeProviderRequest } from './provider-fetch.js'; import { readSSEStream } from './sse-reader.js'; @@ -83,15 +84,15 @@ interface AnthropicStreamEvent { * Available Anthropic models */ const ANTHROPIC_MODELS = [ - 'claude-sonnet-4-20250514', - 'claude-3-5-sonnet-20241022', - 'claude-3-5-haiku-20241022', - 'claude-3-opus-20240229', - 'claude-3-sonnet-20240229', - 'claude-3-haiku-20240307', + 'claude-sonnet-4-6', + 'claude-sonnet-4-5-20250929', + 'claude-opus-4-8', + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-haiku-4-5-20251001', ] as const; -const DEFAULT_MODEL = 'claude-sonnet-4-20250514'; +const DEFAULT_MODEL = 'claude-sonnet-4-6'; const API_VERSION = '2023-06-01'; /** @@ -134,9 +135,7 @@ export class AnthropicProvider implements LLMProvider { async chat(messages: Message[], options?: ChatOptions): Promise { const model = options?.model ?? this.defaultModel; - // Extract system message - const systemMessage = messages.find((m) => m.role === 'system'); - const chatMessages = messages.filter((m) => m.role !== 'system'); + const { systemContent, chatMessages } = splitSystemMessage(messages); const requestBody: Record = { model, @@ -144,8 +143,8 @@ export class AnthropicProvider implements LLMProvider { max_tokens: options?.maxTokens ?? 4096, }; - if (systemMessage) { - requestBody['system'] = systemMessage.content; + if (systemContent !== undefined) { + requestBody['system'] = systemContent; } if (options?.temperature !== undefined) { @@ -160,11 +159,14 @@ export class AnthropicProvider implements LLMProvider { requestBody['tools'] = this.convertTools(options.tools); } - const response = await this.makeRequest( - '/v1/messages', - requestBody, - options?.signal - ); + const response = await makeProviderRequest({ + url: `${this.baseUrl}/v1/messages`, + headers: this.getHeaders(), + body: requestBody, + timeout: this.timeout, + signal: options?.signal, + providerName: 'Anthropic', + }); return this.convertResponse(response); } @@ -175,9 +177,7 @@ export class AnthropicProvider implements LLMProvider { ): AsyncGenerator { const model = options?.model ?? this.defaultModel; - // Extract system message - const systemMessage = messages.find((m) => m.role === 'system'); - const chatMessages = messages.filter((m) => m.role !== 'system'); + const { systemContent, chatMessages } = splitSystemMessage(messages); const requestBody: Record = { model, @@ -186,8 +186,8 @@ export class AnthropicProvider implements LLMProvider { stream: true, }; - if (systemMessage) { - requestBody['system'] = systemMessage.content; + if (systemContent !== undefined) { + requestBody['system'] = systemContent; } if (options?.temperature !== undefined) { @@ -234,19 +234,23 @@ export class AnthropicProvider implements LLMProvider { if (event.type === 'content_block_stop') { if (toolId && toolName) { + const accumulatedArgs = toolArgs || '{}'; + let args: unknown = accumulatedArgs; try { - const args = JSON.parse(toolArgs || '{}') as Record; - yield { - toolCall: { - id: toolId, - name: toolName, - arguments: args, - }, - done: false, - }; + args = JSON.parse(accumulatedArgs) as unknown; } catch { - // Invalid JSON, skip + // Preserve the raw accumulated string. The shared tool envelope + // rejects non-object arguments as a protocol error without + // executing the proposed call. } + yield { + toolCall: { + id: toolId, + name: toolName, + arguments: args, + }, + done: false, + }; toolId = ''; toolName = ''; toolArgs = ''; @@ -258,7 +262,11 @@ export class AnthropicProvider implements LLMProvider { return; } } catch { - // Invalid JSON, skip line + // Invalid JSON, skip line — a systematically malformed stream would + // otherwise fail silently, so leave a trail when debugging + if (process.env['DEBUG']) { + console.debug('[Anthropic] Skipped malformed SSE chunk'); + } } } @@ -301,9 +309,21 @@ export class AnthropicProvider implements LLMProvider { }); } } else { + const content: AnthropicContent[] = []; + if (msg.content) { + content.push({ type: 'text', text: msg.content }); + } + for (const toolCall of msg.toolCalls ?? []) { + content.push({ + type: 'tool_use', + id: toolCall.id, + name: toolCall.name, + input: toolCall.arguments, + }); + } result.push({ role: msg.role as 'user' | 'assistant', - content: msg.content, + content: content.length > 0 ? content : msg.content, }); } } @@ -389,24 +409,6 @@ export class AnthropicProvider implements LLMProvider { }; } - /** - * Make API request - */ - private async makeRequest( - endpoint: string, - body: Record, - signal?: AbortSignal - ): Promise { - return makeProviderRequest({ - url: `${this.baseUrl}${endpoint}`, - headers: this.getHeaders(), - body, - timeout: this.timeout, - signal, - providerName: 'Anthropic', - }); - } - } /** diff --git a/src/chat/providers/gemini.test.ts b/src/chat/providers/gemini.test.ts index 5178d3e..24beeb6 100644 --- a/src/chat/providers/gemini.test.ts +++ b/src/chat/providers/gemini.test.ts @@ -41,17 +41,20 @@ describe('C1: Gemini API key not in URL', () => { }); it('sends API key via x-goog-api-key header in chatStream()', async () => { + const reader = { + read: vi.fn() + .mockResolvedValueOnce({ + done: false, + value: new TextEncoder().encode( + 'data: {"candidates":[{"content":{"parts":[{"text":"Hi"}],"role":"model"},"finishReason":"STOP"}]}\n\n' + ), + }) + .mockResolvedValueOnce({ done: true, value: undefined }), + cancel: vi.fn().mockResolvedValue(undefined), + releaseLock: vi.fn(), + }; const mockBody = { - getReader: () => ({ - read: vi.fn() - .mockResolvedValueOnce({ - done: false, - value: new TextEncoder().encode( - 'data: {"candidates":[{"content":{"parts":[{"text":"Hi"}],"role":"model"},"finishReason":"STOP"}]}\n\n' - ), - }) - .mockResolvedValueOnce({ done: true, value: undefined }), - }), + getReader: () => reader, }; mockFetch.mockResolvedValueOnce({ diff --git a/src/chat/providers/gemini.ts b/src/chat/providers/gemini.ts index 1c14a62..a02091b 100644 --- a/src/chat/providers/gemini.ts +++ b/src/chat/providers/gemini.ts @@ -14,6 +14,7 @@ import { type StreamChunk, type ToolCall, registerProvider, + splitSystemMessage, } from './provider.js'; import { makeProviderRequest } from './provider-fetch.js'; import { readSSEStream } from './sse-reader.js'; @@ -33,12 +34,15 @@ type GeminiPart = | { text: string } | { functionCall: { + id?: string; name: string; args: Record; }; + thoughtSignature?: string; } | { functionResponse: { + id?: string; name: string; response: Record; }; @@ -78,13 +82,13 @@ interface GeminiResponse { * Available Gemini models */ const GEMINI_MODELS = [ - 'gemini-2.0-flash-exp', - 'gemini-1.5-pro', - 'gemini-1.5-flash', - 'gemini-1.5-flash-8b', + 'gemini-3.5-flash', + 'gemini-3.1-pro-preview', + 'gemini-3-flash-preview', + 'gemini-3.1-flash-lite', ] as const; -const DEFAULT_MODEL = 'gemini-1.5-flash'; +const DEFAULT_MODEL = 'gemini-3.5-flash'; /** * Gemini provider implementation @@ -127,18 +131,16 @@ export class GeminiProvider implements LLMProvider { async chat(messages: Message[], options?: ChatOptions): Promise { const model = options?.model ?? this.defaultModel; - // Extract system message - const systemMessage = messages.find((m) => m.role === 'system'); - const chatMessages = messages.filter((m) => m.role !== 'system'); + const { systemContent, chatMessages } = splitSystemMessage(messages); const requestBody: Record = { contents: this.convertMessages(chatMessages), }; // System instruction - if (systemMessage) { + if (systemContent !== undefined) { requestBody['systemInstruction'] = { - parts: [{ text: systemMessage.content }], + parts: [{ text: systemContent }], }; } @@ -162,12 +164,14 @@ export class GeminiProvider implements LLMProvider { requestBody['tools'] = [this.convertTools(options.tools)]; } - const endpoint = `/models/${model}:generateContent`; - const response = await this.makeRequest( - endpoint, - requestBody, - options?.signal - ); + const response = await makeProviderRequest({ + url: `${this.baseUrl}/models/${model}:generateContent`, + headers: this.getHeaders(), + body: requestBody, + timeout: this.timeout, + signal: options?.signal, + providerName: 'Gemini', + }); return this.convertResponse(response, model); } @@ -178,17 +182,15 @@ export class GeminiProvider implements LLMProvider { ): AsyncGenerator { const model = options?.model ?? this.defaultModel; - // Extract system message - const systemMessage = messages.find((m) => m.role === 'system'); - const chatMessages = messages.filter((m) => m.role !== 'system'); + const { systemContent, chatMessages } = splitSystemMessage(messages); const requestBody: Record = { contents: this.convertMessages(chatMessages), }; - if (systemMessage) { + if (systemContent !== undefined) { requestBody['systemInstruction'] = { - parts: [{ text: systemMessage.content }], + parts: [{ text: systemContent }], }; } @@ -209,10 +211,7 @@ export class GeminiProvider implements LLMProvider { for await (const data of readSSEStream({ url: `${this.baseUrl}/models/${model}:streamGenerateContent?alt=sse`, - headers: { - 'Content-Type': 'application/json', - 'x-goog-api-key': this.apiKey, - }, + headers: this.getHeaders(), body: requestBody, signal: options?.signal, providerName: 'Gemini', @@ -231,6 +230,9 @@ export class GeminiProvider implements LLMProvider { id: `fc_${Date.now()}`, name: part.functionCall.name, arguments: part.functionCall.args, + ...(part.thoughtSignature !== undefined + ? { thoughtSignature: part.thoughtSignature } + : {}), }, done: false, }; @@ -242,7 +244,11 @@ export class GeminiProvider implements LLMProvider { return; } } catch { - // Invalid JSON, skip line + // Invalid JSON, skip line — a systematically malformed stream would + // otherwise fail silently, so leave a trail when debugging + if (process.env['DEBUG']) { + console.debug('[Gemini] Skipped malformed SSE chunk'); + } } } @@ -260,21 +266,45 @@ export class GeminiProvider implements LLMProvider { if (msg.role === 'tool') { // Function response + const functionResponse: { + id?: string; + name: string; + response: Record; + } = { + name: msg.toolName ?? 'unknown', + response: this.parseToolResponse(msg.content), + }; + if (msg.toolCallId !== undefined) { + functionResponse.id = msg.toolCallId; + } result.push({ role: 'user', parts: [ { - functionResponse: { - name: msg.toolName ?? 'unknown', - response: this.parseToolResponse(msg.content), - }, + functionResponse, }, ], }); } else { + const parts: GeminiPart[] = []; + if (msg.content) { + parts.push({ text: msg.content }); + } + for (const toolCall of msg.toolCalls ?? []) { + parts.push({ + functionCall: { + id: toolCall.id, + name: toolCall.name, + args: toolCall.arguments, + }, + ...(toolCall.thoughtSignature !== undefined + ? { thoughtSignature: toolCall.thoughtSignature } + : {}), + }); + } result.push({ role: msg.role === 'assistant' ? 'model' : 'user', - parts: [{ text: msg.content }], + parts: parts.length > 0 ? parts : [{ text: msg.content }], }); } } @@ -329,9 +359,12 @@ export class GeminiProvider implements LLMProvider { content += part.text; } else if ('functionCall' in part) { toolCalls.push({ - id: `fc_${Date.now()}_${toolCalls.length}`, + id: part.functionCall.id ?? `fc_${Date.now()}_${toolCalls.length}`, name: part.functionCall.name, arguments: part.functionCall.args, + ...(part.thoughtSignature !== undefined + ? { thoughtSignature: part.thoughtSignature } + : {}), }); } } @@ -339,7 +372,10 @@ export class GeminiProvider implements LLMProvider { return { content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - finishReason: this.convertFinishReason(candidate.finishReason), + finishReason: + toolCalls.length > 0 && candidate.finishReason === 'STOP' + ? 'tool_calls' + : this.convertFinishReason(candidate.finishReason), usage: response.usageMetadata ? { promptTokens: response.usageMetadata.promptTokenCount, @@ -370,26 +406,14 @@ export class GeminiProvider implements LLMProvider { } /** - * Make API request + * Get request headers */ - private async makeRequest( - endpoint: string, - body: Record, - signal?: AbortSignal - ): Promise { - return makeProviderRequest({ - url: `${this.baseUrl}${endpoint}`, - headers: { - 'Content-Type': 'application/json', - 'x-goog-api-key': this.apiKey, - }, - body, - timeout: this.timeout, - signal, - providerName: 'Gemini', - }); + private getHeaders(): Record { + return { + 'Content-Type': 'application/json', + 'x-goog-api-key': this.apiKey, + }; } - } /** diff --git a/src/chat/providers/openai-compatible.ts b/src/chat/providers/openai-compatible.ts index d20a315..c29df25 100644 --- a/src/chat/providers/openai-compatible.ts +++ b/src/chat/providers/openai-compatible.ts @@ -16,6 +16,12 @@ import { type ToolCall, } from './provider.js'; import { readSSEStream } from './sse-reader.js'; +import { + assertNoRedirect, + MAX_PROVIDER_ERROR_BODY_BYTES, + readBoundedResponseText, + sanitizeProviderErrorBody, +} from './provider-fetch.js'; /** * OpenAI-compatible API message format @@ -252,25 +258,32 @@ export abstract class OpenAICompatibleProvider implements LLMProvider { // Final chunk if (choice.finish_reason === 'tool_calls') { for (const [, tc] of toolCalls) { + let args: unknown = tc.arguments; try { - const args = JSON.parse(tc.arguments) as Record; - yield { - toolCall: { - id: tc.id, - name: tc.name, - arguments: args, - }, - done: false, - }; + args = JSON.parse(tc.arguments) as unknown; } catch { - // Invalid JSON, skip + // Preserve the raw accumulated string. The shared tool envelope + // rejects non-object arguments as a protocol error without + // executing the proposed call. } + yield { + toolCall: { + id: tc.id, + name: tc.name, + arguments: args, + }, + done: false, + }; } yield { done: true }; return; } } catch { - // Invalid JSON, skip line + // Invalid JSON, skip line — a systematically malformed stream would + // otherwise fail silently, so leave a trail when debugging + if (process.env['DEBUG']) { + console.debug(`[${this.name}] Skipped malformed SSE chunk`); + } } } @@ -291,6 +304,18 @@ export abstract class OpenAICompatibleProvider implements LLMProvider { base.tool_call_id = msg.toolCallId; } + if (msg.role === 'assistant' && msg.toolCalls) { + base.content = msg.content || null; + base.tool_calls = msg.toolCalls.map((toolCall) => ({ + id: toolCall.id, + type: 'function' as const, + function: { + name: toolCall.name, + arguments: JSON.stringify(toolCall.arguments), + }, + })); + } + return base; }); } @@ -369,11 +394,11 @@ export abstract class OpenAICompatibleProvider implements LLMProvider { /** * Parse tool call arguments JSON */ - protected parseArguments(args: string): Record { + protected parseArguments(args: string): unknown { try { - return JSON.parse(args) as Record; + return JSON.parse(args) as unknown; } catch { - return {}; + return args; } } @@ -406,11 +431,20 @@ export abstract class OpenAICompatibleProvider implements LLMProvider { headers: this.getHeaders(), body: JSON.stringify(body), signal: combinedSignal, + redirect: 'manual', }); + assertNoRedirect(response, this.name); + if (!response.ok) { - const error = await response.text(); - throw new Error(`${this.name} API error: ${response.status} ${error}`); + const error = await readBoundedResponseText( + response, + MAX_PROVIDER_ERROR_BODY_BYTES, + combinedSignal, + ); + throw new Error( + `${this.name} API error: ${response.status} ${sanitizeProviderErrorBody(error)}` + ); } return (await response.json()) as T; diff --git a/src/chat/providers/provider-fetch.test.ts b/src/chat/providers/provider-fetch.test.ts new file mode 100644 index 0000000..344b3ee --- /dev/null +++ b/src/chat/providers/provider-fetch.test.ts @@ -0,0 +1,140 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + assertNoRedirect, + makeProviderRequest, + readBoundedResponseText, + sanitizeProviderErrorBody, +} from './provider-fetch.js'; + +// Mock fetch globally +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +describe('sanitizeProviderErrorBody', () => { + it('strips terminal control characters', () => { + expect(sanitizeProviderErrorBody('\x1b]0;Injected\x07failure')).toBe('failure'); + }); + + it('redacts values of sensitive-looking keys in JSON-shaped bodies', () => { + const body = '{"error":"bad request","api_key":"sk-live-12345","nested":{"authToken": "abc"},"count":2}'; + const sanitized = sanitizeProviderErrorBody(body); + + expect(sanitized).not.toContain('sk-live-12345'); + expect(sanitized).not.toContain('abc"'); + expect(sanitized).toContain('"api_key":"[REDACTED]"'); + expect(sanitized).toContain('"authToken": "[REDACTED]"'); + expect(sanitized).toContain('"error":"bad request"'); + expect(sanitized).toContain('"count":2'); + }); + + it('truncates response bodies to 500 characters', () => { + const output = sanitizeProviderErrorBody('x'.repeat(501)); + + expect(output).toHaveLength(503); + expect(output).toBe(`${'x'.repeat(500)}...`); + }); +}); + +describe('readBoundedResponseText', () => { + it('stops reading and cancels after the byte limit', async () => { + let cancelled = false; + const response = new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(12)); + controller.enqueue(new Uint8Array(12)); + }, + cancel() { + cancelled = true; + }, + })); + + const text = await readBoundedResponseText(response, 16); + + expect(Buffer.byteLength(text)).toBe(16); + expect(cancelled).toBe(true); + }); + + it('cancels a stalled error body when its signal aborts', async () => { + let cancelled = false; + const controller = new AbortController(); + const response = new Response(new ReadableStream({ + start() { + // Never produce a chunk. + }, + cancel() { + cancelled = true; + }, + })); + + const read = readBoundedResponseText(response, 16, controller.signal); + controller.abort(); + + await expect(read).rejects.toThrow(/aborted/i); + expect(cancelled).toBe(true); + }); +}); + +describe('assertNoRedirect', () => { + it('throws on a redirect response, naming the redirect and the location', () => { + const response = { + status: 302, + headers: { get: () => 'https://elsewhere.example' }, + } as unknown as Response; + + expect(() => assertNoRedirect(response, 'openai')).toThrow( + /redirect.*elsewhere\.example/i + ); + }); + + it('does not throw on a success response', () => { + const response = { + status: 200, + headers: { get: () => null }, + } as unknown as Response; + + expect(() => assertNoRedirect(response, 'openai')).not.toThrow(); + }); + + it('does not throw on a not-found response', () => { + const response = { + status: 404, + headers: { get: () => null }, + } as unknown as Response; + + expect(() => assertNoRedirect(response, 'openai')).not.toThrow(); + }); +}); + +describe('makeProviderRequest', () => { + beforeEach(() => { + mockFetch.mockReset(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('rejects a redirect response and requests fetch without following it', async () => { + mockFetch.mockResolvedValueOnce({ + status: 307, + ok: false, + headers: new Headers({ location: 'https://elsewhere.example' }), + }); + + await expect( + makeProviderRequest({ + url: 'https://api.openai.com/v1/chat/completions', + headers: { authorization: 'Bearer test-key' }, + body: {}, + timeout: 5000, + providerName: 'openai', + }) + ).rejects.toThrow(/redirect/i); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.openai.com/v1/chat/completions', + expect.objectContaining({ redirect: 'manual' }) + ); + }); +}); diff --git a/src/chat/providers/provider-fetch.ts b/src/chat/providers/provider-fetch.ts index da89ed5..3f7c37b 100644 --- a/src/chat/providers/provider-fetch.ts +++ b/src/chat/providers/provider-fetch.ts @@ -6,6 +6,111 @@ */ import { stripControlChars } from '../../utils/terminal-sanitizer.js'; +import { isSensitiveKey } from '../../utils/redaction.js'; +import type { ReadableStreamReadResult } from 'node:stream/web'; + +export const MAX_PROVIDER_ERROR_BODY_BYTES = 16 * 1024; + +/** + * Redact values of sensitive-looking keys in JSON-shaped error text, e.g. + * {"api_key":"secret"}. Regex-based rather than JSON.parse so it also works + * on truncated or almost-JSON bodies; key sensitivity comes from the shared + * redaction list. + */ +function redactJsonLikeSecrets(text: string): string { + // Value alternatives: a JSON string, or a bare scalar (number/bool/null). + // Object/array openers are deliberately excluded so a non-sensitive key + // with an object value doesn't swallow the nested keys inside it. + return text.replace( + /"([^"\\]{1,64})"(\s*:\s*)("(?:[^"\\]|\\.)*"|[^,{}[\]\s"]+)/g, + (match, key: string, sep: string) => + isSensitiveKey(key) ? `"${key}"${sep}"[REDACTED]"` : match + ); +} + +export function sanitizeProviderErrorBody(errorText: string): string { + const sanitized = redactJsonLikeSecrets(stripControlChars(errorText)); + return sanitized.length > 500 ? sanitized.slice(0, 500) + '...' : sanitized; +} + +/** + * Reject redirect responses on provider requests. Following a redirect would + * re-send the Authorization header (the user's API key) to whatever origin + * the response names — same policy as the Dashboard transport's manual + * redirect handling. All provider fetches use `redirect: 'manual'` and call + * this on the response. + */ +export function assertNoRedirect(response: Response, providerName: string): void { + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + const target = location ? ` to "${sanitizeProviderErrorBody(location)}"` : ''; + throw new Error( + `${providerName} API error: unexpected redirect (${response.status})${target} — provider requests never follow redirects` + ); + } +} + +export async function readBoundedResponseText( + response: Response, + maxBytes = MAX_PROVIDER_ERROR_BODY_BYTES, + signal?: AbortSignal +): Promise { + if (!response.body) { + const text = await response.text(); + return Buffer.from(text, 'utf8').subarray(0, maxBytes).toString('utf8'); + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + + try { + while (totalBytes < maxBytes) { + const { done, value } = await readWithAbort(reader, signal); + if (done) break; + + const remaining = maxBytes - totalBytes; + const chunk = value.byteLength > remaining ? value.subarray(0, remaining) : value; + chunks.push(chunk); + totalBytes += chunk.byteLength; + + if (value.byteLength > remaining || totalBytes >= maxBytes) { + await reader.cancel().catch(() => {}); + break; + } + } + } catch (error) { + await reader.cancel().catch(() => {}); + throw error; + } finally { + reader.releaseLock(); + } + + return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)), totalBytes).toString('utf8'); +} + +function readWithAbort( + reader: ReadableStreamDefaultReader, + signal?: AbortSignal +): Promise> { + if (!signal) return reader.read(); + if (signal.aborted) return Promise.reject(new Error('Provider response read aborted')); + + return new Promise>((resolve, reject) => { + const onAbort = (): void => reject(new Error('Provider response read aborted')); + signal.addEventListener('abort', onAbort, { once: true }); + reader.read().then( + (result) => { + signal.removeEventListener('abort', onAbort); + resolve(result); + }, + (error: unknown) => { + signal.removeEventListener('abort', onAbort); + reject(error); + }, + ); + }); +} export async function makeProviderRequest(options: { url: string; @@ -28,15 +133,21 @@ export async function makeProviderRequest(options: { headers: options.headers, body: JSON.stringify(options.body), signal: combinedSignal, + redirect: 'manual', }); + assertNoRedirect(response, options.providerName); + if (!response.ok) { - const errorText = await response.text(); + const errorText = await readBoundedResponseText( + response, + MAX_PROVIDER_ERROR_BODY_BYTES, + combinedSignal, + ); // SECURITY: Strip control characters and truncate to prevent exfiltration // of large payloads from untrusted API error bodies - const sanitized = stripControlChars(errorText); - const truncated = sanitized.length > 500 ? sanitized.slice(0, 500) + '...' : sanitized; - throw new Error(`${options.providerName} API error: ${response.status} ${truncated}`); + const sanitized = sanitizeProviderErrorBody(errorText); + throw new Error(`${options.providerName} API error: ${response.status} ${sanitized}`); } return (await response.json()) as T; diff --git a/src/chat/providers/provider.test.ts b/src/chat/providers/provider.test.ts index c89ebe5..93770d2 100644 --- a/src/chat/providers/provider.test.ts +++ b/src/chat/providers/provider.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it } from 'vitest'; -import { resolveProviderSelection } from './provider.js'; +import { abilityToTool, resolveProviderSelection, validateProviderBaseUrl } from './provider.js'; +import { ConfigError } from '../../utils/errors.js'; describe('resolveProviderSelection', () => { afterEach(() => { @@ -50,4 +51,180 @@ describe('resolveProviderSelection', () => { expect(result.source).toBe('auto'); expect(result.warnings[0]).toMatch(/Ignoring unsupported LLM provider/); }); + + it.each(['file:///tmp/provider', 'ftp://provider.example.com', 'not-a-url'])( + 'rejects an unsafe custom base URL before provider creation: %s', + (baseUrl) => { + expect(() => resolveProviderSelection({ + flagProvider: 'local', + apiKey: 'test-key', + baseUrl, + })).toThrow(/base URL/i); + }, + ); + + it.each(['http://127.0.0.1:11434/v1', 'https://provider.example.com/v1'])( + 'accepts an HTTP(S) custom base URL: %s', + (baseUrl) => { + const result = resolveProviderSelection({ + flagProvider: 'local', + apiKey: 'test-key', + baseUrl, + }); + expect(result.config.baseUrl).toBe(baseUrl); + }, + ); + + it('warns when a custom base URL overrides a hosted provider endpoint', () => { + const result = resolveProviderSelection({ + flagProvider: 'openai', + apiKey: 'sk-test-openai', + baseUrl: 'https://proxy.example', + }); + + expect(result.warnings).toEqual([ + expect.stringMatching(/Custom base URL overrides.*proxy\.example/), + ]); + }); +}); + +describe('validateProviderBaseUrl', () => { + it('rejects cleartext HTTP for a hosted provider', () => { + expect(() => + validateProviderBaseUrl('http://evil.example.com', { provider: 'openai' }) + ).toThrow(ConfigError); + }); + + it('accepts HTTPS for a hosted provider and warns about the host', () => { + const warnings: string[] = []; + + const result = validateProviderBaseUrl('https://proxy.corp.example', { + provider: 'openai', + warnings, + }); + + expect(result).toBe('https://proxy.corp.example'); + expect(warnings).toEqual([ + expect.stringMatching(/proxy\.corp\.example/), + ]); + expect(warnings[0]).toMatch(/openai/); + }); + + it('accepts cleartext HTTP to localhost for the local provider without warning', () => { + const warnings: string[] = []; + + const result = validateProviderBaseUrl('http://localhost:8080/v1', { + provider: 'local', + warnings, + }); + + expect(result).toBe('http://localhost:8080/v1'); + expect(warnings).toEqual([]); + }); + + it('accepts cleartext HTTP to private-network hosts for the local provider', () => { + expect( + validateProviderBaseUrl('http://192.168.1.50:8080', { provider: 'local' }) + ).toBe('http://192.168.1.50:8080'); + expect( + validateProviderBaseUrl('http://172.20.0.5', { provider: 'local' }) + ).toBe('http://172.20.0.5'); + }); + + it('rejects cleartext HTTP outside the 172.16-31 private range for the local provider', () => { + expect(() => + validateProviderBaseUrl('http://172.32.0.1', { provider: 'local' }) + ).toThrow(ConfigError); + }); + + it('rejects cleartext HTTP to a public host for the local provider', () => { + expect(() => + validateProviderBaseUrl('http://myserver.example.com', { provider: 'local' }) + ).toThrow(ConfigError); + }); + + it('accepts HTTPS with no provider context (back-compat)', () => { + expect(validateProviderBaseUrl('https://anything.example')).toBe( + 'https://anything.example' + ); + }); +}); + +describe('abilityToTool', () => { + const EMPTY_OBJECT_SCHEMA = { type: 'object', properties: {} }; + + it('passes a valid object schema through unchanged', () => { + const schema = { + type: 'object', + properties: { site_id: { type: 'integer' } }, + required: ['site_id'], + }; + + const tool = abilityToTool('mainwp/get-site-v1', 'Get a site', schema); + + expect(tool.parameters).toEqual(schema); + }); + + it('defaults to an empty object schema when input schema is undefined', () => { + const tool = abilityToTool('core/no-input', 'No input', undefined); + + expect(tool.parameters).toEqual(EMPTY_OBJECT_SCHEMA); + }); + + it('normalizes a PHP empty-array schema to an empty object schema', () => { + const tool = abilityToTool( + 'core/get-environment-info', + 'Env info', + [] as unknown as Record + ); + + expect(tool.parameters).toEqual(EMPTY_OBJECT_SCHEMA); + }); + + it('coerces a nullable top-level type array to plain object', () => { + const schema = { + type: ['object', 'null'], + properties: { page: { type: 'integer' } }, + }; + + const tool = abilityToTool('mainwp/list-sites-v1', 'List sites', schema); + + expect(tool.parameters['type']).toBe('object'); + expect(tool.parameters['properties']).toEqual(schema.properties); + // Original schema object must not be mutated + expect(schema.type).toEqual(['object', 'null']); + }); + + it('normalizes nested PHP empty-array properties to empty objects', () => { + const schema = { + type: ['object', 'null'], + properties: [] as unknown as Record, + additionalProperties: false, + }; + + const tool = abilityToTool('mainwp/get-network-snapshot-v1', 'Snapshot', schema); + + expect(tool.parameters).toEqual({ + type: 'object', + properties: {}, + additionalProperties: false, + }); + }); + + it('keeps nested type arrays and legal empty-array defaults intact', () => { + const schema = { + type: 'object', + properties: { + tag_ids: { + type: ['array', 'null'], + items: { type: 'integer' }, + default: [], + }, + }, + }; + + const tool = abilityToTool('mainwp/count-sites-v1', 'Count sites', schema); + + expect(tool.parameters).toEqual(schema); + }); }); diff --git a/src/chat/providers/provider.ts b/src/chat/providers/provider.ts index 85141b9..b5ecb60 100644 --- a/src/chat/providers/provider.ts +++ b/src/chat/providers/provider.ts @@ -5,6 +5,11 @@ * Supported: OpenAI, Anthropic, Gemini, OpenRouter, Local (OpenAI-compatible) */ +import { sanitizeInputSchema } from '../../validation/sanitize-schema.js'; +import { ConfigError } from '../../utils/errors.js'; + +export { sanitizeInputSchema } from '../../validation/sanitize-schema.js'; + /** * Message role in conversation */ @@ -16,6 +21,14 @@ export type MessageRole = 'system' | 'user' | 'assistant' | 'tool'; export interface Message { role: MessageRole; content: string; + /** Native assistant tool calls that must precede matching tool results */ + toolCalls?: Array<{ + id: string; + name: string; + arguments: Record; + /** Opaque Gemini thought signature pass-through; other providers ignore it. */ + thoughtSignature?: string; + }>; /** Tool call ID (for tool responses) */ toolCallId?: string; /** Tool name (for tool responses) */ @@ -37,7 +50,10 @@ export interface ToolDefinition { export interface ToolCall { id: string; name: string; - arguments: Record; + /** Kept unknown until the envelope parser proves it is an object */ + arguments: unknown; + /** Opaque Gemini thought signature pass-through; other providers ignore it. */ + thoughtSignature?: string; } /** @@ -331,7 +347,10 @@ export function resolveProviderSelection(options: { const envConfig = getProviderConfigFromEnv(selectedName) ?? {}; const apiKey = options.apiKey ?? envConfig.apiKey ?? ''; - const baseUrl = options.baseUrl ?? envConfig.baseUrl; + const baseUrl = validateProviderBaseUrl(options.baseUrl ?? envConfig.baseUrl, { + provider: selectedName, + warnings, + }); return { name: selectedName, @@ -347,6 +366,82 @@ export function resolveProviderSelection(options: { }; } +/** + * Loopback/private-network hostnames where cleartext HTTP to a local LLM is + * an accepted tradeoff. Everything else must use TLS. + */ +function isPrivateHostname(hostname: string): boolean { + const host = hostname.toLowerCase(); + return ( + host === 'localhost' || + host === '[::1]' || + host === '::1' || + host.endsWith('.localhost') || + /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host) || + /^10\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host) || + /^192\.168\.\d{1,3}\.\d{1,3}$/.test(host) || + /^172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}$/.test(host) + ); +} + +/** + * Validate a custom provider base URL. + * + * Policy (AGENTS.md: API keys go "only to the fixed provider origin", local + * endpoints carry "TLS expectations"): + * - Hosted providers (openai, anthropic, gemini, openrouter): a base-URL + * override must be HTTPS — the API key would otherwise cross the network in + * cleartext — and always draws a warning naming the host the key will be + * sent to. HTTP endpoints belong on the `local` provider. + * - Local provider: HTTP is allowed only for loopback/private-network hosts. + */ +export function validateProviderBaseUrl( + baseUrl: string | undefined, + context?: { provider?: ProviderName; warnings?: string[] } +): string | undefined { + if (baseUrl === undefined) return undefined; + + const normalized = baseUrl.trim(); + let parsed: URL; + try { + parsed = new URL(normalized); + } catch { + throw new ConfigError('Invalid provider base URL. Use an absolute HTTP(S) URL.'); + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new ConfigError('Invalid provider base URL scheme. Only HTTP and HTTPS are supported.'); + } + + const provider = context?.provider; + const isHostedProvider = provider !== undefined && provider !== 'local'; + + if (parsed.protocol === 'http:') { + if (isHostedProvider) { + throw new ConfigError( + `Cleartext HTTP base URL is not allowed for the ${provider} provider — the API key and chat data would cross the network unencrypted.`, + undefined, + 'Use an https:// URL, or use --provider local for a local OpenAI-compatible endpoint.' + ); + } + if (!isPrivateHostname(parsed.hostname)) { + throw new ConfigError( + `Cleartext HTTP base URL to a non-private host ("${parsed.hostname}") is not allowed.`, + undefined, + 'Use https://, or point at a localhost/private-network address.' + ); + } + } + + if (isHostedProvider && context?.warnings) { + context.warnings.push( + `Custom base URL overrides the fixed ${provider} endpoint — the ${provider} API key and chat data will be sent to ${parsed.host}.` + ); + } + + return normalized; +} + /** * Auto-detect configured provider from environment */ @@ -364,6 +459,24 @@ export function detectConfiguredProvider(): string | undefined { return undefined; } +/** + * Split the system message from the chat messages. + * + * Providers that carry the system prompt out-of-band (Anthropic's `system` + * field, Gemini's `systemInstruction`) share this instead of re-implementing + * the extraction. + */ +export function splitSystemMessage(messages: Message[]): { + systemContent: string | undefined; + chatMessages: Message[]; +} { + const systemMessage = messages.find((m) => m.role === 'system'); + return { + systemContent: systemMessage?.content, + chatMessages: messages.filter((m) => m.role !== 'system'), + }; +} + /** * Convert ability schema to tool definition */ @@ -375,6 +488,6 @@ export function abilityToTool( return { name, description, - parameters: inputSchema ?? { type: 'object', properties: {} }, + parameters: sanitizeInputSchema(inputSchema), }; } diff --git a/src/chat/providers/sse-reader.test.ts b/src/chat/providers/sse-reader.test.ts new file mode 100644 index 0000000..5031c67 --- /dev/null +++ b/src/chat/providers/sse-reader.test.ts @@ -0,0 +1,101 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + readSSEStream, + SSE_IDLE_TIMEOUT_MS, + SSE_MAX_DURATION_MS, +} from './sse-reader.js'; + +describe('readSSEStream bounds', () => { + beforeEach(() => { + vi.stubGlobal('fetch', vi.fn()); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it('rejects an oversized unterminated SSE line', async () => { + vi.mocked(fetch).mockResolvedValue(new Response( + `data: ${'x'.repeat(1024 * 1024 + 1)}`, + { status: 200 }, + )); + + const stream = readSSEStream({ + url: 'https://provider.example.com/stream', + headers: {}, + body: {}, + providerName: 'TestProvider', + }); + + await expect(stream.next()).rejects.toThrow(/buffer limit/i); + }); + + it('rejects when no stream data arrives before the idle timeout', async () => { + vi.useFakeTimers(); + vi.mocked(fetch).mockResolvedValue(new Response(new ReadableStream({ + start() { + // Never produce a chunk. + }, + }), { status: 200 })); + + const stream = readSSEStream({ + url: 'https://provider.example.com/stream', + headers: {}, + body: {}, + providerName: 'TestProvider', + }); + const next = stream.next(); + const rejection = expect(next).rejects.toThrow(/idle timeout/i); + await vi.advanceTimersByTimeAsync(SSE_IDLE_TIMEOUT_MS + 1); + + await rejection; + }); + + it('applies an abort signal before response headers arrive', async () => { + const controller = new AbortController(); + vi.mocked(fetch).mockImplementation((_url, init) => new Promise((_resolve, reject) => { + const signal = init?.signal; + signal?.addEventListener('abort', () => reject(new Error('fetch aborted')), { once: true }); + })); + + const stream = readSSEStream({ + url: 'https://provider.example.com/stream', + headers: {}, + body: {}, + providerName: 'TestProvider', + signal: controller.signal, + }); + const next = stream.next(); + const rejection = expect(next).rejects.toThrow(/aborted/i); + controller.abort(); + + await rejection; + }); + + it('rejects a periodically active stream at the maximum duration', async () => { + vi.useFakeTimers(); + let controller: ReadableStreamDefaultController; + vi.mocked(fetch).mockResolvedValue(new Response(new ReadableStream({ + start(streamController) { + controller = streamController; + }, + }), { status: 200 })); + + const stream = readSSEStream({ + url: 'https://provider.example.com/stream', + headers: {}, + body: {}, + providerName: 'TestProvider', + }); + + for (let elapsed = 0; elapsed < SSE_MAX_DURATION_MS; elapsed += 30_000) { + const next = stream.next(); + controller!.enqueue(new TextEncoder().encode('data: {}\n')); + await expect(next).resolves.toMatchObject({ value: '{}', done: false }); + await vi.advanceTimersByTimeAsync(30_000); + } + + await expect(stream.next()).rejects.toThrow(/maximum duration/i); + }); +}); diff --git a/src/chat/providers/sse-reader.ts b/src/chat/providers/sse-reader.ts index 79c3dd4..ff6ec9c 100644 --- a/src/chat/providers/sse-reader.ts +++ b/src/chat/providers/sse-reader.ts @@ -6,6 +6,16 @@ * for provider-specific interpretation. */ +import { + assertNoRedirect, + readBoundedResponseText, + sanitizeProviderErrorBody, +} from './provider-fetch.js'; + +export const MAX_SSE_LINE_BUFFER_BYTES = 1024 * 1024; +export const SSE_IDLE_TIMEOUT_MS = 60_000; +export const SSE_MAX_DURATION_MS = 5 * 60_000; + /** * Make an SSE streaming request and yield raw JSON strings from "data: " lines. * @@ -19,21 +29,28 @@ export async function* readSSEStream(options: { signal?: AbortSignal | undefined; providerName: string; }): AsyncGenerator { + const deadline = Date.now() + SSE_MAX_DURATION_MS; + const durationSignal = AbortSignal.timeout(SSE_MAX_DURATION_MS); + const combinedSignal = options.signal + ? AbortSignal.any([options.signal, durationSignal]) + : durationSignal; const fetchOptions: RequestInit = { method: 'POST', headers: options.headers, body: JSON.stringify(options.body), + signal: combinedSignal, + redirect: 'manual', }; - if (options.signal) { - fetchOptions.signal = options.signal; - } - const response = await fetch(options.url, fetchOptions); + assertNoRedirect(response, options.providerName); + if (!response.ok) { - const error = await response.text(); - throw new Error(`${options.providerName} API error: ${response.status} ${error}`); + const error = await readBoundedResponseText(response, undefined, combinedSignal); + throw new Error( + `${options.providerName} API error: ${response.status} ${sanitizeProviderErrorBody(error)}` + ); } if (!response.body) { @@ -44,17 +61,83 @@ export async function* readSSEStream(options: { const decoder = new TextDecoder(); let buffer = ''; - while (true) { - const { done, value } = await reader.read(); - if (done) break; + try { + while (true) { + const { done, value } = await readWithIdleTimeout( + reader, + options.providerName, + options.signal, + deadline + ); + if (done) break; + if (!value) continue; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n'); - buffer = lines.pop() ?? ''; + for (const line of lines) { + if (Buffer.byteLength(line, 'utf8') > MAX_SSE_LINE_BUFFER_BYTES) { + throw new Error(`${options.providerName} SSE line buffer limit exceeded`); + } + if (!line.startsWith('data: ')) continue; + yield line.slice(6); + } - for (const line of lines) { - if (!line.startsWith('data: ')) continue; - yield line.slice(6); + if (Buffer.byteLength(buffer, 'utf8') > MAX_SSE_LINE_BUFFER_BYTES) { + throw new Error(`${options.providerName} SSE line buffer limit exceeded`); + } } + } catch (error) { + void reader.cancel().catch(() => {}); + throw error; + } finally { + reader.releaseLock(); } } + +function readWithIdleTimeout( + reader: ReadableStreamDefaultReader, + providerName: string, + signal: AbortSignal | undefined, + deadline: number +): Promise<{ done: boolean; value?: Uint8Array }> { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new Error(`${providerName} SSE stream aborted`)); + return; + } + + const remainingDuration = deadline - Date.now(); + if (remainingDuration <= 0) { + reject(new Error(`${providerName} SSE maximum duration exceeded`)); + return; + } + + const timeoutMs = Math.min(SSE_IDLE_TIMEOUT_MS, remainingDuration); + const timeoutMessage = remainingDuration <= SSE_IDLE_TIMEOUT_MS + ? `${providerName} SSE maximum duration exceeded` + : `${providerName} SSE idle timeout exceeded`; + const timeoutId = setTimeout(() => { + reject(new Error(timeoutMessage)); + }, timeoutMs); + const onAbort = (): void => { + clearTimeout(timeoutId); + reject(new Error(`${providerName} SSE stream aborted`)); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + + reader.read().then( + (result) => { + clearTimeout(timeoutId); + signal?.removeEventListener('abort', onAbort); + resolve(result); + }, + (error: unknown) => { + clearTimeout(timeoutId); + signal?.removeEventListener('abort', onAbort); + reject(error); + } + ); + }); +} diff --git a/src/chat/providers/streamed-tool-arguments.test.ts b/src/chat/providers/streamed-tool-arguments.test.ts new file mode 100644 index 0000000..977d099 --- /dev/null +++ b/src/chat/providers/streamed-tool-arguments.test.ts @@ -0,0 +1,76 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +let streamData: string[] = []; +vi.mock('./sse-reader.js', () => ({ + readSSEStream: async function* (): AsyncGenerator { + for (const chunk of streamData) yield chunk; + }, +})); + +import { OpenAIProvider } from './openai.js'; +import { AnthropicProvider } from './anthropic.js'; +import type { StreamChunk } from './provider.js'; + +async function collect(stream: AsyncGenerator): Promise { + const chunks: StreamChunk[] = []; + for await (const chunk of stream) chunks.push(chunk); + return chunks; +} + +describe('streamed malformed tool arguments', () => { + beforeEach(() => { + streamData = []; + }); + + it('surfaces raw malformed OpenAI arguments for protocol rejection', async () => { + streamData = [ + JSON.stringify({ + id: 'response-1', + model: 'test-model', + choices: [{ + index: 0, + delta: { + tool_calls: [{ + index: 0, + id: 'call_bad', + function: { name: 'mainwp__list-sites-v1', arguments: '{bad' }, + }], + }, + finish_reason: null, + }], + }), + JSON.stringify({ + id: 'response-1', + model: 'test-model', + choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }], + }), + ]; + + const chunks = await collect(new OpenAIProvider({ apiKey: 'test-key' }).chatStream([])); + + expect(chunks).toContainEqual(expect.objectContaining({ + toolCall: expect.objectContaining({ arguments: '{bad' }), + })); + }); + + it('surfaces raw malformed Anthropic arguments for protocol rejection', async () => { + streamData = [ + JSON.stringify({ + type: 'content_block_start', + content_block: { type: 'tool_use', id: 'call_bad', name: 'mainwp__list-sites-v1' }, + }), + JSON.stringify({ + type: 'content_block_delta', + delta: { type: 'input_json_delta', partial_json: '{bad' }, + }), + JSON.stringify({ type: 'content_block_stop' }), + JSON.stringify({ type: 'message_stop' }), + ]; + + const chunks = await collect(new AnthropicProvider({ apiKey: 'test-key' }).chatStream([])); + + expect(chunks).toContainEqual(expect.objectContaining({ + toolCall: expect.objectContaining({ arguments: '{bad' }), + })); + }); +}); diff --git a/src/chat/providers/tool-protocol.test.ts b/src/chat/providers/tool-protocol.test.ts new file mode 100644 index 0000000..3292751 --- /dev/null +++ b/src/chat/providers/tool-protocol.test.ts @@ -0,0 +1,382 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ChatEngine } from '../chat-engine.js'; +import type { Ability, ExecutionOptions, ExecutionResult } from '../../core/abilities-executor.js'; +import type { LLMProvider } from './provider.js'; +import { OpenAIProvider } from './openai.js'; +import { AnthropicProvider } from './anthropic.js'; +import { GeminiProvider } from './gemini.js'; + +vi.mock('../../utils/audit-logger.js', () => ({ + logDestructiveActionSafe: vi.fn(async () => undefined), +})); + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +const readonlyAbility: Ability = { + name: 'mainwp/list-sites-v1', + label: 'List sites', + description: 'List sites', + category: 'sites', + input_schema: { type: 'object', properties: {} }, + meta: { + annotations: { readonly: true, destructive: false, idempotent: true }, + }, +}; + +const destructiveAbility: Ability = { + name: 'mainwp/delete-site-v1', + label: 'Delete site', + description: 'Delete a site', + category: 'sites', + input_schema: { + type: 'object', + properties: { site_id: { type: 'integer' } }, + required: ['site_id'], + }, + meta: { + annotations: { readonly: false, destructive: true, idempotent: false }, + }, +}; + +type WireProvider = 'openai' | 'anthropic' | 'gemini'; + +function okJson(data: unknown): object { + return { + ok: true, + json: async () => data, + }; +} + +function createProvider(name: WireProvider): LLMProvider { + switch (name) { + case 'openai': + return new OpenAIProvider({ apiKey: 'test-key' }); + case 'anthropic': + return new AnthropicProvider({ apiKey: 'test-key' }); + case 'gemini': + return new GeminiProvider({ apiKey: 'test-key' }); + } +} + +function toolResponse(name: WireProvider, toolName: string, id: string): object { + switch (name) { + case 'openai': + return { + id: 'response-1', + model: 'test-model', + choices: [{ + index: 0, + message: { + role: 'assistant', + content: null, + tool_calls: [{ + id, + type: 'function', + function: { name: toolName, arguments: toolName.includes('delete') ? '{"site_id":123}' : '{}' }, + }], + }, + finish_reason: 'tool_calls', + }], + }; + case 'anthropic': + return { + id: 'response-1', + type: 'message', + role: 'assistant', + content: [{ + type: 'tool_use', + id, + name: toolName, + input: toolName.includes('delete') ? { site_id: 123 } : {}, + }], + model: 'test-model', + stop_reason: 'tool_use', + usage: { input_tokens: 1, output_tokens: 1 }, + }; + case 'gemini': + return { + candidates: [{ + content: { + role: 'model', + parts: [{ + functionCall: { + id, + name: toolName, + args: toolName.includes('delete') ? { site_id: 123 } : {}, + }, + }], + }, + finishReason: 'STOP', + }], + }; + } +} + +function answerResponse(name: WireProvider): object { + switch (name) { + case 'openai': + return { + id: 'response-2', + model: 'test-model', + choices: [{ + index: 0, + message: { role: 'assistant', content: '{"answer":"Done"}' }, + finish_reason: 'stop', + }], + }; + case 'anthropic': + return { + id: 'response-2', + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: '{"answer":"Done"}' }], + model: 'test-model', + stop_reason: 'end_turn', + usage: { input_tokens: 1, output_tokens: 1 }, + }; + case 'gemini': + return { + candidates: [{ + content: { role: 'model', parts: [{ text: '{"answer":"Done"}' }] }, + finishReason: 'STOP', + }], + }; + } +} + +function createExecutor( + ability: Ability, + executeHandler?: ( + options?: ExecutionOptions + ) => ExecutionResult +): { + listAbilities: ReturnType; + getAbility: ReturnType; + execute: ReturnType; +} { + return { + listAbilities: vi.fn(async () => [ability]), + getAbility: vi.fn(async (name: string) => name === ability.name ? ability : undefined), + execute: vi.fn(async ( + _name: string, + _input: Record, + options?: ExecutionOptions + ) => executeHandler?.(options) ?? { success: true, data: { ok: true } }), + }; +} + +function requestBody(callIndex: number): Record { + const request = mockFetch.mock.calls[callIndex]?.[1] as { body: string } | undefined; + if (!request) throw new Error(`Missing fetch call ${callIndex}`); + return JSON.parse(request.body) as Record; +} + +function assertAliasedDeclaration(provider: WireProvider, body: Record): void { + const tools = body['tools'] as Array>; + let name: string; + if (provider === 'openai') { + name = ((tools[0]?.['function'] as Record)?.['name']) as string; + } else if (provider === 'anthropic') { + name = tools[0]?.['name'] as string; + } else { + const declarations = tools[0]?.['functionDeclarations'] as Array>; + name = declarations[0]?.['name'] as string; + } + expect(name).toBe('mainwp__list-sites-v1'); + expect(name).not.toContain('/'); +} + +function assertCallResultPair( + provider: WireProvider, + body: Record, + id: string, + alias: string +): void { + if (provider === 'openai') { + const messages = body['messages'] as Array>; + const assistant = messages.find((message) => message['role'] === 'assistant' && message['tool_calls']); + const tool = messages.find((message) => message['role'] === 'tool'); + const call = (assistant?.['tool_calls'] as Array>)?.[0]; + expect(call?.['id']).toBe(id); + expect((call?.['function'] as Record)?.['name']).toBe(alias); + expect(tool?.['tool_call_id']).toBe(id); + return; + } + + if (provider === 'anthropic') { + const messages = body['messages'] as Array>; + const blocks = messages.flatMap((message) => + Array.isArray(message['content']) ? message['content'] as Array> : [] + ); + expect(blocks).toContainEqual(expect.objectContaining({ + type: 'tool_use', id, name: alias, + })); + expect(blocks).toContainEqual(expect.objectContaining({ + type: 'tool_result', tool_use_id: id, + })); + return; + } + + const contents = body['contents'] as Array>; + const parts = contents.flatMap((content) => content['parts'] as Array>); + expect(parts).toContainEqual({ + functionCall: expect.objectContaining({ id, name: alias }), + }); + expect(parts).toContainEqual({ + functionResponse: expect.objectContaining({ id, name: alias }), + }); +} + +describe.each(['openai', 'anthropic', 'gemini'])('%s native tool protocol', (providerName) => { + afterEach(() => { + mockFetch.mockReset(); + }); + + it('aliases declarations and preserves the native call block on continuation', async () => { + const callId = `call_${providerName}_readonly`; + mockFetch + .mockResolvedValueOnce(okJson(toolResponse(providerName, 'mainwp__list-sites-v1', callId))) + .mockResolvedValueOnce(okJson(answerResponse(providerName))); + const executor = createExecutor(readonlyAbility); + const engine = new ChatEngine({ + provider: createProvider(providerName), + executor: executor as never, + }); + + await engine.sendMessage('List sites'); + + assertAliasedDeclaration(providerName, requestBody(0)); + assertCallResultPair( + providerName, + requestBody(1), + callId, + 'mainwp__list-sites-v1' + ); + }); + + it('preserves the original native call id after destructive approval', async () => { + const callId = `call_${providerName}_delete`; + mockFetch + .mockResolvedValueOnce(okJson(toolResponse(providerName, 'mainwp__delete-site-v1', callId))) + .mockResolvedValueOnce(okJson(answerResponse(providerName))); + const executor = createExecutor(destructiveAbility, (options) => + options?.dryRun + ? { success: true, data: { affected: [{ id: 123 }] } } + : { success: true, data: { deleted: true } } + ); + const engine = new ChatEngine({ + provider: createProvider(providerName), + executor: executor as never, + }); + + await engine.sendMessage('Delete site 123'); + await engine.sendMessage('yes'); + await engine.sendMessage('Summarize the result'); + + assertCallResultPair( + providerName, + requestBody(1), + callId, + 'mainwp__delete-site-v1' + ); + }); +}); + +describe('Gemini thought signatures', () => { + afterEach(() => { + mockFetch.mockReset(); + }); + + it('replays a function-call thought signature unchanged on continuation', async () => { + const callId = 'call_gemini_signed'; + const response = toolResponse('gemini', 'mainwp__list-sites-v1', callId) as { + candidates: Array<{ content: { parts: Array> } }>; + }; + response.candidates[0]!.content.parts[0]!['thoughtSignature'] = 'sig-required'; + mockFetch + .mockResolvedValueOnce(okJson(response)) + .mockResolvedValueOnce(okJson(answerResponse('gemini'))); + const engine = new ChatEngine({ + provider: createProvider('gemini'), + executor: createExecutor(readonlyAbility) as never, + }); + + await engine.sendMessage('List sites'); + + const contents = requestBody(1)['contents'] as Array>; + const parts = contents.flatMap((content) => + content['parts'] as Array> + ); + expect(parts).toContainEqual({ + functionCall: expect.objectContaining({ id: callId }), + thoughtSignature: 'sig-required', + }); + }); +}); + +describe('provider model defaults', () => { + it('advertises the active Anthropic models and defaults to Sonnet 4.6', () => { + const provider = new AnthropicProvider({ apiKey: 'test-key' }); + expect(provider.getDefaultModel()).toBe('claude-sonnet-4-6'); + expect(provider.getModels()).toEqual([ + 'claude-sonnet-4-6', + 'claude-sonnet-4-5-20250929', + 'claude-opus-4-8', + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-haiku-4-5-20251001', + ]); + }); + + it('advertises active Gemini text models and defaults to Gemini 3.5 Flash', () => { + const provider = new GeminiProvider({ apiKey: 'test-key' }); + expect(provider.getDefaultModel()).toBe('gemini-3.5-flash'); + expect(provider.getModels()).toEqual([ + 'gemini-3.5-flash', + 'gemini-3.1-pro-preview', + 'gemini-3-flash-preview', + 'gemini-3.1-flash-lite', + ]); + }); +}); + +describe('OpenAI malformed native arguments', () => { + afterEach(() => { + mockFetch.mockReset(); + }); + + it('routes invalid argument JSON through the parse fallback without execution', async () => { + mockFetch.mockResolvedValueOnce(okJson({ + id: 'response-invalid', + model: 'test-model', + choices: [{ + index: 0, + message: { + role: 'assistant', + content: null, + tool_calls: [{ + id: 'call_invalid_json', + type: 'function', + function: { + name: 'mainwp__list-sites-v1', + arguments: '{not-json', + }, + }], + }, + finish_reason: 'tool_calls', + }], + })); + const executor = createExecutor(readonlyAbility); + const engine = new ChatEngine({ + provider: createProvider('openai'), + executor: executor as never, + maxParseRetries: 0, + }); + + const responses = await engine.sendMessage('List sites'); + + expect(responses[0]?.type).toBe('error'); + expect(executor.execute).not.toHaveBeenCalled(); + }); +}); diff --git a/src/chat/system-prompt.test.ts b/src/chat/system-prompt.test.ts new file mode 100644 index 0000000..8c9bf7a --- /dev/null +++ b/src/chat/system-prompt.test.ts @@ -0,0 +1,55 @@ +/** + * Tests for system prompt generation. + * + * The LLM-facing safety labels must come from SafetyController.classify() + * (including the destructive-name override), never raw annotations, so the + * prompt can never call an ability "readonly" that runtime treats as + * destructive. + */ + +import { describe, it, expect } from 'vitest'; +import { buildConfiguredPrompt } from './system-prompt.js'; +import type { Ability } from '../core/abilities-executor.js'; + +const readonlyAbility: Ability = { + name: 'mainwp/list-sites-v1', + label: 'List Sites', + description: 'List all connected sites', + category: 'sites', + meta: { + annotations: { readonly: true, destructive: false, idempotent: true }, + }, +}; + +// Destructive NAME but annotated readonly — a server under-reporting +// destructiveness. classify() upgrades this to destructive. +const mislabelledResetAbility: Ability = { + name: 'mainwp/reset-site-v1', + label: 'Reset Site', + description: 'Reset a site to defaults', + category: 'sites', + meta: { + annotations: { readonly: true, destructive: false, idempotent: true }, + }, +}; + +describe('system prompt safety labels', () => { + it('labels honestly-annotated readonly abilities as readonly', () => { + const prompt = buildConfiguredPrompt([readonlyAbility]); + expect(prompt).toContain('**mainwp/list-sites-v1** [readonly, idempotent]'); + expect(prompt).not.toContain('Destructive Actions Warning'); + }); + + it('labels a destructive-named ability DESTRUCTIVE even when annotated readonly', () => { + const prompt = buildConfiguredPrompt([mislabelledResetAbility]); + + // Tag line must show DESTRUCTIVE, not readonly (classify() downgrades + // readonly when the destructive override applies) + expect(prompt).toContain('**mainwp/reset-site-v1** [DESTRUCTIVE, idempotent]'); + expect(prompt).not.toContain('[readonly, idempotent]: Reset a site'); + + // And it must appear in the destructive-actions warning list + expect(prompt).toContain('Destructive Actions Warning'); + expect(prompt).toMatch(/Destructive Actions Warning[\s\S]*mainwp\/reset-site-v1/); + }); +}); diff --git a/src/chat/system-prompt.ts b/src/chat/system-prompt.ts index 017ad2e..6f6777b 100644 --- a/src/chat/system-prompt.ts +++ b/src/chat/system-prompt.ts @@ -8,6 +8,7 @@ */ import type { Ability } from '../core/abilities-executor.js'; +import { getSafetyController } from '../core/safety-controller.js'; /** * Core system prompt content @@ -90,12 +91,15 @@ The system handles all actual execution through the Abilities API.`; * Format ability for inclusion in system prompt */ function formatAbility(ability: Ability): string { - const annotations = ability.meta?.annotations; + // Labels come from SafetyController.classify(), not raw annotations, so the + // LLM-facing text always matches the runtime safety classification + // (including the destructive-name override). + const classification = getSafetyController().classify(ability); const tags: string[] = []; - if (annotations?.readonly) tags.push('readonly'); - if (annotations?.destructive) tags.push('DESTRUCTIVE'); - if (annotations?.idempotent) tags.push('idempotent'); + if (classification.isReadOnly) tags.push('readonly'); + if (classification.isDestructive) tags.push('DESTRUCTIVE'); + if (classification.isIdempotent) tags.push('idempotent'); const tagStr = tags.length > 0 ? ` [${tags.join(', ')}]` : ''; @@ -130,8 +134,10 @@ function buildAbilitiesSection(abilities: Ability[]): string { sections.push(''); } - // Add destructive actions reminder - const destructive = abilities.filter((a) => a.meta?.annotations?.destructive); + // Add destructive actions reminder (same classification source as runtime) + const destructive = abilities.filter( + (a) => getSafetyController().classify(a).isDestructive + ); if (destructive.length > 0) { sections.push('\n## Destructive Actions Warning\n'); sections.push( @@ -165,8 +171,6 @@ export interface SystemPromptConfig { includeSchemas: boolean; /** Maximum messages to keep in context (excluding system prompt). undefined = no limit */ maxContextMessages?: number; - /** Maximum estimated tokens in context. Reserved for future use. */ - maxContextTokens?: number; } /** diff --git a/src/chat/tool-envelope.test.ts b/src/chat/tool-envelope.test.ts new file mode 100644 index 0000000..adff62b --- /dev/null +++ b/src/chat/tool-envelope.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from 'vitest'; +import { parseResponse } from './tool-envelope.js'; +import type { LLMResponse, ToolCall } from './providers/provider.js'; + +function contentResponse(content: string): LLMResponse { + return { + content, + finishReason: 'stop', + model: 'test-model', + }; +} + +function expectToolCall( + content: string, + tool: string, + input: Record +): void { + expect(parseResponse(contentResponse(content)).response).toEqual({ + type: 'tool', + tool, + input, + }); +} + +describe('parseResponse', () => { + it('parses a tool call from a fenced json block', () => { + expectToolCall( + '```json\n{"tool":"list-sites-v1","input":{"page":1}}\n```', + 'list-sites-v1', + { page: 1 } + ); + }); + + it('parses a tool call from a fenced block without a language tag', () => { + expectToolCall( + '```\n{"tool":"list-sites-v1","input":{"page":2}}\n```', + 'list-sites-v1', + { page: 2 } + ); + }); + + it('parses JSON after prose containing an earlier brace pair', () => { + expectToolCall( + 'I\'ll update {site} now. {"tool":"update-site-v1","input":{"site_id":7}}', + 'update-site-v1', + { site_id: 7 } + ); + }); + + it('parses JSON followed by prose', () => { + expectToolCall( + '{"tool":"list-sites-v1","input":{}} I can explain the result next.', + 'list-sites-v1', + {} + ); + }); + + it('handles nested objects, braces in strings, and escaped quotes', () => { + const input = { + metadata: { + template: '{site}', + text: 'b}c', + quote: 'say "hello"', + }, + }; + + expectToolCall( + `Ready. ${JSON.stringify({ tool: 'update-site-v1', input })}`, + 'update-site-v1', + input + ); + }); + + it('treats a response with no JSON as a plain answer', () => { + const content = 'No matching sites were found.'; + + expect(parseResponse(contentResponse(content)).response).toEqual({ + type: 'answer', + answer: content, + }); + }); + + it('treats prose containing non-JSON braces as an answer, not a protocol error', () => { + const content = 'The config uses { key: value } format for site entries.'; + + expect(parseResponse(contentResponse(content)).response).toEqual({ + type: 'answer', + answer: content, + }); + }); + + it('keeps malformed content that leads with "{" on the retryable error path', () => { + const result = parseResponse(contentResponse('{"type": "tool_call", broken')); + + expect(result.response.type).toBe('error'); + expect(result.response).toMatchObject({ retryable: true }); + }); + + it('treats a malformed tool envelope after prose as retryable, not an answer', () => { + const result = parseResponse( + contentResponse('I will call it now: {"tool": "delete-site-v1", "input":') + ); + + expect(result.response.type).toBe('error'); + expect(result.response).toMatchObject({ retryable: true }); + }); + + it('treats a malformed answer envelope after prose as retryable, not an answer', () => { + const result = parseResponse( + contentResponse('Here is my reply: {"answer": "the site is') + ); + + expect(result.response.type).toBe('error'); + expect(result.response).toMatchObject({ retryable: true }); + }); + + it('rejects native responses containing multiple tool calls', () => { + const toolCalls: ToolCall[] = [ + { id: 'call_1', name: 'list-sites-v1', arguments: {} }, + { id: 'call_2', name: 'list-sites-v1', arguments: {} }, + ]; + + const result = parseResponse({ + content: '', + toolCalls, + finishReason: 'tool_calls', + model: 'test-model', + }); + + expect(result.response).toEqual({ + type: 'error', + error: 'Expected exactly one tool call, received 2', + retryable: true, + }); + }); +}); diff --git a/src/chat/tool-envelope.ts b/src/chat/tool-envelope.ts index 843425f..aae9c88 100644 --- a/src/chat/tool-envelope.ts +++ b/src/chat/tool-envelope.ts @@ -16,7 +16,13 @@ import type { Ability } from '../core/abilities-executor.js'; * Parsed response types */ export type ParsedResponse = - | { type: 'tool'; tool: string; input: Record; id?: string } + | { + type: 'tool'; + tool: string; + input: Record; + id?: string; + thoughtSignature?: string; + } | { type: 'answer'; answer: string } | { type: 'error'; error: string; retryable: boolean }; @@ -43,6 +49,8 @@ export interface ParserOptions { validateToolExists?: boolean; /** Whether to validate input against schema */ validateInput?: boolean; + /** Protocol-safe tool name to real ability name */ + toolAliases?: ReadonlyMap; } /** @@ -53,10 +61,54 @@ const JSON_PATTERNS = [ /```json\s*\n?([\s\S]*?)\n?```/, // Code block without language /```\s*\n?([\s\S]*?)\n?```/, - // Raw JSON object - /(\{[\s\S]*\})/, ]; +function extractFirstJsonObject(text: string): string | null { + for (let start = 0; start < text.length; start++) { + if (text[start] !== '{') { + continue; + } + + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = start; index < text.length; index++) { + const character = text[index]; + + if (inString) { + if (escaped) { + escaped = false; + } else if (character === '\\') { + escaped = true; + } else if (character === '"') { + inString = false; + } + continue; + } + + if (character === '"') { + inString = true; + } else if (character === '{') { + depth++; + } else if (character === '}') { + depth--; + if (depth === 0) { + const candidate = text.slice(start, index + 1); + try { + JSON.parse(candidate); + return candidate; + } catch { + break; + } + } + } + } + } + + return null; +} + /** * Parse LLM response to extract tool call or answer */ @@ -64,8 +116,32 @@ export function parseResponse( response: LLMResponse, options: ParserOptions = {} ): ParseResult { + if ( + response.finishReason === 'length' || + response.finishReason === 'content_filter' + ) { + return protocolError( + `Cannot process a response with finish reason "${response.finishReason}"`, + response.content + ); + } + // First, check for native function calling if (response.toolCalls && response.toolCalls.length > 0) { + if (response.finishReason !== 'tool_calls') { + return protocolError( + `Tool calls require finish reason "tool_calls", received "${response.finishReason}"`, + JSON.stringify(response.toolCalls) + ); + } + + if (response.toolCalls.length !== 1) { + return protocolError( + `Expected exactly one tool call, received ${response.toolCalls.length}`, + JSON.stringify(response.toolCalls) + ); + } + const firstToolCall = response.toolCalls[0]; if (firstToolCall) { return parseNativeToolCall(firstToolCall, options); @@ -83,7 +159,15 @@ function parseNativeToolCall( toolCall: ToolCall, options: ParserOptions ): ParseResult { - const validation = validateToolCall(toolCall.name, toolCall.arguments, options); + if (!isObjectInput(toolCall.arguments)) { + return protocolError( + `Tool input for "${toolCall.name}" must be a JSON object`, + JSON.stringify(toolCall) + ); + } + + const toolName = resolveToolName(toolCall.name, options); + const validation = validateToolCall(toolName, toolCall.arguments, options); if (validation) { return { @@ -97,9 +181,12 @@ function parseNativeToolCall( return { response: { type: 'tool', - tool: toolCall.name, + tool: toolName, input: toolCall.arguments, id: toolCall.id, + ...(toolCall.thoughtSignature !== undefined + ? { thoughtSignature: toolCall.thoughtSignature } + : {}), }, rawContent: JSON.stringify(toolCall), nativeFunctionCall: true, @@ -129,10 +216,38 @@ function parseContentJson( } } - // If no pattern matched, try the whole content + // If no fence matched, find the first balanced, parseable JSON object. if (!jsonStr) { - jsonStr = trimmed; attempts++; + jsonStr = extractFirstJsonObject(trimmed); + } + + // Preserve whole-response JSON parsing for non-object JSON values. + if (!jsonStr) { + try { + JSON.parse(trimmed); + jsonStr = trimmed; + attempts++; + } catch { + // Only content that LOOKS like an attempted envelope goes to the + // retryable protocol-error path: it leads with "{", or it carries an + // envelope key after prose (a truncated `Deleting: {"tool": ...` must + // retry, not pass as an answer). Prose that merely contains braces — + // "the config uses { key: value } format" — is an answer; the + // balanced-object scan above already extracted any real embedded JSON. + const looksLikeEnvelopeAttempt = /"(?:tool|answer)"\s*:/.test(trimmed); + if (!trimmed.startsWith('{') && !looksLikeEnvelopeAttempt) { + return { + response: { type: 'answer', answer: trimmed }, + rawContent: content, + nativeFunctionCall: false, + attempts, + }; + } + + jsonStr = trimmed; + attempts++; + } } // Parse JSON @@ -168,6 +283,19 @@ function parseContentJson( const obj = parsed as Record; + if ('answer' in obj && 'tool' in obj) { + return { + response: { + type: 'error', + error: 'Response cannot contain both "answer" and "tool" properties', + retryable: true, + }, + rawContent: content, + nativeFunctionCall: false, + attempts, + }; + } + // Check for answer format if ('answer' in obj && typeof obj['answer'] === 'string') { return { @@ -180,11 +308,20 @@ function parseContentJson( // Check for tool format if ('tool' in obj && typeof obj['tool'] === 'string') { - const toolName = obj['tool']; - const input = - typeof obj['input'] === 'object' && obj['input'] !== null - ? (obj['input'] as Record) - : {}; + const toolName = resolveToolName(obj['tool'], options); + if (!isObjectInput(obj['input'])) { + return { + response: { + type: 'error', + error: `Tool input for "${toolName}" must be a JSON object`, + retryable: true, + }, + rawContent: content, + nativeFunctionCall: false, + attempts, + }; + } + const input = obj['input']; const validation = validateToolCall(toolName, input, options); @@ -219,6 +356,23 @@ function parseContentJson( }; } +function protocolError(error: string, rawContent: string): ParseResult { + return { + response: { type: 'error', error, retryable: true }, + rawContent, + nativeFunctionCall: true, + attempts: 1, + }; +} + +function isObjectInput(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function resolveToolName(name: string, options: ParserOptions): string { + return options.toolAliases?.get(name) ?? name; +} + /** * Validate tool call against known abilities */ @@ -351,4 +505,3 @@ Or answer: } \`\`\``; } - diff --git a/src/commands/abilities/list.ts b/src/commands/abilities/list.ts index 2ea4b68..402b722 100644 --- a/src/commands/abilities/list.ts +++ b/src/commands/abilities/list.ts @@ -8,7 +8,7 @@ import { Flags } from '@oclif/core'; import { BaseCommand, commonFlags } from '../../lib/base-command.js'; import { formatTable, formatHeading } from '../../output/formatter.js'; import { color, colors } from '../../utils/colors.js'; -import { stripControlChars } from '../../utils/terminal-sanitizer.js'; +import { sanitizeSingleLine } from '../../utils/terminal-sanitizer.js'; import type { Ability } from '../../core/abilities-executor.js'; /** @@ -154,7 +154,7 @@ export default class AbilitiesList extends BaseCommand { for (let j = 0; j < catAbilities.length; j++) { lines.push(tableLines[j + 2] ?? ''); const ability = catAbilities[j]!; - const safeName = stripControlChars(ability.name); + const safeName = sanitizeSingleLine(ability.name); const shortName = safeName.split('/').pop() ?? safeName; const hint = buildUsageHint(shortName, ability); lines.push(color(` ${hint}`, colors.dim)); diff --git a/src/commands/abilities/run.ts b/src/commands/abilities/run.ts index 29098ed..a90d6f0 100644 --- a/src/commands/abilities/run.ts +++ b/src/commands/abilities/run.ts @@ -18,7 +18,7 @@ import { formatHeading, formatKeyValue, } from '../../output/formatter.js'; -import { InputError, MutualExclusionError } from '../../utils/errors.js'; +import { InputError, MutualExclusionError, UnknownOutcomeError } from '../../utils/errors.js'; import { getSafetyController, type PreviewResult } from '../../core/safety-controller.js'; import { getSchemaValidator } from '../../validation/schema-validator.js'; import { getInputSanitizer } from '../../validation/input-sanitizer.js'; @@ -26,6 +26,8 @@ import { promptForConfirmation, isInteractive } from '../../utils/prompt.js'; import { logDestructiveActionSafe } from '../../utils/audit-logger.js'; import type { WatchResult } from '../../core/batch-manager.js'; import { APIError } from '../../utils/errors.js'; +import type { Ability } from '../../core/abilities-executor.js'; +import { executeAbilityWithPolicy } from '../../core/execute-ability-with-policy.js'; export default class AbilitiesRun extends BaseCommand { static description = 'Execute an ability'; @@ -118,13 +120,33 @@ export default class AbilitiesRun extends BaseCommand { // Resolve input from --input, --input-file, or stdin const rawInput = await this.resolveInput(flags.input, flags['input-file']); - // Parse input JSON - let input: Record; + // Parse input JSON. The raw input never goes into the error message: it + // can carry secrets (a password pasted into a malformed payload) that + // would otherwise land in stderr, CI logs, or the --json envelope. + let parsed: unknown; try { - input = JSON.parse(rawInput) as Record; - } catch { - throw new InputError(`Invalid JSON input: ${rawInput}`); + parsed = JSON.parse(rawInput) as unknown; + } catch (error) { + const position = error instanceof Error + ? /at position (\d+)/.exec(error.message)?.[1] + : undefined; + throw new InputError( + `Invalid JSON input${position ? ` (parse error at position ${position})` : ''}`, + undefined, + 'Check the JSON passed via --input, --input-file, or stdin. Use --input-file for complex payloads.' + ); + } + + // Abilities take named parameters; an array or primitive would otherwise + // slip through to the Dashboard when the ability declares no schema. + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new InputError( + 'Input JSON must be an object of ability parameters', + undefined, + 'Pass a JSON object, e.g. --input \'{"site_id": 5}\'.' + ); } + let input: Record = parsed as Record; // Sanitize input input = inputSanitizer.sanitize(input); @@ -169,11 +191,11 @@ export default class AbilitiesRun extends BaseCommand { if (dryRun || !shouldExecute) { // Preview mode — --dry-run always previews, regardless of ability classification - await this.executePreview(ability.name, input, dryRun); + await this.executePreview(ability, input); } else if (confirm && safetyController.requiresSafetyFlow(ability)) { // Destructive execution with confirmation await this.executeDestructive({ - abilityName: ability.name, + ability, input, force: flags.force, wait: flags.wait, @@ -182,7 +204,7 @@ export default class AbilitiesRun extends BaseCommand { } else { // Direct execution (read-only or non-destructive) await this.executeDirect({ - abilityName: ability.name, + ability, input, wait: flags.wait, waitTimeout: flags['wait-timeout'], @@ -194,21 +216,20 @@ export default class AbilitiesRun extends BaseCommand { * Execute preview (dry_run mode) */ private async executePreview( - abilityName: string, - input: Record, - _dryRun?: boolean + ability: Ability, + input: Record ): Promise { const executor = await this.getExecutor(); + const abilityName = ability.name; - const result = await executor.execute(abilityName, input, { dryRun: true }); + const result = await executeAbilityWithPolicy(executor, ability, input, { dryRun: true }); if (!result.success) { throw new InputError(result.error?.message ?? 'Preview failed', result.error); } const safetyController = getSafetyController(); - const ability = await executor.getAbility(abilityName); - const preview = safetyController.formatPreviewResult(ability!, input, result); + const preview = safetyController.formatPreviewResult(ability, input, result); this.output( { @@ -225,32 +246,75 @@ export default class AbilitiesRun extends BaseCommand { * Execute destructive ability with confirmation */ private async executeDestructive(opts: { - abilityName: string; + ability: Ability; input: Record; force: boolean; wait?: boolean; waitTimeout?: number; }): Promise { - const { abilityName, input, force, wait, waitTimeout } = opts; + const { ability, input, force, wait, waitTimeout } = opts; + const abilityName = ability.name; const executor = await this.getExecutor(); const safetyController = getSafetyController(); - // Get preview data first for audit logging (gracefully handle failures) + // Preview is mandatory and fail-closed (plan.md §2.1): a destructive + // execution must never proceed without a successful dry_run the operator + // has seen. --force skips the prompt below, never this preview. let preview: PreviewResult | undefined; + let previewFailure: unknown; try { - const ability = await executor.getAbility(abilityName); - const previewResult = await executor.execute(abilityName, input, { dryRun: true }); - if (previewResult.success && ability) { + const previewResult = await executeAbilityWithPolicy( + executor, + ability, + input, + { dryRun: true } + ); + if (previewResult.success) { preview = safetyController.formatPreviewResult(ability, input, previewResult); + } else { + previewFailure = previewResult.error ?? new Error('dry_run returned no result'); } - } catch { - // Preview failure is non-fatal - continue without preview data in audit + } catch (error) { + previewFailure = error; } - // Helper to build preview metadata for audit entries (spread-friendly) - const previewMeta = preview - ? { preview: { summary: preview.summary, affectedCount: preview.affected.length } } - : {}; + if (!preview) { + const reason = getInputSanitizer().sanitizeErrorMessage( + previewFailure instanceof Error + ? previewFailure.message + : ((previewFailure as { message?: string } | undefined)?.message ?? 'unknown error') + ); + await logDestructiveActionSafe({ + abilityName, + userDecision: 'declined', + execution: { success: false, error: `Preview failed: ${reason}` }, + input, + }); + // Only the sanitized reason goes into details: the raw previewFailure + // could carry an unsanitized upstream payload into the --json envelope + // (and an Error instance would serialize to {} anyway). + throw new APIError( + 'PREVIEW_FAILED', + `Preview (dry_run) failed for "${abilityName}": ${reason}. Destructive execution refused.`, + undefined, + { reason } + ); + } + + // Show the preview before any approval decision. In JSON mode it goes to + // stderr so stdout stays a single clean envelope (preview data is also + // included in the final envelope below). + const previewText = this.formatPreviewOutput(preview); + if (this.jsonOutput) { + this.logToStderr(previewText); + } else if (!this.quietMode) { + this.log(previewText); + } + + // Preview metadata for audit entries (spread-friendly) + const previewMeta = { + preview: { summary: preview.summary, affectedCount: preview.affected.length }, + }; // In non-interactive mode, require --force or fail if (!isInteractive() && !force) { @@ -282,8 +346,48 @@ export default class AbilitiesRun extends BaseCommand { } } - // Execute with confirm - const result = await executor.execute(abilityName, input, { confirm: true }); + // Record the approval BEFORE dispatching the confirm call. If the process + // dies or the transport fails mid-confirm, this entry is the only durable + // evidence that an approved destructive action may have reached the + // Dashboard. + await logDestructiveActionSafe({ + abilityName, + ...previewMeta, + userDecision: 'approved', + stage: 'dispatch', + input, + }); + + // Execute with confirm. A throw here (timeout, connection reset, response + // parse failure) arrives AFTER dispatch was initiated: the outcome is + // unknown, not a plain network failure. Fail closed: audit the uncertainty + // and surface it as OUTCOME_UNKNOWN. Never auto-retry the confirm. + let result: Awaited>; + try { + result = await executeAbilityWithPolicy( + executor, + ability, + input, + { confirm: true } + ); + } catch (error) { + const reason = getInputSanitizer().sanitizeErrorMessage( + error instanceof Error ? error.message : String(error) + ); + await logDestructiveActionSafe({ + abilityName, + ...previewMeta, + userDecision: 'approved', + execution: { success: false, error: reason, outcomeUnknown: true }, + input, + }); + throw new UnknownOutcomeError( + `Confirm call for "${abilityName}" failed after dispatch: ${reason}. ` + + 'The Dashboard may or may not have executed the action.', + { reason }, + 'Verify the Dashboard state (e.g. with a read-only ability) before retrying. Do not re-run with --confirm until you have confirmed the action did not complete.' + ); + } // Build execution result for audit const executionResult: { success: boolean; error?: string } = { @@ -323,8 +427,8 @@ export default class AbilitiesRun extends BaseCommand { { mode: 'batch', ability: abilityName, - jobId: result.jobId, ...result, + preview, }, () => this.formatBatchOutput(abilityName, result.jobId!) ); @@ -336,6 +440,7 @@ export default class AbilitiesRun extends BaseCommand { mode: 'execute', ability: abilityName, ...result, + preview, }, () => this.formatExecutionOutput(abilityName, result.data) ); @@ -345,14 +450,15 @@ export default class AbilitiesRun extends BaseCommand { * Execute directly (read-only or non-destructive) */ private async executeDirect(opts: { - abilityName: string; + ability: Ability; input: Record; wait?: boolean; waitTimeout?: number; }): Promise { - const { abilityName, input, wait, waitTimeout } = opts; + const { ability, input, wait, waitTimeout } = opts; + const abilityName = ability.name; const executor = await this.getExecutor(); - const result = await executor.execute(abilityName, input); + const result = await executeAbilityWithPolicy(executor, ability, input); if (!result.success) { throw new APIError( @@ -375,7 +481,6 @@ export default class AbilitiesRun extends BaseCommand { { mode: 'batch', ability: abilityName, - jobId: result.jobId, ...result, }, () => this.formatBatchOutput(abilityName, result.jobId!) @@ -408,27 +513,24 @@ export default class AbilitiesRun extends BaseCommand { }); if (watchResult.timedOut) { - // Output partial results and throw API error for exit code 4 - this.output( - { - mode: 'batch', - ability: abilityName, - jobId, - timedOut: true, - ...watchResult.status, - elapsed_ms: watchResult.elapsed, - }, - () => formatWarning(`Batch job ${jobId} timed out after ${timeoutSeconds}s (partial results returned)`) - ); + // Human mode surfaces partial results before the error line. JSON mode + // must emit exactly ONE document, so the partial status travels in the + // error envelope's details instead of a preceding success envelope. + if (!this.jsonOutput) { + this.output( + {}, + () => formatWarning(`Batch job ${jobId} timed out after ${timeoutSeconds}s (partial results returned)`) + ); + } throw new APIError( 'BATCH_TIMEOUT', `Batch job timed out after ${timeoutSeconds}s`, undefined, - { jobId, partialStatus: watchResult.status } + { jobId, partialStatus: watchResult.status, elapsed_ms: watchResult.elapsed } ); } - // Job completed (or failed) + // Job reached a terminal status const data = { mode: 'batch', ability: abilityName, @@ -438,6 +540,29 @@ export default class AbilitiesRun extends BaseCommand { elapsed_ms: watchResult.elapsed, }; + // Non-completed terminal statuses map to exit code 4. Human mode prints + // the result details first; JSON mode emits only the error envelope + // (single-document contract), carrying the status in details. + if ( + watchResult.status.status === 'failed' || + watchResult.status.status === 'partial' || + watchResult.status.status === 'cancelled' + ) { + if (!this.jsonOutput) { + this.output(data, () => this.formatWatchResultOutput(abilityName, jobId, watchResult)); + } + throw new APIError( + watchResult.status.status === 'failed' + ? 'BATCH_FAILED' + : watchResult.status.status === 'partial' + ? 'BATCH_PARTIAL' + : 'BATCH_CANCELLED', + `Batch job ${jobId} finished with status "${watchResult.status.status}"`, + undefined, + { jobId, status: watchResult.status, elapsed_ms: watchResult.elapsed } + ); + } + this.output(data, () => this.formatWatchResultOutput(abilityName, jobId, watchResult)); } diff --git a/src/commands/chat.test.ts b/src/commands/chat.test.ts index 63e759b..8df04e0 100644 --- a/src/commands/chat.test.ts +++ b/src/commands/chat.test.ts @@ -7,6 +7,7 @@ import { describe, it, expect, vi } from 'vitest'; import { stripControlChars } from '../utils/terminal-sanitizer.js'; +import { formatResponse } from './chat.js'; // We test the sanitization integration by verifying the functions // used in chat.ts correctly handle malicious content. The actual @@ -84,6 +85,37 @@ describe('Chat Output Sanitization', () => { }); }); + describe('formatResponse error rendering', () => { + it('prefixes the ability name when an error carries tool', () => { + const rendered = formatResponse({ + type: 'error', + error: 'Resource not found', + tool: 'mainwp/delete-site-v1', + }); + + expect(rendered).toBe('[mainwp/delete-site-v1] Error: Resource not found'); + }); + + it('renders engine-level errors without a tool prefix', () => { + const rendered = formatResponse({ + type: 'error', + error: 'Provider unavailable', + }); + + expect(rendered).toBe('Error: Provider unavailable'); + }); + + it('sanitizes the tool name in the error prefix', () => { + const rendered = formatResponse({ + type: 'error', + error: 'boom', + tool: 'delete\x1b[10C(hidden)', + }); + + expect(rendered).toBe('[delete(hidden)] Error: boom'); + }); + }); + describe('M2: formatPreview sanitization', () => { it('sanitizes preview summary', () => { const maliciousSummary = '\x1b[2JWill delete 5 sites\x1b[H'; diff --git a/src/commands/chat.ts b/src/commands/chat.ts index e345b50..bdcc0a2 100644 --- a/src/commands/chat.ts +++ b/src/commands/chat.ts @@ -20,6 +20,8 @@ import { import type { PreviewResult } from '../core/safety-controller.js'; import { isInteractive } from '../utils/prompt.js'; import { stripControlChars } from '../utils/terminal-sanitizer.js'; +import { getInputSanitizer } from '../validation/input-sanitizer.js'; +import { APIError, UnknownOutcomeError, type MainWPCTLError } from '../utils/errors.js'; // Import providers to register them import '../chat/providers/index.js'; @@ -58,9 +60,9 @@ function formatPreview(preview: PreviewResult): string { } /** - * Format chat response for display + * Format chat response for display (exported for tests) */ -function formatResponse(response: ChatResponse): string { +export function formatResponse(response: ChatResponse): string { switch (response.type) { case 'message': return stripControlChars(response.content); @@ -77,8 +79,10 @@ function formatResponse(response: ChatResponse): string { case 'preview': return formatPreview(response.preview); - case 'error': - return `Error: ${stripControlChars(response.error)}`; + case 'error': { + const message = `Error: ${stripControlChars(response.error)}`; + return response.tool ? `[${stripControlChars(response.tool)}] ${message}` : message; + } } } @@ -132,6 +136,7 @@ export default class ChatCommand extends BaseCommand { }), 'max-context-messages': Flags.integer({ description: 'Maximum messages to keep in context (default: 20, 0 = unlimited)', + min: 0, }), stream: Flags.boolean({ description: 'Enable streaming responses (progressive output)', @@ -275,31 +280,73 @@ export default class ChatCommand extends BaseCommand { await this.runInteractive(); } + /** + * Map a terminal chat response to the error that should decide the process + * outcome, or undefined when the turn succeeded. One-shot chat must not + * exit 0 when the turn ended in a failure — CI would read a failed MainWP + * operation as success. + */ + private static terminalFailure( + response: ChatResponse | undefined + ): MainWPCTLError | undefined { + if (!response) return undefined; + + if (response.type === 'error') { + // An unknown destructive outcome keeps its identity (and exit 3): + // downgrading it to a generic chat error would hide the one failure + // an operator must reconcile before retrying anything. + if (response.code === 'OUTCOME_UNKNOWN') { + return new UnknownOutcomeError(response.error, response); + } + return new APIError('CHAT_ERROR', response.error, undefined, response); + } + + if (response.type === 'tool_result' && !response.result.success) { + return new APIError( + response.result.error?.code ?? 'ABILITY_EXECUTION_ERROR', + response.result.error?.message ?? 'Tool execution failed', + undefined, + { tool: response.tool, error: response.result.error } + ); + } + + return undefined; + } + /** * Handle a single message (non-interactive mode) */ private async handleSingleMessage(message: string): Promise { const responses = await this.chatEngine!.sendMessage(message); + // Select the terminal-state response. + // preview/error: singular (loop breaks after producing one), so find() is correct. + // tool_result: multiple can accumulate in multi-step turns, so findLast() + // ensures we return the final outcome, not an intermediate step. + const terminal = + responses.find((response) => response.type === 'preview') ?? + responses.find((response) => response.type === 'error') ?? + responses.findLast((response) => response.type === 'tool_result') ?? + responses.at(-1); + + const failure = ChatCommand.terminalFailure(terminal); + if (this.jsonOutput) { - // Select the terminal-state response for JSON output. - // preview/error: singular (loop breaks after producing one), so find() is correct. - // tool_result: multiple can accumulate in multi-step turns, so findLast() - // ensures we return the final outcome, not an intermediate step. - const jsonResponse = - responses.find((response) => response.type === 'preview') ?? - responses.find((response) => response.type === 'error') ?? - responses.findLast((response) => response.type === 'tool_result') ?? - responses.at(-1); - - if (jsonResponse) { - this.log(JSON.stringify(jsonResponse, null, 2)); + // Failures go through catch(): documented error envelope + non-zero exit. + if (failure) throw failure; + // Success wraps in the documented {success, data, error, meta} envelope + // instead of printing a bare ChatResponse. + if (terminal) { + this.output(terminal); } - return; } for (const response of responses) { + // The failing terminal response is printed by catch() below — printing + // it here too would duplicate the error line. + if (failure && response === terminal) continue; + // Add newline after streamed content (streaming doesn't include final newline) if (this.isStreaming && response.type === 'message') { this.log(''); // Blank line after streamed content @@ -313,13 +360,13 @@ export default class ChatCommand extends BaseCommand { // If preview is pending, we can't continue in non-interactive if (response.type === 'preview') { - if (!this.jsonOutput) { - this.log('\nDestructive action requires approval.'); - this.log('Run in interactive mode to approve.'); - } + this.log('\nDestructive action requires approval.'); + this.log('Run in interactive mode to approve.'); return; } } + + if (failure) throw failure; } /** @@ -380,17 +427,6 @@ export default class ChatCommand extends BaseCommand { return; } - // Cancel pending preview - if ( - pendingPreview && - (trimmed.toLowerCase() === 'cancel' || trimmed.toLowerCase() === 'no') - ) { - this.chatEngine!.cancelPendingPreview(); - this.log('Operation cancelled.'); - prompt(); - return; - } - try { const responses = await this.chatEngine!.sendMessage(trimmed); @@ -413,9 +449,13 @@ export default class ChatCommand extends BaseCommand { this.log('This is a destructive action. Type "yes" to approve or "no" to cancel.'); } } catch (error) { - this.logToStderr( - `Error: ${stripControlChars(error instanceof Error ? error.message : String(error))}` + // Provider/transport errors can echo untrusted response fragments; + // apply the same secret/path redaction the --json path gets before + // the message reaches the terminal. + const sanitized = getInputSanitizer().sanitizeErrorMessage( + error instanceof Error ? error.message : String(error) ); + this.logToStderr(`Error: ${stripControlChars(sanitized)}`); } this.log(''); diff --git a/src/commands/config/show.ts b/src/commands/config/show.ts index 9992293..d18ed16 100644 --- a/src/commands/config/show.ts +++ b/src/commands/config/show.ts @@ -27,8 +27,10 @@ import { resolveProviderSelection, type ProviderSelectionSource, } from '../../chat/providers/provider.js'; -import { maskPassword, maskApiKey } from '../../utils/format.js'; +import { maskPassword, maskApiKey, maskUrlUserinfo } from '../../utils/format.js'; import { color, colors } from '../../utils/colors.js'; +import { formatDivider, formatSection, formatStatusIcon } from '../../output/formatter.js'; +import { sanitizeSingleLine } from '../../utils/terminal-sanitizer.js'; /** * Configuration display structure @@ -41,6 +43,7 @@ interface ConfigDisplay { skipSSLVerification: boolean; skipSSLVerificationSource: 'profile' | 'settings' | 'default'; allowInsecureHttp: boolean; + /** Best-effort guess: when keychain and env both hold the same password, this reports 'environment' since the two sources can't be distinguished. */ credentialsSource: 'keychain' | 'environment' | 'none'; credentialsMasked: string | null; }; @@ -107,7 +110,7 @@ export default class ConfigShowCommand extends BaseCommand { if (this.jsonOutput) { this.output(configDisplay); } else { - this.displayConfig(configDisplay, flags.verbose); + await this.displayConfig(configDisplay, flags.verbose); } } @@ -192,7 +195,8 @@ export default class ConfigShowCommand extends BaseCommand { return { active: activeProfile.name, - dashboardUrl: activeProfile.dashboardUrl, + // Mask userinfo from profiles stored before intake rejection existed + dashboardUrl: maskUrlUserinfo(activeProfile.dashboardUrl), username: activeProfile.username, skipSSLVerification: activeProfile.skipSSLVerification ?? this.settings.skipSSLVerification, @@ -263,71 +267,77 @@ export default class ConfigShowCommand extends BaseCommand { /** * Display configuration in human-readable format */ - private displayConfig(config: ConfigDisplay, verbose: boolean): void { + private async displayConfig(config: ConfigDisplay, verbose: boolean): Promise { this.log('\n MainWP Control CLI - Configuration\n'); - this.log(' ' + '─'.repeat(40)); + this.log(formatDivider()); // Profile Configuration Section - this.log(`\n ${color('Profile Configuration', colors.bold)}`); + const profileRows: string[] = []; if (config.profile.active) { - this.log(` Active Profile: ${color(config.profile.active, colors.green)}`); - this.log(` Dashboard URL: ${config.profile.dashboardUrl}`); - this.log(` Username: ${config.profile.username}`); - this.log( + // Config-file values are user-editable on disk — sanitize before display. + profileRows.push(` Active Profile: ${color(sanitizeSingleLine(config.profile.active), colors.green)}`); + profileRows.push(` Dashboard URL: ${sanitizeSingleLine(config.profile.dashboardUrl ?? '')}`); + profileRows.push(` Username: ${sanitizeSingleLine(config.profile.username ?? '')}`); + profileRows.push( ` SSL Verify: ${ config.profile.skipSSLVerification ? color('Disabled', colors.yellow) : color('Enabled', colors.green) }${this.describeSSLSource(config.profile.skipSSLVerificationSource)}` ); - this.log( + profileRows.push( ` HTTP Allowed: ${ config.profile.allowInsecureHttp ? color('Enabled (insecure)', colors.yellow) : color('Disabled', colors.green) }` ); if (config.profile.credentialsSource === 'none') { - this.log(` Credentials: ${color('✗ Not found', colors.red)}`); - this.log(` ${color('Run `mainwpcontrol login` or set MAINWP_APP_PASSWORD', colors.gray)}`); + profileRows.push(` Credentials: ${color('✗ Not found', colors.red)}`); + profileRows.push(` ${color('Run `mainwpcontrol login` or set MAINWP_APP_PASSWORD', colors.gray)}`); } else { const sourceLabel = config.profile.credentialsSource === 'keychain' ? 'Stored in keychain' : 'From environment variable'; - this.log( - ` Credentials: ${color('✓', colors.green)} ${sourceLabel} (${config.profile.credentialsMasked})` + profileRows.push( + ` Credentials: ${formatStatusIcon('pass')} ${sourceLabel} (${sanitizeSingleLine(config.profile.credentialsMasked ?? '')})` ); } } else { - this.log(` ${color('No active profile configured', colors.yellow)}`); - this.log(` ${color('Run `mainwpcontrol login` or `mainwpcontrol profile use `', colors.gray)}`); + profileRows.push(` ${color('No active profile configured', colors.yellow)}`); + profileRows.push(` ${color('Run `mainwpcontrol login` or `mainwpcontrol profile use `', colors.gray)}`); + } + this.log(formatSection('Profile Configuration', profileRows)); - // Show available profiles count - this.showAvailableProfilesHint(); + // Show available profiles count right after the profile section. + // Awaited (not fire-and-forget) so its output lands in deterministic order. + if (!config.profile.active) { + await this.showAvailableProfilesHint(); } // LLM Provider Section - this.log(`\n ${color('LLM Provider', colors.bold)}`); + const llmRows: string[] = []; if (config.llmProvider.configured) { - this.log(` Provider: ${color(config.llmProvider.name!, colors.green)}`); - this.log(` API Key: ${config.llmProvider.apiKeyMasked}`); - this.log(` Source: ${config.llmProvider.source}`); - this.log(` Status: ${color('✓ Configured', colors.green)}`); + llmRows.push(` Provider: ${color(config.llmProvider.name!, colors.green)}`); + llmRows.push(` API Key: ${sanitizeSingleLine(config.llmProvider.apiKeyMasked ?? '')}`); + llmRows.push(` Source: ${config.llmProvider.source}`); + llmRows.push(` Status: ${color('✓ Configured', colors.green)}`); } else if (config.llmProvider.name) { - this.log(` Provider: ${color(config.llmProvider.name, colors.yellow)}`); - this.log(` Source: ${config.llmProvider.source}`); - this.log(` Status: ${color('⚠ API key not set', colors.yellow)}`); + llmRows.push(` Provider: ${color(config.llmProvider.name, colors.yellow)}`); + llmRows.push(` Source: ${config.llmProvider.source}`); + llmRows.push(` Status: ${color('⚠ API key not set', colors.yellow)}`); } else { - this.log(` ${color('No LLM provider configured', colors.yellow)}`); - this.log(` ${color('Chat mode requires one of these environment variables:', colors.gray)}`); + llmRows.push(` ${color('No LLM provider configured', colors.yellow)}`); + llmRows.push(` ${color('Chat mode requires one of these environment variables:', colors.gray)}`); for (const [name, envConfig] of Object.entries(PROVIDER_ENV_VARS)) { - this.log(` ${color(` ${name}: ${envConfig.key}`, colors.gray)}`); + llmRows.push(` ${color(` ${name}: ${envConfig.key}`, colors.gray)}`); } } for (const warning of config.llmProvider.warnings) { - this.log(` ${color(warning, colors.yellow)}`); + llmRows.push(` ${color(sanitizeSingleLine(warning), colors.yellow)}`); } + this.log(formatSection('LLM Provider', llmRows)); // Settings Section - this.log(`\n ${color('Settings', colors.bold)}`); + const settingsRows: string[] = []; const hasSettings = Object.keys(config.settings).length > 0; if (hasSettings || verbose) { @@ -339,24 +349,24 @@ export default class ConfigShowCommand extends BaseCommand { const allowInsecureHttp = config.effectiveSettings.allowInsecureHttp; const skipSSLVerification = config.effectiveSettings.skipSSLVerification; - this.log( + settingsRows.push( ` Timeout: ${timeout}ms${config.settings.timeout === undefined ? color(' (default)', colors.gray) : ''}` ); - this.log( + settingsRows.push( ` Debug Mode: ${debug ? 'Enabled' : 'Disabled'}${ config.settings.debug === undefined ? color(' (default)', colors.gray) : '' }` ); - this.log( + settingsRows.push( ` Chat Context: ${chatContextMessages} messages${ config.settings.chatContextMessages === undefined ? color(' (default)', colors.gray) : '' }` ); if (defaultProvider !== undefined || verbose) { - this.log( + settingsRows.push( ` Default LLM: ${ defaultProvider ?? color('Auto-detect', colors.gray) }` @@ -364,11 +374,11 @@ export default class ConfigShowCommand extends BaseCommand { } if (chatContextTokens !== undefined && verbose) { - this.log(` Token Limit: ${chatContextTokens}`); + settingsRows.push(` Token Limit: ${chatContextTokens}`); } if (allowInsecureHttp || verbose) { - this.log( + settingsRows.push( ` Allow HTTP: ${ allowInsecureHttp ? color('Enabled (insecure)', colors.yellow) : color('Disabled', colors.green) }` @@ -376,7 +386,7 @@ export default class ConfigShowCommand extends BaseCommand { } if (skipSSLVerification || verbose) { - this.log( + settingsRows.push( ` Global SSL Skip: ${ skipSSLVerification ? color('Enabled (advanced)', colors.yellow) : color('Disabled', colors.green) }` @@ -384,20 +394,24 @@ export default class ConfigShowCommand extends BaseCommand { } if (!hasSettings && verbose) { - this.log(` ${color('Using all default settings', colors.gray)}`); + settingsRows.push(` ${color('Using all default settings', colors.gray)}`); } } else { - this.log(` ${color('Using default settings (use -v for details)', colors.gray)}`); + settingsRows.push(` ${color('Using default settings (use -v for details)', colors.gray)}`); } + this.log(formatSection('Settings', settingsRows)); // Configuration Files Section - this.log(`\n ${color('Configuration Files', colors.bold)}`); - this.log(` Config Dir: ${config.paths.configDir}`); - this.log(` Profiles: ${config.paths.profilesFile}`); - this.log(` Settings: ${config.paths.settingsFile}`); - this.log(` Audit Log: ${config.paths.auditLog}`); - - this.log('\n ' + '─'.repeat(40) + '\n'); + this.log( + formatSection('Configuration Files', [ + ` Config Dir: ${sanitizeSingleLine(config.paths.configDir)}`, + ` Profiles: ${sanitizeSingleLine(config.paths.profilesFile)}`, + ` Settings: ${sanitizeSingleLine(config.paths.settingsFile)}`, + ` Audit Log: ${sanitizeSingleLine(config.paths.auditLog)}`, + ]) + ); + + this.log('\n' + formatDivider() + '\n'); } private describeSSLSource(source: ConfigDisplay['profile']['skipSSLVerificationSource']): string { @@ -419,7 +433,7 @@ export default class ConfigShowCommand extends BaseCommand { const profileStore = getProfileStore(); const profiles = await profileStore.list(); if (profiles.length > 0) { - this.log(` ${color(`${profiles.length} profile(s) available: ${profiles.map((p) => p.name).join(', ')}`, colors.gray)}`); + this.log(` ${color(`${profiles.length} profile(s) available: ${sanitizeSingleLine(profiles.map((p) => p.name).join(', '))}`, colors.gray)}`); } } catch { // Ignore errors diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 83ab561..c6ac270 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -20,8 +20,15 @@ import { resolveProviderSelection, } from '../chat/providers/provider.js'; import { ExitCode } from '../utils/exit-codes.js'; -import { maskPassword, maskApiKey } from '../utils/format.js'; +import { + maskPassword, + maskApiKey, + maskUrlUserinfo, + maskUrlUserinfoInText, +} from '../utils/format.js'; import { color, colors } from '../utils/colors.js'; +import { formatDivider, formatStatusIcon, getStatusColor } from '../output/formatter.js'; +import { sanitizeSingleLine, stripControlChars } from '../utils/terminal-sanitizer.js'; /** * Check result @@ -204,7 +211,8 @@ export default class DoctorCommand extends BaseCommand { name: 'Active Profile', status: 'pass', message: `Active: ${activeProfile.name}`, - details: activeProfile.dashboardUrl, + // Mask userinfo from profiles stored before intake rejection existed + details: maskUrlUserinfo(activeProfile.dashboardUrl), }; } catch (error) { return { @@ -295,7 +303,8 @@ export default class DoctorCommand extends BaseCommand { } catch (error) { const message = error instanceof Error ? error.message : String(error); - let details = message; + // Fetch errors can echo the full request URL, credentials included + let details = maskUrlUserinfoInText(message); if (message.includes('ECONNREFUSED')) { details = 'Connection refused. Is the Dashboard running?'; } else if (message.includes('ENOTFOUND')) { @@ -434,24 +443,27 @@ export default class DoctorCommand extends BaseCommand { */ private displayReport(report: DoctorReport, verbose: boolean): void { this.log('\n MainWP Control CLI - System Check\n'); - this.log(' ' + '─'.repeat(40)); + this.log(formatDivider()); for (const check of report.checks) { - const icon = this.getStatusIcon(check.status); - const statusColor = this.getStatusColorCode(check.status); + const icon = formatStatusIcon(check.status); + const statusColor = getStatusColor(check.status); + // Sanitize at the display boundary: message/details can carry + // error-derived or config-derived text (the --json path gets the + // same treatment via the envelope's sanitizeForTerminal). this.log(` ${icon} ${check.name}`); - this.log(` ${color(check.message, statusColor)}`); + this.log(` ${color(sanitizeSingleLine(check.message), statusColor)}`); if (verbose && check.details) { - const detailLines = check.details.split('\n'); + const detailLines = stripControlChars(check.details).split('\n'); for (const line of detailLines) { this.log(` ${color(line, colors.gray)}`); } } } - this.log(' ' + '─'.repeat(40)); + this.log(formatDivider()); // Summary this.log( @@ -468,31 +480,4 @@ export default class DoctorCommand extends BaseCommand { } } - /** - * Get status icon - */ - private getStatusIcon(status: CheckResult['status']): string { - switch (status) { - case 'pass': - return color('✓', colors.green); - case 'warn': - return color('⚠', colors.yellow); - case 'fail': - return color('✗', colors.red); - } - } - - /** - * Get status color code string - */ - private getStatusColorCode(status: CheckResult['status']): string { - switch (status) { - case 'pass': - return colors.green; - case 'warn': - return colors.yellow; - case 'fail': - return colors.red; - } - } } diff --git a/src/commands/jobs/watch.test.ts b/src/commands/jobs/watch.test.ts index 6c7a271..03eddb5 100644 --- a/src/commands/jobs/watch.test.ts +++ b/src/commands/jobs/watch.test.ts @@ -5,7 +5,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; // Mock modules before importing -vi.mock('../../core/batch-manager.js', () => ({ +vi.mock('../../core/batch-manager.js', async (importOriginal) => ({ + ...(await importOriginal()), createBatchManager: vi.fn(), })); @@ -13,6 +14,37 @@ vi.mock('../../core/batch-manager.js', () => ({ import { createBatchManager } from '../../core/batch-manager.js'; import type { JobStatus, WatchResult, WatchOptions } from '../../core/batch-manager.js'; import { formatProgressBar, formatElapsed } from '../../output/formatter.js'; +import { successOutput } from '../../output/json-envelope.js'; +// Import the real helpers from watch.ts instead of re-implementing them — +// see chat.test.ts for the same pattern. +import JobsWatch, { isTerminalStatus, RESULTS_PREVIEW_LIMIT } from './watch.js'; + +/** + * Create a JobsWatch instance with a mocked oclif Config and captured log + * output, so we can drive its real private outputResult()/formatHumanOutput() + * methods instead of duplicating their logic in assertions. + */ +function createWatchCommand(): { command: JobsWatch; log: ReturnType } { + const mockConfig = { + root: '/mock/root', + bin: 'mainwpcontrol', + name: 'mainwpcontrol', + version: '1.0.0', + pjson: { name: 'mainwpcontrol', version: '1.0.0' }, + dataDir: '/mock/data', + cacheDir: '/mock/cache', + configDir: '/mock/config', + findCommand: vi.fn(), + runCommand: vi.fn(), + runHook: vi.fn(), + }; + + const command = new JobsWatch([], mockConfig as never); + const log = vi.fn(); + command.log = log; + + return { command, log }; +} describe('jobs watch command', () => { let mockWatchJob: ReturnType; @@ -54,17 +86,13 @@ describe('jobs watch command', () => { describe('status display', () => { it('identifies terminal statuses correctly', () => { - const isTerminalStatus = (status: string): boolean => { - return status === 'completed' || status === 'failed' || status === 'partial'; - }; - expect(isTerminalStatus('completed')).toBe(true); expect(isTerminalStatus('failed')).toBe(true); expect(isTerminalStatus('partial')).toBe(true); + expect(isTerminalStatus('cancelled')).toBe(true); expect(isTerminalStatus('pending')).toBe(false); expect(isTerminalStatus('running')).toBe(false); }); - }); describe('generator consumption', () => { @@ -102,7 +130,10 @@ describe('jobs watch command', () => { }); describe('JSON output', () => { - it('structures JSON output correctly', () => { + it('outputs the real success envelope via outputResult()', () => { + const { command, log } = createWatchCommand(); + (command as any).jsonOutput = true; + const result: WatchResult = { status: { id: 'job_123', @@ -116,23 +147,19 @@ describe('jobs watch command', () => { elapsed: 5000, }; - // JSON output structure - const jsonOutput = { - job_id: result.status.id, - status: result.status.status, - progress: result.status.progress, - total: result.status.total, - processed: result.status.processed, - results: result.status.results, - errors: result.status.errors, - timed_out: result.timedOut, - elapsed_ms: result.elapsed, - }; + (command as any).outputResult('job_123', result); + + expect(log).toHaveBeenCalledTimes(1); + const parsed = JSON.parse(log.mock.calls[0]![0] as string); - expect(jsonOutput.job_id).toBe('job_123'); - expect(jsonOutput.status).toBe('completed'); - expect(jsonOutput.timed_out).toBe(false); - expect(jsonOutput.elapsed_ms).toBe(5000); + expect(parsed).toEqual( + successOutput({ + job_id: 'job_123', + ...result.status, + timedOut: false, + elapsed_ms: 5000, + }) + ); }); }); @@ -158,22 +185,50 @@ describe('jobs watch command', () => { }); describe('results preview', () => { - it('limits results preview to 5 items', () => { - const RESULTS_PREVIEW_LIMIT = 5; - const results = Array.from({ length: 10 }, (_, i) => ({ id: i })); - const preview = results.slice(0, RESULTS_PREVIEW_LIMIT); + it('limits results preview to RESULTS_PREVIEW_LIMIT via the real formatHumanOutput path', () => { + const { command, log } = createWatchCommand(); - expect(preview).toHaveLength(5); - expect(results.length - preview.length).toBe(5); // "and 5 more..." + const results = Array.from({ length: RESULTS_PREVIEW_LIMIT + 5 }, (_, i) => ({ + id: i, + name: `site-${i}`, + })); + const result: WatchResult = { + status: { id: 'job_123', status: 'completed', results }, + timedOut: false, + elapsed: 5000, + }; + + (command as any).outputResult('job_123', result); + + const output = log.mock.calls[0]![0] as string; + + for (const item of results.slice(0, RESULTS_PREVIEW_LIMIT)) { + expect(output).toContain(`- ${item.name}`); + } + expect(output).toContain(`... and ${results.length - RESULTS_PREVIEW_LIMIT} more`); + + const excludedItem = results[results.length - 1]!; + expect(output).not.toContain(`- ${excludedItem.name}`); }); - it('shows all results when under limit', () => { - const RESULTS_PREVIEW_LIMIT = 5; - const results = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const preview = results.slice(0, RESULTS_PREVIEW_LIMIT); + it('shows all results when under the limit', () => { + const { command, log } = createWatchCommand(); - expect(preview).toHaveLength(3); - expect(preview).toEqual(results); + const results = [{ id: 1, name: 'site-1' }, { id: 2, name: 'site-2' }]; + const result: WatchResult = { + status: { id: 'job_123', status: 'completed', results }, + timedOut: false, + elapsed: 5000, + }; + + (command as any).outputResult('job_123', result); + + const output = log.mock.calls[0]![0] as string; + + for (const item of results) { + expect(output).toContain(`- ${item.name}`); + } + expect(output).not.toContain('more'); }); }); }); diff --git a/src/commands/jobs/watch.ts b/src/commands/jobs/watch.ts index 336a232..1d2c6be 100644 --- a/src/commands/jobs/watch.ts +++ b/src/commands/jobs/watch.ts @@ -17,12 +17,17 @@ import { formatElapsed, } from '../../output/formatter.js'; import { safeString } from '../../utils/terminal-sanitizer.js'; +import { APIError } from '../../utils/errors.js'; +import { errorOutput } from '../../output/json-envelope.js'; import { + isTerminalStatus, type BatchManager, type JobStatus, type WatchResult, } from '../../core/batch-manager.js'; +export { isTerminalStatus }; + /** Progress bar width in characters */ const PROGRESS_BAR_WIDTH = 30; @@ -30,7 +35,7 @@ const PROGRESS_BAR_WIDTH = 30; const TERMINAL_LINE_WIDTH = 80; /** Maximum number of result items to preview */ -const RESULTS_PREVIEW_LIMIT = 5; +export const RESULTS_PREVIEW_LIMIT = 5; export default class JobsWatch extends BaseCommand { static description = 'Monitor batch job status'; @@ -93,15 +98,19 @@ export default class JobsWatch extends BaseCommand { // Set up abort controller for graceful shutdown const controller = new AbortController(); - const handleSignal = () => { + let signalExitCode: 130 | 143 | undefined; + const handleSignal = (exitCode: 130 | 143) => { + signalExitCode = exitCode; controller.abort(); - if (!flags.json && !flags['no-progress']) { + if (!this.jsonOutput && !flags['no-progress']) { this.log('\nAborted by user.'); } }; + const handleSIGINT = () => handleSignal(130); + const handleSIGTERM = () => handleSignal(143); - process.on('SIGINT', handleSignal); - process.on('SIGTERM', handleSignal); + process.on('SIGINT', handleSIGINT); + process.on('SIGTERM', handleSIGTERM); try { // Watch the job @@ -109,15 +118,62 @@ export default class JobsWatch extends BaseCommand { maxWait: flags.timeout * 1000, initialDelay: flags['initial-delay'], maxDelay: flags['max-delay'], - showProgress: !flags['no-progress'] && !flags.json, + showProgress: !flags['no-progress'] && !this.jsonOutput, signal: controller.signal, }); + if (signalExitCode !== undefined) { + const error = new APIError( + 'CANCELLED', + 'Job watch cancelled by signal', + undefined, + { jobId: args.id } + ); + if (this.jsonOutput) { + this.log(JSON.stringify(errorOutput(error), null, 2)); + } else { + this.logToStderr(formatErrorText(error.message)); + } + this.exit(signalExitCode); + } + + // Non-success outcomes: human mode prints result details before the + // error; JSON mode must emit exactly ONE document, so only the error + // envelope is printed (status travels in its details). + const failedOutcome = result.timedOut + ? new APIError( + 'BATCH_TIMEOUT', + `Batch job ${args.id} timed out`, + undefined, + { jobId: args.id, partialStatus: result.status } + ) + : result.status.status === 'failed' || + result.status.status === 'partial' || + result.status.status === 'cancelled' + ? new APIError( + result.status.status === 'failed' + ? 'BATCH_FAILED' + : result.status.status === 'cancelled' + ? 'BATCH_CANCELLED' + : 'BATCH_PARTIAL', + `Batch job ${args.id} finished with status "${result.status.status}"`, + undefined, + { jobId: args.id, status: result.status } + ) + : undefined; + + if (failedOutcome) { + if (!this.jsonOutput) { + this.outputResult(args.id, result); + } + throw failedOutcome; + } + // Output final result this.outputResult(args.id, result); } finally { - process.off('SIGINT', handleSignal); - process.off('SIGTERM', handleSignal); + process.off('SIGINT', handleSIGINT); + process.off('SIGTERM', handleSIGTERM); } } @@ -203,20 +259,17 @@ export default class JobsWatch extends BaseCommand { return Math.round((status.processed / status.total) * 100); } - // Estimate based on status - switch (status.status) { - case 'pending': - return 0; - case 'running': - return 50; - case 'completed': - return 100; - case 'failed': - case 'partial': - return status.progress ?? 0; - default: - return 0; + if (status.status === 'completed') { + return 100; + } + + if (isTerminalStatus(status.status)) { + // failed or partial + return status.progress ?? 0; } + + // Estimate based on non-terminal status + return status.status === 'running' ? 50 : 0; } /** @@ -266,14 +319,16 @@ export default class JobsWatch extends BaseCommand { // Header if (timedOut) { lines.push(formatWarning(`Job ${jobId} timed out after ${formatElapsed(elapsed)}`)); + } else if (!isTerminalStatus(status.status)) { + lines.push(`Job ${jobId}: ${status.status}`); } else if (status.status === 'completed') { lines.push(formatSuccess(`Job ${jobId} completed`)); } else if (status.status === 'failed') { lines.push(formatErrorText(`Job ${jobId} failed`)); - } else if (status.status === 'partial') { - lines.push(formatWarning(`Job ${jobId} partially completed`)); + } else if (status.status === 'cancelled') { + lines.push(formatWarning(`Job ${jobId} cancelled`)); } else { - lines.push(`Job ${jobId}: ${status.status}`); + lines.push(formatWarning(`Job ${jobId} partially completed`)); } lines.push(''); diff --git a/src/commands/login.ts b/src/commands/login.ts index 3392d62..7c17512 100644 --- a/src/commands/login.ts +++ b/src/commands/login.ts @@ -6,12 +6,14 @@ import { Flags } from '@oclif/core'; import { BaseCommand, commonFlags } from '../lib/base-command.js'; -import { getProfileStore, type Profile } from '../config/profile-store.js'; +import { getProfileStore, validateDashboardUrl, type Profile } from '../config/profile-store.js'; import { getKeychain } from '../config/keychain.js'; import { createHttpClient } from '../core/http-client.js'; import { formatSuccess, formatWarning, formatInfo } from '../output/formatter.js'; import { AuthError, InputError } from '../utils/errors.js'; import { promptForInput, promptForPassword, isInteractive } from '../utils/prompt.js'; +import { sanitizeSingleLine } from '../utils/terminal-sanitizer.js'; +import { maskUrlUserinfo } from '../utils/format.js'; export default class Login extends BaseCommand { static description = 'Authenticate with a MainWP Dashboard'; @@ -101,6 +103,11 @@ export default class Login extends BaseCommand { } normalizedUrl = normalizedUrl.replace(/\/+$/, ''); + // Reject malformed URLs (embedded credentials included) before the + // connection test — undici otherwise fails first with an opaque + // NetworkError and the user never sees the real reason. + validateDashboardUrl(normalizedUrl, { rejectUserinfo: true }); + // Generate profile name from URL if not provided const profileName = flags.name ?? new URL(normalizedUrl).hostname; @@ -162,11 +169,48 @@ export default class Login extends BaseCommand { }; const profileStore = getProfileStore(); - await profileStore.save(profile); - - // Store password in keychain const keychain = getKeychain(); - const keychainResult = await keychain.set(profileName, password); + const previousProfile = await profileStore.get(profileName); + // Rollback must restore only what the keychain actually held (the + // MAINWP_APP_PASSWORD env fallback must never be persisted), and an + // unreadable keychain must abort before the credential is overwritten: + // treating a failed read as "nothing stored" would make a later profile- + // save failure "roll back" by deleting a credential that still exists. + let previousCredential: string | undefined; + if (previousProfile) { + const stored = await keychain.getStored(profileName); + if (stored.status === 'error') { + throw new AuthError( + `Cannot read the existing keychain credential for profile "${profileName}": ${stored.error}`, + undefined, + 'Unlock the system keychain and retry. The stored credential was left untouched.' + ); + } + previousCredential = stored.status === 'found' ? stored.password : undefined; + } + // Attempt credential storage before publishing the profile. A thrown + // keychain failure cannot leave a profile that was only half-created. + // Supported keychain-unavailable environments still receive the existing + // explicit warning and MAINWP_APP_PASSWORD fallback behavior below. + const keychainResult = await keychain.set(profileName, password, normalizedUrl); + try { + await profileStore.save(profile); + } catch (error) { + if (keychainResult.stored) { + const rollbackResult = previousCredential + ? await keychain.set(profileName, previousCredential) + : await keychain.delete(profileName); + const rollbackSucceeded = 'stored' in rollbackResult + ? rollbackResult.stored + : rollbackResult.deleted; + if (!rollbackSucceeded) { + this.logToStderr(formatWarning( + 'Profile save failed and the keychain credential rollback also failed.' + )); + } + } + throw error; + } // Set as active await profileStore.setActive(profileName); @@ -175,15 +219,16 @@ export default class Login extends BaseCommand { this.output( { profile: profileName, - url: normalizedUrl, + // Defense in depth: intake rejection should make masking a no-op here + url: maskUrlUserinfo(normalizedUrl), username, credentialStorage: keychainResult.location, }, () => { const lines = [ - formatSuccess(`Logged in as ${username}`), - ` Profile: ${profileName}`, - ` Dashboard: ${normalizedUrl}`, + formatSuccess(`Logged in as ${sanitizeSingleLine(username)}`), + ` Profile: ${sanitizeSingleLine(profileName)}`, + ` Dashboard: ${sanitizeSingleLine(maskUrlUserinfo(normalizedUrl))}`, ]; if (keychainResult.stored) { @@ -192,7 +237,7 @@ export default class Login extends BaseCommand { lines.push(''); lines.push(formatWarning('Credentials NOT saved to keychain.')); if (keychainResult.error) { - lines.push(` Reason: ${keychainResult.error}`); + lines.push(` Reason: ${sanitizeSingleLine(keychainResult.error)}`); } lines.push( ' Future commands must continue receiving MAINWP_APP_PASSWORD because plaintext credentials are not stored locally.' diff --git a/src/commands/profile/delete.ts b/src/commands/profile/delete.ts index e3e44fd..6c07528 100644 --- a/src/commands/profile/delete.ts +++ b/src/commands/profile/delete.ts @@ -61,17 +61,36 @@ export default class ProfileDelete extends BaseCommand { // Delete credentials from keychain const keychain = getKeychain(); - await keychain.delete(args.name); + const credentialDeletion = await keychain.delete(args.name); // Remove profile from profile store (handles active profile switching automatically) await profileStore.remove(args.name); + // "No credential existed" is the goal state, not a failure — profiles + // authenticated via MAINWP_APP_PASSWORD never had a keychain entry. + const credentialsHandled = credentialDeletion.deleted || credentialDeletion.notFound === true; + this.output( { deleted: args.name, - message: 'Profile and credentials deleted successfully', + credentialsDeleted: credentialDeletion.deleted, + message: credentialDeletion.deleted + ? 'Profile and credentials deleted successfully' + : credentialsHandled + ? 'Profile deleted; no stored credentials found' + : 'Profile deleted, but keychain credential removal failed', + ...(credentialDeletion.error && !credentialsHandled + ? { credentialWarning: credentialDeletion.error } + : {}), }, - () => formatSuccess(`Deleted profile: ${args.name}`) + () => credentialsHandled + ? formatSuccess(`Deleted profile: ${args.name}`) + : [ + formatSuccess(`Deleted profile: ${args.name}`), + formatWarning( + `Keychain credential removal failed${credentialDeletion.error ? `: ${credentialDeletion.error}` : '.'}` + ), + ].join('\n') ); } } diff --git a/src/config/atomic-write.test.ts b/src/config/atomic-write.test.ts index b2a4fc6..93fe505 100644 --- a/src/config/atomic-write.test.ts +++ b/src/config/atomic-write.test.ts @@ -5,7 +5,7 @@ * in both profile-store and settings save paths. */ -import { describe, it, expect, vi, afterEach } from 'vitest'; +import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest'; // Mock fs.promises with all needed methods vi.mock('node:fs', () => ({ @@ -15,6 +15,7 @@ vi.mock('node:fs', () => ({ rename: vi.fn(), unlink: vi.fn().mockResolvedValue(undefined), mkdir: vi.fn().mockResolvedValue(undefined), + open: vi.fn(), }, })); @@ -22,6 +23,18 @@ import { promises as fs } from 'node:fs'; import { saveSettings, clearSettingsCache } from './settings.js'; describe('Atomic write temp file cleanup', () => { + beforeEach(() => { + // atomicWriteFile writes through an exclusive fs.open handle; delegate + // the handle's writeFile to fs.writeFile as (path, content) so the + // existing call-shape assertions hold. + vi.mocked(fs.open).mockImplementation((tmpPath) => Promise.resolve({ + writeFile: (content: unknown) => + (fs.writeFile as unknown as (p: unknown, c: unknown) => Promise)(tmpPath, content), + sync: () => Promise.resolve(), + close: () => Promise.resolve(), + } as unknown as import('node:fs').promises.FileHandle)); + }); + afterEach(() => { clearSettingsCache(); vi.restoreAllMocks(); diff --git a/src/config/fs-utils.test.ts b/src/config/fs-utils.test.ts new file mode 100644 index 0000000..f1631ee --- /dev/null +++ b/src/config/fs-utils.test.ts @@ -0,0 +1,38 @@ +import { promises as fs } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('node:crypto', () => ({ + randomBytes: () => Buffer.from('fixed-temp-suffix'), +})); + +import { atomicWriteFile } from './fs-utils.js'; + +describe('atomicWriteFile', () => { + const createdDirectories: string[] = []; + + afterEach(async () => { + await Promise.all(createdDirectories.splice(0).map((dir) => fs.rm(dir, { + recursive: true, + force: true, + }))); + }); + + it.skipIf(process.platform === 'win32')( + 'refuses to follow an existing symlink at the randomized temporary path', + async () => { + const dir = await fs.mkdtemp(join(tmpdir(), 'mainwpcontrol-fs-utils-')); + createdDirectories.push(dir); + const target = join(dir, 'target'); + const filePath = join(dir, 'settings.json'); + const suffix = Buffer.from('fixed-temp-suffix').toString('hex'); + const tmpPath = `${filePath}.${suffix}.tmp`; + await fs.writeFile(target, 'unchanged', 'utf8'); + await fs.symlink(target, tmpPath); + + await expect(atomicWriteFile(filePath, 'replacement')).rejects.toMatchObject({ code: 'EEXIST' }); + await expect(fs.readFile(target, 'utf8')).resolves.toBe('unchanged'); + }, + ); +}); diff --git a/src/config/fs-utils.ts b/src/config/fs-utils.ts index c973616..3aba207 100644 --- a/src/config/fs-utils.ts +++ b/src/config/fs-utils.ts @@ -5,6 +5,7 @@ */ import { promises as fs } from 'node:fs'; +import { randomBytes } from 'node:crypto'; import { dirname } from 'node:path'; /** @@ -16,19 +17,29 @@ import { dirname } from 'node:path'; */ export async function atomicWriteFile(filePath: string, content: string): Promise { const dir = dirname(filePath); - const tmpPath = `${filePath}.tmp`; + const tmpPath = `${filePath}.${randomBytes(12).toString('hex')}.tmp`; await fs.mkdir(dir, { recursive: true, mode: 0o700 }); - await fs.writeFile(tmpPath, content, { - encoding: 'utf-8', - mode: 0o600, - }); - + let temporaryFileCreated = false; try { + // Exclusive create + explicit fsync before rename: after a crash the + // renamed file must contain the new content, not a zero-length shell. + // (Parent-directory fsync is deliberately omitted — it is not portable + // to Windows and the worst case there is the old file surviving whole.) + const handle = await fs.open(tmpPath, 'wx', 0o600); + temporaryFileCreated = true; + try { + await handle.writeFile(content, 'utf-8'); + await handle.sync(); + } finally { + await handle.close(); + } await fs.rename(tmpPath, filePath); } catch (error) { - await fs.unlink(tmpPath).catch(() => {}); + if (temporaryFileCreated) { + await fs.unlink(tmpPath).catch(() => {}); + } throw error; } } diff --git a/src/config/keychain.test.ts b/src/config/keychain.test.ts new file mode 100644 index 0000000..9dfcb0a --- /dev/null +++ b/src/config/keychain.test.ts @@ -0,0 +1,295 @@ +/** + * Tests for keychain error handling + * + * Keytar is native code and can reject with non-Error values (null, + * undefined, strings). These tests pin the warn-and-continue behavior: + * a bad rejection value must never crash delete() or set(). + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +// Mock keytar so loadKeytar() picks up controllable functions +vi.mock('keytar', () => ({ + setPassword: vi.fn(), + getPassword: vi.fn(), + deletePassword: vi.fn(), +})); + +import * as keytar from 'keytar'; +import { Keychain, canonicalDashboardIdentity } from './keychain.js'; +import { AuthError } from '../utils/errors.js'; + +describe('Keychain error normalization', () => { + let errorSpy: ReturnType; + + beforeEach(() => { + delete process.env['MAINWPCONTROL_NO_KEYTAR']; + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it.each([ + ['null', null, 'null'], + ['undefined', undefined, 'undefined'], + ['a string', 'keychain locked', 'keychain locked'], + ])('delete() reports failure without crashing when keytar rejects with %s', async (_label, rejection, expected) => { + vi.mocked(keytar.deletePassword).mockRejectedValue(rejection); + + await expect(new Keychain().delete('default')).resolves.toEqual({ + deleted: false, + error: expected, + }); + expect(errorSpy).not.toHaveBeenCalled(); + }); + + it('delete() reports the message when keytar rejects with an Error', async () => { + vi.mocked(keytar.deletePassword).mockRejectedValue(new Error('access denied')); + + await expect(new Keychain().delete('default')).resolves.toEqual({ + deleted: false, + error: 'access denied', + }); + }); + + it('delete() marks a missing credential as notFound, not a failure', async () => { + vi.mocked(keytar.deletePassword).mockResolvedValue(false); + + await expect(new Keychain().delete('default')).resolves.toEqual({ + deleted: false, + notFound: true, + error: 'No matching keychain credential was found', + }); + }); + + it('getStored() ignores MAINWP_APP_PASSWORD while get() falls back to it', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue(null); + vi.stubEnv('MAINWP_APP_PASSWORD', 'env-secret'); + + const keychain = new Keychain(); + await expect(keychain.getStored('default')).resolves.toEqual({ status: 'not-found' }); + await expect(keychain.get('default')).resolves.toBe('env-secret'); + + vi.unstubAllEnvs(); + }); + + it('getStored() reports a read error distinctly from not-found', async () => { + vi.mocked(keytar.getPassword).mockRejectedValue(new Error('keychain locked')); + + await expect(new Keychain().getStored('default')).resolves.toEqual({ + status: 'error', + error: 'keychain locked', + }); + }); + + it('getStored() returns the persisted credential when present', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue('stored-secret'); + + await expect(new Keychain().getStored('default')).resolves.toEqual({ + status: 'found', + password: 'stored-secret', + }); + }); + + it('delete() redacts paths and bounds keytar errors', async () => { + vi.mocked(keytar.deletePassword).mockRejectedValue( + new Error(`/Users/tester/.config/mainwpcontrol ${'x'.repeat(1000)}`), + ); + + const result = await new Keychain().delete('default'); + + expect(result.deleted).toBe(false); + expect(result.error).not.toContain('/Users/tester'); + expect(result.error?.length).toBeLessThanOrEqual(500); + }); + + it('delete() reports failure when keytar is unavailable', async () => { + vi.resetModules(); + process.env['MAINWPCONTROL_NO_KEYTAR'] = '1'; + + const { Keychain: FreshKeychain } = await import('./keychain.js'); + + await expect(new FreshKeychain().delete('default')).resolves.toEqual({ + deleted: false, + error: 'Keychain (keytar) is not available', + }); + }); + + it('set() redacts paths and bounds keytar errors', async () => { + vi.mocked(keytar.setPassword).mockRejectedValue( + new Error(`/Users/tester/.config/mainwpcontrol ${'x'.repeat(1000)}`), + ); + + const result = await new Keychain().set('default', 'secret'); + + expect(result.stored).toBe(false); + expect(result.error).not.toContain('/Users/tester'); + expect(result.error?.length).toBeLessThanOrEqual(500); + }); + + it('set() returns a failure result when keytar rejects with a non-Error', async () => { + vi.mocked(keytar.setPassword).mockRejectedValue(null); + + const result = await new Keychain().set('default', 'secret'); + + expect(result).toEqual({ stored: false, location: 'none', error: 'null' }); + }); +}); + +describe('Keychain identity binding', () => { + beforeEach(() => { + delete process.env['MAINWPCONTROL_NO_KEYTAR']; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('canonicalDashboardIdentity', () => { + it('strips a trailing slash', () => { + expect(canonicalDashboardIdentity('https://dash.example.com/wp/')).toBe( + 'https://dash.example.com/wp' + ); + }); + + it('preserves a non-default port', () => { + expect(canonicalDashboardIdentity('https://dash.example.com:8443')).toBe( + 'https://dash.example.com:8443' + ); + }); + + it('strips multiple trailing slashes', () => { + expect(canonicalDashboardIdentity('https://dash.example.com/wp///')).toBe( + 'https://dash.example.com/wp' + ); + }); + + it('leaves a bare host unchanged', () => { + expect(canonicalDashboardIdentity('https://dash.example.com')).toBe( + 'https://dash.example.com' + ); + }); + }); + + it('set() with a dashboardUrl stores a v1 JSON envelope', async () => { + vi.mocked(keytar.setPassword).mockResolvedValue(undefined); + + await new Keychain().set('default', 'pw', 'https://dash.example.com'); + + const [, , payload] = vi.mocked(keytar.setPassword).mock.calls[0]!; + expect(JSON.parse(payload)).toEqual({ + v: 1, + password: 'pw', + identity: 'https://dash.example.com', + }); + }); + + it('set() without a dashboardUrl stores the raw string verbatim (rollback path)', async () => { + vi.mocked(keytar.setPassword).mockResolvedValue(undefined); + + await new Keychain().set('default', 'pw'); + + const [, , payload] = vi.mocked(keytar.setPassword).mock.calls[0]!; + expect(payload).toBe('pw'); + }); + + it('get() with a matching expectedDashboardUrl returns the inner password', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue( + JSON.stringify({ v: 1, password: 'pw', identity: 'https://dash.example.com' }) + ); + + await expect( + new Keychain().get('default', 'https://dash.example.com') + ).resolves.toBe('pw'); + }); + + it('get() throws AuthError mentioning both identities when the expected URL changed', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue( + JSON.stringify({ v: 1, password: 'pw', identity: 'https://old.example.com' }) + ); + + const keychain = new Keychain(); + await expect( + keychain.get('default', 'https://new.example.com') + ).rejects.toBeInstanceOf(AuthError); + await expect( + keychain.get('default', 'https://new.example.com') + ).rejects.toMatchObject({ + message: expect.stringContaining('https://old.example.com'), + }); + await expect( + keychain.get('default', 'https://new.example.com') + ).rejects.toMatchObject({ + message: expect.stringContaining('https://new.example.com'), + }); + }); + + it('getOrThrow() throws AuthError mentioning both identities when the expected URL changed', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue( + JSON.stringify({ v: 1, password: 'pw', identity: 'https://old.example.com' }) + ); + + const keychain = new Keychain(); + await expect( + keychain.getOrThrow('default', 'https://new.example.com') + ).rejects.toBeInstanceOf(AuthError); + await expect( + keychain.getOrThrow('default', 'https://new.example.com') + ).rejects.toMatchObject({ + message: expect.stringContaining('https://old.example.com'), + }); + await expect( + keychain.getOrThrow('default', 'https://new.example.com') + ).rejects.toMatchObject({ + message: expect.stringContaining('https://new.example.com'), + }); + }); + + it('get() refuses a legacy unbound entry for an authenticated request', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue('abcd efgh'); + vi.mocked(keytar.setPassword).mockResolvedValue(undefined); + + // An unbound password cannot be safely bound to whatever URL the profile + // currently holds — the fix is a one-time re-login, never auto-binding. + await expect( + new Keychain().get('default', 'https://dash.example.com') + ).rejects.toBeInstanceOf(AuthError); + await expect( + new Keychain().get('default', 'https://dash.example.com') + ).rejects.toMatchObject({ + hint: expect.stringContaining('login'), + }); + expect(vi.mocked(keytar.setPassword)).not.toHaveBeenCalled(); + }); + + it('get() without an expectedDashboardUrl still reads a legacy entry (display paths)', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue('abcd efgh'); + vi.mocked(keytar.setPassword).mockResolvedValue(undefined); + + await expect(new Keychain().get('default')).resolves.toBe('abcd efgh'); + expect(vi.mocked(keytar.setPassword)).not.toHaveBeenCalled(); + }); + + it('get() falls back to MAINWP_APP_PASSWORD without identity-checking the env var', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue(null); + vi.stubEnv('MAINWP_APP_PASSWORD', 'env-secret'); + + await expect( + new Keychain().get('default', 'https://dash.example.com') + ).resolves.toBe('env-secret'); + + vi.unstubAllEnvs(); + }); + + it('get() treats a "{"-prefixed non-envelope payload as a legacy raw password', async () => { + vi.mocked(keytar.getPassword).mockResolvedValue('{not valid json'); + + // Legacy semantics apply: readable without a URL, refused with one. + await expect(new Keychain().get('default')).resolves.toBe('{not valid json'); + await expect( + new Keychain().get('default', 'https://dash.example.com') + ).rejects.toBeInstanceOf(AuthError); + }); +}); diff --git a/src/config/keychain.ts b/src/config/keychain.ts index 7885105..7751bc7 100644 --- a/src/config/keychain.ts +++ b/src/config/keychain.ts @@ -10,6 +10,8 @@ */ import { AuthError } from '../utils/errors.js'; +import { sanitizeErrorMessage } from '../utils/error-sanitizer.js'; +import { sanitizeSingleLine } from '../utils/terminal-sanitizer.js'; /** * Service name for keychain entries @@ -26,6 +28,21 @@ const ENV_VAR = 'MAINWP_APP_PASSWORD'; * dialog, this prevents the CLI from hanging indefinitely. */ const KEYTAR_TIMEOUT_MS = 5_000; +const MAX_KEYCHAIN_ERROR_LENGTH = 500; + +/** + * Keytar is native code and can reject with non-Error values; a blind + * `(error as Error).message` throws on null/undefined and turns a + * warn-and-continue path into a crash. + */ +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function sanitizeKeychainError(error: unknown): string { + return sanitizeErrorMessage(sanitizeSingleLine(errorMessage(error))) + .slice(0, MAX_KEYCHAIN_ERROR_LENGTH); +} function withTimeout(promise: Promise, ms: number): Promise { return new Promise((resolve, reject) => { @@ -87,6 +104,63 @@ async function loadKeytar(): Promise { } } +/** + * Canonical Dashboard identity used to bind a stored credential to the + * destination it was saved for: scheme + host(:port) + normalized base path. + * Profile names are user-facing selectors, not an authorization boundary + * (AGENTS.md) — this is the boundary. + */ +export function canonicalDashboardIdentity(dashboardUrl: string): string { + const parsed = new URL(dashboardUrl); + const path = parsed.pathname.replace(/\/+$/, ''); + return `${parsed.protocol}//${parsed.host}${path}`; +} + +/** + * Stored credential envelope (format v1). Legacy entries are the bare + * application password; new entries bind the password to the Dashboard + * identity they were saved for, so editing profiles.json cannot silently + * redirect a stored credential to a different host. + */ +interface StoredCredentialV1 { + v: 1; + password: string; + identity: string; +} + +function encodeCredential(password: string, dashboardUrl: string): string { + const envelope: StoredCredentialV1 = { + v: 1, + password, + identity: canonicalDashboardIdentity(dashboardUrl), + }; + return JSON.stringify(envelope); +} + +/** + * Decode a stored keychain payload. WordPress application passwords never + * start with "{", so a JSON-looking payload that fails to parse as a v1 + * envelope is treated as a legacy bare password rather than rejected. + */ +function decodeCredential(raw: string): { password: string; identity?: string } { + if (raw.startsWith('{')) { + try { + const parsed = JSON.parse(raw) as Partial; + if ( + parsed !== null && + parsed.v === 1 && + typeof parsed.password === 'string' && + typeof parsed.identity === 'string' + ) { + return { password: parsed.password, identity: parsed.identity }; + } + } catch { + // Fall through: treat as legacy raw secret + } + } + return { password: raw }; +} + /** * Result of a credential storage operation */ @@ -99,6 +173,23 @@ export interface KeychainSetResult { error?: string; } +export interface KeychainDeleteResult { + deleted: boolean; + /** True when no credential existed to delete — the goal state already holds. */ + notFound?: boolean; + error?: string; +} + +/** + * Result of a persisted-only credential read. Distinguishes "nothing stored" + * from "could not read" — callers making destructive decisions (rollback, + * overwrite) must not treat a failed read as an empty keychain. + */ +export type KeychainReadResult = + | { status: 'found'; password: string } + | { status: 'not-found' } + | { status: 'error'; error: string }; + /** * Keychain class */ @@ -112,23 +203,33 @@ export class Keychain { } /** - * Store a credential + * Store a credential. + * + * When `dashboardUrl` is provided the password is stored bound to that + * Dashboard's canonical identity; retrieval with an expected URL then + * refuses to release the credential to a different destination. Omit + * `dashboardUrl` only to restore a previously read raw payload verbatim + * (rollback). * * @returns Result indicating whether storage succeeded and where credentials are stored */ - async set(profileName: string, password: string): Promise { + async set( + profileName: string, + password: string, + dashboardUrl?: string + ): Promise { const kt = await loadKeytar(); + const payload = dashboardUrl ? encodeCredential(password, dashboardUrl) : password; if (kt) { try { - await withTimeout(kt.setPassword(SERVICE_NAME, profileName, password), KEYTAR_TIMEOUT_MS); + await withTimeout(kt.setPassword(SERVICE_NAME, profileName, payload), KEYTAR_TIMEOUT_MS); return { stored: true, location: 'keychain' }; } catch (error) { - const errorMessage = (error as Error).message; return { stored: false, location: 'none', - error: errorMessage, + error: sanitizeKeychainError(error), }; } } @@ -141,21 +242,70 @@ export class Keychain { } /** - * Retrieve a credential + * Read the persisted keychain credential only — no MAINWP_APP_PASSWORD + * fallback (the env var must never masquerade as a stored credential). + * + * An unavailable keytar reads as not-found: nothing can be stored or + * deleted through it either, so no overwrite/rollback hazard exists. */ - async get(profileName: string): Promise { - // First try keytar + async getStored(profileName: string): Promise { const kt = await loadKeytar(); - if (kt) { - try { - const password = await withTimeout(kt.getPassword(SERVICE_NAME, profileName), KEYTAR_TIMEOUT_MS); - if (password) { - return password; + if (!kt) { + return { status: 'not-found' }; + } + + try { + const password = await withTimeout(kt.getPassword(SERVICE_NAME, profileName), KEYTAR_TIMEOUT_MS); + return password ? { status: 'found', password } : { status: 'not-found' }; + } catch (error) { + return { status: 'error', error: sanitizeKeychainError(error) }; + } + } + + /** + * Retrieve a credential for authentication: keytar first, then the + * MAINWP_APP_PASSWORD environment variable. Keytar read errors fall + * through to the env var. + * + * When `expectedDashboardUrl` is provided and the stored credential is + * identity-bound, a mismatch throws instead of releasing the password — + * a hand-edited profiles.json must not redirect a stored credential to a + * different host. Legacy (unbound) entries are refused for authenticated + * use when an expected URL is provided; a one-time `login` re-binds them. + * Without an expected URL they still read, for display paths. The env var + * is per-invocation operator input and is not identity-checked. + */ + async get( + profileName: string, + expectedDashboardUrl?: string + ): Promise { + const stored = await this.getStored(profileName); + if (stored.status === 'found') { + const decoded = decodeCredential(stored.password); + if (expectedDashboardUrl && decoded.identity) { + const expected = canonicalDashboardIdentity(expectedDashboardUrl); + if (decoded.identity !== expected) { + throw new AuthError( + `The stored credential for profile "${profileName}" was saved for ${decoded.identity}, but the profile now points to ${expected}. Refusing to send it.`, + undefined, + 'The profile URL changed after login. Run `mainwpcontrol login` to re-authenticate against the new URL.' + ); } - } catch { - // Keytar failed, fall through to env var } + if (expectedDashboardUrl && !decoded.identity) { + // Legacy unbound entry: refuse authenticated use. Binding it to the + // profile's current URL would just bless whatever the file says at + // first use — an unknown password cannot be safely bound without + // independently proving the destination. A one-time re-login binds + // it with the user seeing and providing the URL. + throw new AuthError( + `The stored credential for profile "${profileName}" predates credential-identity binding and cannot be safely used.`, + undefined, + 'One-time upgrade: run `mainwpcontrol login` to re-authenticate and bind the credential to your Dashboard URL.' + ); + } + return decoded.password; } // Fallback to environment variable @@ -170,25 +320,37 @@ export class Keychain { /** * Delete a credential */ - async delete(profileName: string): Promise { + async delete(profileName: string): Promise { const kt = await loadKeytar(); if (kt) { try { - await withTimeout(kt.deletePassword(SERVICE_NAME, profileName), KEYTAR_TIMEOUT_MS); + const deleted = await withTimeout( + kt.deletePassword(SERVICE_NAME, profileName), + KEYTAR_TIMEOUT_MS, + ); + return deleted + ? { deleted: true } + : { deleted: false, notFound: true, error: 'No matching keychain credential was found' }; } catch (error) { - if (process.stderr.isTTY) { - console.error(`Warning: Failed to remove credentials from keychain: ${(error as Error).message}`); - } + return { + deleted: false, + error: sanitizeKeychainError(error), + }; } } + + return { + deleted: false, + error: 'Keychain (keytar) is not available', + }; } /** * Get credential or throw */ - async getOrThrow(profileName: string): Promise { - const password = await this.get(profileName); + async getOrThrow(profileName: string, expectedDashboardUrl?: string): Promise { + const password = await this.get(profileName, expectedDashboardUrl); if (!password) { throw new AuthError( diff --git a/src/config/profile-store.test.ts b/src/config/profile-store.test.ts new file mode 100644 index 0000000..ec3278c --- /dev/null +++ b/src/config/profile-store.test.ts @@ -0,0 +1,104 @@ +import { promises as fs } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ProfileStore, type Profile } from './profile-store.js'; + +const baseProfile: Profile = { + name: 'test', + dashboardUrl: 'https://dashboard.example.com', + username: 'admin', + createdAt: '2026-07-13T00:00:00.000Z', +}; + +describe('ProfileStore URL validation', () => { + const originalXdgConfigHome = process.env['XDG_CONFIG_HOME']; + let tempRoot: string; + + beforeEach(async () => { + tempRoot = await fs.mkdtemp(join(tmpdir(), 'mainwp-profile-store-')); + process.env['XDG_CONFIG_HOME'] = tempRoot; + }); + + afterEach(async () => { + if (originalXdgConfigHome === undefined) { + delete process.env['XDG_CONFIG_HOME']; + } else { + process.env['XDG_CONFIG_HOME'] = originalXdgConfigHome; + } + await fs.rm(tempRoot, { recursive: true, force: true }); + }); + + it.each([ + 'https://embedded@dashboard.example.com', + 'https://embedded:secret@dashboard.example.com', + ])('rejects dashboard URLs with embedded userinfo: %s', async (dashboardUrl) => { + const store = new ProfileStore(); + + await expect(store.save({ ...baseProfile, dashboardUrl })).rejects.toMatchObject({ + message: 'Embedded credentials in the dashboard URL are not supported', + hint: expect.stringMatching(/--username.*password prompt/i), + }); + }); + + async function writeProfilesFile(skipSSLVerification: unknown): Promise { + const configDir = join(tempRoot, 'mainwpcontrol'); + await fs.mkdir(configDir, { recursive: true }); + await fs.writeFile( + join(configDir, 'profiles.json'), + JSON.stringify({ + activeProfile: baseProfile.name, + profiles: [{ ...baseProfile, skipSSLVerification }], + }) + ); + } + + it.each([ + ['false', 'string "false"'], + ['true', 'string "true"'], + ])('coerces non-boolean skipSSLVerification (%s) to false and warns', async (rawValue) => { + await writeProfilesFile(rawValue); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const store = new ProfileStore(); + + const profile = await store.get(baseProfile.name); + + expect(profile?.skipSSLVerification).toBe(false); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining('Ignoring invalid skipSSLVerification for profile "test"; expected a boolean.') + ); + + errorSpy.mockRestore(); + }); + + it('preserves a valid boolean skipSSLVerification without warning', async () => { + await writeProfilesFile(true); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const store = new ProfileStore(); + + const profile = await store.get(baseProfile.name); + + expect(profile?.skipSSLVerification).toBe(true); + expect(errorSpy).not.toHaveBeenCalled(); + + errorSpy.mockRestore(); + }); + + it('leaves skipSSLVerification absent when not set', async () => { + const configDir = join(tempRoot, 'mainwpcontrol'); + await fs.mkdir(configDir, { recursive: true }); + await fs.writeFile( + join(configDir, 'profiles.json'), + JSON.stringify({ activeProfile: baseProfile.name, profiles: [baseProfile] }) + ); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const store = new ProfileStore(); + + const profile = await store.get(baseProfile.name); + + expect(profile?.skipSSLVerification).toBeUndefined(); + expect(errorSpy).not.toHaveBeenCalled(); + + errorSpy.mockRestore(); + }); +}); diff --git a/src/config/profile-store.ts b/src/config/profile-store.ts index 3457d83..fc97eb3 100644 --- a/src/config/profile-store.ts +++ b/src/config/profile-store.ts @@ -9,6 +9,7 @@ import { join } from 'node:path'; import { ConfigError } from '../utils/errors.js'; import { getConfigDir } from './settings.js'; import { atomicWriteFile } from './fs-utils.js'; +import { sanitizeSingleLine } from '../utils/terminal-sanitizer.js'; /** * Profile data (credentials stored separately in keychain) @@ -79,42 +80,68 @@ async function saveProfilesFile(data: ProfilesFile): Promise { await atomicWriteFile(path, JSON.stringify(data, null, 2)); } +/** + * Validate a Dashboard URL's format and protocol + * + * `rejectUserinfo` is set only on intake paths (login, save): legacy profiles + * already on disk with embedded credentials must keep loading so their + * URLs can be masked at display instead of bricking the config. + */ +export function validateDashboardUrl( + url: string, + options: { rejectUserinfo?: boolean } = {} +): void { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + // Never echo the malformed URL: it can embed credentials + // (https://user:pass@host) that would land in terminal output and logs. + throw new ConfigError( + 'Invalid Dashboard URL format', + undefined, + 'URL must include protocol (http:// or https://) and hostname' + ); + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new ConfigError( + `Invalid URL protocol: ${parsed.protocol}. Must be http or https`, + undefined, + 'Only HTTP and HTTPS protocols are supported' + ); + } + + // SECURITY: Reject rather than silently strip — the user should know + // their pasted URL carried credentials. + if (options.rejectUserinfo && (parsed.username || parsed.password)) { + throw new ConfigError( + 'Embedded credentials in the dashboard URL are not supported', + undefined, + 'Pass the username with --username and enter the password at the password prompt' + ); + } + + // HTTP warning is emitted at login time via formatWarning, not here +} + /** * Profile store class */ export class ProfileStore { private data: ProfilesFile | null = null; - /** - * Validate a URL format and protocol - */ - private validateUrl(url: string): void { - let parsed: URL; - try { - parsed = new URL(url); - } catch { - throw new ConfigError( - `Invalid Dashboard URL format: ${url}`, - undefined, - 'URL must include protocol (http:// or https://) and hostname' - ); - } - - if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { - throw new ConfigError( - `Invalid URL protocol: ${parsed.protocol}. Must be http or https`, - undefined, - 'Only HTTP and HTTPS protocols are supported' - ); - } - - // HTTP warning is emitted at login time via formatWarning, not here + private validateUrl(url: string, options: { rejectUserinfo?: boolean } = {}): void { + validateDashboardUrl(url, options); } /** * Validate a profile's required fields and URL format */ - private validateProfile(profile: Profile): void { + private validateProfile( + profile: Profile, + options: { rejectUserinfo?: boolean } = {} + ): void { const validationHint = 'Run `mainwpcontrol login` to create a valid profile'; if (!profile.name || profile.name.trim().length === 0) { @@ -149,7 +176,21 @@ export class ProfileStore { ); } - this.validateUrl(profile.dashboardUrl); + this.validateUrl(profile.dashboardUrl, options); + + // SECURITY: a non-boolean skipSSLVerification (e.g. the string "false") + // is truthy and would silently disable TLS verification downstream. + // Fail closed: coerce to false and warn, mirroring readBooleanSetting() + // in settings.ts. + if ( + profile.skipSSLVerification !== undefined && + typeof profile.skipSSLVerification !== 'boolean' + ) { + console.error( + `Warning: Ignoring invalid skipSSLVerification for profile "${sanitizeSingleLine(profile.name)}"; expected a boolean. Falling back to false.` + ); + profile.skipSSLVerification = false; + } } /** @@ -175,6 +216,10 @@ export class ProfileStore { data.activeProfile && !data.profiles.some((p) => p.name === data.activeProfile) ) { + console.error( + `Warning: Active profile "${sanitizeSingleLine(data.activeProfile)}" no longer exists. ` + + `Falling back to "${sanitizeSingleLine(data.profiles[0]?.name ?? 'none')}".` + ); data.activeProfile = data.profiles[0]?.name; } } @@ -255,8 +300,9 @@ export class ProfileStore { * Save a profile (create or update) */ async save(profile: Profile): Promise { - // Validate profile before saving - this.validateProfile(profile); + // Validate profile before saving; intake is the only place userinfo + // URLs are rejected outright (legacy stored profiles are masked instead) + this.validateProfile(profile, { rejectUserinfo: true }); const data = await this.ensureLoaded(); diff --git a/src/core/abilities-executor.test.ts b/src/core/abilities-executor.test.ts index 34fca75..d1f2ea1 100644 --- a/src/core/abilities-executor.test.ts +++ b/src/core/abilities-executor.test.ts @@ -77,6 +77,53 @@ describe('AbilitiesExecutor', () => { }, }, }, + { + // String-typed annotation from a buggy/hostile server. SafetyController + // validates annotations with strict boolean checks and treats this as + // unset; transport must resolve it the same way (POST, never GET). + name: 'mainwp/get-stats-v1', + label: 'Get Stats', + description: 'Site statistics', + category: 'sites', + meta: { + annotations: { + readonly: 'true', + destructive: false, + idempotent: true, + }, + }, + } as unknown as Ability, + { + // One malformed boolean invalidates the annotation set. A hostile + // readonly:true value must not select GET when policy fails closed. + name: 'mainwp/get-malformed-v1', + label: 'Get Malformed', + description: 'Malformed annotation fixture', + category: 'sites', + meta: { + annotations: { + readonly: true, + destructive: false, + idempotent: 'false', + }, + }, + } as unknown as Ability, + { + // Contradictory/skewed annotations: destructive NAME but readonly:true. + // Used to verify transport (HTTP method) resolves destructiveness the + // same way policy does, and never routes this out as GET. + name: 'mainwp/reset-site-v1', + label: 'Reset Site', + description: 'Reset a site to defaults', + category: 'sites', + meta: { + annotations: { + readonly: true, + destructive: false, + idempotent: true, + }, + }, + }, ]; beforeEach(() => { @@ -137,6 +184,81 @@ describe('AbilitiesExecutor', () => { expect(byShort).toBeDefined(); expect(byFull?.name).toBe(byShort?.name); }); + + it('skips malformed discovery entries and invalid ability names with warnings', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockGet.mockResolvedValueOnce({ + data: [ + null, + [], + { name: '' }, + { name: 'missing-namespace-v1' }, + mockAbilities[0], + ], + }); + + const abilities = await executor.listAbilities(); + + expect(abilities).toEqual([mockAbilities[0]]); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('invalid ability')); + }); + + it('refuses case-variant ability names so they cannot evade destructive classification', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockGet.mockResolvedValueOnce({ + data: [ + { ...mockAbilities[0], name: 'Mainwp/Delete-Site-V1' }, + mockAbilities[0], + ], + }); + + const abilities = await executor.listAbilities(); + + expect(abilities).toEqual([mockAbilities[0]]); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('invalid ability')); + }); + + it('keeps the first duplicate full name and warns', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockGet.mockResolvedValueOnce({ + data: [ + mockAbilities[0], + { ...mockAbilities[0], label: 'Duplicate' }, + ], + }); + + await executor.listAbilities(); + + expect((await executor.getAbility('mainwp/list-sites-v1'))?.label).toBe('List Sites'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('duplicate ability')); + }); + + it('removes colliding short aliases while preserving both full names', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const first = { ...mockAbilities[0], name: 'alpha/shared-v1' }; + const second = { ...mockAbilities[0], name: 'beta/shared-v1' }; + mockGet.mockResolvedValueOnce({ data: [first, second] }); + + await executor.listAbilities(); + + expect(await executor.getAbility('alpha/shared-v1')).toEqual(first); + expect(await executor.getAbility('beta/shared-v1')).toEqual(second); + expect(await executor.getAbility('shared-v1')).toBeUndefined(); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('ambiguous short alias')); + }); + + it('rejects discovery that exceeds the configured page cap', async () => { + mockGet.mockResolvedValue({ + data: [], + headers: new Headers({ 'x-wp-totalpages': '999' }), + }); + + await expect(executor.listAbilities()).rejects.toMatchObject({ + code: 'INVALID_RESPONSE', + }); + + expect(mockGet).toHaveBeenCalledTimes(1); + }); }); describe('getAbility', () => { @@ -218,6 +340,57 @@ describe('AbilitiesExecutor', () => { expect(mockDelete).toHaveBeenCalledOnce(); }); + it('never uses GET for a destructive-named ability marked readonly (transport/policy consistency)', async () => { + mockPost.mockResolvedValueOnce({ data: { success: true } }); + + // reset-site-v1 is annotated readonly:true but its name is known-destructive. + // Transport must resolve destructiveness the same way SafetyController does, + // so this never goes out as GET. It also must not go out as DELETE: the + // annotations are distrusted here, so their `idempotent` flag cannot + // pick the method — the safe write default is POST. + await executor.execute('reset-site-v1', { site_id: 1 }, { confirm: true }); + + // The run request went out as POST; a readonly GET run would have left + // mockPost uncalled. + expect(mockPost).toHaveBeenCalledOnce(); + expect(mockDelete).not.toHaveBeenCalled(); + // The only GET is the abilities-list fetch — the run itself never uses GET. + expect(mockGet).toHaveBeenCalledTimes(1); + expect(String(mockGet.mock.calls[0]?.[0])).not.toContain('/run'); + }); + + it('treats non-boolean annotation values as unset for method selection', async () => { + mockPost.mockResolvedValueOnce({ data: { success: true } }); + + // get-stats-v1 carries readonly: "true" (string). SafetyController's + // strict boolean validation ignores it, so transport must too: + // the run goes out as POST, never a readonly GET. + await executor.execute('get-stats-v1', {}); + + expect(mockPost).toHaveBeenCalledOnce(); + // The only GET was the abilities-list fetch, not a readonly run. + expect(mockGet).toHaveBeenCalledOnce(); + }); + + it('uses POST when any annotation field is malformed despite readonly true', async () => { + mockPost.mockResolvedValueOnce({ data: { success: true } }); + + await executor.execute('get-malformed-v1', {}); + + expect(mockPost).toHaveBeenCalledOnce(); + expect(mockGet).toHaveBeenCalledOnce(); + }); + + it('rejects a request with both dryRun and confirm set', async () => { + await expect( + executor.execute('delete-site-v1', { site_id: 1 }, { dryRun: true, confirm: true }) + ).rejects.toThrow(/cannot both be set/); + + // No run request of any kind is emitted with both flags. + expect(mockPost).not.toHaveBeenCalled(); + expect(mockDelete).not.toHaveBeenCalled(); + }); + it('adds dry_run flag when dryRun option is true', async () => { mockPost.mockResolvedValueOnce({ data: { @@ -296,6 +469,50 @@ describe('AbilitiesExecutor', () => { { id: 2, name: 'Site 2' }, ]); }); + + it('normalizes the Dashboard queued envelope and exposes its job id', async () => { + mockGet.mockResolvedValueOnce({ + data: { + queued: true, + job_id: 'sync_123', + status_url: 'https://dashboard.local/wp-json/mainwp/v2/jobs/sync_123', + sites_queued: 10, + }, + }); + + const result = await executor.execute('list-sites-v1', {}); + + expect(result.success).toBe(true); + expect(result.jobId).toBe('sync_123'); + }); + + it('extracts a queued job id from a wrapped data envelope', async () => { + mockGet.mockResolvedValueOnce({ + data: { + success: true, + data: { queued: true, job_id: 'sync_456' }, + }, + }); + + const result = await executor.execute('list-sites-v1', {}); + + expect(result.success).toBe(true); + expect(result.jobId).toBe('sync_456'); + }); + + it.each(['bad\njob', 'x'.repeat(513)])( + 'rejects an unsafe queued job id', + async (jobId) => { + mockGet.mockResolvedValueOnce({ + data: { queued: true, job_id: jobId }, + }); + + await expect(executor.execute('list-sites-v1', {})).resolves.toMatchObject({ + success: false, + error: { code: 'INVALID_RESPONSE' }, + }); + }, + ); }); describe('getCategories', () => { diff --git a/src/core/abilities-executor.ts b/src/core/abilities-executor.ts index d801119..b144b33 100644 --- a/src/core/abilities-executor.ts +++ b/src/core/abilities-executor.ts @@ -8,6 +8,8 @@ import { HttpClient, type HttpClientConfig, createHttpClient } from './http-client.js'; import { APIError, InputError } from '../utils/errors.js'; import { getInputSanitizer } from '../validation/input-sanitizer.js'; +import { isKnownDestructiveName } from './safety-controller.js'; +import { validateJobId } from './job-id.js'; /** * Ability annotation metadata @@ -64,6 +66,13 @@ export interface ExecutionResult { */ type AbilitiesListResponse = Ability[]; +const MAX_DISCOVERY_PAGES = 20; +// Deliberately case-SENSITIVE: the Dashboard registers lowercase names, and +// the destructive-name override (DESTRUCTIVE_NAME_PATTERNS) matches lowercase. +// A case-variant like "Mainwp/Delete-Site-V1" is refused at discovery rather +// than normalized, so it can never evade classification or alias a cache key. +const ABILITY_NAME_PATTERN = /^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._-]*-v[1-9]\d*$/; + /** * Abilities Executor class */ @@ -73,6 +82,8 @@ export class AbilitiesExecutor { private abilitiesCache: Map | null = null; private cacheExpiry = 0; private readonly cacheTTL = 5 * 60 * 1000; // 5 minutes + /** In-flight cache fill, shared by concurrent callers to avoid a fetch stampede. */ + private cacheFillPromise: Promise | null = null; constructor(config: HttpClientConfig) { this.httpClient = createHttpClient(config); @@ -122,8 +133,9 @@ export class AbilitiesExecutor { ); } - // Build the request body - const body = this.buildRequestBody(input, options); + // Merge execution options into user input — the single place control + // flags are applied. Both request paths below format this same output. + const params = this.buildEffectiveParams(input, options); // Determine HTTP method based on annotations const method = this.getHttpMethod(ability, options); @@ -140,17 +152,20 @@ export class AbilitiesExecutor { if (method === 'GET') { // For GET requests, nest input under input[key] (WordPress REST style). - // Control flags (dry_run, confirm) stay top-level. - const queryString = this.buildGetQueryString(input, options); + const queryString = this.buildGetQueryString(params); const url = queryString ? `${endpoint}?${queryString}` : endpoint; response = await this.httpClient.get>(url, requestOptions); } else if (method === 'DELETE') { - const queryString = this.buildGetQueryString(input, options); + const queryString = this.buildGetQueryString(params); const url = queryString ? `${endpoint}?${queryString}` : endpoint; response = await this.httpClient.delete>(url, requestOptions); } else { // POST with JSON body - response = await this.httpClient.post>(endpoint, body, requestOptions); + response = await this.httpClient.post>( + endpoint, + { input: params }, + requestOptions + ); } return this.normalizeResponse(response.data); @@ -191,14 +206,32 @@ export class AbilitiesExecutor { } /** - * Ensure abilities cache is populated + * Ensure abilities cache is populated. + * Concurrent callers share one in-flight fetch instead of each starting + * their own paginated fetch (plausible in chat's tool-calling loop). */ private async ensureCache(): Promise { if (this.abilitiesCache && Date.now() < this.cacheExpiry) { return; } - this.abilitiesCache = new Map(); + if (this.cacheFillPromise) { + return this.cacheFillPromise; + } + + this.cacheFillPromise = this.fillCache().finally(() => { + this.cacheFillPromise = null; + }); + + return this.cacheFillPromise; + } + + /** + * Fetch all ability pages and populate the cache. + */ + private async fillCache(): Promise { + const cache = new Map(); + const aliasOwners = new Map(); // Fetch all pages — API returns Ability[] with WP pagination headers. let page = 1; @@ -209,28 +242,64 @@ export class AbilitiesExecutor { `${this.baseEndpoint}/abilities?per_page=100&page=${page}` ); - const abilities = Array.isArray(response.data) + const abilities: unknown[] = Array.isArray(response.data) ? response.data - : (response.data as Record)['abilities'] as Ability[] ?? []; + : this.asRecord(response.data)?.['abilities'] instanceof Array + ? this.asRecord(response.data)?.['abilities'] as unknown[] + : []; + + for (const entry of abilities) { + if (!this.isPlainObject(entry) || + typeof entry['name'] !== 'string' || + !ABILITY_NAME_PATTERN.test(entry['name'])) { + console.error('Warning: Discovery skipped an invalid ability entry.'); + continue; + } - for (const ability of abilities) { - this.abilitiesCache.set(ability.name, ability); + const ability = entry as unknown as Ability; + if (cache.has(ability.name)) { + console.error(`Warning: Discovery ignored duplicate ability "${ability.name}".`); + continue; + } + cache.set(ability.name, ability); const shortName = this.getShortName(ability.name); if (shortName !== ability.name) { - this.abilitiesCache.set(shortName, ability); + const existingOwner = aliasOwners.get(shortName); + if (existingOwner === undefined) { + aliasOwners.set(shortName, ability.name); + cache.set(shortName, ability); + } else if (existingOwner !== null) { + aliasOwners.set(shortName, null); + cache.delete(shortName); + console.error( + `Warning: Discovery removed ambiguous short alias "${shortName}". Use full ability names.` + ); + } } } // Read WP pagination header for total pages const wpTotalPages = response.headers?.get?.('x-wp-totalpages'); - if (wpTotalPages) { - totalPages = parseInt(wpTotalPages, 10) || 1; + if (page === 1 && wpTotalPages) { + const declaredPages = Number.parseInt(wpTotalPages, 10); + if (Number.isInteger(declaredPages) && declaredPages > 0) { + if (declaredPages > MAX_DISCOVERY_PAGES) { + throw new APIError( + 'INVALID_RESPONSE', + `Discovery declared ${declaredPages} pages, exceeding the ${MAX_DISCOVERY_PAGES}-page limit.` + ); + } + totalPages = declaredPages; + } else { + console.error('Warning: Discovery returned an invalid pagination header.'); + } } page++; } while (page <= totalPages); + this.abilitiesCache = cache; this.cacheExpiry = Date.now() + this.cacheTTL; } @@ -251,17 +320,32 @@ export class AbilitiesExecutor { } /** - * Build request body with execution options. - * The WP Abilities API expects: { input: { ...userInput, dry_run?, confirm? } } + * Merge execution options into user input, producing the final param set. + * Both the POST body and the GET/DELETE query-string path format this same + * output — this is the single place dry_run/confirm/user_confirmed are applied. + * + * SECURITY: strips any dry_run/confirm/user_confirmed present in user/LLM + * input before applying the ones from `options`. These control flags are + * set exclusively by execution options (CLI flags or the safety flow) — + * preserve this strip exactly, do not weaken it. */ - private buildRequestBody( + private buildEffectiveParams( input: Record, options?: ExecutionOptions ): Record { + // INVARIANT: dry_run and confirm are mutually exclusive. Callers enforce + // this upstream (SafetyController.validateExecutionFlags, oclif exclusive + // flags); assert here too so no code path can emit a request carrying both. + if (options?.dryRun && options?.confirm) { + throw new InputError( + 'dry_run and confirm cannot both be set', + undefined, + 'This is an internal error — preview and execute are separate steps.' + ); + } + const merged = { ...input }; - // SECURITY: Strip control flags from user/LLM-provided input. - // These are set exclusively from execution options (CLI flags or safety flow). delete merged['dry_run']; delete merged['confirm']; delete merged['user_confirmed']; @@ -275,7 +359,7 @@ export class AbilitiesExecutor { merged['user_confirmed'] = true; } - return { input: merged }; + return merged; } /** @@ -286,14 +370,31 @@ export class AbilitiesExecutor { _options?: ExecutionOptions ): 'GET' | 'POST' | 'DELETE' { const annotations = ability.meta?.annotations; - - // If readonly, use GET - if (annotations?.readonly) { + const annotationsAreValid = + typeof annotations?.readonly === 'boolean' && + typeof annotations.destructive === 'boolean' && + typeof annotations.idempotent === 'boolean'; + + // Resolve destructiveness the same way SafetyController does — annotations + // OR a known-destructive name — so transport never disagrees with policy. + // A name-destructive ability must never go out as GET (readonly transport), + // even if a hostile/buggy server marks it readonly. + // Strict === true matches SafetyController.validateAnnotations(): a + // non-boolean annotation value (e.g. readonly: "true" from a buggy or + // hostile server) must not be treated as set. + const annotatedDestructive = annotationsAreValid && annotations.destructive; + const destructive = annotatedDestructive || isKnownDestructiveName(ability.name); + + // Read-only (and not name-destructive) → GET + if (annotationsAreValid && annotations.readonly && !destructive) { return 'GET'; } - // If destructive and idempotent, use DELETE - if (annotations?.destructive && annotations?.idempotent) { + // Destructive and idempotent → DELETE. Only when the annotations + // themselves say destructive: if destructiveness came from the name + // override, the annotations are already distrusted, so `idempotent` + // from the same source must not pick the method — fall through to POST. + if (annotatedDestructive && annotations.idempotent) { return 'DELETE'; } @@ -303,21 +404,16 @@ export class AbilitiesExecutor { /** * Build query string for GET/DELETE requests with WordPress-style nesting. - * User input goes under input[key]=value; control flags stay top-level. + * `params` is already the merged output of buildEffectiveParams(), so + * dry_run/confirm/user_confirmed (if present) are formatted the same as + * any other param — no separate control-flag handling needed here. */ - private buildGetQueryString( - input: Record, - options?: ExecutionOptions, - ): string { + private buildGetQueryString(params: Record): string { const parts: string[] = []; - // SECURITY: Control flags managed exclusively by execution options - const controlFlags = ['dry_run', 'confirm', 'user_confirmed']; - // All params go under input[key] — the API treats input as a single object. - for (const [key, value] of Object.entries(input)) { + for (const [key, value] of Object.entries(params)) { if (value === undefined || value === null) continue; - if (controlFlags.includes(key)) continue; const ek = encodeURIComponent(key); if (Array.isArray(value)) { @@ -332,12 +428,6 @@ export class AbilitiesExecutor { } } - if (options?.dryRun) parts.push('input[dry_run]=true'); - if (options?.confirm) { - parts.push('input[confirm]=true'); - parts.push('input[user_confirmed]=true'); - } - return parts.join('&'); } @@ -346,23 +436,49 @@ export class AbilitiesExecutor { * Normalize API response to ExecutionResult */ private normalizeResponse(data: unknown): ExecutionResult { + const record = this.asRecord(data); + const wrappedData = this.asRecord(record?.['data']); + const queuedEnvelope = record?.['queued'] === true ? record : wrappedData; + const queuedJobId = queuedEnvelope?.['queued'] === true + ? validateJobId(queuedEnvelope['job_id']) + : undefined; + // If already in expected format, return as-is if ( - typeof data === 'object' && - data !== null && - 'success' in data && - typeof (data as Record)['success'] === 'boolean' + record && + typeof record['success'] === 'boolean' ) { - return data as ExecutionResult; + return { + ...(data as ExecutionResult), + ...(queuedJobId ? { jobId: queuedJobId } : {}), + }; } // Wrap raw data in success response return { success: true, data: data as T, + ...(queuedJobId ? { jobId: queuedJobId } : {}), }; } + private asRecord(value: unknown): Record | undefined { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return undefined; + } + + return value as Record; + } + + private isPlainObject(value: unknown): value is Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; + } + /** * Clear the abilities cache */ diff --git a/src/core/batch-manager.test.ts b/src/core/batch-manager.test.ts index 0e80d2a..7b2cc1c 100644 --- a/src/core/batch-manager.test.ts +++ b/src/core/batch-manager.test.ts @@ -93,19 +93,101 @@ describe('BatchManager', () => { ['completed', 'completed'], ['failed', 'failed'], ['partial', 'partial'], - ['unknown', 'pending'], // Defaults to pending + ['cancelled', 'cancelled'], ]; for (const [input, expected] of statusMappings) { + const jobId = `job_${input}`; mockGet.mockResolvedValueOnce({ - data: { job_id: 'job', status: input }, + data: { job_id: jobId, status: input }, }); - const status = await manager.getJobStatus('job'); + const status = await manager.getJobStatus(jobId); expect(status.status).toBe(expected); } }); + it('rejects unknown or missing status values', async () => { + mockGet + .mockResolvedValueOnce({ data: { job_id: 'job', status: 'mystery' } }) + .mockResolvedValueOnce({ data: { job_id: 'job' } }); + + await expect(manager.getJobStatus('job')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + await expect(manager.getJobStatus('job')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + + it('rejects a response for a different job id', async () => { + mockGet.mockResolvedValue({ + data: { job_id: 'job_other', status: 'running' }, + }); + + await expect(manager.getJobStatus('job_expected')).rejects.toMatchObject({ + code: 'INVALID_RESPONSE', + }); + }); + + it('rejects unsafe requested and response job ids', async () => { + await expect(manager.getJobStatus('bad\njob')).rejects.toMatchObject({ + code: 'INVALID_RESPONSE', + }); + expect(mockGet).not.toHaveBeenCalled(); + + mockGet.mockResolvedValue({ + data: { job_id: 'bad\njob', status: 'running' }, + }); + await expect(manager.getJobStatus('job')).rejects.toMatchObject({ + code: 'INVALID_RESPONSE', + }); + }); + + it.each([ + ['negative progress', { progress: -1 }], + ['progress above 100', { progress: 101 }], + ['non-finite progress', { progress: Number.NaN }], + ['negative total', { total: -1 }], + ['negative processed', { processed: -1 }], + ['processed above total', { processed: 2, total: 1 }], + ])('rejects invalid numeric status data: %s', async (_label, fields) => { + mockGet.mockResolvedValue({ + data: { job_id: 'job', status: 'running', ...fields }, + }); + + await expect(manager.getJobStatus('job')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + + it.each(['results', 'errors'])('rejects oversized %s arrays', async (field) => { + mockGet.mockResolvedValue({ + data: { + job_id: 'job', + status: 'running', + [field]: Array.from({ length: 10_001 }, () => null), + }, + }); + + await expect(manager.getJobStatus('job')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + + it.each(['results', 'errors'])('rejects non-array %s fields', async (field) => { + mockGet.mockResolvedValue({ + data: { + job_id: 'job', + status: 'running', + [field]: 'invalid', + }, + }); + + await expect(manager.getJobStatus('job')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + + it('rejects regression after a terminal status was observed', async () => { + mockGet + .mockResolvedValueOnce({ data: { job_id: 'job', status: 'completed' } }) + .mockResolvedValueOnce({ data: { job_id: 'job', status: 'running' } }); + + await expect(manager.getJobStatus('job')).resolves.toMatchObject({ status: 'completed' }); + await expect(manager.getJobStatus('job')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + it('throws on invalid response', async () => { mockGet.mockResolvedValue({ data: null }); @@ -122,6 +204,15 @@ describe('BatchManager', () => { }); describe('watchJob', () => { + it('rejects an invalid job ID before polling or building a placeholder', async () => { + const generator = manager.watchJob('job\x1b[2Jmalicious'); + + await expect(generator.next()).rejects.toMatchObject({ + code: 'INVALID_RESPONSE', + }); + expect(mockGet).not.toHaveBeenCalled(); + }); + it('yields status updates until completed', async () => { mockGet .mockResolvedValueOnce({ diff --git a/src/core/batch-manager.ts b/src/core/batch-manager.ts index d7882bc..1da5feb 100644 --- a/src/core/batch-manager.ts +++ b/src/core/batch-manager.ts @@ -8,11 +8,12 @@ import { HttpClient, type HttpClientConfig, createHttpClient } from './http-client.js'; import { APIError, NetworkError } from '../utils/errors.js'; import { ExponentialBackoff } from '../utils/retry.js'; +import { validateJobId } from './job-id.js'; /** * Job status types */ -export type JobStatusType = 'pending' | 'running' | 'completed' | 'failed' | 'partial'; +export type JobStatusType = 'pending' | 'running' | 'completed' | 'failed' | 'partial' | 'cancelled'; /** * Job status response from API @@ -82,12 +83,24 @@ const DEFAULTS = { multiplier: 2, }; +const MAX_STATUS_ARRAY_LENGTH = 10_000; + /** * Batch Manager class */ +/** + * Check if a job status is terminal (job finished, no further polling). + * Shared with `jobs watch` so the two never drift. + */ +export function isTerminalStatus(status: string): boolean { + return status === 'completed' || status === 'failed' || + status === 'partial' || status === 'cancelled'; +} + export class BatchManager { private readonly httpClient: HttpClient; private readonly baseEndpoint = '/wp-json/wp-abilities/v1'; + private readonly terminalJobs = new Set(); constructor(config: HttpClientConfig) { this.httpClient = createHttpClient(config); @@ -103,6 +116,9 @@ export class BatchManager { jobId: string, options: WatchOptions = {} ): AsyncGenerator { + // Validate up front: a timeout/abort before the first poll builds a + // placeholder status from this ID, which must never carry a raw value. + const validatedJobId = validateJobId(jobId); const maxWait = options.maxWait ?? DEFAULTS.maxWait; const startTime = Date.now(); @@ -131,7 +147,7 @@ export class BatchManager { // Fetch current status try { - const status = await this.getJobStatus(jobId, options.signal); + const status = await this.getJobStatus(validatedJobId, options.signal); lastStatus = status; // Yield the status update @@ -143,7 +159,7 @@ export class BatchManager { } // Check if job is complete - if (this.isTerminalStatus(status.status)) { + if (isTerminalStatus(status.status)) { break; } @@ -182,11 +198,11 @@ export class BatchManager { // If we don't have a status, create a placeholder if (!lastStatus) { lastStatus = { - id: jobId, + id: validatedJobId, status: timedOut ? 'partial' : 'failed', errors: [{ message: timedOut ? 'Polling timed out' : 'Polling aborted' }], }; - } else if (timedOut && !this.isTerminalStatus(lastStatus.status)) { + } else if (timedOut && !isTerminalStatus(lastStatus.status)) { // Mark as partial if timed out while still running lastStatus = { ...lastStatus, @@ -225,28 +241,36 @@ export class BatchManager { * Get the current status of a batch job */ async getJobStatus(jobId: string, signal?: AbortSignal): Promise { + const validatedJobId = validateJobId(jobId); const endpoint = `${this.baseEndpoint}/abilities/mainwp/get-batch-job-status-v1/run`; - const qs = `input[job_id]=${encodeURIComponent(jobId)}`; + const qs = `input[job_id]=${encodeURIComponent(validatedJobId)}`; const response = await this.httpClient.get( `${endpoint}?${qs}`, signal ? { signal } : undefined ); - return this.normalizeJobStatus(response.data); + const status = this.normalizeJobStatus(response.data, validatedJobId); + if (this.terminalJobs.has(validatedJobId) && !this.isServerTerminalStatus(status.status)) { + throw new APIError( + 'INVALID_RESPONSE', + 'Job status regressed from a terminal state' + ); + } + if (this.isServerTerminalStatus(status.status)) { + this.terminalJobs.add(validatedJobId); + } + return status; } - /** - * Check if a status is terminal (job finished) - */ - private isTerminalStatus(status: JobStatusType): boolean { - return status === 'completed' || status === 'failed' || status === 'partial'; + private isServerTerminalStatus(status: JobStatusType): boolean { + return status === 'completed' || status === 'failed' || status === 'cancelled'; } /** * Normalize API response to JobStatus, unwrapping success envelope if present */ - private normalizeJobStatus(data: unknown): JobStatus { + private normalizeJobStatus(data: unknown, requestedJobId: string): JobStatus { if (typeof data !== 'object' || data === null) { throw new APIError('INVALID_RESPONSE', 'Invalid job status response'); } @@ -263,19 +287,36 @@ export class BatchManager { fields = inner as Record; } - const id = String(fields['job_id'] ?? fields['id'] ?? ''); - if (!id) { + const rawId = fields['job_id'] ?? fields['id']; + if (rawId === undefined) { throw new APIError('INVALID_RESPONSE', 'Job status response missing job ID'); } + const responseJobId = validateJobId(rawId); + if (responseJobId !== requestedJobId) { + throw new APIError( + 'INVALID_RESPONSE', + `Job status response ID mismatch for requested job ${requestedJobId}` + ); + } + + const progress = this.parseStatusNumber(fields, 'progress', 100); + const total = this.parseStatusNumber(fields, 'total'); + const processed = this.parseStatusNumber(fields, 'processed'); + if (processed !== undefined && total !== undefined && processed > total) { + throw new APIError('INVALID_RESPONSE', 'Job status processed count exceeds total'); + } + + const results = this.parseStatusArray(fields, 'results'); + const rawErrors = this.parseStatusArray(fields, 'errors'); return { - id, + id: responseJobId, status: this.parseJobStatus(fields['status']), - progress: typeof fields['progress'] === 'number' ? fields['progress'] : undefined, - total: typeof fields['total'] === 'number' ? fields['total'] : undefined, - processed: typeof fields['processed'] === 'number' ? fields['processed'] : undefined, - results: Array.isArray(fields['results']) ? fields['results'] : undefined, - errors: this.parseJobErrors(fields['errors']), + progress, + total, + processed, + results, + errors: this.parseJobErrors(rawErrors), created_at: typeof fields['created_at'] === 'string' ? fields['created_at'] : undefined, completed_at: typeof fields['completed_at'] === 'string' ? fields['completed_at'] : undefined, }; @@ -286,7 +327,7 @@ export class BatchManager { */ private parseJobStatus(value: unknown): JobStatusType { if (typeof value !== 'string') { - return 'pending'; + throw new APIError('INVALID_RESPONSE', 'Job status response has an invalid status'); } const status = value.toLowerCase(); @@ -297,6 +338,7 @@ export class BatchManager { case 'completed': case 'failed': case 'partial': + case 'cancelled': return status; case 'processing': case 'in_progress': @@ -307,8 +349,44 @@ export class BatchManager { case 'error': return 'failed'; default: - return 'pending'; + throw new APIError('INVALID_RESPONSE', 'Job status response has an unknown status'); + } + } + + private parseStatusNumber( + fields: Record, + key: 'progress' | 'total' | 'processed', + maximum?: number + ): number | undefined { + const value = fields[key]; + if (value === undefined) return undefined; + if ( + typeof value !== 'number' || + !Number.isFinite(value) || + value < 0 || + (maximum !== undefined && value > maximum) + ) { + throw new APIError('INVALID_RESPONSE', `Job status ${key} is invalid`); + } + return value; + } + + private parseStatusArray( + fields: Record, + key: 'results' | 'errors' + ): unknown[] | undefined { + const value = fields[key]; + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new APIError('INVALID_RESPONSE', `Job status ${key} is not an array`); + } + if (value.length > MAX_STATUS_ARRAY_LENGTH) { + throw new APIError( + 'INVALID_RESPONSE', + `Job status ${key} exceeds the ${MAX_STATUS_ARRAY_LENGTH}-item limit` + ); } + return value; } /** diff --git a/src/core/execute-ability-with-policy.test.ts b/src/core/execute-ability-with-policy.test.ts new file mode 100644 index 0000000..a3298b1 --- /dev/null +++ b/src/core/execute-ability-with-policy.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { + AbilitiesExecutor, + Ability, + ExecutionOptions, + ExecutionResult, +} from './abilities-executor.js'; +import { executeAbilityWithPolicy } from './execute-ability-with-policy.js'; +import { ConfirmationRequiredError, MutualExclusionError } from '../utils/errors.js'; + +function createAbility(name: string, destructive: boolean): Ability { + return { + name, + label: name, + description: name, + category: 'test', + meta: { + annotations: { + destructive, + readonly: !destructive, + idempotent: false, + }, + }, + }; +} + +function createExecutor(result: ExecutionResult = { success: true }): { + executor: AbilitiesExecutor; + execute: ReturnType; +} { + const execute = vi.fn().mockResolvedValue(result); + return { + executor: { execute } as unknown as AbilitiesExecutor, + execute, + }; +} + +describe('executeAbilityWithPolicy', () => { + it('rejects dryRun and confirm together before execution', async () => { + const { executor, execute } = createExecutor(); + + await expect( + executeAbilityWithPolicy( + executor, + createAbility('list-sites-v1', false), + {}, + { dryRun: true, confirm: true } + ) + ).rejects.toBeInstanceOf(MutualExclusionError); + expect(execute).not.toHaveBeenCalled(); + }); + + it('rejects destructive execution without confirm unless it is a dry run', async () => { + const { executor, execute } = createExecutor(); + + await expect( + executeAbilityWithPolicy(executor, createAbility('delete-site-v1', true), {}) + ).rejects.toBeInstanceOf(ConfirmationRequiredError); + expect(execute).not.toHaveBeenCalled(); + }); + + it('passes non-destructive abilities through', async () => { + const result = { success: true, data: { sites: [] } }; + const { executor, execute } = createExecutor(result); + const ability = createAbility('list-sites-v1', false); + + await expect(executeAbilityWithPolicy(executor, ability, { page: 2 })).resolves.toBe(result); + expect(execute).toHaveBeenCalledWith('list-sites-v1', { page: 2 }); + }); + + it.each([ + { dryRun: true }, + { confirm: true }, + ])('forwards execution options faithfully: %j', async (options) => { + const { executor, execute } = createExecutor(); + const ability = createAbility('delete-site-v1', true); + + await executeAbilityWithPolicy(executor, ability, { site_id: 1 }, options); + + expect(execute).toHaveBeenCalledWith('delete-site-v1', { site_id: 1 }, options); + }); +}); diff --git a/src/core/execute-ability-with-policy.ts b/src/core/execute-ability-with-policy.ts new file mode 100644 index 0000000..305f529 --- /dev/null +++ b/src/core/execute-ability-with-policy.ts @@ -0,0 +1,38 @@ +/** + * Shared policy-enforcing choke point for ability execution. + * + * UX decisions such as prompting and preview rendering remain with callers. + */ + +import type { + AbilitiesExecutor, + Ability, + ExecutionOptions, + ExecutionResult, +} from './abilities-executor.js'; +import { getSafetyController } from './safety-controller.js'; + +export async function executeAbilityWithPolicy( + executor: AbilitiesExecutor, + ability: Ability, + input: Record, + options?: ExecutionOptions +): Promise> { + const safetyController = getSafetyController(); + + // This shared gate classifies the ability, rejects mutually exclusive flags, + // and refuses destructive execution unless the call is a preview or carries + // explicit confirmation. AbilitiesExecutor keeps its own transport-level + // control-flag enforcement as defense in depth. + safetyController.validateExecutionFlags( + ability, + options?.dryRun, + options?.confirm + ); + + if (options === undefined) { + return executor.execute(ability.name, input); + } + + return executor.execute(ability.name, input, options); +} diff --git a/src/core/http-client.test.ts b/src/core/http-client.test.ts index 6e850f2..a376e99 100644 --- a/src/core/http-client.test.ts +++ b/src/core/http-client.test.ts @@ -121,7 +121,7 @@ describe('HttpClient Redirect Security', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{"success":true}'), }); @@ -148,7 +148,7 @@ describe('HttpClient Redirect Security', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{}'), }); @@ -361,7 +361,7 @@ describe('HttpClient SSL Configuration', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{}'), }); @@ -384,7 +384,7 @@ describe('HttpClient SSL Configuration', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{}'), }); @@ -420,7 +420,10 @@ describe('HttpClient Response Size Checking', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers({ 'content-length': 'abc' }), + headers: new Headers({ + 'content-length': 'abc', + 'content-type': 'application/json', + }), text: () => Promise.resolve('{"ok":true}'), }); @@ -430,21 +433,17 @@ describe('HttpClient Response Size Checking', () => { expect(response.data).toEqual({ ok: true }); }); - it('skips post-read body check when Content-Length already validated', async () => { - // Content-Length is 50, which is under the 100 limit. - // Body is also under limit. No error should occur. + it('rejects an oversized body when Content-Length understates its size', async () => { mockFetch.mockResolvedValueOnce({ status: 200, ok: true, statusText: 'OK', headers: new Headers({ 'content-length': '50' }), - text: () => Promise.resolve('x'.repeat(50)), + text: () => Promise.resolve('x'.repeat(200)), }); const client = createHttpClient(baseConfig); - const response = await client.get('/test'); - - expect(response.status).toBe(200); + await expect(client.get('/test')).rejects.toThrow(/Response too large/); }); it('rejects oversized Content-Length before reading body', async () => { @@ -472,6 +471,86 @@ describe('HttpClient Response Size Checking', () => { const client = createHttpClient(baseConfig); await expect(client.get('/test')).rejects.toThrow(/Response too large/); }); + + it('stops reading a streamed body as soon as the byte limit is exceeded', async () => { + let cancelled = false; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(60)); + controller.enqueue(new Uint8Array(60)); + controller.enqueue(new Uint8Array(60)); + controller.close(); + }, + cancel() { + cancelled = true; + }, + }); + mockFetch.mockResolvedValueOnce(new Response(body, { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const client = createHttpClient(baseConfig); + await expect(client.get('/test')).rejects.toThrow(/Response too large/); + expect(cancelled).toBe(true); + }); +}); + +describe('HttpClient Success Response Validation', () => { + const baseConfig: HttpClientConfig = { + baseUrl: 'https://dashboard.example.com', + username: 'admin', + appPassword: 'test-password', + }; + + beforeEach(() => { + mockFetch.mockReset(); + }); + + it('rejects an empty successful response', async () => { + mockFetch.mockResolvedValueOnce(new Response('', { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const client = createHttpClient(baseConfig); + await expect(client.get('/test')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + + it('rejects a non-JSON successful response', async () => { + mockFetch.mockResolvedValueOnce(new Response('login', { + status: 200, + headers: { 'content-type': 'text/html' }, + })); + + const client = createHttpClient(baseConfig); + await expect(client.get('/test')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + + it('rejects malformed JSON in a successful response', async () => { + mockFetch.mockResolvedValueOnce(new Response('{bad json', { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const client = createHttpClient(baseConfig); + await expect(client.get('/test')).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }); + }); + + it('keeps the request timeout active while consuming the response body', async () => { + const body = new ReadableStream({ + start() { + // Intentionally never enqueue or close. + }, + }); + mockFetch.mockResolvedValueOnce(new Response(body, { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const client = createHttpClient({ ...baseConfig, timeout: 10 }); + await expect(client.get('/test')).rejects.toThrow(/Request timed out/); + }); }); describe('HttpClient sanitizeErrorData — Recursive Redaction', () => { @@ -598,7 +677,7 @@ describe('HttpClient buildUrl Origin Validation', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{}'), }); @@ -633,7 +712,7 @@ describe('HttpClient buildUrl Origin Validation', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{"ok":true}'), }); @@ -649,7 +728,7 @@ describe('HttpClient buildUrl Origin Validation', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{}'), }); @@ -785,7 +864,7 @@ describe('HttpClient Manual Redirect Mode', () => { status: 200, ok: true, statusText: 'OK', - headers: new Headers(), + headers: new Headers({ 'content-type': 'application/json' }), text: () => Promise.resolve('{}'), }); @@ -801,3 +880,51 @@ describe('HttpClient Manual Redirect Mode', () => { expect(fetchCall[1].redirect).toBe('manual'); }); }); + +describe('HttpClient AbortError Attribution', () => { + const baseConfig: HttpClientConfig = { + baseUrl: 'https://dashboard.example.com', + username: 'admin', + appPassword: 'test-password', + }; + + beforeEach(() => { + mockFetch.mockReset(); + }); + + function abortError(): Error { + const error = new Error('This operation was aborted'); + error.name = 'AbortError'; + return error; + } + + it('reports "Request cancelled" when the caller signal aborted', async () => { + mockFetch.mockRejectedValueOnce(abortError()); + + const controller = new AbortController(); + controller.abort(); + + const client = createHttpClient(baseConfig); + await expect( + client.get('/test', { signal: controller.signal }) + ).rejects.toThrow('Request cancelled'); + }); + + it('reports "Request timed out" when the abort was not caller-initiated', async () => { + mockFetch.mockRejectedValueOnce(abortError()); + + const client = createHttpClient(baseConfig); + await expect(client.get('/test')).rejects.toThrow('Request timed out'); + }); + + it('reports "Request timed out" when a caller signal exists but never aborted', async () => { + mockFetch.mockRejectedValueOnce(abortError()); + + const controller = new AbortController(); + + const client = createHttpClient(baseConfig); + await expect( + client.get('/test', { signal: controller.signal }) + ).rejects.toThrow('Request timed out'); + }); +}); diff --git a/src/core/http-client.ts b/src/core/http-client.ts index 93d045a..815e23b 100644 --- a/src/core/http-client.ts +++ b/src/core/http-client.ts @@ -8,6 +8,7 @@ import { createRequire } from 'node:module'; import { Agent } from 'undici'; import { NetworkError, TLSError, APIError, AuthError } from '../utils/errors.js'; +import { redactSensitiveKeys } from '../utils/redaction.js'; const require = createRequire(import.meta.url); const { version: PKG_VERSION } = require('../../package.json') as { version: string }; @@ -190,17 +191,22 @@ export class HttpClient { const response = await fetch(url, fetchOptions); - clearTimeout(timeoutId); - // SECURITY: Handle redirects manually - only follow same-origin if (this.isRedirect(response.status)) { + clearTimeout(timeoutId); return this.handleRedirect(response, method, body, options, redirectCount); } - // Check response size via Content-Length header (pre-read guard) + // Check response size via Content-Length header (pre-read guard). + // REPORT-ONLY: this trusts the server-reported Content-Length; a server + // that lies about it still gets fully buffered by the post-read check + // below. Acceptable here since the only server we talk to is the + // trusted Dashboard the operator configured, not an arbitrary origin. const contentLength = response.headers.get('content-length'); const parsedContentLength = contentLength ? parseInt(contentLength, 10) : NaN; if (!isNaN(parsedContentLength) && parsedContentLength > this.maxResponseSize) { + controller.abort(); + void response.body?.cancel().catch(() => {}); throw new NetworkError( `Response too large: ${parsedContentLength} bytes`, undefined, @@ -208,31 +214,44 @@ export class HttpClient { ); } - // Parse response - const text = await response.text(); - - // Post-read body length check (only when Content-Length was absent or unparseable) - if (isNaN(parsedContentLength) && text.length > this.maxResponseSize) { - throw new NetworkError( - `Response too large: ${text.length} bytes`, - undefined, - 'Response is too large. Check the Dashboard logs or try a simpler query' - ); - } + const text = await this.readResponseBody(response, controller, effectiveSignal); let data: T; - try { - // SECURITY: Strip __proto__ and constructor keys to prevent prototype - // pollution from untrusted API responses - data = text ? (JSON.parse(text, (key, value) => { - if (key === '__proto__' || key === 'constructor') { - return undefined; - } - return value; - }) as T) : ({} as T); - } catch { - // If not JSON, wrap as string - data = text as unknown as T; + if (response.ok) { + const contentType = response.headers.get('content-type')?.toLowerCase() ?? ''; + const mediaType = contentType.split(';', 1)[0]?.trim() ?? ''; + if (mediaType !== 'application/json' && !mediaType.endsWith('+json')) { + throw new APIError( + 'INVALID_RESPONSE', + 'Dashboard returned a successful response without a JSON content type', + response.status + ); + } + if (text.trim().length === 0) { + throw new APIError( + 'INVALID_RESPONSE', + 'Dashboard returned an empty successful response', + response.status + ); + } + + try { + data = this.parseJson(text); + } catch { + throw new APIError( + 'INVALID_RESPONSE', + 'Dashboard returned malformed JSON in a successful response', + response.status + ); + } + } else { + try { + data = text ? this.parseJson(text) : ({} as T); + } catch { + // Preserve existing non-2xx behavior: raw bodies are sanitized by + // handleHttpError before they are exposed through a typed error. + data = text as unknown as T; + } } // Handle HTTP errors @@ -247,11 +266,102 @@ export class HttpClient { data, }; } catch (error) { + // Distinguish caller cancellation from our own timeout: AbortSignal.any() + // erases which signal fired, so check the caller's signal directly. + throw this.normalizeError(error, options?.signal?.aborted === true); + } finally { clearTimeout(timeoutId); - throw this.normalizeError(error); } } + private async readResponseBody( + response: Response, + controller: AbortController, + signal: AbortSignal + ): Promise { + if (!response.body) { + const text = await response.text(); + const byteLength = Buffer.byteLength(text, 'utf8'); + if (byteLength > this.maxResponseSize) { + controller.abort(); + throw this.responseTooLarge(byteLength); + } + return text; + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + + try { + while (true) { + const { done, value } = await this.readChunk(reader, signal); + if (done) break; + if (!value) continue; + + totalBytes += value.byteLength; + if (totalBytes > this.maxResponseSize) { + controller.abort(); + await reader.cancel().catch(() => {}); + throw this.responseTooLarge(totalBytes); + } + chunks.push(value); + } + } catch (error) { + void reader.cancel().catch(() => {}); + throw error; + } finally { + reader.releaseLock(); + } + + return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)), totalBytes).toString('utf8'); + } + + private readChunk( + reader: ReadableStreamDefaultReader, + signal: AbortSignal + ): Promise<{ done: boolean; value?: Uint8Array }> { + if (signal.aborted) { + return Promise.reject(new DOMException('The operation was aborted', 'AbortError')); + } + + return new Promise((resolve, reject) => { + const onAbort = (): void => { + reject(new DOMException('The operation was aborted', 'AbortError')); + }; + signal.addEventListener('abort', onAbort, { once: true }); + reader.read().then( + (result) => { + signal.removeEventListener('abort', onAbort); + resolve(result); + }, + (error: unknown) => { + signal.removeEventListener('abort', onAbort); + reject(error); + } + ); + }); + } + + private responseTooLarge(byteLength: number): NetworkError { + return new NetworkError( + `Response too large: ${byteLength} bytes`, + undefined, + 'Response is too large. Check the Dashboard logs or try a simpler query' + ); + } + + private parseJson(text: string): T { + // SECURITY: Strip __proto__ and constructor keys to prevent prototype + // pollution from untrusted API responses. + return JSON.parse(text, (key, value) => { + if (key === '__proto__' || key === 'constructor') { + return undefined; + } + return value; + }) as T; + } + /** * Check if status code is a redirect */ @@ -272,6 +382,9 @@ export class HttpClient { options: RequestOptions | undefined, redirectCount: number ): Promise> { + // The redirect response's body is never read; release the connection + // before following (or refusing) the redirect. + void response.body?.cancel().catch(() => {}); if (redirectCount >= HttpClient.MAX_REDIRECTS) { throw new NetworkError( 'Too many redirects', @@ -425,12 +538,15 @@ export class HttpClient { /** * Normalize errors to MainWPCTLError types */ - private normalizeError(error: unknown): Error { + private normalizeError(error: unknown, cancelled = false): Error { if (!(error instanceof Error)) { return new NetworkError(String(error)); } if (error.name === 'AbortError') { + if (cancelled) { + return new NetworkError('Request cancelled'); + } return new NetworkError( 'Request timed out', undefined, @@ -470,27 +586,7 @@ export class HttpClient { * Sanitize error data to prevent credential leaks */ private sanitizeErrorData(data: unknown): unknown { - if (typeof data !== 'object' || data === null) return data; - if (Array.isArray(data)) return data.map(item => this.sanitizeErrorData(item)); - - const sanitized: Record = {}; - // Substring matching catches camelCase, snake_case, and header variants - // (e.g., accessToken, private_key, set-cookie, refreshToken) - const sensitiveSubstrings = [ - 'password', 'token', 'secret', 'authorization', 'cookie', - 'apikey', 'api_key', 'bearer', 'credential', 'private_key', - 'signing_key', - ]; - - for (const [key, value] of Object.entries(data)) { - const keyLower = key.toLowerCase(); - if (sensitiveSubstrings.some(s => keyLower.includes(s))) { - sanitized[key] = '[REDACTED]'; - } else { - sanitized[key] = this.sanitizeErrorData(value); - } - } - return sanitized; + return redactSensitiveKeys(data); } } diff --git a/src/core/job-id.ts b/src/core/job-id.ts new file mode 100644 index 0000000..8611b4b --- /dev/null +++ b/src/core/job-id.ts @@ -0,0 +1,27 @@ +import { APIError } from '../utils/errors.js'; + +const MAX_JOB_ID_BYTES = 512; + +function hasUnsafeCharacters(value: string): boolean { + return [...value].some((character) => { + const codePoint = character.codePointAt(0)!; + return codePoint <= 0x1f || + (codePoint >= 0x7f && codePoint <= 0x9f) || + (codePoint >= 0x202a && codePoint <= 0x202e) || + (codePoint >= 0x2066 && codePoint <= 0x2069); + }); +} + +export function validateJobId(value: unknown): string { + if ( + typeof value !== 'string' || + value.length === 0 || + value.trim() !== value || + Buffer.byteLength(value, 'utf8') > MAX_JOB_ID_BYTES || + hasUnsafeCharacters(value) + ) { + throw new APIError('INVALID_RESPONSE', 'Job response contains an invalid job ID'); + } + + return value; +} diff --git a/src/core/safety-controller.test.ts b/src/core/safety-controller.test.ts index ecfa569..1866192 100644 --- a/src/core/safety-controller.test.ts +++ b/src/core/safety-controller.test.ts @@ -217,7 +217,7 @@ describe('Golden Test: Safety Classification', () => { expect(classification.requiresSafetyFlow).toBe(true); }); - it('handles abilities without annotations (defaults to safe)', () => { + it('handles abilities without annotations as destructive', () => { const ability: Ability = { name: 'legacy-ability-v1', label: 'Legacy Ability', @@ -227,9 +227,9 @@ describe('Golden Test: Safety Classification', () => { }; const classification = controller.classify(ability); - expect(classification.isDestructive).toBe(false); + expect(classification.isDestructive).toBe(true); expect(classification.isReadOnly).toBe(false); - expect(classification.requiresSafetyFlow).toBe(false); + expect(classification.requiresSafetyFlow).toBe(true); }); }); @@ -296,7 +296,7 @@ describe('Annotation Validation (F2)', () => { controller = new SafetyController(); }); - it('falls back to safe defaults for non-boolean annotation values', () => { + it('treats non-boolean annotation values as destructive', () => { const ability: Ability = { name: 'bad-annotations-v1', label: 'Bad', @@ -313,11 +313,10 @@ describe('Annotation Validation (F2)', () => { const classification = controller.classify(ability); - // All non-boolean → fall back to defaults (false) - expect(classification.isDestructive).toBe(false); + expect(classification.isDestructive).toBe(true); expect(classification.isReadOnly).toBe(false); expect(classification.isIdempotent).toBe(false); - expect(classification.requiresSafetyFlow).toBe(false); + expect(classification.requiresSafetyFlow).toBe(true); }); it('warns on contradictory annotations and requires safety flow', () => { @@ -340,7 +339,7 @@ describe('Annotation Validation (F2)', () => { errorSpy.mockRestore(); }); - it('missing/undefined annotation fields produce safe defaults', () => { + it('missing/undefined annotation fields require the safety flow', () => { const ability: Ability = { name: 'no-annotations-v1', label: 'None', @@ -351,10 +350,10 @@ describe('Annotation Validation (F2)', () => { const classification = controller.classify(ability); - expect(classification.isDestructive).toBe(false); + expect(classification.isDestructive).toBe(true); expect(classification.isReadOnly).toBe(false); expect(classification.isIdempotent).toBe(false); - expect(classification.requiresSafetyFlow).toBe(false); + expect(classification.requiresSafetyFlow).toBe(true); }); }); @@ -407,6 +406,34 @@ describe('M6: Known-destructive pattern defense-in-depth', () => { expect(classification.requiresSafetyFlow).toBe(true); }); + it.each(['update-site-plugins-v1', 'activate-site-theme-v1'])( + 'forces destructive classification for %s when annotations under-report it', + (name) => { + const ability = createTestAbility(name, { + destructive: false, + readonly: false, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(true); + expect(classification.requiresSafetyFlow).toBe(true); + } + ); + + it.each(['get-site-v1', 'sync-sites-v1'])( + 'keeps %s non-destructive', + (name) => { + const ability = createTestAbility(name, { + destructive: false, + readonly: false, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(false); + expect(classification.requiresSafetyFlow).toBe(false); + } + ); + it('does not force destructive for non-matching ability names', () => { const ability = createTestAbility('list-sites-v1', { destructive: false, @@ -427,6 +454,163 @@ describe('M6: Known-destructive pattern defense-in-depth', () => { expect(classification.isDestructive).toBe(true); expect(classification.requiresSafetyFlow).toBe(true); }); + + it('forces destructive classification for reset-* patterns', () => { + const ability = createTestAbility('mainwp/reset-site-v1', { + destructive: false, + readonly: true, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(true); + expect(classification.requiresSafetyFlow).toBe(true); + }); + + it('forces destructive classification for restore-* patterns', () => { + const ability = createTestAbility('restore-backup-v1', { + destructive: false, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(true); + expect(classification.requiresSafetyFlow).toBe(true); + }); + + it('forces destructive classification for rollback-* patterns', () => { + const ability = createTestAbility('mainwp/rollback-plugin-v1', { + destructive: false, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(true); + expect(classification.requiresSafetyFlow).toBe(true); + }); + + it('forces destructive classification for wipe-* patterns', () => { + const ability = createTestAbility('wipe-site-v1', { + destructive: false, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(true); + expect(classification.requiresSafetyFlow).toBe(true); + }); + + it('forces destructive classification for purge-* patterns', () => { + const ability = createTestAbility('mainwp/purge-cache-v1', { + destructive: false, + readonly: true, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(true); + expect(classification.requiresSafetyFlow).toBe(true); + }); + + it('forces destructive classification for uninstall-* patterns', () => { + const ability = createTestAbility('uninstall-plugin-v1', { + destructive: false, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(true); + expect(classification.requiresSafetyFlow).toBe(true); + }); + + it('does not force destructive for generic update-* patterns', () => { + const ability = createTestAbility('update-dashboard-settings-v1', { + destructive: false, + readonly: true, + }); + + const classification = controller.classify(ability); + expect(classification.isDestructive).toBe(false); + expect(classification.requiresSafetyFlow).toBe(false); + }); +}); + +describe('formatPreviewResult: fail closed on absent preview evidence', () => { + let controller: SafetyController; + + beforeEach(() => { + controller = new SafetyController(); + }); + + it('data undefined: honest "no data" warning, affected [], requiresApproval true', () => { + const ability = createTestAbility('delete-site-v1', { destructive: true }); + const result = controller.formatPreviewResult(ability, {}, { + success: true, + }); + + expect(result.affected).toEqual([]); + expect(result.summary).toContain('Preview returned no data'); + expect(result.summary).not.toContain('No items would be affected'); + expect(result.requiresApproval).toBe(true); + }); + + it('data {}: same "no data" warning path as undefined data', () => { + const ability = createTestAbility('delete-site-v1', { destructive: true }); + const result = controller.formatPreviewResult(ability, {}, { + success: true, + data: {}, + }); + + expect(result.affected).toEqual([]); + expect(result.summary).toContain('Preview returned no data'); + expect(result.summary).not.toContain('No items would be affected'); + expect(result.requiresApproval).toBe(true); + }); + + it('data in an unrecognized shape: "unrecognized format" warning, raw payload as affected', () => { + const ability = createTestAbility('delete-site-v1', { destructive: true }); + const data = { foo: 'bar' }; + const result = controller.formatPreviewResult(ability, {}, { + success: true, + data, + }); + + expect(result.summary).toContain('unrecognized format'); + expect(result.affected).toEqual([data]); + expect(result.requiresApproval).toBe(true); + }); + + it('data {affected: []}: recognized-but-empty summary "No items would be ."', () => { + const ability = createTestAbility('delete-site-v1', { destructive: true }); + const result = controller.formatPreviewResult(ability, {}, { + success: true, + data: { affected: [] }, + }); + + expect(result.summary).toBe('No items would be deleted.'); + expect(result.affected).toEqual([]); + expect(result.requiresApproval).toBe(true); + }); + + it('data {affected: [{id: 1}]}: counts 1 item, affected contains the item', () => { + const ability = createTestAbility('delete-site-v1', { destructive: true }); + const item = { id: 1 }; + const result = controller.formatPreviewResult(ability, {}, { + success: true, + data: { affected: [item] }, + }); + + expect(result.summary).toBe('1 item would be deleted.'); + expect(result.affected).toEqual([item]); + expect(result.requiresApproval).toBe(true); + }); + + it('data {preview: {...}}: single-item preview still works', () => { + const ability = createTestAbility('delete-site-v1', { destructive: true }); + const preview = { name: 'site-1' }; + const result = controller.formatPreviewResult(ability, {}, { + success: true, + data: { preview }, + }); + + expect(result.summary).toBe('1 item would be deleted.'); + expect(result.affected).toEqual([preview]); + expect(result.requiresApproval).toBe(true); + }); }); describe('ACTION_VERBS substring ordering', () => { diff --git a/src/core/safety-controller.ts b/src/core/safety-controller.ts index 804aeb5..ba03956 100644 --- a/src/core/safety-controller.ts +++ b/src/core/safety-controller.ts @@ -61,24 +61,58 @@ export type ExecutionIntent = * 2. dry_run and confirm are MUTUALLY EXCLUSIVE * 3. Safety check happens BEFORE any network call */ -/** Default annotations for abilities without explicit metadata */ -const DEFAULT_ANNOTATIONS: AbilityAnnotations = { - readonly: false, - destructive: false, - idempotent: false, -}; +/** + * Known-destructive ability name patterns. + * + * Defense-in-depth: an ability whose name matches is treated as destructive + * regardless of what the API reports, so a compromised or buggy server cannot + * downgrade a destructive ability to bypass the safety flow. Intentionally + * verb-conservative — each verb is unambiguously destructive on its own; we do + * not add generic verbs like `update-` that are frequently non-destructive, + * since that would force preview+confirm on safe abilities and erode trust. + * + * Exported so transport (HTTP-method selection in AbilitiesExecutor) resolves + * destructiveness the same way policy does, instead of trusting raw annotations. + */ +const DESTRUCTIVE_NAME_PATTERNS = [ + /^(?:mainwp\/)?delete-/, + /^(?:mainwp\/)?disconnect-/, + /^(?:mainwp\/)?suspend-/, + /^(?:mainwp\/)?deactivate-/, + /^(?:mainwp\/)?remove-/, + /^(?:mainwp\/)?run-updates-/, + /^(?:mainwp\/)?update-all-/, + /^(?:mainwp\/)?update-site-/, + /^(?:mainwp\/)?activate-/, + /^(?:mainwp\/)?reset-/, + /^(?:mainwp\/)?restore-/, + /^(?:mainwp\/)?rollback-/, + /^(?:mainwp\/)?wipe-/, + /^(?:mainwp\/)?purge-/, + /^(?:mainwp\/)?uninstall-/, +]; + +/** + * Whether an ability name matches a known-destructive pattern. + * Single source of truth for the name-based destructive override, shared by + * safety classification and HTTP-method selection. + */ +export function isKnownDestructiveName(name: string): boolean { + return DESTRUCTIVE_NAME_PATTERNS.some((pattern) => pattern.test(name)); +} export class SafetyController { /** - * Classify an ability's safety requirements + * Classify an ability's safety requirements. * - * Safety classification derives ONLY from ability annotations. - * No heuristics are permitted. + * Classification is the MORE RESTRICTIVE of the API annotations and a + * conservative name-based destructive override (see DESTRUCTIVE_NAME_PATTERNS): + * an ability is destructive if its annotations say so OR its name matches. + * The name override is deliberate defense-in-depth against a server that + * under-reports destructiveness; it never downgrades, only upgrades. */ classify(ability: Ability): SafetyClassification { - const annotations = this.validateAnnotations( - ability.meta?.annotations ?? DEFAULT_ANNOTATIONS - ); + const annotations = this.validateAnnotations(ability.meta?.annotations); // SECURITY: Defense-in-depth — force destructive classification for // abilities whose names match known-destructive patterns, regardless @@ -95,39 +129,33 @@ export class SafetyController { }; } - /** - * Known-destructive ability name patterns. - * These abilities require the safety flow regardless of API-reported annotations. - */ - private static readonly DESTRUCTIVE_PATTERNS = [ - /^(?:mainwp\/)?delete-/, - /^(?:mainwp\/)?disconnect-/, - /^(?:mainwp\/)?suspend-/, - /^(?:mainwp\/)?deactivate-/, - /^(?:mainwp\/)?remove-/, - /^(?:mainwp\/)?run-updates-/, - /^(?:mainwp\/)?update-all-/, - ]; - private isKnownDestructivePattern(name: string): boolean { - return SafetyController.DESTRUCTIVE_PATTERNS.some(pattern => pattern.test(name)); + return isKnownDestructiveName(name); } /** * Validate annotation fields and resolve contradictions. * - * - Non-boolean values fall back to safe defaults. + * - Missing or non-boolean values fail closed as destructive. * - Contradictory annotations (destructive + readonly) → warn and treat as destructive. */ - private validateAnnotations(annotations: AbilityAnnotations): AbilityAnnotations { - const defaults = DEFAULT_ANNOTATIONS; + private validateAnnotations(annotations: unknown): AbilityAnnotations { + if (typeof annotations !== 'object' || annotations === null || Array.isArray(annotations)) { + return { destructive: true, readonly: false, idempotent: false }; + } + + const record = annotations as Record; + if ( + typeof record['destructive'] !== 'boolean' || + typeof record['readonly'] !== 'boolean' || + typeof record['idempotent'] !== 'boolean' + ) { + return { destructive: true, readonly: false, idempotent: false }; + } - const destructive = typeof annotations.destructive === 'boolean' - ? annotations.destructive : defaults.destructive; - let readonly_ = typeof annotations.readonly === 'boolean' - ? annotations.readonly : defaults.readonly; - const idempotent = typeof annotations.idempotent === 'boolean' - ? annotations.idempotent : defaults.idempotent; + const destructive = record['destructive']; + let readonly_ = record['readonly']; + const idempotent = record['idempotent']; // Contradictory: both destructive and readonly — treat as destructive (safe default) if (destructive && readonly_) { @@ -245,6 +273,23 @@ export class SafetyController { const data = apiResult.data as Record | undefined; const affected = this.extractAffectedItems(data); + // Unrecognized response shape: never tell the operator "no items would + // be affected" when we simply couldn't read the preview — show the raw + // data and say so, since a falsely reassuring summary right before a + // destructive confirm is worse than an honest "unknown". + if (affected === null) { + const noData = data === undefined || Object.keys(data).length === 0; + return { + affected: noData ? [] : [data], + summary: noData + ? 'Preview returned no data — the ability did not report what would be affected. Review the request carefully before approving.' + : 'Preview returned data in an unrecognized format — review the raw response below before approving.', + requiresApproval: true, + abilityName: ability.name, + input, + }; + } + return { affected, summary: this.generatePreviewSummary(ability, affected), @@ -255,11 +300,18 @@ export class SafetyController { } /** - * Extract affected items from API preview response + * Extract affected items from API preview response. + * + * Returns null when the response carries no positive preview evidence — + * either data in none of the recognized shapes, or no data at all. Callers + * must distinguish "the preview showed zero items" (a recognized-but-empty + * array) from "the preview showed nothing" (null): only the former may be + * summarized as "no items would be affected". */ - private extractAffectedItems(data: Record | undefined): unknown[] { + private extractAffectedItems(data: Record | undefined): unknown[] | null { + // Absent data is not evidence that nothing would be affected. if (!data) { - return []; + return null; } // Common patterns for affected items @@ -281,7 +333,9 @@ export class SafetyController { return [data['preview']]; } - return []; + // Data present but in no recognized shape — and an empty object is the + // same lack of evidence as no data. + return null; } /** Ability name keywords → past-tense action verbs */ @@ -320,4 +374,3 @@ export function getSafetyController(): SafetyController { } return instance; } - diff --git a/src/lib/base-command.test.ts b/src/lib/base-command.test.ts new file mode 100644 index 0000000..3cd80ce --- /dev/null +++ b/src/lib/base-command.test.ts @@ -0,0 +1,69 @@ +/** + * Unit tests for BaseCommand's debug-context redaction. + * + * debugLog() ships its context to stderr under --debug; these tests pin + * that sensitive values are redacted wherever they sit in the structure, + * including inside arrays. + */ + +import { describe, it, expect } from 'vitest'; +import type { Config } from '@oclif/core'; +import { BaseCommand } from './base-command.js'; + +class TestCommand extends BaseCommand { + async run(): Promise {} +} + +function redact(context: Record): Record { + const cmd = new TestCommand([], {} as Config); + // Private method reached via index access — pins the redaction behavior + // without spinning up the full oclif lifecycle. + return ( + cmd as unknown as { + redactDebugContext(c: Record): Record; + } + ).redactDebugContext(context); +} + +describe('BaseCommand debug-context redaction', () => { + it('redacts sensitive keys at the top level', () => { + const result = redact({ username: 'admin', appPassword: 's3cr3t' }); + + expect(result['username']).toBe('admin'); + expect(result['appPassword']).toBe('[REDACTED]'); + }); + + it('redacts sensitive keys in nested objects', () => { + const result = redact({ config: { token: 'abc123', url: 'https://x.test' } }); + + expect(result['config']).toEqual({ token: '[REDACTED]', url: 'https://x.test' }); + }); + + it('redacts sensitive keys inside arrays of objects', () => { + const result = redact({ + sites: [ + { name: 'one', apiKey: 'abc123' }, + { name: 'two', password: 'def456' }, + ], + }); + + expect(result['sites']).toEqual([ + { name: 'one', apiKey: '[REDACTED]' }, + { name: 'two', password: '[REDACTED]' }, + ]); + }); + + it('truncates long strings, including inside arrays', () => { + const long = 'a'.repeat(400); + const result = redact({ body: long, items: [long] }); + + expect(result['body']).toBe(`${'a'.repeat(297)}...`); + expect(result['items']).toEqual([`${'a'.repeat(297)}...`]); + }); + + it('passes primitives and short strings through unchanged', () => { + const result = redact({ count: 3, ok: true, note: 'short', missing: null }); + + expect(result).toEqual({ count: 3, ok: true, note: 'short', missing: null }); + }); +}); diff --git a/src/lib/base-command.ts b/src/lib/base-command.ts index 580a9e3..85d3a79 100644 --- a/src/lib/base-command.ts +++ b/src/lib/base-command.ts @@ -18,10 +18,12 @@ import { } from '../config/settings.js'; import { createAbilitiesExecutor, type AbilitiesExecutor } from '../core/abilities-executor.js'; import { createBatchManager, type BatchManager } from '../core/batch-manager.js'; -import { isMainWPCTLError, ConfigError } from '../utils/errors.js'; +import type { HttpClientConfig } from '../core/http-client.js'; +import { isMainWPCTLError, ConfigError, InputError } from '../utils/errors.js'; import { successOutput, errorOutput } from '../output/json-envelope.js'; import { ExitCode } from '../utils/exit-codes.js'; import { formatError, formatWarning } from '../output/formatter.js'; +import { isSensitiveKey } from '../utils/redaction.js'; /** * Common flags available to all commands @@ -109,6 +111,12 @@ export abstract class BaseCommand extends Command { */ private batchManagerInstance: BatchManager | undefined; + /** + * Cached HTTP client config, so the keychain is only looked up once per + * process even if a command uses both getExecutor() and getBatchManager(). + */ + private clientConfig: HttpClientConfig | undefined; + /** * Whether this command needs a profile to be loaded. * Override to return false for commands like `login` that don't need a profile. @@ -122,6 +130,11 @@ export abstract class BaseCommand extends Command { * Call this at the start of each command's run() method. */ protected async initCommon(flags: CommonFlags): Promise { + // Provisional, before anything that can throw: a settings-load failure + // must still honor an explicit --json so catch() emits the envelope on + // stdout instead of prose-only stderr. + this.jsonOutput = flags.json ?? false; + // Load and validate settings this.rawSettings = await loadSettings(); const resolved = resolveSettings(this.rawSettings); @@ -182,11 +195,13 @@ export abstract class BaseCommand extends Command { } /** - * Get the AbilitiesExecutor instance + * Build (and cache) the HTTP client config for the current profile. + * Resolves the keychain password once per process — getExecutor() and + * getBatchManager() both call this instead of hitting the keychain themselves. */ - protected async getExecutor(): Promise { - if (this.executor) { - return this.executor; + private async buildClientConfig(): Promise { + if (this.clientConfig) { + return this.clientConfig; } if (!this.currentProfile) { @@ -198,16 +213,35 @@ export abstract class BaseCommand extends Command { } const keychain = getKeychain(); - const password = await keychain.getOrThrow(this.currentProfile.name); - this.executor = createAbilitiesExecutor({ + // Identity-bound read: refuses the credential if the profile's URL no + // longer matches the Dashboard the credential was saved for. + const appPassword = await keychain.getOrThrow( + this.currentProfile.name, + this.currentProfile.dashboardUrl + ); + this.clientConfig = { baseUrl: this.currentProfile.dashboardUrl, username: this.currentProfile.username, - appPassword: password, + appPassword, ...this.getTransportConfig(), - }); + }; + + return this.clientConfig; + } + + /** + * Get the AbilitiesExecutor instance + */ + protected async getExecutor(): Promise { + if (this.executor) { + return this.executor; + } + + const config = await this.buildClientConfig(); + this.executor = createAbilitiesExecutor(config); this.debugLog('Initialized abilities executor', { - profile: this.currentProfile.name, + profile: this.currentProfile?.name, timeoutMs: this.settings.timeout, allowInsecureHttp: this.settings.allowInsecureHttp, skipSSLVerification: this.getTransportConfig().skipSSLVerification, @@ -224,25 +258,11 @@ export abstract class BaseCommand extends Command { return this.batchManagerInstance; } - if (!this.currentProfile) { - throw new ConfigError( - 'No profile loaded', - undefined, - 'This is an internal error. Please report this issue.' - ); - } - - const keychain = getKeychain(); - const appPassword = await keychain.getOrThrow(this.currentProfile.name); - this.batchManagerInstance = createBatchManager({ - baseUrl: this.currentProfile.dashboardUrl, - username: this.currentProfile.username, - appPassword, - ...this.getTransportConfig(), - }); + const config = await this.buildClientConfig(); + this.batchManagerInstance = createBatchManager(config); this.debugLog('Initialized batch manager', { - profile: this.currentProfile.name, + profile: this.currentProfile?.name, timeoutMs: this.settings.timeout, allowInsecureHttp: this.settings.allowInsecureHttp, skipSSLVerification: this.getTransportConfig().skipSSLVerification, @@ -291,30 +311,50 @@ export abstract class BaseCommand extends Command { return this.explicitDebugMode || !this.quietMode; } - private redactDebugContext(context: Record): Record { - const sensitiveKeys = ['password', 'secret', 'token', 'authorization', 'cookie', 'apikey']; + private static readonly MAX_DEBUG_DEPTH = 32; + + private redactDebugContext( + context: Record, + depth = 0, + ancestors = new WeakSet() + ): Record { const redacted: Record = {}; for (const [key, value] of Object.entries(context)) { - if (sensitiveKeys.some(s => key.toLowerCase() === s)) { - redacted[key] = '[REDACTED]'; - continue; - } + redacted[key] = isSensitiveKey(key) ? '[REDACTED]' : this.redactDebugValue(value, depth, ancestors); + } - if (typeof value === 'string' && value.length > 300) { - redacted[key] = `${value.slice(0, 297)}...`; - continue; - } + return redacted; + } - if (value && typeof value === 'object' && !Array.isArray(value)) { - redacted[key] = this.redactDebugContext(value as Record); - continue; - } + /** + * Redact a single debug-context value: truncate long strings, recurse into + * arrays and objects. Kept separate from redactSensitiveKeys() because + * debug output also truncates — delegating would lose that for nested data. + * Depth-capped with ancestor tracking, matching the shared sanitizers: + * cycles truncate, legitimately shared references survive. + */ + private redactDebugValue(value: unknown, depth = 0, ancestors = new WeakSet()): unknown { + if (typeof value === 'string' && value.length > 300) { + return `${value.slice(0, 297)}...`; + } - redacted[key] = value; + if (value && typeof value === 'object') { + if (depth >= BaseCommand.MAX_DEBUG_DEPTH || ancestors.has(value)) { + return '[truncated]'; + } + ancestors.add(value); + try { + if (Array.isArray(value)) { + return value.map((item) => this.redactDebugValue(item, depth + 1, ancestors)); + } + return this.redactDebugContext(value as Record, depth + 1, ancestors); + } finally { + ancestors.delete(value); + } } - return redacted; + return value; } /** @@ -341,6 +381,26 @@ export abstract class BaseCommand extends Command { * Handle errors with appropriate exit codes */ protected async catch(err: Error & { exitCode?: number; oclif?: { exit?: number } }): Promise { + // oclif flag/arg parse failures (CLIParseError subclasses all carry a + // `parse` property, e.g. FailedFlagValidationError from `exclusive` + // flags) are user input errors → exit 1. Handled before the generic + // oclif re-throw below, whose CLIError default exit of 2 would land + // them in the auth/config bucket. + if ('parse' in err) { + const rawJsonRequested = process.argv.some( + (argument) => argument === '--json' || argument.startsWith('--json=') + ); + if (rawJsonRequested) { + // Wrap so the envelope code matches the exit code: a bare oclif parse + // error is not a MainWPCTLError and would be labeled INTERNAL_ERROR. + this.log(JSON.stringify(errorOutput(new InputError(err.message)), null, 2)); + } else { + this.logToStderr(formatError(err)); + } + this.exit(ExitCode.INPUT_ERROR); + return; + } + // Re-throw oclif exit errors to preserve their exit code if (err.oclif && typeof err.oclif.exit === 'number') { throw err; diff --git a/src/output/formatter.test.ts b/src/output/formatter.test.ts index af8db1f..f5329ce 100644 --- a/src/output/formatter.test.ts +++ b/src/output/formatter.test.ts @@ -3,7 +3,46 @@ */ import { describe, it, expect } from 'vitest'; -import { formatWarning } from './formatter.js'; +import { + formatError, + formatWarning, + formatKeyValue, + formatTable, + formatList, + formatPreview, + formatDivider, + formatSection, + formatStatusIcon, + getStatusColor, +} from './formatter.js'; +import { colors } from '../utils/colors.js'; +import { InputError } from '../utils/errors.js'; + +describe('formatError credential redaction', () => { + it.each([ + ['Bearer token', 'Request failed with Bearer abc123secret', 'abc123secret', 'Bearer [REDACTED]'], + ['credential URL', 'Request failed at https://user:pass@host/x', 'user:pass', '[URL_WITH_CREDENTIALS]'], + ])('redacts %s credentials from Error messages', (_label, message, secret, marker) => { + const output = formatError(new Error(message)); + + expect(output).not.toContain(secret); + expect(output).toContain(marker); + }); + + it('redacts credentials from error details and hints', () => { + const output = formatError( + new InputError( + 'Request failed with Bearer message-secret', + { endpoint: 'https://detail-user:detail-pass@host/x' }, + 'Retry with Bearer hint-secret' + ) + ); + + expect(output).not.toContain('message-secret'); + expect(output).not.toContain('detail-user:detail-pass'); + expect(output).not.toContain('hint-secret'); + }); +}); describe('M5: formatWarning sanitization', () => { it('strips escape sequences from warning messages', () => { @@ -29,4 +68,60 @@ describe('M5: formatWarning sanitization', () => { expect(result).toContain(clean); }); + + it('collapses warning messages to one terminal row', () => { + expect(formatWarning('first\r\nsecond\tvalue')).toContain( + 'first second value' + ); + }); +}); + +describe('single-row formatter sanitization', () => { + it('collapses keys, table cells, list items, and preview actions', () => { + expect(formatKeyValue('multi\nline', 'value')).toContain('multi line:'); + expect(formatTable(['head\ner'], [['cell\r\nvalue']])).toContain('head er'); + expect(formatTable(['head\ner'], [['cell\r\nvalue']])).toContain('cell value'); + expect(formatList(['list\nitem'])).toContain('list item'); + expect(formatPreview('delete\nsite', [])).toContain('delete site'); + }); +}); + +describe('formatDivider', () => { + it('renders a 40-character divider by default, matching doctor/config-show reports', () => { + expect(formatDivider()).toBe(' ' + '─'.repeat(40)); + }); + + it('honors a custom width', () => { + expect(formatDivider(10)).toBe(' ' + '─'.repeat(10)); + }); +}); + +describe('formatSection', () => { + it('joins a title and pre-formatted rows into a single block', () => { + const result = formatSection('Settings', [' Timeout: 1000ms', ' Debug: Disabled']); + + expect(result).toContain('Settings'); + expect(result).toContain(' Timeout: 1000ms'); + expect(result).toContain(' Debug: Disabled'); + }); + + it('renders a title with no rows', () => { + const result = formatSection('Empty Section', []); + + expect(result).toContain('Empty Section'); + }); +}); + +describe('formatStatusIcon / getStatusColor', () => { + it('maps pass/warn/fail to distinct icons', () => { + expect(formatStatusIcon('pass')).toContain('✓'); + expect(formatStatusIcon('warn')).toContain('⚠'); + expect(formatStatusIcon('fail')).toContain('✗'); + }); + + it('maps pass/warn/fail to their color codes', () => { + expect(getStatusColor('pass')).toBe(colors.green); + expect(getStatusColor('warn')).toBe(colors.yellow); + expect(getStatusColor('fail')).toBe(colors.red); + }); }); diff --git a/src/output/formatter.ts b/src/output/formatter.ts index 9cd8db0..8385fbf 100644 --- a/src/output/formatter.ts +++ b/src/output/formatter.ts @@ -3,7 +3,12 @@ */ import { isMainWPCTLError } from '../utils/errors.js'; -import { stripControlChars, sanitizeForTerminal, safeString } from '../utils/terminal-sanitizer.js'; +import { + sanitizeForTerminal, + sanitizeSingleLine, + safeString, +} from '../utils/terminal-sanitizer.js'; +import { sanitizeErrorMessage, sanitizeErrorValue } from '../utils/error-sanitizer.js'; import { colors, color } from '../utils/colors.js'; /** @@ -17,17 +22,24 @@ export function formatSuccess(message: string): string { * Format an error message */ export function formatError(error: Error | string): string { - const message = error instanceof Error ? stripControlChars(error.message) : stripControlChars(error); + // Single-line: hostile error text must not inject CR/LF and fake + // subsequent output lines (anti-spoofing, same rule as other terminal fields). + const message = sanitizeErrorMessage( + sanitizeSingleLine(error instanceof Error ? error.message : error) + ); let output = color('✗ Error: ', colors.red, colors.bold) + message; if (isMainWPCTLError(error)) { if (error.details) { // Sanitize error details before display (untrusted API data) - const sanitizedDetails = sanitizeForTerminal(error.details); + const sanitizedDetails = sanitizeErrorValue(sanitizeForTerminal(error.details)); output += '\n' + color(' Details: ', colors.dim) + JSON.stringify(sanitizedDetails); } if (error.hint) { - output += '\n' + color('💡 ' + stripControlChars(error.hint), colors.dim); + output += '\n' + color( + '💡 ' + sanitizeErrorMessage(sanitizeSingleLine(error.hint)), + colors.dim + ); } } @@ -38,7 +50,7 @@ export function formatError(error: Error | string): string { * Format a warning message */ export function formatWarning(message: string): string { - return color('⚠ Warning: ', colors.yellow) + stripControlChars(message); + return color('⚠ Warning: ', colors.yellow) + sanitizeSingleLine(message); } /** @@ -55,12 +67,59 @@ export function formatHeading(text: string): string { return color(text, colors.bold, colors.cyan); } +/** + * Status for pass/warn/fail style reports (doctor, config show) + */ +export type StatusKind = 'pass' | 'warn' | 'fail'; + +/** + * Format a colored status icon for pass/warn/fail states + */ +export function formatStatusIcon(status: StatusKind): string { + switch (status) { + case 'pass': + return color('✓', colors.green); + case 'warn': + return color('⚠', colors.yellow); + case 'fail': + return color('✗', colors.red); + } +} + +/** + * Get the color code associated with a pass/warn/fail status + */ +export function getStatusColor(status: StatusKind): string { + switch (status) { + case 'pass': + return colors.green; + case 'warn': + return colors.yellow; + case 'fail': + return colors.red; + } +} + +/** + * Format a fixed-width horizontal divider used by report-style commands + */ +export function formatDivider(width = 40): string { + return ' ' + '─'.repeat(width); +} + +/** + * Format a titled section from pre-formatted rows + */ +export function formatSection(title: string, rows: string[]): string { + return [`\n ${color(title, colors.bold)}`, ...rows].join('\n'); +} + /** * Format a key-value pair */ export function formatKeyValue(key: string, value: unknown): string { // Sanitize both key and value (may contain untrusted API data) - const safeKey = stripControlChars(key); + const safeKey = sanitizeSingleLine(key); const valueStr = safeString(value); return color(safeKey + ': ', colors.dim) + valueStr; } @@ -77,8 +136,8 @@ export function formatTable( } // Sanitize all table data (may contain untrusted API data) - const safeHeaders = headers.map((h) => stripControlChars(h)); - const safeRows = rows.map((row) => row.map((cell) => stripControlChars(cell ?? ''))); + const safeHeaders = headers.map((h) => sanitizeSingleLine(h)); + const safeRows = rows.map((row) => row.map((cell) => sanitizeSingleLine(cell ?? ''))); // Calculate column widths using sanitized data const widths = safeHeaders.map((h, i) => { @@ -110,7 +169,7 @@ export function formatTable( */ export function formatList(items: string[], bullet = '•'): string { // Sanitize list items (may contain untrusted API data) - return items.map((item) => ` ${bullet} ${stripControlChars(item)}`).join('\n'); + return items.map((item) => ` ${bullet} ${sanitizeSingleLine(item)}`).join('\n'); } /** @@ -124,7 +183,7 @@ export function formatPreview( const sanitizedItems = sanitizeForTerminal(affectedItems); const lines = [ - color('Preview: ', colors.yellow, colors.bold) + stripControlChars(action), + color('Preview: ', colors.yellow, colors.bold) + sanitizeSingleLine(action), '', color('Affected items:', colors.dim), JSON.stringify(sanitizedItems, null, 2), diff --git a/src/output/json-envelope.test.ts b/src/output/json-envelope.test.ts index e3f1f17..c12254a 100644 --- a/src/output/json-envelope.test.ts +++ b/src/output/json-envelope.test.ts @@ -132,6 +132,56 @@ describe('Golden Test: JSON Output Parses Cleanly', () => { }); describe('Golden Test: Error Code Propagation', () => { + it.each([ + ['Bearer token', 'Request failed with Bearer abc123secret', 'abc123secret', 'Bearer [REDACTED]'], + ['credential URL', 'Request failed at https://user:pass@host/x', 'user:pass', '[URL_WITH_CREDENTIALS]'], + ])('redacts %s credentials from Error messages', (_label, message, secret, marker) => { + const output = errorOutput(new Error(message)); + + expect(output.error?.message).not.toContain(secret); + expect(output.error?.message).toContain(marker); + }); + + it('emits a stable envelope for cyclic error details instead of overflowing', () => { + const details: Record = { endpoint: 'https://host/x' }; + details['self'] = details; + + const output = errorOutput(new InputError('Bad input', details)); + const parsed = JSON.parse(formatJSON(output)); + + expect(parsed.success).toBe(false); + expect(parsed.error.code).toBe('INPUT_ERROR'); + expect(parsed.error.details.self).toBe('[TRUNCATED]'); + }); + + it('emits a stable envelope for deeply nested error details instead of overflowing', () => { + let deep: unknown = 'leaf'; + for (let index = 0; index < 100_000; index++) { + deep = { nested: deep }; + } + + const output = errorOutput(new InputError('Bad input', { deep })); + const serialized = formatJSON(output); + + expect(JSON.parse(serialized).success).toBe(false); + expect(serialized).toContain('[TRUNCATED]'); + expect(serialized).not.toContain('leaf'); + }); + + it('redacts credentials from error details and hints', () => { + const output = errorOutput( + new InputError( + 'Request failed with Bearer message-secret', + { endpoint: 'https://detail-user:detail-pass@host/x' }, + 'Retry with Bearer hint-secret' + ) + ); + + expect(JSON.stringify(output.error)).not.toContain('message-secret'); + expect(JSON.stringify(output.error)).not.toContain('detail-user:detail-pass'); + expect(JSON.stringify(output.error)).not.toContain('hint-secret'); + }); + it('propagates MainWPCTLError codes correctly', () => { const inputError = new InputError('Bad input'); const networkError = new NetworkError('Connection failed'); diff --git a/src/output/json-envelope.ts b/src/output/json-envelope.ts index 162d837..58d0097 100644 --- a/src/output/json-envelope.ts +++ b/src/output/json-envelope.ts @@ -6,6 +6,7 @@ import { isMainWPCTLError, type MainWPCTLError } from '../utils/errors.js'; import { sanitizeForTerminal, stripControlChars } from '../utils/terminal-sanitizer.js'; +import { sanitizeErrorMessage, sanitizeErrorValue } from '../utils/error-sanitizer.js'; /** * Stable CLI output envelope @@ -64,21 +65,23 @@ export function errorOutput( if (isMainWPCTLError(error)) { errorBody = { code: error.code, - message: stripControlChars(error.message), - details: error.details ? sanitizeForTerminal(error.details) : undefined, + message: sanitizeErrorMessage(stripControlChars(error.message)), + details: error.details + ? sanitizeErrorValue(sanitizeForTerminal(error.details)) + : undefined, }; if (error.hint) { - errorBody.hint = stripControlChars(error.hint); + errorBody.hint = sanitizeErrorMessage(stripControlChars(error.hint)); } } else if (error instanceof Error) { errorBody = { code: 'INTERNAL_ERROR', - message: stripControlChars(error.message), + message: sanitizeErrorMessage(stripControlChars(error.message)), }; } else { errorBody = { code: 'INTERNAL_ERROR', - message: stripControlChars(String(error)), + message: sanitizeErrorMessage(stripControlChars(String(error))), }; } diff --git a/src/utils/audit-logger.permissions.test.ts b/src/utils/audit-logger.permissions.test.ts new file mode 100644 index 0000000..e990886 --- /dev/null +++ b/src/utils/audit-logger.permissions.test.ts @@ -0,0 +1,47 @@ +import { promises as fs } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { AuditLogger } from './audit-logger.js'; + +describe('AuditLogger permissions', () => { + const originalXdgConfigHome = process.env['XDG_CONFIG_HOME']; + let tempRoot: string | undefined; + + afterEach(async () => { + if (originalXdgConfigHome === undefined) { + delete process.env['XDG_CONFIG_HOME']; + } else { + process.env['XDG_CONFIG_HOME'] = originalXdgConfigHome; + } + + if (tempRoot) { + await fs.rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + // POSIX-only: Windows has no mode bits for chmod to set — it only toggles + // the read-only flag, and stat reports 0o666 regardless. The self-heal + // contract this test pins cannot be expressed there. + it.skipIf(process.platform === 'win32')('self-heals loose config directory and audit log permissions', async () => { + tempRoot = await fs.mkdtemp(join(tmpdir(), 'mainwp-audit-')); + process.env['XDG_CONFIG_HOME'] = tempRoot; + + const configDir = join(tempRoot, 'mainwpcontrol'); + const logPath = join(configDir, 'audit.log'); + await fs.mkdir(configDir, { mode: 0o755 }); + await fs.chmod(configDir, 0o755); + await fs.writeFile(logPath, '', { mode: 0o644 }); + await fs.chmod(logPath, 0o644); + + await new AuditLogger().logDestructiveAction({ + abilityName: 'mainwp/delete-site-v1', + userDecision: 'approved', + input: { site_id: 123 }, + }); + + expect((await fs.stat(configDir)).mode & 0o777).toBe(0o700); + expect((await fs.stat(logPath)).mode & 0o777).toBe(0o600); + }); +}); diff --git a/src/utils/audit-logger.test.ts b/src/utils/audit-logger.test.ts index 489b0a3..b66b23f 100644 --- a/src/utils/audit-logger.test.ts +++ b/src/utils/audit-logger.test.ts @@ -10,18 +10,25 @@ const MOCK_LOG = join('/mock/config', 'audit.log'); // Mock dependencies before importing the module under test const mockMkdir = vi.fn(); -const mockAppendFile = vi.fn(); -const mockAccess = vi.fn(); +const mockChmod = vi.fn(); const mockStat = vi.fn(); const mockUnlink = vi.fn(); const mockRename = vi.fn(); const mockOpen = vi.fn(); +const mockHandleChmod = vi.fn(); +const mockHandleWriteFile = vi.fn(); +const mockHandleClose = vi.fn(); vi.mock('node:fs', () => ({ + constants: { + O_APPEND: 1, + O_CREAT: 2, + O_WRONLY: 4, + O_NOFOLLOW: 8, + }, promises: { mkdir: (...args: unknown[]) => mockMkdir(...args), - appendFile: (...args: unknown[]) => mockAppendFile(...args), - access: (...args: unknown[]) => mockAccess(...args), + chmod: (...args: unknown[]) => mockChmod(...args), stat: (...args: unknown[]) => mockStat(...args), unlink: (...args: unknown[]) => mockUnlink(...args), rename: (...args: unknown[]) => mockRename(...args), @@ -57,9 +64,16 @@ describe('AuditLogger', () => { // Default: file exists, not needing rotation mockMkdir.mockResolvedValue(undefined); + mockChmod.mockResolvedValue(undefined); mockStat.mockResolvedValue({ size: 100 }); - mockAccess.mockResolvedValue(undefined); - mockAppendFile.mockResolvedValue(undefined); + mockHandleChmod.mockResolvedValue(undefined); + mockHandleWriteFile.mockResolvedValue(undefined); + mockHandleClose.mockResolvedValue(undefined); + mockOpen.mockResolvedValue({ + chmod: mockHandleChmod, + writeFile: mockHandleWriteFile, + close: mockHandleClose, + }); logger = new AuditLogger(); }); @@ -93,9 +107,13 @@ describe('AuditLogger', () => { it('writes NDJSON line with correct structure', async () => { await logger.logDestructiveAction(baseInput); - expect(mockAppendFile).toHaveBeenCalledTimes(1); - const [path, content] = mockAppendFile.mock.calls[0]!; - expect(path).toBe(MOCK_LOG); + expect(mockHandleWriteFile).toHaveBeenCalledTimes(1); + const [content] = mockHandleWriteFile.mock.calls[0]!; + expect(mockOpen).toHaveBeenCalledWith( + MOCK_LOG, + process.platform === 'win32' ? 7 : 15, + 0o600, + ); const entry = JSON.parse(content.trim()); expect(entry.timestamp).toBe('2026-03-18T12:00:00.000Z'); @@ -112,7 +130,7 @@ describe('AuditLogger', () => { preview: { summary: 'Delete 1 site', affectedCount: 1 }, }); - const entry = JSON.parse(mockAppendFile.mock.calls[0]![1].trim()); + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); expect(entry.preview).toEqual({ summary: 'Delete 1 site', affectedCount: 1 }); }); @@ -122,7 +140,7 @@ describe('AuditLogger', () => { execution: { success: true }, }); - const entry = JSON.parse(mockAppendFile.mock.calls[0]![1].trim()); + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); expect(entry.execution).toEqual({ success: true }); }); @@ -132,14 +150,40 @@ describe('AuditLogger', () => { execution: { success: false, error: 'Site not found' }, }); - const entry = JSON.parse(mockAppendFile.mock.calls[0]![1].trim()); + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); expect(entry.execution).toEqual({ success: false, error: 'Site not found' }); }); + it('bounds an oversized preview summary with a visible marker', async () => { + await logger.logDestructiveAction({ + ...baseInput, + preview: { summary: 'y'.repeat(10_000), affectedCount: 3 }, + }); + + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); + expect(entry.preview.affectedCount).toBe(3); + expect(entry.preview.summary.endsWith('[TRUNCATED]')).toBe(true); + expect(Buffer.byteLength(entry.preview.summary, 'utf8')) + .toBeLessThanOrEqual(2 * 1024 + '[TRUNCATED]'.length); + }); + + it('bounds an oversized execution error with a visible marker', async () => { + await logger.logDestructiveAction({ + ...baseInput, + execution: { success: false, error: 'z'.repeat(10_000) }, + }); + + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); + expect(entry.execution.success).toBe(false); + expect(entry.execution.error.endsWith('[TRUNCATED]')).toBe(true); + expect(Buffer.byteLength(entry.execution.error, 'utf8')) + .toBeLessThanOrEqual(2 * 1024 + '[TRUNCATED]'.length); + }); + it('omits preview and execution when not provided', async () => { await logger.logDestructiveAction(baseInput); - const entry = JSON.parse(mockAppendFile.mock.calls[0]![1].trim()); + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); expect(entry).not.toHaveProperty('preview'); expect(entry).not.toHaveProperty('execution'); }); @@ -150,7 +194,7 @@ describe('AuditLogger', () => { userDecision: 'declined', }); - const entry = JSON.parse(mockAppendFile.mock.calls[0]![1].trim()); + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); expect(entry.userDecision).toBe('declined'); }); @@ -163,10 +207,38 @@ describe('AuditLogger', () => { }); expect(mockRedactSensitive).toHaveBeenCalledWith({ site_id: 123, password: 'secret' }); - const entry = JSON.parse(mockAppendFile.mock.calls[0]![1].trim()); + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); expect(entry.input.password).toBe('[REDACTED]'); }); + it('bounds oversized redacted input and records an explicit truncation marker', async () => { + mockRedactSensitive.mockReturnValueOnce({ payload: 'x'.repeat(20_000) }); + + await logger.logDestructiveAction({ + ...baseInput, + input: { payload: 'x'.repeat(20_000) }, + }); + + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); + expect(Buffer.byteLength(JSON.stringify(entry.input), 'utf8')).toBeLessThanOrEqual(8 * 1024); + expect(entry.inputTruncated).toEqual({ + marker: 'TRUNCATED', + originalBytes: 20_014, + limitBytes: 8 * 1024, + }); + }); + + it('leaves no replacement characters when truncation splits a multi-byte character', async () => { + const payload = 'a' + '😀'.repeat(5_000); + mockRedactSensitive.mockReturnValueOnce({ p: payload }); + + await logger.logDestructiveAction({ ...baseInput, input: { p: payload } }); + + const entry = JSON.parse(mockHandleWriteFile.mock.calls[0]![0].trim()); + expect(entry.inputTruncated?.marker).toBe('TRUNCATED'); + expect(entry.input.serializedPrefix.endsWith('�')).toBe(false); + }); + it('creates config directory with restricted permissions', async () => { await logger.logDestructiveAction(baseInput); @@ -174,17 +246,49 @@ describe('AuditLogger', () => { recursive: true, mode: 0o700, }); + expect(mockChmod).toHaveBeenCalledWith('/mock/config', 0o700); }); - it('creates log file with 0o600 permissions when it does not exist', async () => { - mockAccess.mockRejectedValueOnce(new Error('ENOENT')); - const mockFd = { close: vi.fn().mockResolvedValue(undefined) }; - mockOpen.mockResolvedValueOnce(mockFd); + it('warns but still writes when a permission repair fails', async () => { + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockChmod.mockRejectedValueOnce(new Error('EPERM: operation not permitted')); + + await logger.logDestructiveAction(baseInput); + + expect(mockHandleWriteFile).toHaveBeenCalledTimes(1); + expect(consoleError).toHaveBeenCalledWith( + expect.stringContaining('could not restrict permissions'), + ); + consoleError.mockRestore(); + }); + it('opens the log atomically in append mode and restricts permissions', async () => { await logger.logDestructiveAction(baseInput); - expect(mockOpen).toHaveBeenCalledWith(MOCK_LOG, 'w', 0o600); - expect(mockFd.close).toHaveBeenCalled(); + expect(mockOpen).toHaveBeenCalledWith( + MOCK_LOG, + process.platform === 'win32' ? 7 : 15, + 0o600, + ); + expect(mockHandleChmod).toHaveBeenCalledWith(0o600); + expect(mockHandleClose).toHaveBeenCalled(); + }); + + it('continues logging when directory permission self-healing fails', async () => { + mockChmod.mockRejectedValueOnce(new Error('EPERM')); + + await expect(logger.logDestructiveAction(baseInput)).resolves.not.toThrow(); + + expect(mockHandleWriteFile).toHaveBeenCalledTimes(1); + }); + + it('continues logging when file permission self-healing fails', async () => { + mockHandleChmod.mockRejectedValueOnce(new Error('EPERM')); + + await expect(logger.logDestructiveAction(baseInput)).resolves.not.toThrow(); + + expect(mockHandleWriteFile).toHaveBeenCalledTimes(1); + expect(mockHandleClose).toHaveBeenCalled(); }); }); diff --git a/src/utils/audit-logger.ts b/src/utils/audit-logger.ts index 46940cb..2182594 100644 --- a/src/utils/audit-logger.ts +++ b/src/utils/audit-logger.ts @@ -11,7 +11,7 @@ * Sensitive data (passwords, tokens, API keys) is automatically redacted. */ -import { promises as fs } from 'node:fs'; +import { constants as fsConstants, promises as fs } from 'node:fs'; import { join } from 'node:path'; import { getConfigDir } from '../config/settings.js'; import { getInputSanitizer } from '../validation/input-sanitizer.js'; @@ -26,6 +26,15 @@ const MAX_LOG_SIZE = 10 * 1024 * 1024; */ const MAX_ROTATIONS = 5; +const MAX_SERIALIZED_INPUT_BYTES = 8 * 1024; + +/** + * Byte cap for free-text entry fields (preview summary, execution error). + * These strings can be derived from Dashboard responses, so an unbounded + * value could bloat the audit log the same way unbounded input could. + */ +const MAX_FREE_TEXT_BYTES = 2 * 1024; + /** * Audit log filename */ @@ -39,20 +48,41 @@ export interface AuditEntry { timestamp: string; /** Name of the ability executed */ abilityName: string; - /** Preview information (optional - not available in CLI executeDestructive path) */ + /** Preview information (absent when the preview itself failed) */ preview?: { summary: string; affectedCount: number; }; /** User's decision */ userDecision: 'approved' | 'declined'; - /** Execution result (only present when approved) */ + /** + * Present on the entry written immediately before the confirm call is + * dispatched. A 'dispatch' entry with no later matching result entry means + * the process died or errored mid-confirm — the action may have executed. + */ + stage?: 'dispatch'; + /** + * Execution result when approved. On a declined entry this instead records + * why the flow was aborted before the user could approve (e.g. + * "Preview failed: ..." from the fail-closed preview gate). + */ execution?: { success: boolean; error?: string; + /** + * True when the confirm call failed at the transport layer after + * dispatch: the Dashboard may or may not have executed the action. + */ + outcomeUnknown?: boolean; }; /** Input parameters (redacted of sensitive data) */ input: Record; + /** Present when input was bounded before writing the entry. */ + inputTruncated?: { + marker: 'TRUNCATED'; + originalBytes: number; + limitBytes: number; + }; } /** @@ -65,9 +95,11 @@ export interface LogDestructiveActionInput { affectedCount: number; }; userDecision: 'approved' | 'declined'; + stage?: 'dispatch'; execution?: { success: boolean; error?: string; + outcomeUnknown?: boolean; }; input: Record; } @@ -79,6 +111,33 @@ export function getAuditLogPath(): string { return join(getConfigDir(), AUDIT_LOG_FILENAME); } +/** + * Cap a free-text field at MAX_FREE_TEXT_BYTES, cutting on a byte boundary + * and dropping any trailing replacement characters from a split multi-byte + * sequence. Over-limit values end with a visible [TRUNCATED] marker. + */ +function boundText(text: string): string { + if (Buffer.byteLength(text, 'utf8') <= MAX_FREE_TEXT_BYTES) { + return text; + } + const truncated = Buffer.from(text, 'utf8') + .subarray(0, MAX_FREE_TEXT_BYTES) + .toString('utf8') + .replace(/�+$/, ''); + return `${truncated}[TRUNCATED]`; +} + +/** + * A failed permission repair must not abort the audit write, but it also + * must not pass silently: the log may be left readable by other users. + */ +function warnPermissionRepairFailed(path: string, error: unknown): void { + console.error( + `Warning: [AuditLogger] could not restrict permissions on ${path}: ` + + (error instanceof Error ? error.message : String(error)) + ); +} + /** * Audit Logger class */ @@ -98,6 +157,9 @@ export class AuditLogger { // Create directory with restricted permissions (owner only) const dir = getConfigDir(); await fs.mkdir(dir, { recursive: true, mode: 0o700 }); + await fs.chmod(dir, 0o700).catch((error: unknown) => { + warnPermissionRepairFailed(dir, error); + }); // Check if rotation is needed if (await this.shouldRotate(logPath)) { @@ -106,46 +168,90 @@ export class AuditLogger { // Redact sensitive data from input const redactedInput = this.inputSanitizer.redactSensitive(params.input); + const boundedInput = this.boundInput(redactedInput); // Build audit entry const entry: AuditEntry = { timestamp: new Date().toISOString(), abilityName: params.abilityName, userDecision: params.userDecision, - input: redactedInput, + input: boundedInput.input, }; + if (boundedInput.truncated) { + entry.inputTruncated = boundedInput.truncated; + } // Add optional fields + if (params.stage) { + entry.stage = params.stage; + } if (params.preview) { - entry.preview = params.preview; + entry.preview = { + summary: boundText(params.preview.summary), + affectedCount: params.preview.affectedCount, + }; } if (params.execution) { - entry.execution = params.execution; + entry.execution = { ...params.execution }; + if (entry.execution.error !== undefined) { + entry.execution.error = boundText(entry.execution.error); + } } // Format as NDJSON line const line = JSON.stringify(entry) + '\n'; - // Ensure file exists with proper permissions before appending - await this.ensureLogFile(logPath); - - // Append to log file - await fs.appendFile(logPath, line, 'utf-8'); + // Open atomically in append mode and self-heal existing file permissions. + const noFollow = process.platform === 'win32' ? 0 : (fsConstants.O_NOFOLLOW ?? 0); + const appendFlags = fsConstants.O_APPEND | fsConstants.O_CREAT | + fsConstants.O_WRONLY | noFollow; + const handle = await fs.open(logPath, appendFlags, 0o600); + try { + await handle.chmod(0o600).catch((error: unknown) => { + warnPermissionRepairFailed(logPath, error); + }); + await handle.writeFile(line, 'utf-8'); + } finally { + await handle.close(); + } } - /** - * Ensure log file exists with proper permissions - * - * SECURITY: Creates file with 0o600 (owner read/write only) if it doesn't exist. - */ - private async ensureLogFile(logPath: string): Promise { - try { - await fs.access(logPath); - } catch { - // File doesn't exist, create with restricted permissions - const fd = await fs.open(logPath, 'w', 0o600); - await fd.close(); + private boundInput(input: Record): { + input: Record; + truncated?: AuditEntry['inputTruncated']; + } { + const serialized = JSON.stringify(input); + const originalBytes = Buffer.byteLength(serialized, 'utf8'); + if (originalBytes <= MAX_SERIALIZED_INPUT_BYTES) { + return { input }; } + + const serializedBytes = Buffer.from(serialized, 'utf8'); + let prefixBytes = MAX_SERIALIZED_INPUT_BYTES; + let bounded: Record; + do { + bounded = { + // Cutting on a byte boundary can split a multi-byte sequence; drop + // the resulting trailing replacement characters. + serializedPrefix: serializedBytes + .subarray(0, prefixBytes) + .toString('utf8') + .replace(/�+$/, ''), + }; + if (Buffer.byteLength(JSON.stringify(bounded), 'utf8') <= MAX_SERIALIZED_INPUT_BYTES) { + break; + } + prefixBytes = Math.floor(prefixBytes * 0.75); + } while (prefixBytes > 0); + + return { + input: bounded!, + truncated: { + marker: 'TRUNCATED', + originalBytes, + limitBytes: MAX_SERIALIZED_INPUT_BYTES, + }, + }; } /** diff --git a/src/utils/error-sanitizer.test.ts b/src/utils/error-sanitizer.test.ts new file mode 100644 index 0000000..de2040d --- /dev/null +++ b/src/utils/error-sanitizer.test.ts @@ -0,0 +1,86 @@ +/** + * Tests for pure error sanitizers. + * + * Error details can carry parsed API responses — hostile input. These tests + * pin credential redaction coverage and the structural guards that keep + * sanitization from overflowing the stack. + */ + +import { describe, it, expect } from 'vitest'; +import { sanitizeErrorMessage, sanitizeErrorValue } from './error-sanitizer.js'; + +describe('sanitizeErrorMessage', () => { + it('redacts user-and-password credentialed URLs', () => { + expect(sanitizeErrorMessage('failed: https://admin:secret@dashboard.example.com/wp-json')).toBe( + 'failed: [URL_WITH_CREDENTIALS]' + ); + }); + + it('redacts username-only credentialed URLs', () => { + expect(sanitizeErrorMessage('failed: https://alice@dashboard.example.com')).toBe( + 'failed: [URL_WITH_CREDENTIALS]' + ); + }); + + it('leaves URLs without userinfo unchanged', () => { + const message = 'failed: https://dashboard.example.com/wp-json?page=1'; + expect(sanitizeErrorMessage(message)).toBe(message); + }); + + it('redacts sensitive query-string parameter values', () => { + expect( + sanitizeErrorMessage('failed: https://dashboard.example.com/cb?access_token=abc123&page=2') + ).toBe('failed: https://dashboard.example.com/cb?access_token=[REDACTED]&page=2'); + }); +}); + +describe('sanitizeErrorValue', () => { + it('sanitizes strings nested in arrays and objects', () => { + expect( + sanitizeErrorValue({ + urls: ['https://admin:secret@dashboard.example.com'], + }) + ).toEqual({ urls: ['[URL_WITH_CREDENTIALS]'] }); + }); + + it('redacts values under sensitive keys outright', () => { + expect( + sanitizeErrorValue({ + password: 'hunter2', + api_key: { nested: 'secret' }, + note: 'kept', + }) + ).toEqual({ + password: '[REDACTED]', + api_key: '[REDACTED]', + note: 'kept', + }); + }); + + it('terminates on cyclic structures instead of overflowing the stack', () => { + const cyclic: Record = { name: 'outer' }; + cyclic['self'] = cyclic; + + expect(sanitizeErrorValue(cyclic)).toEqual({ + name: 'outer', + self: '[TRUNCATED]', + }); + }); + + it('truncates beyond the depth limit instead of recursing indefinitely', () => { + let deep: unknown = 'leaf'; + for (let index = 0; index < 50; index++) { + deep = { nested: deep }; + } + + const sanitized = JSON.stringify(sanitizeErrorValue(deep)); + expect(sanitized).toContain('[TRUNCATED]'); + expect(sanitized).not.toContain('leaf'); + }); + + it('passes primitives through unchanged', () => { + expect(sanitizeErrorValue(42)).toBe(42); + expect(sanitizeErrorValue(null)).toBe(null); + expect(sanitizeErrorValue(true)).toBe(true); + }); +}); diff --git a/src/utils/error-sanitizer.ts b/src/utils/error-sanitizer.ts new file mode 100644 index 0000000..308bac5 --- /dev/null +++ b/src/utils/error-sanitizer.ts @@ -0,0 +1,86 @@ +/** + * Pure sanitizers for error messages and structured error details. + */ + +import { isSensitiveKey } from './redaction.js'; + +const PATH_PATTERNS = [ + /\/Users\/[^/\s]+/g, + /\/home\/[^/\s]+/g, + /C:\\Users\\[^\\]+/gi, + /\.config\/mainwpcontrol/g, +]; + +export function sanitizeErrorMessage(message: string): string { + let sanitized = message; + + for (const pattern of PATH_PATTERNS) { + sanitized = sanitized.replace(pattern, '[PATH]'); + } + + // Password is optional: `https://alice@host` still leaks a username. + sanitized = sanitized.replace( + /https?:\/\/[^\s@/]+(?::[^\s@]*)?@[^\s]+/g, + '[URL_WITH_CREDENTIALS]' + ); + sanitized = sanitized.replace( + /Basic\s+[A-Za-z0-9+/]+=*/gi, + 'Basic [REDACTED]' + ); + sanitized = sanitized.replace( + /Bearer\s+[A-Za-z0-9._-]+/gi, + 'Bearer [REDACTED]' + ); + + // Query-string parameters whose key is on the shared sensitive list + // (access_token, api_key, ...) — a URL like ?access_token=... carries the + // credential outside the userinfo form handled above. + sanitized = sanitized.replace( + /([?&])([^=&\s"']{1,64})=([^&\s"']+)/g, + (match, sep: string, key: string) => + isSensitiveKey(key) ? `${sep}${key}=[REDACTED]` : match + ); + + return sanitized; +} + +// Error details can carry parsed API responses (hostile input): a deeply +// nested payload or a locally-attached cyclic structure must not overflow +// the stack while being sanitized. The WeakSet tracks the current ancestor +// path (not all visited objects) so legitimately shared references survive. +const MAX_SANITIZE_DEPTH = 8; + +export function sanitizeErrorValue( + value: unknown, + depth = 0, + path: WeakSet = new WeakSet() +): unknown { + if (typeof value === 'string') { + return sanitizeErrorMessage(value); + } + + if (value !== null && typeof value === 'object') { + if (path.has(value) || depth >= MAX_SANITIZE_DEPTH) { + return '[TRUNCATED]'; + } + path.add(value); + + const result = Array.isArray(value) + ? value.map((item) => sanitizeErrorValue(item, depth + 1, path)) + : Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + sanitizeErrorMessage(key), + // A sensitive key's value is a credential wherever it appears in + // hostile error details — redact it outright instead of recursing. + isSensitiveKey(key) + ? '[REDACTED]' + : sanitizeErrorValue(item, depth + 1, path), + ]) + ); + + path.delete(value); + return result; + } + + return value; +} diff --git a/src/utils/errors.ts b/src/utils/errors.ts index b838a37..4c39110 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -116,6 +116,22 @@ export class TLSError extends MainWPCTLError { } } +/** + * Unknown outcome after a dispatched destructive confirm (exit code 3) + * + * Thrown when the confirm call failed at the transport layer after the CLI + * initiated dispatch: the Dashboard may or may not have executed the action. + * Callers must never auto-retry the confirm in response to this error. + */ +export class UnknownOutcomeError extends MainWPCTLError { + readonly exitCode = ExitCode.NETWORK_ERROR; + readonly code = 'OUTCOME_UNKNOWN'; + + constructor(message: string, details?: unknown, hint?: string) { + super(message, details, hint); + } +} + /** * API error (exit code 4) */ diff --git a/src/utils/format.test.ts b/src/utils/format.test.ts index 45315e4..d3287ee 100644 --- a/src/utils/format.test.ts +++ b/src/utils/format.test.ts @@ -3,7 +3,14 @@ */ import { describe, it, expect } from 'vitest'; -import { maskSecret, maskPassword, maskApiKey, type MaskOptions } from './format.js'; +import { + maskSecret, + maskPassword, + maskApiKey, + maskUrlUserinfo, + maskUrlUserinfoInText, + type MaskOptions, +} from './format.js'; describe('maskSecret', () => { describe('with default options', () => { @@ -113,3 +120,83 @@ describe('maskApiKey', () => { expect(maskApiKey('sk-ant-api03-xxxxxxxxxxxxxx')).toBe('sk-ant...xxxx'); }); }); + +describe('maskUrlUserinfo', () => { + it('masks embedded username and password', () => { + expect(maskUrlUserinfo('https://admin:secret@dashboard.example.com/path')).toBe( + 'https://***:***@dashboard.example.com/path' + ); + }); + + it('masks a username when no password is present', () => { + expect(maskUrlUserinfo('https://admin@dashboard.example.com')).toBe( + 'https://***:***@dashboard.example.com' + ); + }); + + it('masks the full userinfo when the password contains "@"', () => { + expect(maskUrlUserinfo('https://admin:p@ssw@rd@dashboard.example.com/path')).toBe( + 'https://***:***@dashboard.example.com/path' + ); + }); + + it('does not consume past the query string when it contains "@"', () => { + expect(maskUrlUserinfo('https://admin:secret@dashboard.example.com?to=a@b')).toBe( + 'https://***:***@dashboard.example.com?to=a@b' + ); + }); + + it('returns URLs without userinfo unchanged', () => { + const url = 'https://dashboard.example.com/path?site=1'; + expect(maskUrlUserinfo(url)).toBe(url); + }); + + it('returns invalid URL input unchanged', () => { + const url = 'not a valid URL'; + expect(maskUrlUserinfo(url)).toBe(url); + }); + + it('fails closed when a newline in the userinfo defeats the masking regex', () => { + // new URL() strips \n before detecting credentials, but the raw string + // keeps it, so the whitespace-excluding replace cannot match. + const result = maskUrlUserinfo('https://admin:sec\nret@dashboard.example.com/path'); + expect(result).toBe('[URL_WITH_CREDENTIALS_REDACTED]'); + expect(result).not.toContain('sec'); + }); + + it('fails closed when a tab in the userinfo defeats the masking regex', () => { + const result = maskUrlUserinfo('https://admin:sec\tret@dashboard.example.com'); + expect(result).toBe('[URL_WITH_CREDENTIALS_REDACTED]'); + }); + + it('fails closed when leading whitespace defeats the anchored regex', () => { + // Leading whitespace is trimmed by the parser but the regex is anchored, + // so the replace fails and the fail-closed path must catch it too. + const result = maskUrlUserinfo(' https://admin:secret@dashboard.example.com'); + expect(result).toBe('[URL_WITH_CREDENTIALS_REDACTED]'); + expect(result).not.toContain('secret'); + }); +}); + +describe('maskUrlUserinfoInText', () => { + it('masks credentialed URLs embedded in error messages', () => { + expect( + maskUrlUserinfoInText( + 'Request cannot be constructed from a URL that includes credentials: https://legacy:secret@dashboard.example.com/wp-json/route?page=1' + ) + ).toBe( + 'Request cannot be constructed from a URL that includes credentials: https://***:***@dashboard.example.com/wp-json/route?page=1' + ); + }); + + it('leaves text without credentialed URLs unchanged', () => { + const text = 'Connection refused for https://dashboard.example.com (mail admin@example.com)'; + expect(maskUrlUserinfoInText(text)).toBe(text); + }); + + it('masks the full userinfo when the password contains "@"', () => { + expect( + maskUrlUserinfoInText('fetch failed: https://legacy:p@ss@dashboard.example.com/wp-json timed out') + ).toBe('fetch failed: https://***:***@dashboard.example.com/wp-json timed out'); + }); +}); diff --git a/src/utils/format.ts b/src/utils/format.ts index 61720cf..0796fdb 100644 --- a/src/utils/format.ts +++ b/src/utils/format.ts @@ -90,3 +90,66 @@ export function maskApiKey(apiKey: string): string { minLength: 10, }); } + +/** + * Mask userinfo (username/password) embedded in a URL. + * + * SECURITY: Profiles saved before userinfo rejection was added may still carry + * `user:pass@` in the stored dashboard URL; every display path must mask it. + * + * Returns the input unchanged when it is not a parseable URL or has no + * userinfo. String replacement (not URL re-serialization) keeps the rest of + * the URL byte-for-byte identical — no trailing-slash normalization. + * + * @param url - The URL to mask + * @returns The URL with userinfo replaced by `***:***@`, the input unchanged + * when it has no userinfo, or `[URL_WITH_CREDENTIALS_REDACTED]` when userinfo + * was detected but could not be isolated in the raw string + * + * @example + * ```ts + * maskUrlUserinfo('https://admin:secret@example.com') // 'https://***:***@example.com' + * maskUrlUserinfo('https://example.com/path') // unchanged + * ``` + */ +export function maskUrlUserinfo(url: string): string { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return url; + } + + if (!parsed.username && !parsed.password) { + return url; + } + + // Greedy through the LAST @ in the authority: a password containing "@" + // must not leak its tail. `?`/`#`/`/` bound the authority section. + const masked = url.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/?#\s]*@/i, '$1***:***@'); + + // The parser saw userinfo the regex could not isolate: WHATWG parsing + // strips tab/newline and trims C0 controls before detecting credentials, + // so a raw string containing them slips past the whitespace-excluding + // regex. Fail closed rather than echo the credentials. + if (masked === url) { + return '[URL_WITH_CREDENTIALS_REDACTED]'; + } + + return masked; +} + +/** + * Mask userinfo in any URLs embedded within arbitrary text. + * + * SECURITY: Error messages (e.g. fetch failures) can echo a full request URL + * including embedded credentials from a legacy profile. + * + * @param text - Text that may contain credentialed URLs + * @returns The text with each `scheme://user:pass@` replaced by `scheme://***:***@` + */ +export function maskUrlUserinfoInText(text: string): string { + // Greedy through the LAST @ before a path/query/fragment or whitespace, so + // passwords containing "@" mask fully instead of leaking after the first @. + return text.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/?#]+@/gi, '$1***:***@'); +} diff --git a/src/utils/prompt.ts b/src/utils/prompt.ts index e42e91d..d8306eb 100644 --- a/src/utils/prompt.ts +++ b/src/utils/prompt.ts @@ -136,6 +136,9 @@ export async function promptForPassword(question: string): Promise { stdin.removeListener('data', onData); rl.close(); process.stdout.write('\n'); + // 130 = 128 + SIGINT(2), the standard Unix convention for Ctrl-C. + // Intentionally outside the documented 0-5 exit code contract — + // see README's Exit Codes table for the carve-out. process.exit(130); break; diff --git a/src/utils/redaction.test.ts b/src/utils/redaction.test.ts new file mode 100644 index 0000000..c088590 --- /dev/null +++ b/src/utils/redaction.test.ts @@ -0,0 +1,138 @@ +/** + * Tests for the shared sensitive-key redaction utility + */ + +import { describe, it, expect } from 'vitest'; +import { isSensitiveKey, redactSensitiveKeys } from './redaction.js'; + +describe('isSensitiveKey', () => { + it('detects the superset terms', () => { + expect(isSensitiveKey('password')).toBe(true); + expect(isSensitiveKey('secret')).toBe(true); + expect(isSensitiveKey('token')).toBe(true); + expect(isSensitiveKey('authorization')).toBe(true); + expect(isSensitiveKey('auth')).toBe(true); + expect(isSensitiveKey('cookie')).toBe(true); + expect(isSensitiveKey('apikey')).toBe(true); + expect(isSensitiveKey('api_key')).toBe(true); + expect(isSensitiveKey('bearer')).toBe(true); + expect(isSensitiveKey('credential')).toBe(true); + expect(isSensitiveKey('private_key')).toBe(true); + expect(isSensitiveKey('signing_key')).toBe(true); + expect(isSensitiveKey('encryption_key')).toBe(true); + }); + + it('is case-insensitive', () => { + expect(isSensitiveKey('PASSWORD')).toBe(true); + expect(isSensitiveKey('Secret')).toBe(true); + expect(isSensitiveKey('ApiKey')).toBe(true); + }); + + it('detects compound keys regardless of separator style', () => { + expect(isSensitiveKey('apiToken')).toBe(true); + expect(isSensitiveKey('appPassword')).toBe(true); + expect(isSensitiveKey('refreshToken')).toBe(true); + expect(isSensitiveKey('X-Api-Key')).toBe(true); + expect(isSensitiveKey('api-key')).toBe(true); + expect(isSensitiveKey('signing-key')).toBe(true); + expect(isSensitiveKey('encryptionKey')).toBe(true); + expect(isSensitiveKey('set-cookie')).toBe(true); + expect(isSensitiveKey('private-key')).toBe(true); + }); + + it('does not flag non-sensitive keys', () => { + expect(isSensitiveKey('site_id')).toBe(false); + expect(isSensitiveKey('name')).toBe(false); + expect(isSensitiveKey('url')).toBe(false); + expect(isSensitiveKey('description')).toBe(false); + expect(isSensitiveKey('username')).toBe(false); + }); +}); + +describe('redactSensitiveKeys', () => { + it('redacts sensitive top-level keys and leaves others intact', () => { + const data = { + apiToken: 'abc123', + appPassword: 'def456', + site_id: 42, + }; + + const redacted = redactSensitiveKeys(data) as Record; + + expect(redacted.apiToken).toBe('[REDACTED]'); + expect(redacted.appPassword).toBe('[REDACTED]'); + expect(redacted.site_id).toBe(42); + }); + + it('redacts nested sensitive fields', () => { + const data = { + config: { + refreshToken: 'secret-value', + name: 'test', + }, + }; + + const redacted = redactSensitiveKeys(data) as { config: Record }; + + expect(redacted.config.refreshToken).toBe('[REDACTED]'); + expect(redacted.config.name).toBe('test'); + }); + + it('redacts sensitive fields inside arrays', () => { + const data = [{ 'X-Api-Key': 'xyz' }, { name: 'ok' }]; + + const redacted = redactSensitiveKeys(data) as Record[]; + + expect(redacted[0]?.['X-Api-Key']).toBe('[REDACTED]'); + expect(redacted[1]?.['name']).toBe('ok'); + }); + + it('passes through non-object values unchanged', () => { + expect(redactSensitiveKeys('a string')).toBe('a string'); + expect(redactSensitiveKeys(42)).toBe(42); + expect(redactSensitiveKeys(null)).toBe(null); + expect(redactSensitiveKeys(undefined)).toBe(undefined); + }); + + it('terminates on cyclic structures instead of overflowing the stack', () => { + const cyclic: Record = { name: 'outer' }; + cyclic['self'] = cyclic; + + expect(redactSensitiveKeys(cyclic)).toEqual({ + name: 'outer', + self: '[TRUNCATED]', + }); + }); + + it('truncates beyond the depth limit instead of recursing indefinitely', () => { + let deep: unknown = { password: 'leaf' }; + for (let index = 0; index < 50; index++) { + deep = { nested: deep }; + } + + const serialized = JSON.stringify(redactSensitiveKeys(deep)); + expect(serialized).toContain('[TRUNCATED]'); + expect(serialized).not.toContain('leaf'); + }); + + it('keeps a hostile __proto__ key as an ordinary data property', () => { + const input = JSON.parse('{"__proto__": {"polluted": true}, "password": "x"}') as unknown; + + const result = redactSensitiveKeys(input) as Record; + + expect(result['password']).toBe('[REDACTED]'); + expect(result['__proto__']).toEqual({ polluted: true }); + expect(Object.getPrototypeOf(result)).toBeNull(); + expect(({} as Record)['polluted']).toBeUndefined(); + }); + + it('keeps legitimately shared (non-cyclic) references intact', () => { + const shared = { host: 'example.com' }; + const data = { first: shared, second: shared }; + + expect(redactSensitiveKeys(data)).toEqual({ + first: { host: 'example.com' }, + second: { host: 'example.com' }, + }); + }); +}); diff --git a/src/utils/redaction.ts b/src/utils/redaction.ts new file mode 100644 index 0000000..aa44048 --- /dev/null +++ b/src/utils/redaction.ts @@ -0,0 +1,96 @@ +/** + * Shared sensitive-key redaction for mainwpcontrol + * + * Single source of truth for "does this key name look sensitive" used by + * http-client's error sanitization, base-command's debug logging, and + * input-sanitizer's error/log redaction. Consolidates three previously + * independent term lists into one superset. + */ + +/** + * Case-insensitive substrings that mark a key as sensitive. Keys are + * normalized (lowercased, `-`/`_` stripped) before matching, so this list + * also catches separator variants (api-key, api_key, apikey) and compound + * keys (apiToken, appPassword, X-Api-Key) without needing per-variant entries. + * + * TRADEOFF: this is plain substring matching, not word-boundary-aware, so a + * key like "author" would also match "auth". That mirrors the pre-existing + * behavior of the http-client and input-sanitizer implementations this util + * replaces. Over-redaction (hiding a value that wasn't actually sensitive) is + * the safe failure mode for error/debug output, so the tradeoff is accepted + * rather than building a full boundary-aware matcher. No key in this + * codebase's API surface collides with it today. + */ +const SENSITIVE_KEY_SUBSTRINGS = [ + // 'auth' also substring-matches 'authorization'; normalization folds + // 'api_key'/'api-key' into 'apikey' — don't re-add those spellings. + 'password', 'secret', 'token', 'auth', 'cookie', + 'apikey', 'bearer', 'credential', 'private_key', + 'signing_key', 'encryption_key', +] as const; + +/** + * Normalize a key for matching: lowercase, strip `-`/`_` separators. + */ +function normalizeKey(key: string): string { + return key.toLowerCase().replace(/[-_]/g, ''); +} + +/** Pre-normalized match terms (separators stripped once, not per key). */ +const NORMALIZED_TERMS = SENSITIVE_KEY_SUBSTRINGS.map((s) => normalizeKey(s)); + +/** + * Check if a key name appears to reference sensitive data. + */ +export function isSensitiveKey(key: string): boolean { + const normalized = normalizeKey(key); + return NORMALIZED_TERMS.some((term) => normalized.includes(term)); +} + +/** + * Depth bound for redaction traversal. Comfortably above input-sanitizer's + * depth cap (10) so legitimate audit input is never truncated, while a + * hostile or accidentally cyclic/deep structure terminates instead of + * overflowing the stack. + */ +const MAX_REDACT_DEPTH = 32; + +/** + * Recursively redact values whose key matches {@link isSensitiveKey}. + * Non-object values pass through unchanged; arrays are mapped element-wise. + * Cycles and nesting past MAX_REDACT_DEPTH collapse to '[TRUNCATED]'; the + * WeakSet tracks the current ancestor path (not all visited objects) so + * legitimately shared references survive. + */ +export function redactSensitiveKeys( + value: unknown, + depth = 0, + path: WeakSet = new WeakSet() +): unknown { + if (value === null || typeof value !== 'object') { + return value; + } + + if (path.has(value) || depth >= MAX_REDACT_DEPTH) { + return '[TRUNCATED]'; + } + path.add(value); + + let result: unknown; + if (Array.isArray(value)) { + result = value.map((item) => redactSensitiveKeys(item, depth + 1, path)); + } else { + // Null prototype so a hostile "__proto__" key lands as an ordinary + // data property instead of rewriting the accumulator's prototype. + const redacted: Record = Object.create(null) as Record; + for (const [key, val] of Object.entries(value)) { + redacted[key] = isSensitiveKey(key) + ? '[REDACTED]' + : redactSensitiveKeys(val, depth + 1, path); + } + result = redacted; + } + + path.delete(value); + return result; +} diff --git a/src/utils/retry.test.ts b/src/utils/retry.test.ts index ad1df91..d80853b 100644 --- a/src/utils/retry.test.ts +++ b/src/utils/retry.test.ts @@ -180,39 +180,5 @@ describe('ExponentialBackoff', () => { }); }); - describe('getDelayForAttempt', () => { - it('returns 0 for attempt 0', () => { - const backoff = new ExponentialBackoff({ - initialDelay: 1000, - }); - expect(backoff.getDelayForAttempt(0)).toBe(0); - }); - - it('calculates correct delay for each attempt', () => { - const backoff = new ExponentialBackoff({ - initialDelay: 1000, - maxDelay: 30000, - multiplier: 2, - }); - - expect(backoff.getDelayForAttempt(1)).toBe(1000); - expect(backoff.getDelayForAttempt(2)).toBe(2000); - expect(backoff.getDelayForAttempt(3)).toBe(4000); - expect(backoff.getDelayForAttempt(4)).toBe(8000); - expect(backoff.getDelayForAttempt(5)).toBe(16000); - expect(backoff.getDelayForAttempt(6)).toBe(30000); // Capped - expect(backoff.getDelayForAttempt(7)).toBe(30000); // Still capped - }); - - it('does not modify internal state', () => { - const backoff = new ExponentialBackoff({ - initialDelay: 1000, - }); - - backoff.getDelayForAttempt(5); - expect(backoff.delay).toBe(1000); - expect(backoff.attempts).toBe(0); - }); - }); }); diff --git a/src/utils/retry.ts b/src/utils/retry.ts index b89c9be..2b0a9dd 100644 --- a/src/utils/retry.ts +++ b/src/utils/retry.ts @@ -118,14 +118,5 @@ export class ExponentialBackoff { return true; } - - /** - * Get delay for a specific retry attempt (without advancing state) - */ - getDelayForAttempt(attempt: number): number { - if (attempt === 0) return 0; - const delay = this.initialDelay * Math.pow(this.multiplier, attempt - 1); - return Math.min(delay, this.maxDelay); - } } diff --git a/src/utils/terminal-sanitizer.test.ts b/src/utils/terminal-sanitizer.test.ts index 182d140..5af07b8 100644 --- a/src/utils/terminal-sanitizer.test.ts +++ b/src/utils/terminal-sanitizer.test.ts @@ -11,11 +11,20 @@ import { describe, it, expect } from 'vitest'; import { stripControlChars, + sanitizeSingleLine, sanitizeForTerminal, safeString, containsEscapeSequences, } from './terminal-sanitizer.js'; +describe('sanitizeSingleLine', () => { + it('strips terminal escapes and collapses CR, LF, and tabs to one space', () => { + const unsafe = '\x1b[31mprovider\x1b[0m\r\n\tinjected'; + + expect(sanitizeSingleLine(unsafe)).toBe('provider injected'); + }); +}); + describe('stripControlChars', () => { describe('ANSI CSI sequences', () => { it('strips color codes', () => { @@ -88,6 +97,13 @@ describe('stripControlChars', () => { }); }); + describe('bidirectional controls', () => { + it('strips RLO and isolate controls that could visually reorder output', () => { + expect(stripControlChars('safe‮detrevni‬ end')).toBe('safedetrevni end'); + expect(stripControlChars('a⁦b⁧c⁨d⁩e')).toBe('abcde'); + }); + }); + describe('C1 control characters', () => { it('strips C1 control range (0x80-0x9F)', () => { const c1 = 'before\x80\x90\x9Fafter'; @@ -214,6 +230,17 @@ describe('sanitizeForTerminal', () => { }); }); + it('keeps a hostile __proto__ key as an ordinary data property', () => { + const input = JSON.parse('{"__proto__": {"polluted": true}, "safe": "ok"}') as unknown; + + const result = sanitizeForTerminal(input) as Record; + + expect(result['safe']).toBe('ok'); + expect(result['__proto__']).toEqual({ polluted: true }); + expect(Object.getPrototypeOf(result)).toBeNull(); + expect(({} as Record)['polluted']).toBeUndefined(); + }); + it('handles deeply nested structures', () => { const input = { level1: { @@ -234,6 +261,27 @@ describe('sanitizeForTerminal', () => { }, }); }); + + it('terminates on cyclic structures instead of overflowing the stack', () => { + const cyclic: Record = { name: 'outer' }; + cyclic['self'] = cyclic; + + expect(sanitizeForTerminal(cyclic)).toEqual({ + name: 'outer', + self: '[TRUNCATED]', + }); + }); + + it('truncates beyond the depth limit instead of recursing indefinitely', () => { + let deep: unknown = 'leaf'; + for (let index = 0; index < 100_000; index++) { + deep = { nested: deep }; + } + + const sanitized = JSON.stringify(sanitizeForTerminal(deep)); + expect(sanitized).toContain('[TRUNCATED]'); + expect(sanitized).not.toContain('leaf'); + }); }); describe('safeString', () => { diff --git a/src/utils/terminal-sanitizer.ts b/src/utils/terminal-sanitizer.ts index ed1395e..8cd5bb4 100644 --- a/src/utils/terminal-sanitizer.ts +++ b/src/utils/terminal-sanitizer.ts @@ -48,6 +48,13 @@ const ESCAPE_PATTERNS = { */ const C0_UNSAFE = /[\x00-\x08\x0b\x0c\x0e-\x1f]/g; +/** + * Unicode bidirectional and isolate controls (U+202A–U+202E, U+2066–U+2069). + * Hostile names could otherwise visually reorder terminal output and spoof + * copy-pasteable commands. job-id validation rejects the same range. + */ +const BIDI_CONTROLS = /[‪-‮⁦-⁩]/g; + /** * Strip all ANSI escape sequences and control characters from a string. * @@ -80,6 +87,7 @@ export function stripControlChars(str: string): string { // Remove control characters result = result.replace(ESCAPE_PATTERNS.c1, ''); result = result.replace(C0_UNSAFE, ''); + result = result.replace(BIDI_CONTROLS, ''); // Remove any remaining bare ESC characters result = result.replace(/\x1b/g, ''); @@ -87,6 +95,25 @@ export function stripControlChars(str: string): string { return result; } +/** + * Sanitize untrusted text for a single terminal output line. + * + * Removes terminal control sequences, then replaces any run of line-breaking + * or horizontal-tab characters with one space to prevent line injection. + */ +export function sanitizeSingleLine(str: string): string { + return stripControlChars(str).replace(/[\r\n\t]+/g, ' '); +} + +/** + * Sanitized values can originate from hostile API responses: the traversal + * is depth-bounded so deep nesting cannot overflow the stack, and the + * current ancestor path is tracked so cycles terminate. Tracking the path + * (not all visited objects) keeps legitimately shared references intact — + * command envelopes do reuse objects across fields. + */ +const MAX_SANITIZE_DEPTH = 64; + /** * Recursively sanitize a value for safe terminal output. * @@ -96,10 +123,21 @@ export function stripControlChars(str: string): string { * - Objects: recursively sanitizes each value * - Other types: converted to string and sanitized * + * Cyclic or deeper-than-bound structures are replaced with '[TRUNCATED]' + * rather than overflowing the stack. + * * @param value - The value to sanitize * @returns A sanitized copy of the value (original is not modified) */ export function sanitizeForTerminal(value: unknown): unknown { + return sanitizeForTerminalBounded(value, 0, new WeakSet()); +} + +function sanitizeForTerminalBounded( + value: unknown, + depth: number, + path: WeakSet +): unknown { if (value === null || value === undefined) { return value; } @@ -112,18 +150,29 @@ export function sanitizeForTerminal(value: unknown): unknown { return value; } - if (Array.isArray(value)) { - return value.map((item) => sanitizeForTerminal(item)); - } - if (typeof value === 'object') { - const sanitized: Record = {}; - for (const [key, val] of Object.entries(value)) { - // Sanitize both keys and values - const sanitizedKey = stripControlChars(key); - sanitized[sanitizedKey] = sanitizeForTerminal(val); + if (path.has(value) || depth >= MAX_SANITIZE_DEPTH) { + return '[TRUNCATED]'; } - return sanitized; + path.add(value); + + let result: unknown; + if (Array.isArray(value)) { + result = value.map((item) => sanitizeForTerminalBounded(item, depth + 1, path)); + } else { + // Null prototype so a hostile "__proto__" key lands as an ordinary + // data property instead of rewriting the accumulator's prototype. + const sanitized: Record = Object.create(null) as Record; + for (const [key, val] of Object.entries(value)) { + // Sanitize both keys and values + const sanitizedKey = stripControlChars(key); + sanitized[sanitizedKey] = sanitizeForTerminalBounded(val, depth + 1, path); + } + result = sanitized; + } + + path.delete(value); + return result; } // For other types (functions, symbols, etc.), convert to string and sanitize diff --git a/src/validation/input-sanitizer.test.ts b/src/validation/input-sanitizer.test.ts index 841be05..919b336 100644 --- a/src/validation/input-sanitizer.test.ts +++ b/src/validation/input-sanitizer.test.ts @@ -3,7 +3,8 @@ */ import { describe, it, expect } from 'vitest'; -import { InputSanitizer } from './input-sanitizer.js'; +import { InputSanitizer, DEFAULT_LIMITS } from './input-sanitizer.js'; +import { InputError } from '../utils/errors.js'; describe('InputSanitizer — isSensitiveKey', () => { const sanitizer = new InputSanitizer(); @@ -69,3 +70,112 @@ describe('InputSanitizer — redactSensitive', () => { expect(config.name).toBe('test'); }); }); + +describe('InputSanitizer — sanitize() enforcement', () => { + const sanitizer = new InputSanitizer(); + + it('rejects a serialized input larger than maxInputSize', () => { + const oversized = { value: 'x'.repeat(DEFAULT_LIMITS.maxInputSize + 1) }; + + expect(() => sanitizer.sanitize(oversized)).toThrow(InputError); + expect(() => sanitizer.sanitize(oversized)).toThrow(/Input size exceeds limit/); + }); + + it('rejects a string longer than maxStringLength', () => { + const input = { value: 'x'.repeat(DEFAULT_LIMITS.maxStringLength + 1) }; + + expect(() => sanitizer.sanitize(input)).toThrow(InputError); + expect(() => sanitizer.sanitize(input)).toThrow(/maximum length/); + }); + + it('rejects nesting deeper than maxObjectDepth', () => { + let nested: Record = { leaf: true }; + for (let i = 0; i < DEFAULT_LIMITS.maxObjectDepth + 5; i++) { + nested = { child: nested }; + } + + expect(() => sanitizer.sanitize(nested)).toThrow(InputError); + expect(() => sanitizer.sanitize(nested)).toThrow(/maximum depth/); + }); + + it('rejects an array with more than maxArrayElements items', () => { + const input = { + items: Array.from({ length: DEFAULT_LIMITS.maxArrayElements + 1 }, (_, i) => i), + }; + + expect(() => sanitizer.sanitize(input)).toThrow(InputError); + expect(() => sanitizer.sanitize(input)).toThrow(/maximum elements/); + }); + + it('rejects an object with more than maxObjectKeys keys', () => { + const input: Record = {}; + for (let i = 0; i < DEFAULT_LIMITS.maxObjectKeys + 1; i++) { + input[`key${i}`] = i; + } + + expect(() => sanitizer.sanitize(input)).toThrow(InputError); + expect(() => sanitizer.sanitize(input)).toThrow(/maximum keys/); + }); + + it('accepts input within all limits', () => { + const input = { name: 'site-1', tags: ['a', 'b'], meta: { nested: true } }; + + expect(sanitizer.sanitize(input)).toBe(input); + }); + + // SECURITY: keys with PHP query-structural brackets can canonicalize on the + // server to alias a control flag (e.g. `confirm]` -> input.confirm) past the + // exact-name strip in AbilitiesExecutor. Reject them at the input boundary. + it('rejects a key containing a "]" bracket (control-flag canonicalization)', () => { + const input = { 'confirm]': true } as Record; + + expect(() => sanitizer.sanitize(input)).toThrow(InputError); + expect(() => sanitizer.sanitize(input)).toThrow(/Invalid characters in input key/); + }); + + it('rejects a key containing a "[" bracket', () => { + const input = { 'foo[bar': 1 } as Record; + + expect(() => sanitizer.sanitize(input)).toThrow(/Invalid characters in input key/); + }); + + it('rejects a bracketed key nested inside an object', () => { + const input = { outer: { 'dry_run]': true } } as Record; + + expect(() => sanitizer.sanitize(input)).toThrow(InputError); + }); +}); + +describe('InputSanitizer — sanitizeErrorMessage', () => { + const sanitizer = new InputSanitizer(); + + it('redacts credentials embedded in a URL', () => { + const message = 'Failed to connect to https://admin:s3cr3t@dashboard.example.com/api'; + const sanitized = sanitizer.sanitizeErrorMessage(message); + + expect(sanitized).not.toContain('admin:s3cr3t'); + expect(sanitized).toContain('[URL_WITH_CREDENTIALS]'); + }); + + it('redacts Bearer tokens', () => { + const message = 'Request failed: Authorization: Bearer abc123.def456-token'; + const sanitized = sanitizer.sanitizeErrorMessage(message); + + expect(sanitized).not.toContain('abc123.def456-token'); + expect(sanitized).toContain('Bearer [REDACTED]'); + }); + + it('redacts absolute filesystem paths', () => { + const message = "ENOENT: no such file or directory, open '/Users/alice/.config/mainwpcontrol/settings.json'"; + const sanitized = sanitizer.sanitizeErrorMessage(message); + + expect(sanitized).not.toContain('/Users/alice'); + expect(sanitized).toContain('[PATH]'); + }); + + it('leaves clean messages unchanged', () => { + const message = 'Ability mainwp/list-sites-v1 returned no results'; + + expect(sanitizer.sanitizeErrorMessage(message)).toBe(message); + }); +}); diff --git a/src/validation/input-sanitizer.ts b/src/validation/input-sanitizer.ts index 0e42be6..30a2263 100644 --- a/src/validation/input-sanitizer.ts +++ b/src/validation/input-sanitizer.ts @@ -7,6 +7,8 @@ */ import { InputError } from '../utils/errors.js'; +import { isSensitiveKey as isSensitiveKeyShared, redactSensitiveKeys } from '../utils/redaction.js'; +import { sanitizeErrorMessage as sanitizeErrorMessageShared } from '../utils/error-sanitizer.js'; /** * Default limits for input sanitization @@ -35,34 +37,6 @@ export interface SanitizeOptions { maxInputSize?: number; } -/** - * Patterns that indicate sensitive data - */ -const SENSITIVE_PATTERNS = [ - /password/i, - /secret/i, - /token/i, - /api[_-]?key/i, - /auth/i, - /credential/i, - /private[_-]?key/i, - /bearer/i, - /signing[_-]?key/i, - /encryption[_-]?key/i, -]; - -/** - * Patterns for redacting file paths - */ -const PATH_PATTERNS = [ - // Absolute paths - /\/Users\/[^/\s]+/g, - /\/home\/[^/\s]+/g, - /C:\\Users\\[^\\]+/gi, - // Config directories - /\.config\/mainwpcontrol/g, -]; - /** * Input Sanitizer class */ @@ -155,6 +129,20 @@ export class InputSanitizer { ); } for (const key of keys) { + // SECURITY: reject keys containing PHP query-structural characters. + // On the GET/DELETE transport, a key like `confirm]` serializes to + // `input[confirm%5D]=…`, which PHP url-decodes and parses back to + // `input.confirm`, aliasing a control flag past the exact-name strip + // in AbilitiesExecutor.buildEffectiveParams. Plain field names never + // contain brackets, so rejecting them closes the canonicalization + // hole for control flags and every other field. + if (key.includes('[') || key.includes(']')) { + throw new InputError( + `Invalid characters in input key at "${path}.${key}"`, + { path, key }, + 'Input property names cannot contain "[" or "]" characters' + ); + } this.validateValue((value as Record)[key], depth + 1, `${path}.${key}`); } } @@ -164,73 +152,21 @@ export class InputSanitizer { * Check if a key name appears to contain sensitive data */ isSensitiveKey(key: string): boolean { - return SENSITIVE_PATTERNS.some((pattern) => pattern.test(key)); + return isSensitiveKeyShared(key); } /** * Redact sensitive values in an object (for logging/errors) */ redactSensitive(data: Record): Record { - return this.redactValue(data) as Record; - } - - /** - * Recursively redact sensitive values - */ - private redactValue(value: unknown): unknown { - if (value === null || value === undefined) { - return value; - } - - if (Array.isArray(value)) { - return value.map((item) => this.redactValue(item)); - } - - if (typeof value === 'object') { - const result: Record = {}; - for (const [key, val] of Object.entries(value)) { - if (this.isSensitiveKey(key)) { - result[key] = '[REDACTED]'; - } else { - result[key] = this.redactValue(val); - } - } - return result; - } - - return value; + return redactSensitiveKeys(data) as Record; } /** * Sanitize error message to remove sensitive paths and data */ sanitizeErrorMessage(message: string): string { - let sanitized = message; - - // Redact file paths - for (const pattern of PATH_PATTERNS) { - sanitized = sanitized.replace(pattern, '[PATH]'); - } - - // Redact URLs with credentials - sanitized = sanitized.replace( - /https?:\/\/[^:]+:[^@]+@[^\s]+/g, - '[URL_WITH_CREDENTIALS]' - ); - - // Redact base64 that might be auth headers - sanitized = sanitized.replace( - /Basic\s+[A-Za-z0-9+/]+=*/gi, - 'Basic [REDACTED]' - ); - - // Redact bearer tokens - sanitized = sanitized.replace( - /Bearer\s+[A-Za-z0-9._-]+/gi, - 'Bearer [REDACTED]' - ); - - return sanitized; + return sanitizeErrorMessageShared(message); } /** @@ -267,4 +203,3 @@ export function getInputSanitizer(): InputSanitizer { } return instance; } - diff --git a/src/validation/sanitize-schema.test.ts b/src/validation/sanitize-schema.test.ts new file mode 100644 index 0000000..b8005bd --- /dev/null +++ b/src/validation/sanitize-schema.test.ts @@ -0,0 +1,271 @@ +/** + * Tests for sanitizeInputSchema: PHP empty-array normalization, schema depth + * cap, and pattern-length cap. + */ + +import { describe, it, expect } from 'vitest'; +import { sanitizeInputSchema } from './sanitize-schema.js'; + +function nestedSchema(depth: number): Record { + let schema: Record = { type: 'string' }; + for (let i = 0; i < depth; i++) { + schema = { type: 'object', properties: { child: schema } }; + } + return schema; +} + +function descend(schema: Record, times: number): Record { + let node = schema; + for (let i = 0; i < times; i++) { + node = (node['properties'] as Record)['child'] as Record; + } + return node; +} + +describe('sanitizeInputSchema', () => { + it('passes a normal schema through unchanged', () => { + const input = { + type: 'object', + properties: { + site_id: { type: 'integer' }, + name: { type: 'string' }, + }, + required: ['site_id'], + }; + + expect(sanitizeInputSchema(input)).toEqual(input); + }); + + it('returns the object-schema default for a null schema', () => { + expect(sanitizeInputSchema(null)).toEqual({ type: 'object', properties: {} }); + }); + + it('normalizes PHP empty-array artifacts', () => { + const input = { + type: ['object', 'null'], + properties: [], + }; + + const result = sanitizeInputSchema(input); + + expect(result['type']).toBe('object'); + expect(result['properties']).toEqual({}); + }); + + it('collapses nesting past the depth cap to {}', () => { + const result = sanitizeInputSchema(nestedSchema(33)); + + expect(descend(result, 33)).toEqual({}); + }); + + it('preserves nesting within the depth cap', () => { + const result = sanitizeInputSchema(nestedSchema(5)); + + expect(descend(result, 5)).toEqual({ type: 'string' }); + }); + + it('drops every remotely supplied pattern, safe-looking or not', () => { + const input = { + type: 'object', + properties: { + short: { type: 'string', pattern: '^[a-z]+$' }, + long: { type: 'string', pattern: 'a'.repeat(1001) }, + notString: { type: 'string', pattern: 123 }, + }, + }; + + const result = sanitizeInputSchema(input); + const props = result['properties'] as Record>; + + // Guaranteed-safe policy: no remote regex is ever compiled client-side, + // so even an innocuous-looking pattern is removed. + expect('pattern' in props['short']!).toBe(false); + expect('pattern' in props['long']!).toBe(false); + expect('pattern' in props['notString']!).toBe(false); + }); + + it('drops patternProperties wholesale', () => { + const input = { + type: 'object', + patternProperties: { + '^[a-z]+$': { type: 'string' }, + '^(b+)+$': { type: 'string' }, + }, + }; + + const result = sanitizeInputSchema(input); + + expect('patternProperties' in result).toBe(false); + }); + + it('scrubs patterns reached through draft-07 dependencies (bypass regression)', () => { + const input = { + type: 'object', + dependencies: { + x: { + properties: { + y: { type: 'string', pattern: '^(a+)+$' }, + }, + }, + }, + }; + + const result = sanitizeInputSchema(input); + const dep = (result['dependencies'] as Record>)['x']!; + const y = (dep['properties'] as Record>)['y']!; + + expect('pattern' in y).toBe(false); + }); + + it('scrubs patterns nested under unknown or future keywords', () => { + const input = { + type: 'object', + someFutureKeyword: { + deeper: [{ pattern: '^(a+)+$', patternProperties: { x: {} } }], + }, + }; + + const result = sanitizeInputSchema(input); + const future = result['someFutureKeyword'] as Record; + const inner = (future['deeper'] as Record[])[0]!; + + expect('pattern' in inner).toBe(false); + expect('patternProperties' in inner).toBe(false); + }); + + it('preserves properties literally named pattern and data-carrying keys', () => { + const input = { + type: 'object', + properties: { + pattern: { type: 'string' }, + }, + required: [], + default: { pattern: '^kept$' }, + examples: [{ pattern: '^also-kept$' }], + }; + + const result = sanitizeInputSchema(input); + const props = result['properties'] as Record; + + expect(props['pattern']).toEqual({ type: 'string' }); + expect(result['required']).toEqual([]); + expect(result['default']).toEqual({ pattern: '^kept$' }); + expect(result['examples']).toEqual([{ pattern: '^also-kept$' }]); + }); + + it('applies the depth cap to nesting through unknown keywords', () => { + let schema: Record = { pattern: '^(a+)+$' }; + for (let i = 0; i < 60; i++) { + schema = { someUnknownKeyword: schema }; + } + + const result = sanitizeInputSchema({ type: 'object', extension: schema }); + + let node = result['extension'] as Record; + let depth = 0; + while (node && typeof node === 'object' && 'someUnknownKeyword' in node) { + node = node['someUnknownKeyword'] as Record; + depth++; + } + expect(depth).toBeLessThanOrEqual(33); + expect(node).toEqual({}); + }); + + it('applies the depth cap to nested arrays under unknown keywords', () => { + let value: unknown = 'leaf'; + for (let i = 0; i < 60; i++) { + value = [value]; + } + + const result = sanitizeInputSchema({ type: 'object', extension: { deepArrays: value } }); + + // The array chain is cut off at the cap ({}), never passed through whole. + let node = (result['extension'] as Record)['deepArrays']; + let depth = 0; + while (Array.isArray(node)) { + node = node[0]; + depth++; + } + expect(depth).toBeLessThanOrEqual(33); + expect(node).toEqual({}); + }); + + it.each(['const', 'enum', 'default', 'examples'])( + 'drops %s wholesale when its literal value nests past the depth cap', + (dataKey) => { + let value: unknown = { pattern: 'kept-if-shallow' }; + for (let i = 0; i < 60; i++) { + value = i % 2 === 0 ? [value] : { wrap: value }; + } + + const result = sanitizeInputSchema({ type: 'object', [dataKey]: value }); + + expect(dataKey in result).toBe(false); + } + ); + + it('drops patterns and patternProperties in nested subschemas too', () => { + const input = { + type: 'object', + properties: { + nested: { + type: 'object', + properties: { + inner: { type: 'string', pattern: '^(a+)+$' }, + }, + patternProperties: { '^x': { type: 'string' } }, + }, + }, + }; + + const result = sanitizeInputSchema(input); + const nested = (result['properties'] as Record>)['nested']!; + const inner = (nested['properties'] as Record>)['inner']!; + + expect('pattern' in inner).toBe(false); + expect('patternProperties' in nested).toBe(false); + }); + + it('sanitizes subschemas reached through contains and dependentSchemas', () => { + const input = { + type: 'object', + properties: { + list: { + type: 'array', + contains: { type: 'string', pattern: '^(a+)+$' }, + }, + }, + dependentSchemas: { + list: { properties: { extra: { type: 'string', pattern: 'b'.repeat(1001) } } }, + }, + }; + + const result = sanitizeInputSchema(input); + const props = result['properties'] as Record>; + const contains = props['list']!['contains'] as Record; + expect('pattern' in contains).toBe(false); + + const dependent = (result['dependentSchemas'] as Record>)['list']!; + const extra = (dependent['properties'] as Record>)['extra']!; + expect('pattern' in extra).toBe(false); + }); + + it('applies the depth cap to nesting through contains', () => { + let schema: Record = { type: 'string' }; + for (let i = 0; i < 40; i++) { + schema = { type: 'array', contains: schema }; + } + + const result = sanitizeInputSchema({ type: 'object', properties: { deep: schema } }); + + let node = (result['properties'] as Record>)['deep']!; + let depth = 0; + while (node && typeof node === 'object' && 'contains' in node) { + node = node['contains'] as Record>; + depth++; + } + // The chain is cut off at the cap instead of recursing all 40 levels. + expect(depth).toBeLessThanOrEqual(33); + expect(node).toEqual({}); + }); +}); diff --git a/src/validation/sanitize-schema.ts b/src/validation/sanitize-schema.ts new file mode 100644 index 0000000..fb7291e --- /dev/null +++ b/src/validation/sanitize-schema.ts @@ -0,0 +1,164 @@ +/** + * Normalize a Dashboard-served input schema into valid JSON Schema. + * + * The Dashboard is PHP, and json_encode turns empty associative arrays into + * [], so schemas arrive with inputSchema: [] or properties: []. Providers + * also require the top-level type to be exactly 'object', while the + * Dashboard emits type: ['object', 'null'] for optional input. Returns a + * new object; the input is never mutated. + * + * Remote schemas are hostile input, so two guarantees hold for the WHOLE + * tree, not just known keywords: + * - No remote regex is ever compiled client-side: `pattern` and + * `patternProperties` are deleted from every schema node. No heuristic + * reliably separates safe regexes from catastrophic ones, and a hostile + * pattern can stall the CLI in ajv before any request is sent. Dropping + * them only loosens client-side validation; the Dashboard re-validates. + * - Recursion is bounded: nesting past MAX_SCHEMA_DEPTH collapses to {}, + * with arrays counting against the budget, and data-bearing keys dropped + * wholesale when their literal value nests past the budget. + * + * Both are enforced by walking EVERY object/array value (known schema + * keywords and unknown/future ones like draft-07 `dependencies` alike), so + * the guarantees do not depend on maintaining a complete keyword list. The + * keyword lists below only control which positions get the PHP []→{} + * normalization; the scrub and the depth cap apply everywhere. Only + * data-carrying keys (const, enum, default, examples) are copied verbatim — + * their contents are literal values, not schema, and a key named "pattern" + * inside them must survive. + */ +export function sanitizeInputSchema( + inputSchema: Record | null | undefined +): Record { + if ( + inputSchema === undefined || + inputSchema === null || + Array.isArray(inputSchema) || + typeof inputSchema !== 'object' + ) { + return { type: 'object', properties: {} }; + } + const schema = sanitizeSchemaNode(inputSchema); + if (Array.isArray(schema['type']) && schema['type'].includes('object')) { + schema['type'] = 'object'; + } + return schema; +} + +/** + * Keys whose values are maps of subschemas ({ name: schema }). Their KEYS + * are property names, not schema keywords — a property literally named + * "pattern" must survive. patternProperties is absent because it is dropped + * wholesale. draft-07 `dependencies` entries may also be arrays of property + * names; those pass through the subschema walk unchanged. + */ +const SCHEMA_MAP_KEYS = ['properties', 'definitions', '$defs', 'dependentSchemas', 'dependencies']; +/** Keys whose values are a single subschema (PHP [] means empty object). */ +const SCHEMA_KEYS = [ + 'items', 'additionalItems', 'additionalProperties', 'not', 'if', 'then', 'else', + 'contains', 'propertyNames', 'unevaluatedItems', 'unevaluatedProperties', +]; +/** Keys whose values are literal data, never schema — copied verbatim. */ +const DATA_KEYS = new Set(['const', 'enum', 'default', 'examples']); + +// User input is independently depth-capped at 10 (input-sanitizer.ts), so a schema +// nested past this cap cannot meaningfully validate accepted input. Bounding recursion +// here protects against a hostile schema exhausting the stack. +const MAX_SCHEMA_DEPTH = 32; + +function sanitizeSchemaNode(node: Record, depth = 0): Record { + if (depth > MAX_SCHEMA_DEPTH) return {}; + const out: Record = { ...node }; + delete out['pattern']; + delete out['patternProperties']; + + for (const [key, value] of Object.entries(out)) { + if (DATA_KEYS.has(key)) { + // Literal data is preserved verbatim (no schema-key deletion inside + // it), but it still consumes the depth budget: an over-deep value + // drops the whole keyword instead of being partially rewritten. + if (exceedsDepth(value, depth + 1)) { + delete out[key]; + } + continue; + } + + if (SCHEMA_MAP_KEYS.includes(key)) { + if (Array.isArray(value) && value.length === 0) { + out[key] = {}; + continue; + } + if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + const map: Record = {}; + for (const [prop, sub] of Object.entries(value as Record)) { + map[prop] = sanitizeSubschema(sub, depth + 1); + } + out[key] = map; + continue; + } + } + + if ( + (SCHEMA_KEYS.includes(key) && typeof value !== 'boolean') || + Array.isArray(value) && (key === 'allOf' || key === 'anyOf' || key === 'oneOf' || key === 'prefixItems') + ) { + out[key] = sanitizeSubschema(value, depth + 1); + continue; + } + + // Generic walk for every other object/array value: unknown keywords + // cannot smuggle a regex or unbounded nesting past the sanitizer. No + // []→{} conversion here — outside known subschema positions an empty + // array (e.g. required: []) is legitimate data. + if (value !== null && typeof value === 'object') { + out[key] = sanitizeGenericValue(value, depth + 1); + } + } + + return out; +} + +function sanitizeSubschema(sub: unknown, depth: number): unknown { + if (depth > MAX_SCHEMA_DEPTH) return {}; + if (Array.isArray(sub)) { + // A subschema serialized as [] is PHP's empty object; {} accepts anything. + // Arrays count against the depth budget: nested hostile arrays would + // otherwise recurse this walker without ever hitting the cap. + return sub.length === 0 ? {} : sub.map((s) => sanitizeSubschema(s, depth + 1)); + } + if (sub !== null && typeof sub === 'object') { + return sanitizeSchemaNode(sub as Record, depth); + } + return sub; +} + +function sanitizeGenericValue(value: unknown, depth: number): unknown { + if (depth > MAX_SCHEMA_DEPTH) return {}; + if (Array.isArray(value)) { + return value.map((item) => sanitizeGenericValue(item, depth + 1)); + } + if (value !== null && typeof value === 'object') { + return sanitizeSchemaNode(value as Record, depth); + } + return value; +} + +/** + * True when `value` nests past MAX_SCHEMA_DEPTH counting from `startDepth`. + * Iterative on an explicit heap stack, so measuring an arbitrarily deep + * hostile structure cannot itself exhaust the call stack. Total node count + * is already bounded by the transport's byte cap. + */ +function exceedsDepth(value: unknown, startDepth: number): boolean { + const stack: Array<{ v: unknown; d: number }> = [{ v: value, d: startDepth }]; + while (stack.length > 0) { + const { v, d } = stack.pop()!; + if (v === null || typeof v !== 'object') continue; + if (d > MAX_SCHEMA_DEPTH) return true; + const children = Array.isArray(v) ? v : Object.values(v); + for (const child of children) { + stack.push({ v: child, d: d + 1 }); + } + } + return false; +} diff --git a/src/validation/schema-validator.test.ts b/src/validation/schema-validator.test.ts index e933b77..c72c5c3 100644 --- a/src/validation/schema-validator.test.ts +++ b/src/validation/schema-validator.test.ts @@ -3,6 +3,8 @@ */ import { describe, it, expect, beforeEach } from 'vitest'; +import { APIError } from '../utils/errors.js'; +import { ExitCode } from '../utils/exit-codes.js'; import { SchemaValidator } from './schema-validator.js'; describe('SchemaValidator', () => { @@ -97,4 +99,55 @@ describe('SchemaValidator', () => { expect(valid).toBe(true); expect(input).toEqual(original); }); + + it('accepts a whole schema serialized as a PHP empty array', () => { + const schema = [] as unknown as Record; + + const result = validator.validate({}, schema, 'mainwp/no-input-v1'); + + expect(result.valid).toBe(true); + expect(result.coerced).toEqual({}); + }); + + it('accepts properties serialized as a PHP empty array', () => { + const schema = { + type: 'object', + properties: [] as unknown as Record, + }; + + const result = validator.validate({}, schema, 'mainwp/empty-properties-v1'); + + expect(result.valid).toBe(true); + expect(result.coerced).toEqual({}); + }); + + it('accepts a nullable top-level object type array', () => { + const schema = { + type: ['object', 'null'], + properties: {}, + }; + + const result = validator.validate({}, schema, 'mainwp/nullable-input-v1'); + + expect(result.valid).toBe(true); + expect(result.coerced).toEqual({}); + }); + + it('maps an irreparably invalid ability schema to a typed API error', () => { + const schema = { type: 42 }; + let thrown: unknown; + + try { + validator.validate({}, schema, 'mainwp/broken-schema-v1'); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(APIError); + expect(thrown).toMatchObject({ + code: 'ABILITY_SCHEMA_INVALID', + exitCode: ExitCode.API_ERROR, + }); + expect((thrown as Error).message).toContain('mainwp/broken-schema-v1'); + }); }); diff --git a/src/validation/schema-validator.ts b/src/validation/schema-validator.ts index 757632c..4a0927b 100644 --- a/src/validation/schema-validator.ts +++ b/src/validation/schema-validator.ts @@ -6,7 +6,8 @@ */ import AjvModule, { type ErrorObject, type ValidateFunction } from 'ajv'; -import { SchemaValidationError } from '../utils/errors.js'; +import { APIError, SchemaValidationError } from '../utils/errors.js'; +import { sanitizeInputSchema } from './sanitize-schema.js'; // Handle ESM default export const Ajv = AjvModule.default ?? AjvModule; @@ -131,8 +132,22 @@ export class SchemaValidator { } } - // Compile schema - const compiled = this.ajv.compile(schema); + // Compile the normalized Dashboard schema. If AJV still rejects it, the + // server supplied an ability schema this client cannot safely repair. + let compiled: ValidateFunction; + try { + compiled = this.ajv.compile(sanitizeInputSchema(schema)); + } catch (error) { + const schemaName = schemaId ? `"${schemaId}"` : '(unnamed)'; + const cause = error instanceof Error ? error.message : String(error); + throw new APIError( + 'ABILITY_SCHEMA_INVALID', + `Input schema for ability ${schemaName} is invalid`, + undefined, + { cause }, + 'The Dashboard served an input schema that could not be compiled' + ); + } // Cache if ID provided if (schemaId) { diff --git a/tests/acceptance/agent-run.ts b/tests/acceptance/agent-run.ts new file mode 100644 index 0000000..c267a00 --- /dev/null +++ b/tests/acceptance/agent-run.ts @@ -0,0 +1,1476 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import type { ServerResponse } from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ConfigDir } from '../../src/__tests__/process/fixtures/config-dir.js'; +import { + MockServer, + type RecordedRequest, +} from '../../src/__tests__/process/fixtures/mock-server.js'; +import { + FIXTURE_ABILITIES, + FIXTURE_APP_PASSWORD, + FIXTURE_SITES, + FIXTURE_USERNAME, +} from './fixtures.js'; +import { createArtifacts, type Artifacts } from './lib/artifacts.js'; +import { appendBounded, CommandRunner } from './lib/commands.js'; +import { + resolveAcceptanceCredentials, + type AcceptanceCredentials, +} from './lib/env.js'; +import { getWriteGuardReason, isWriteHostAllowed } from './lib/guards.js'; +import { packAndInstall, type PackedPackage } from './lib/pack.js'; +import { Redactor } from './lib/redact.js'; +import { IndependentVerifier } from './lib/verify.js'; + +type AgentMode = 'packed' | 'source'; +type AgentStatus = 'passed' | 'failed' | 'skipped' | 'unverified'; + +interface AgentRunnerOptions { + mode: AgentMode; + scenarioIds: string[]; + writes: boolean; + list: boolean; + keepConsumer: boolean; + help: boolean; +} + +interface AgentGroundTruth { + count?: number; + siteId?: number; + siteUrl?: string; + siteName?: string; + pluginActive?: boolean; + pluginName?: string; + pluginSlug?: string; + updateSiteUrls?: string[]; + beforeSiteCount?: number; + targetSiteId?: number; + targetSiteUrl?: string; + targetSiteName?: string; +} + +interface AgentScenario { + id: string; + kind: 'read' | 'write'; + target: 'live' | 'fixture'; + task(groundTruth: AgentGroundTruth): string; + expectedAbilities: string[]; + groundTruth(verifier: IndependentVerifier): Promise; + evaluate?: ( + truth: AgentGroundTruth, + collected: CollectedAgentOutput, + verifier: IndependentVerifier, + ) => Promise<{ evaluation: AgentEvaluation; reason?: string }>; +} + +interface EvaluationField { + pass: boolean; + evidence: unknown; +} + +interface AgentEvaluation { + understoodRequest: EvaluationField; + rightCapability: EvaluationField; + rightArguments: EvaluationField; + correctCliResult: EvaluationField; + stateChange: EvaluationField; + faithfulFinalAnswer: EvaluationField; +} + +interface AgentTiming { + process_wall_clock_ms: number | null; + duration_ms: number | null; + duration_api_ms: number | null; + ttft_ms: number | null; + invocation_count: number; +} + +interface AgentResult { + id: string; + status: AgentStatus; + target: 'live' | 'fixture'; + model?: string; + invocations: RecordedCliInvocation[]; + finalText: string; + timing: AgentTiming; + groundTruth?: AgentGroundTruth; + evaluation?: AgentEvaluation; + reason?: string; +} + +interface AgentResultDocument { + runId: string; + mode: AgentMode; + target: 'live'; + totals: Record; + scenarios: AgentResult[]; + artifactAudit: { + passed: boolean; + message: string; + }; + harnessError: string | null; +} + +interface RecordedCliInvocation { + toolUseId?: string; + command: string; + argv: string[]; + parseError?: string; +} + +interface RecordedToolResult { + toolUseId?: string; + content: unknown; + isError?: boolean; +} + +interface CollectedAgentOutput { + invocations: RecordedCliInvocation[]; + toolResults: RecordedToolResult[]; + finalText: string; + model?: string; + durationMs?: number; + durationApiMs?: number; + ttftMs?: number; +} + +interface PreparedCli { + binDir: string; + cwd: string; + packedPackage: PackedPackage | null; + cleanup(): void; +} + +const repoRoot = path.resolve(fileURLToPath(new URL('../..', import.meta.url))); + +// The delete scenario runs against the local fixture Dashboard. A live victim +// is not repeatable: MainWP Child locks to a dashboard key on first connect, +// so every add->delete cycle would leave the child rejecting the next add. +// The mcp reference harness scopes its agent delete scenario the same way. +type AgentFixtureSite = Omit<(typeof FIXTURE_SITES)[number], 'plugins'>; + +function jsonResponse(response: ServerResponse, status: number, body: unknown): void { + const encoded = JSON.stringify(body); + response.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': Buffer.byteLength(encoded), + }); + response.end(encoded); +} + +function fixtureRequestInput(request: RecordedRequest): Record { + if (request.method === 'GET' || request.method === 'DELETE') { + const input: Record = {}; + for (const [key, value] of Object.entries(request.query)) { + const match = key.match(/^input\[([^\]]+)\]$/); + if (!match?.[1]) continue; + input[match[1]] = /^-?\d+$/.test(value) ? Number(value) : value; + } + return input; + } + const body = request.body as Record | undefined; + const input = body?.['input']; + return input && typeof input === 'object' && !Array.isArray(input) + ? input as Record + : {}; +} + +function programAgentFixture(server: MockServer): void { + const sites: AgentFixtureSite[] = FIXTURE_SITES.map( + ({ plugins: _plugins, ...site }) => ({ ...site }), + ); + const findSite = (identifier: unknown): AgentFixtureSite | undefined => { + const normalized = String(identifier ?? '').replace(/\/+$/, '').toLowerCase(); + return sites.find(site => { + const url = site.url.replace(/\/+$/, '').toLowerCase(); + return String(site.id) === normalized + || url === normalized + || new URL(url).hostname === normalized; + }); + }; + + server.reset(); + server.setCredentials(FIXTURE_USERNAME, FIXTURE_APP_PASSWORD); + server.setAbilities(FIXTURE_ABILITIES); + server.addRoute( + 'GET', + '/wp-json/wp-abilities/v1/abilities/mainwp/list-sites-v1/run', + (request, response) => { + const input = fixtureRequestInput(request); + const page = typeof input['page'] === 'number' ? input['page'] : 1; + const perPage = typeof input['per_page'] === 'number' ? input['per_page'] : 20; + const start = (page - 1) * perPage; + jsonResponse(response, 200, { + items: sites.slice(start, start + perPage), + page, + per_page: perPage, + total: sites.length, + }); + }, + ); + server.addRoute( + 'GET', + '/wp-json/wp-abilities/v1/abilities/mainwp/count-sites-v1/run', + (_request, response) => jsonResponse(response, 200, { total: sites.length }), + ); + server.addRoute( + 'GET', + '/wp-json/wp-abilities/v1/abilities/mainwp/get-site-v1/run', + (request, response) => { + const site = findSite(fixtureRequestInput(request)['site_id_or_domain']); + if (!site) { + jsonResponse(response, 404, { + code: 'mainwp_site_not_found', + message: 'The requested synthetic site was not found.', + data: { status: 404 }, + }); + return; + } + jsonResponse(response, 200, site); + }, + ); + server.addRoute( + 'POST', + '/wp-json/wp-abilities/v1/abilities/mainwp/delete-site-v1/run', + (request, response) => { + const input = fixtureRequestInput(request); + const site = findSite(input['site_id_or_domain']); + if (!site) { + jsonResponse(response, 404, { + code: 'mainwp_site_not_found', + message: 'The requested synthetic site was not found.', + data: { status: 404 }, + }); + return; + } + if (input['dry_run'] === true) { + jsonResponse(response, 200, { + dry_run: true, + would_affect: { id: site.id, url: site.url, name: site.name }, + warnings: ['This synthetic site record will be permanently deleted.'], + }); + return; + } + sites.splice(sites.findIndex(candidate => candidate.id === site.id), 1); + jsonResponse(response, 200, { + dry_run: false, + deleted: true, + site: { id: site.id, url: site.url, name: site.name }, + would_affect: {}, + warnings: [], + }); + }, + ); +} + +const agentScenarios: AgentScenario[] = [ + { + id: 'agent-count-sites', + kind: 'read', + target: 'live', + task: () => 'How many sites are currently connected to my MainWP dashboard?', + expectedAbilities: ['mainwp/count-sites-v1', 'mainwp/list-sites-v1'], + groundTruth: async verifier => ({ count: await verifier.countSites() }), + }, + { + id: 'agent-updates', + kind: 'read', + target: 'live', + task: () => 'Which of my sites need plugin updates?', + expectedAbilities: [ + 'mainwp/list-updates-v1', + 'mainwp/list-sites-v1', + 'mainwp/get-site-plugins-v1', + ], + groundTruth: async verifier => { + const updateSiteUrls: string[] = []; + for (const site of await verifier.listSites()) { + const plugins = await verifier.getSitePlugins(site.id); + if (plugins.plugins.some(plugin => Boolean(plugin.update_version))) { + updateSiteUrls.push(site.url); + } + } + return { updateSiteUrls: updateSiteUrls.sort() }; + }, + }, + { + id: 'agent-plugin-active', + kind: 'read', + target: 'live', + task: truth => + `Is the ${truth.pluginName} plugin active on ${truth.siteUrl}? Answer yes or no with the site name.`, + expectedAbilities: [ + 'mainwp/get-site-plugins-v1', + 'mainwp/get-site-v1', + 'mainwp/list-sites-v1', + ], + groundTruth: async verifier => { + const preferred = process.env['MAINWP_CONTROL_ACCEPTANCE_TOGGLE_PLUGIN']; + const sites = await verifier.listSites(); + if (sites.length === 0) throw new Error('No site is available for agent-plugin-active'); + for (const site of sites) { + const plugins = (await verifier.getSitePlugins(site.id)).plugins; + const plugin = preferred + ? plugins.find(candidate => candidate.slug === preferred) + : plugins[0]; + if (plugin?.name) { + return { + siteId: site.id, + siteUrl: site.url, + siteName: site.name, + pluginActive: plugin.active, + pluginName: plugin.name, + pluginSlug: plugin.slug, + }; + } + } + throw new Error('No discoverable plugin was found for agent-plugin-active'); + }, + }, + { + id: 'agent-confirm-delete-site', + kind: 'write', + target: 'fixture', + task: truth => + `Delete the MainWP site named ${truth.targetSiteName} at ${truth.targetSiteUrl} (site ID ${truth.targetSiteId}). This deletion is explicitly authorized. Preview the deletion first. If the preview matches this site, execute it with explicit confirmation and no interactive prompt, then report the outcome.`, + expectedAbilities: ['mainwp/delete-site-v1'], + groundTruth: async verifier => { + const sites = await verifier.listSites(); + const target = sites[0]; + if (!target) throw new Error('No fixture site was available for the delete scenario'); + return { + beforeSiteCount: sites.length, + targetSiteId: target.id, + targetSiteUrl: target.url, + targetSiteName: target.name, + }; + }, + evaluate: evaluateDeleteScenario, + }, +]; + +function requiredValue(argv: string[], index: number, flag: string): string { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${flag} requires a value`); + return value; +} + +function parseArgs(argv: string[]): AgentRunnerOptions { + const options: AgentRunnerOptions = { + mode: 'packed', + scenarioIds: [], + writes: false, + list: false, + keepConsumer: false, + help: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--mode') { + const value = requiredValue(argv, index, arg); + if (value !== 'packed' && value !== 'source') { + throw new Error(`Invalid --mode value: ${value}`); + } + options.mode = value; + index += 1; + } else if (arg === '--scenario') { + options.scenarioIds.push(requiredValue(argv, index, arg)); + index += 1; + } else if (arg === '--writes') { + options.writes = true; + } else if (arg === '--list') { + options.list = true; + } else if (arg === '--keep-consumer') { + options.keepConsumer = true; + } else if (arg === '--help' || arg === '-h') { + options.help = true; + } else { + throw new Error(`Unknown acceptance flag: ${arg}`); + } + } + return options; +} + +function printHelp(): void { + console.log(`Usage: tsx tests/acceptance/agent-run.ts [options] + +Options: + --mode packed|source Run the packed install (default) or repo binary + --scenario Run one scenario; repeat to select multiple + --writes Allow live write scenarios (the delete scenario + uses the local fixture Dashboard and always runs) + --list List registered agent scenarios + --keep-consumer Preserve the packed consumer directory + --help Show this help`); +} + +function selectedScenarios(ids: string[]): AgentScenario[] { + if (ids.length === 0) return agentScenarios; + const byId = new Map(agentScenarios.map(scenario => [scenario.id, scenario])); + const unknown = ids.filter(id => !byId.has(id)); + if (unknown.length > 0) throw new Error(`Unknown agent scenario IDs: ${unknown.join(', ')}`); + return ids.map(id => byId.get(id)!); +} + +function shellDisplay(argv: string[]): string { + return argv.map(value => (/[\s*]/.test(value) ? JSON.stringify(value) : value)).join(' '); +} + +function parseShellArgv(command: string): string[] { + const argv: string[] = []; + let current = ''; + let quote: "'" | '"' | null = null; + let escaped = false; + const pushCurrent = (): void => { + if (current.length > 0) { + argv.push(current); + current = ''; + } + }; + + for (const character of command) { + if (escaped) { + current += character; + escaped = false; + continue; + } + if (character === '\\' && quote !== "'") { + escaped = true; + continue; + } + if (quote) { + if (character === quote) quote = null; + else current += character; + continue; + } + if (character === "'" || character === '"') { + quote = character; + continue; + } + if (/\s/.test(character)) { + pushCurrent(); + continue; + } + if (character === ';' || character === '|' || character === '&') { + pushCurrent(); + break; + } + current += character; + } + if (escaped || quote) throw new Error('Bash command contains an unterminated escape or quote'); + pushCurrent(); + return argv; +} + +function contentBlocks(event: unknown): unknown[] { + if (!event || typeof event !== 'object') return []; + const message = (event as Record)['message']; + if (!message || typeof message !== 'object') return []; + const content = (message as Record)['content']; + return Array.isArray(content) ? content : []; +} + +function collectEvent( + event: unknown, + accumulator: CollectedAgentOutput, + elapsedMs: number, +): void { + if (!event || typeof event !== 'object') return; + const record = event as Record; + if (record['type'] === 'assistant' && accumulator.ttftMs === undefined) { + accumulator.ttftMs = elapsedMs; + } + if (typeof record['model'] === 'string') accumulator.model = record['model']; + const message = record['message']; + if (message && typeof message === 'object') { + const model = (message as Record)['model']; + if (typeof model === 'string') accumulator.model = model; + } + + for (const block of contentBlocks(event)) { + if (!block || typeof block !== 'object') continue; + const content = block as Record; + if (content['type'] === 'tool_use' && content['name'] === 'Bash') { + const input = content['input']; + const command = input && typeof input === 'object' + ? (input as Record)['command'] + : undefined; + if (typeof command === 'string' && command.startsWith('mainwpcontrol')) { + try { + accumulator.invocations.push({ + ...(typeof content['id'] === 'string' ? { toolUseId: content['id'] } : {}), + command, + argv: parseShellArgv(command), + }); + } catch (error) { + accumulator.invocations.push({ + ...(typeof content['id'] === 'string' ? { toolUseId: content['id'] } : {}), + command, + argv: [], + parseError: error instanceof Error ? error.message : String(error), + }); + } + } + } else if (content['type'] === 'tool_result') { + accumulator.toolResults.push({ + ...(typeof content['tool_use_id'] === 'string' + ? { toolUseId: content['tool_use_id'] } + : {}), + content: content['content'], + ...((content['is_error'] === true || content['isError'] === true) + ? { isError: true } + : {}), + }); + } else if (content['type'] === 'text' && typeof content['text'] === 'string') { + accumulator.finalText = content['text']; + } + } + + if (record['type'] === 'result') { + if (typeof record['result'] === 'string') accumulator.finalText = record['result']; + if (typeof record['duration_ms'] === 'number') accumulator.durationMs = record['duration_ms']; + if (typeof record['duration_api_ms'] === 'number') { + accumulator.durationApiMs = record['duration_api_ms']; + } + } +} + +function invocationAbility(invocation: RecordedCliInvocation): string | null { + const [binary, topic, command, ability] = invocation.argv; + if ( + binary !== 'mainwpcontrol' + || topic !== 'abilities' + || command !== 'run' + || !ability + ) { + return null; + } + return ability.includes('/') ? ability : `mainwp/${ability}`; +} + +function hasFlag(invocation: RecordedCliInvocation, flag: string): boolean { + return invocation.argv.includes(flag); +} + +function invocationInput(invocation: RecordedCliInvocation): Record | null { + const index = invocation.argv.findIndex(value => value === '--input' || value === '-i'); + if (index === -1) return {}; + const value = invocation.argv[index + 1]; + if (!value) return null; + try { + const parsed = JSON.parse(value) as unknown; + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? parsed as Record + : null; + } catch { + return null; + } +} + +function targetMatches(input: Record | null, truth: AgentGroundTruth): boolean { + if (!input) return false; + const target = input['site_id_or_domain'] ?? input['site_id']; + return ( + (truth.siteId !== undefined && String(target) === String(truth.siteId)) + || (truth.siteUrl !== undefined && String(target) === truth.siteUrl) + || (truth.targetSiteId !== undefined && String(target) === String(truth.targetSiteId)) + || (truth.targetSiteUrl !== undefined && String(target) === truth.targetSiteUrl) + ); +} + +function flattenStrings(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(flattenStrings); + if (value && typeof value === 'object') { + return Object.values(value as Record).flatMap(flattenStrings); + } + return []; +} + +function resultsForInvocations( + invocations: RecordedCliInvocation[], + toolResults: RecordedToolResult[], +): RecordedToolResult[] { + const ids = new Set( + invocations + .map(invocation => invocation.toolUseId) + .filter((value): value is string => Boolean(value)), + ); + return toolResults.filter(result => Boolean(result.toolUseId && ids.has(result.toolUseId))); +} + +function cliResultsMatchTruth( + truth: AgentGroundTruth, + toolResults: RecordedToolResult[], +): boolean { + const text = `${flattenStrings(toolResults).join('\n')}\n${JSON.stringify(toolResults)}`; + if (truth.count !== undefined) { + return ( + new RegExp(`"total"\\s*:\\s*${truth.count}(?:\\D|$)`).test(text) + || new RegExp(`(?:total|sites?(?: connected)?)\\D{0,20}${truth.count}\\b`, 'i').test(text) + ); + } + if (truth.updateSiteUrls) { + if (truth.updateSiteUrls.length === 0) return /\b(no|none|zero|0)\b/i.test(text); + return truth.updateSiteUrls.every(url => text.includes(url)); + } + if (truth.pluginActive !== undefined && truth.pluginSlug) { + // The agent may filter CLI output through shell pipes, so the captured + // result can carry the plugin's name without its slug; either field tied + // to the expected active value counts as grounded evidence. + const escapeRegExp = (value: string): string => + value.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&'); + const fields = [ + { key: 'slug', value: truth.pluginSlug }, + ...(truth.pluginName ? [{ key: 'name', value: truth.pluginName }] : []), + ]; + const structured = fields.some(({ key, value }) => { + const escaped = escapeRegExp(value); + return ( + new RegExp( + `"${key}"\\s*:\\s*"${escaped}"[^}]*"active"\\s*:\\s*${truth.pluginActive}`, + ).test(text) + || new RegExp( + `"active"\\s*:\\s*${truth.pluginActive}[^}]*"${key}"\\s*:\\s*"${escaped}"`, + ).test(text) + ); + }); + const status = truth.pluginActive ? /\bactive\b/i : /\binactive\b/i; + return structured || (text.includes(truth.pluginSlug) && status.test(text)); + } + return false; +} + +function finalAnswerMatches(truth: AgentGroundTruth, text: string): boolean { + if (truth.count !== undefined) { + return [...text.matchAll(/\b\d+\b/g)].some(match => Number(match[0]) === truth.count); + } + if (truth.updateSiteUrls) { + if (truth.updateSiteUrls.length === 0) return /\b(no|none|zero|0)\b/i.test(text); + const hostnameOf = (url: string): string => { + try { + return new URL(url).hostname; + } catch { + return url; + } + }; + const lower = text.toLowerCase(); + return truth.updateSiteUrls.every(url => lower.includes(hostnameOf(url).toLowerCase())); + } + if (truth.pluginActive !== undefined) { + const answer = text.match(/\b(yes|no)\b/i)?.[1]?.toLowerCase(); + return ( + answer === (truth.pluginActive ? 'yes' : 'no') + && Boolean(truth.siteName && text.toLowerCase().includes(truth.siteName.toLowerCase())) + ); + } + return false; +} + +function evaluateReadScenario( + scenario: AgentScenario, + truth: AgentGroundTruth, + collected: CollectedAgentOutput, +): AgentEvaluation { + const appropriate = collected.invocations.filter(invocation => { + const ability = invocationAbility(invocation); + return ability !== null && scenario.expectedAbilities.includes(ability); + }); + const relevantResults = resultsForInvocations(appropriate, collected.toolResults); + const rightArguments = appropriate.every(invocation => invocationInput(invocation) !== null); + const hasTargetArgument = ( + truth.siteId === undefined + || appropriate.some(invocation => targetMatches(invocationInput(invocation), truth)) + ); + // A CLI agent discovers input schemas by trying, so failed intermediate + // attempts are legitimate; grade the non-error results and the end state. + const successfulResults = relevantResults.filter(result => !result.isError); + const lastResult = relevantResults[relevantResults.length - 1]; + const resultsMatch = cliResultsMatchTruth(truth, successfulResults); + return { + understoodRequest: { + pass: collected.finalText.trim().length > 0, + evidence: collected.finalText, + }, + rightCapability: { + pass: appropriate.length > 0, + evidence: collected.invocations.map(invocation => invocation.argv), + }, + rightArguments: { + pass: appropriate.length > 0 && rightArguments && hasTargetArgument, + evidence: appropriate.map(invocation => invocationInput(invocation)), + }, + correctCliResult: { + pass: successfulResults.length > 0 && resultsMatch && lastResult?.isError !== true, + evidence: { + resultCount: relevantResults.length, + errorCount: relevantResults.length - successfulResults.length, + groundTruthMatched: resultsMatch, + endedWithError: lastResult?.isError === true, + }, + }, + stateChange: { + pass: true, + evidence: 'Not applicable. This agent scenario is read-only.', + }, + faithfulFinalAnswer: { + pass: finalAnswerMatches(truth, collected.finalText), + evidence: { truth, finalText: collected.finalText }, + }, + }; +} + +async function evaluateDeleteScenario( + truth: AgentGroundTruth, + collected: CollectedAgentOutput, + verifier: IndependentVerifier, +): Promise<{ evaluation: AgentEvaluation; reason?: string }> { + if ( + truth.beforeSiteCount === undefined + || truth.targetSiteId === undefined + || !truth.targetSiteName + ) { + throw new Error('Delete scenario ground truth was incomplete'); + } + const deleteInvocations = collected.invocations + .map((invocation, index) => ({ invocation, index })) + .filter(({ invocation }) => invocationAbility(invocation) === 'mainwp/delete-site-v1'); + const targeting = deleteInvocations.filter(({ invocation }) => + targetMatches(invocationInput(invocation), truth)); + const preview = targeting.find(({ invocation }) => hasFlag(invocation, '--dry-run')); + const confirmed = targeting.find(({ invocation, index }) => + Boolean( + preview + && index > preview.index + && hasFlag(invocation, '--confirm') + && hasFlag(invocation, '--force'), + )); + const confirmedResults = confirmed + ? resultsForInvocations([confirmed.invocation], collected.toolResults) + : []; + const after = await verifier.listSites(); + const targetStillPresent = after.some(site => site.id === truth.targetSiteId); + const finalText = collected.finalText.toLowerCase(); + const transcriptPass = Boolean(preview && confirmed); + const evaluation: AgentEvaluation = { + understoodRequest: { + pass: collected.finalText.trim().length > 0, + evidence: collected.finalText, + }, + rightCapability: { + pass: targeting.length >= 2, + evidence: deleteInvocations.map(({ invocation }) => invocation.argv), + }, + rightArguments: { + pass: transcriptPass, + evidence: { + targetSiteId: truth.targetSiteId, + previewIndex: preview?.index, + confirmedIndex: confirmed?.index, + }, + }, + correctCliResult: { + pass: confirmedResults.length > 0 && confirmedResults.every(result => !result.isError), + evidence: confirmedResults, + }, + stateChange: { + pass: after.length === truth.beforeSiteCount - 1 && !targetStillPresent, + evidence: { + beforeCount: truth.beforeSiteCount, + afterCount: after.length, + targetStillPresent, + }, + }, + faithfulFinalAnswer: { + pass: ( + finalText.includes(truth.targetSiteName.toLowerCase()) + && /\b(deleted|removed)\b/.test(finalText) + ), + evidence: collected.finalText, + }, + }; + return { + evaluation, + ...(!preview + ? { reason: 'No target-matching delete invocation used --dry-run.' } + : !confirmed + ? { reason: 'No later target-matching delete invocation used --confirm --force.' } + : {}), + }; +} + +/** + * Wall-clock cap per agent invocation. Agent turns legitimately take + * minutes; a hung claude process must not hang the harness forever. + */ +const CLAUDE_TIMEOUT_MS = 15 * 60_000; + +async function runClaude( + argv: string[], + cwd: string, + env: NodeJS.ProcessEnv, + onLine: (line: string, elapsedMs: number) => void, +): Promise<{ exitCode: number; stdout: string; stderr: string; durationMs: number }> { + const started = performance.now(); + const executable = argv[0]; + if (!executable) throw new Error('Claude argv did not contain an executable'); + const child = spawn(executable, argv.slice(1), { + cwd, + env, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let pending = ''; + child.stdout.on('data', chunk => { + const buffer = Buffer.from(chunk); + appendBounded(stdoutChunks, buffer); + pending += buffer.toString('utf8'); + const lines = pending.split(/\r?\n/); + pending = lines.pop() ?? ''; + for (const line of lines) { + onLine(line, Math.round(performance.now() - started)); + } + }); + child.stderr.on('data', chunk => appendBounded(stderrChunks, Buffer.from(chunk))); + let timedOut = false; + const exitCode = await new Promise((resolve, reject) => { + let forceKill: NodeJS.Timeout | undefined; + const timeout = setTimeout(() => { + timedOut = true; + child.kill('SIGTERM'); + forceKill = setTimeout(() => child.kill('SIGKILL'), 5_000); + }, CLAUDE_TIMEOUT_MS); + child.once('error', error => { + clearTimeout(timeout); + if (forceKill) clearTimeout(forceKill); + reject(error); + }); + child.once('close', code => { + clearTimeout(timeout); + if (forceKill) clearTimeout(forceKill); + resolve(code ?? 1); + }); + }); + if (pending.length > 0) onLine(pending, Math.round(performance.now() - started)); + if (timedOut) { + appendBounded( + stderrChunks, + Buffer.from(`\nHarness: claude run exceeded ${CLAUDE_TIMEOUT_MS}ms and was terminated.\n`), + ); + } + return { + exitCode: timedOut && exitCode === 0 ? 1 : exitCode, + stdout: Buffer.concat(stdoutChunks).toString('utf8'), + stderr: Buffer.concat(stderrChunks).toString('utf8'), + durationMs: Math.round(performance.now() - started), + }; +} + +function emptyTiming(): AgentTiming { + return { + process_wall_clock_ms: null, + duration_ms: null, + duration_api_ms: null, + ttft_ms: null, + invocation_count: 0, + }; +} + +function timingFrom( + collected: CollectedAgentOutput, + processWallClockMs: number, +): AgentTiming { + return { + process_wall_clock_ms: processWallClockMs, + duration_ms: collected.durationMs ?? null, + duration_api_ms: collected.durationApiMs ?? null, + ttft_ms: collected.ttftMs ?? null, + invocation_count: collected.invocations.length, + }; +} + +function totals(results: AgentResult[]): Record { + return { + passed: results.filter(result => result.status === 'passed').length, + failed: results.filter(result => result.status === 'failed').length, + skipped: results.filter(result => result.status === 'skipped').length, + unverified: results.filter(result => result.status === 'unverified').length, + }; +} + +// Unverified scenarios (independent verification could not confirm the +// outcome) must fail the run the same as `failed` ones, or CI reads a run +// with unresolved evidence as green. Skipped scenarios must NOT affect this: +// the live baseline legitimately skips guarded write scenarios. +function computeExitCode( + counts: Record, + artifactAudit: AgentResultDocument['artifactAudit'], + hasHarnessError: boolean, +): number { + return ( + hasHarnessError || counts.failed > 0 || counts.unverified > 0 || !artifactAudit.passed + ) ? 1 : 0; +} + +function resultDocument( + artifacts: Artifacts, + options: AgentRunnerOptions, + results: AgentResult[], + artifactAudit: AgentResultDocument['artifactAudit'], + harnessError: string | null, +): AgentResultDocument { + return { + runId: artifacts.runId, + mode: options.mode, + target: 'live', + totals: totals(results), + scenarios: results, + artifactAudit, + harnessError, + }; +} + +function summaryMarkdown(document: AgentResultDocument): string { + const timed = document.scenarios.filter( + result => result.timing.process_wall_clock_ms !== null, + ); + const slowest = [...timed].sort( + (left, right) => + (right.timing.process_wall_clock_ms ?? 0) + - (left.timing.process_wall_clock_ms ?? 0), + )[0]; + const lines = [ + '# MainWP Control agent acceptance results', + '', + `- Run: ${document.runId}`, + `- Mode: ${document.mode}`, + `- Target: live${(() => { + const fixtureIds = document.scenarios + .filter(result => result.target === 'fixture') + .map(result => result.id); + return fixtureIds.length > 0 ? ` (fixture Dashboard: ${fixtureIds.join(', ')})` : ''; + })()}`, + `- Passed: ${document.totals.passed}`, + `- Failed: ${document.totals.failed}`, + `- Skipped: ${document.totals.skipped}`, + `- Unverified: ${document.totals.unverified}`, + `- Artifact audit: ${document.artifactAudit.passed ? 'passed' : 'failed'} - ${document.artifactAudit.message}`, + `- Exit code: ${computeExitCode(document.totals, document.artifactAudit, document.harnessError !== null)}${document.totals.failed === 0 && document.totals.unverified > 0 ? ' (unverified scenarios present)' : ''}`, + ...(document.harnessError ? [`- Harness error: ${document.harnessError}`] : []), + ...(slowest + ? [ + `- Slowest scenario: ${slowest.id} (${slowest.timing.process_wall_clock_ms} ms wall clock)`, + ] + : ['- Slowest scenario: unavailable (no Claude process completed)']), + '', + '| Scenario | Status | Wall (ms) | Claude duration (ms) | API duration (ms) | TTFT (ms) | CLI calls |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: |', + ...document.scenarios.map(result => { + const timing = result.timing; + return `| ${result.id} | ${result.status} | ${timing.process_wall_clock_ms ?? '-'} | ${timing.duration_ms ?? '-'} | ${timing.duration_api_ms ?? '-'} | ${timing.ttft_ms ?? '-'} | ${timing.invocation_count} |`; + }), + '', + ...document.scenarios + .filter(result => result.reason) + .map(result => `- ${result.id}: ${result.reason}`), + '', + ]; + return `${lines.join('\n')}\n`; +} + +function registerAuditValues( + values: Set, + credentials: AcceptanceCredentials, +): void { + values.add(credentials.username); + values.add(credentials.appPassword); + values.add(credentials.appPassword.replace(/\s+/g, '')); + values.add(new URL(credentials.dashboardUrl).origin); + values.add( + `Basic ${Buffer.from(`${credentials.username}:${credentials.appPassword}`).toString('base64')}`, + ); +} + +function auditArtifacts(runDir: string, auditValues: Set): string[] { + const findings: string[] = []; + const values = [...auditValues].filter(Boolean); + const visit = (directory: string): void => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(fullPath); + continue; + } + const content = fs.readFileSync(fullPath, 'utf8'); + if (values.some(value => content.includes(value))) { + findings.push(path.relative(runDir, fullPath)); + } + } + }; + visit(runDir); + return findings; +} + +async function prepareCli( + options: AgentRunnerOptions, + runner: CommandRunner, + artifacts: Artifacts, +): Promise { + if (options.mode === 'packed') { + const packedPackage = await packAndInstall( + repoRoot, + runner, + artifacts, + options.keepConsumer, + ); + return { + binDir: path.dirname(packedPackage.binPath), + cwd: packedPackage.consumerDir, + packedPackage, + cleanup: () => packedPackage.cleanup(), + }; + } + + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mainwp-control-agent-source-')); + const binDir = path.join(tempRoot, 'bin'); + fs.mkdirSync(binDir); + fs.symlinkSync(path.join(repoRoot, 'bin', 'run.js'), path.join(binDir, 'mainwpcontrol')); + return { + binDir, + cwd: repoRoot, + packedPackage: null, + cleanup: () => { + if (!options.keepConsumer) fs.rmSync(tempRoot, { recursive: true, force: true }); + }, + }; +} + +/** + * Guard shim installed ahead of the real binary on the agent's PATH. The + * agent's allowedTools only permit commands starting with "mainwpcontrol", + * so every invocation resolves to this shim, which enforces the scenario's + * ability allowlist and confirm policy before delegating. Defense in depth + * for live scenarios: grading after the fact does not stop a stray + * destructive call against the testbed. + */ +const GUARD_SHIM_SOURCE = `#!/usr/bin/env node +const { spawnSync } = require('node:child_process'); +const realBin = process.env.MAINWP_ACCEPTANCE_REAL_BIN; +if (!realBin) { + console.error('Acceptance guard: MAINWP_ACCEPTANCE_REAL_BIN is not set.'); + process.exit(3); +} +const allowed = (process.env.MAINWP_ACCEPTANCE_ALLOWED_ABILITIES || '') + .split(',').map((s) => s.trim()).filter(Boolean); +const allowConfirm = process.env.MAINWP_ACCEPTANCE_ALLOW_CONFIRM === '1'; +const argv = process.argv.slice(2); +if (!allowConfirm && argv.some((a) => a === '--confirm' || a === '--force')) { + console.error('Acceptance guard: --confirm/--force are not authorized for this scenario.'); + process.exit(3); +} +const isRun = (argv[0] === 'abilities' && argv[1] === 'run') || argv[0] === 'abilities:run'; +if (isRun) { + const abilityName = argv[0] === 'abilities:run' ? argv[1] : argv[2]; + const ok = typeof abilityName === 'string' && allowed.some( + (full) => full === abilityName || full.endsWith('/' + abilityName) + ); + if (!ok) { + console.error( + 'Acceptance guard: ability "' + (abilityName || '') + '" is not in this scenario allowlist.' + ); + process.exit(3); + } +} +const result = spawnSync(realBin, argv, { stdio: 'inherit' }); +process.exit(result.status === null ? 1 : result.status); +`; + +interface GuardShim { + dir: string; + env: Record; + cleanup(): void; +} + +function createGuardShim(scenario: AgentScenario, realBinDir: string): GuardShim { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mainwp-control-agent-guard-')); + fs.writeFileSync(path.join(dir, 'mainwpcontrol'), GUARD_SHIM_SOURCE, { mode: 0o755 }); + return { + dir, + env: { + MAINWP_ACCEPTANCE_REAL_BIN: path.join(realBinDir, 'mainwpcontrol'), + MAINWP_ACCEPTANCE_ALLOWED_ABILITIES: scenario.expectedAbilities.join(','), + // Confirmed destructive execution is authorized only for the fixture + // write scenario; live scenarios never get --confirm/--force. + MAINWP_ACCEPTANCE_ALLOW_CONFIRM: + scenario.kind === 'write' && scenario.target === 'fixture' ? '1' : '0', + }, + cleanup: () => fs.rmSync(dir, { recursive: true, force: true }), + }; +} + +const agentSystemPrompt = [ + 'A MainWP Dashboard is managed exclusively through the mainwpcontrol CLI,', + 'which is on PATH and already configured with credentials.', + 'Use the Bash tool to run explicit mainwpcontrol subcommands with --json output, for example:', + '`mainwpcontrol abilities list --json` or', + '`mainwpcontrol abilities run --input \'\' --json`.', + 'Destructive abilities accept --dry-run to preview and --confirm --force to execute', + 'without an interactive prompt.', + 'Do not attempt to reach the Dashboard any other way.', +].join(' '); + +async function runAgentAcceptance(options: AgentRunnerOptions): Promise { + if (options.help) { + printHelp(); + return 0; + } + if (options.list) { + for (const scenario of agentScenarios) console.log(scenario.id); + return 0; + } + + const scenarios = selectedScenarios(options.scenarioIds); + const credentials = resolveAcceptanceCredentials(); + const fixtureCredentials: AcceptanceCredentials = { + dashboardUrl: 'http://127.0.0.1', + username: FIXTURE_USERNAME, + appPassword: FIXTURE_APP_PASSWORD, + }; + const redactor = new Redactor({ + ...credentials, + authorization: `Basic ${Buffer.from( + `${credentials.username}:${credentials.appPassword}`, + ).toString('base64')}`, + }); + redactor.add({ + ...fixtureCredentials, + authorization: `Basic ${Buffer.from( + `${fixtureCredentials.username}:${fixtureCredentials.appPassword}`, + ).toString('base64')}`, + }); + const auditValues = new Set(); + registerAuditValues(auditValues, credentials); + registerAuditValues(auditValues, fixtureCredentials); + + const runner = new CommandRunner(); + const artifacts = await createArtifacts( + repoRoot, + redactor, + runner, + options.mode, + 'live', + { + agent: true, + writes: options.writes, + scenarios: options.scenarioIds, + keepConsumer: options.keepConsumer, + }, + '-agent', + ); + // TLS verification stays on unless the Dashboard is the local self-signed + // testbed (same host classes the write guard trusts). + const liveSkipTlsVerify = isWriteHostAllowed(new URL(credentials.dashboardUrl).hostname); + const verifier = new IndependentVerifier(credentials, liveSkipTlsVerify); + const results: AgentResult[] = []; + let preparedCli: PreparedCli | null = null; + let harnessError: unknown; + let artifactAudit: AgentResultDocument['artifactAudit'] = { + passed: true, + message: 'No registered credential or Dashboard-origin values were found.', + }; + + try { + preparedCli = await prepareCli(options, runner, artifacts); + const which = await runner.run(['which', 'claude'], repoRoot, { allowFailure: true }); + const claudeAvailable = which.exitCode === 0; + + for (const scenario of scenarios) { + let truth: AgentGroundTruth | undefined; + let configDir: ConfigDir | null = null; + let mockServer: MockServer | null = null; + let guardShim: GuardShim | null = null; + let scenarioVerifier = verifier; + let scenarioCredentials = credentials; + let result: AgentResult | undefined; + try { + if (scenario.kind === 'write' && scenario.target === 'live') { + const guardReason = getWriteGuardReason(credentials.dashboardUrl, options.writes, 'live'); + if (guardReason) { + result = { + id: scenario.id, + status: 'skipped', + target: scenario.target, + invocations: [], + finalText: '', + timing: emptyTiming(), + reason: guardReason, + }; + continue; + } + } + + if (scenario.target === 'fixture') { + mockServer = new MockServer(); + await mockServer.start(); + programAgentFixture(mockServer); + scenarioCredentials = { + dashboardUrl: mockServer.baseUrl, + username: FIXTURE_USERNAME, + appPassword: FIXTURE_APP_PASSWORD, + }; + scenarioVerifier = new IndependentVerifier(scenarioCredentials, false); + } + + try { + truth = await scenario.groundTruth(scenarioVerifier); + } catch (error) { + result = { + id: scenario.id, + status: 'unverified', + target: scenario.target, + invocations: [], + finalText: '', + timing: emptyTiming(), + reason: `Independent verifier precondition failed: ${error instanceof Error ? error.message : String(error)}`, + }; + continue; + } + + const task = scenario.task(truth); + const argv = [ + 'claude', + '-p', + task, + '--allowedTools', + 'Bash(mainwpcontrol *)', + '--disallowedTools', + 'mcp__*', + '--strict-mcp-config', + // The child's Bash sandbox would block CLI network access to the + // Dashboard host, forcing error-and-retry churn on every command. + '--settings', + '{"sandbox": {"enabled": false}}', + '--append-system-prompt', + agentSystemPrompt, + '--output-format', + 'stream-json', + '--verbose', + '--max-turns', + '20', + ]; + if (!claudeAvailable) { + result = { + id: scenario.id, + status: 'unverified', + target: scenario.target, + invocations: [], + finalText: '', + timing: emptyTiming(), + groundTruth: truth, + reason: `Blocked command: ${shellDisplay(argv)}. The claude CLI was not found.`, + }; + continue; + } + + const insecureHttp = new URL(scenarioCredentials.dashboardUrl).protocol === 'http:'; + configDir = await ConfigDir.create({ + profiles: [{ + name: 'acceptance', + dashboardUrl: scenarioCredentials.dashboardUrl, + username: scenarioCredentials.username, + ...(scenario.target === 'live' && liveSkipTlsVerify + ? { skipSSLVerification: true } + : {}), + }], + activeProfile: 'acceptance', + ...(insecureHttp ? { settings: { allowInsecureHttp: true } } : {}), + }); + const collected: CollectedAgentOutput = { + invocations: [], + toolResults: [], + finalText: '', + }; + // The guard shim shadows the real binary on PATH; allowedTools only + // permits commands starting with "mainwpcontrol", so every CLI call + // goes through the scenario's ability allowlist. + guardShim = createGuardShim(scenario, preparedCli.binDir); + const command = await runClaude( + argv, + preparedCli.cwd, + { + ...process.env, + ...guardShim.env, + PATH: `${guardShim.dir}${path.delimiter}${process.env['PATH'] ?? ''}`, + XDG_CONFIG_HOME: configDir.xdgHome, + MAINWPCONTROL_NO_KEYTAR: '1', + MAINWP_APP_PASSWORD: scenarioCredentials.appPassword, + ...(insecureHttp ? { MAINWP_ALLOW_HTTP: '1' } : {}), + }, + (line, elapsedMs) => { + artifacts.appendJsonLine('agent-transcript.jsonl', { + scenario: scenario.id, + line, + }); + try { + collectEvent(JSON.parse(line) as unknown, collected, elapsedMs); + } catch { + // The raw line is already preserved in the redacted transcript. + } + }, + ); + runner.record({ + argv, + cwd: preparedCli.cwd, + exitCode: command.exitCode, + durationMs: command.durationMs, + stdoutTail: command.stdout.slice(-12_000), + stderrTail: command.stderr.slice(-12_000), + }); + const timing = timingFrom(collected, command.durationMs); + if (command.exitCode !== 0) { + result = { + id: scenario.id, + status: 'unverified', + target: scenario.target, + ...(collected.model ? { model: collected.model } : {}), + invocations: collected.invocations, + finalText: collected.finalText, + timing, + groundTruth: truth, + reason: `Blocked command: ${shellDisplay(argv)}. Exit ${command.exitCode}: ${command.stderr.slice(-2000)}`, + }; + continue; + } + + const evaluated = scenario.evaluate + ? await scenario.evaluate(truth, collected, scenarioVerifier) + : { evaluation: evaluateReadScenario(scenario, truth, collected) }; + const passed = Object.values(evaluated.evaluation).every(field => field.pass); + result = { + id: scenario.id, + status: passed ? 'passed' : 'failed', + target: scenario.target, + ...(collected.model ? { model: collected.model } : {}), + invocations: collected.invocations, + finalText: collected.finalText, + timing, + groundTruth: truth, + evaluation: evaluated.evaluation, + ...(!passed && evaluated.reason ? { reason: evaluated.reason } : {}), + }; + } catch (error) { + result = { + id: scenario.id, + status: 'failed', + target: scenario.target, + invocations: [], + finalText: '', + timing: emptyTiming(), + ...(truth ? { groundTruth: truth } : {}), + reason: error instanceof Error ? error.message : String(error), + }; + } finally { + await configDir?.cleanup().catch(error => { + result = { + ...(result ?? { + id: scenario.id, + target: scenario.target, + invocations: [], + finalText: '', + timing: emptyTiming(), + }), + status: 'failed', + reason: `Config cleanup failed: ${error instanceof Error ? error.message : String(error)}`, + }; + }); + guardShim?.cleanup(); + if (scenarioVerifier !== verifier) { + await scenarioVerifier.close().catch(() => {}); + } + if (mockServer) { + await mockServer.stop().catch(() => {}); + } + if (result) { + results.push(result); + const document = resultDocument( + artifacts, + options, + results, + artifactAudit, + null, + ); + artifacts.writeJson('results.json', document); + artifacts.write('summary.md', summaryMarkdown(document)); + } + } + } + } catch (error) { + harnessError = error; + } finally { + preparedCli?.cleanup(); + await verifier.close().catch(error => { + harnessError ??= error; + }); + } + + const initialFindings = auditArtifacts(artifacts.runDir, auditValues); + if (initialFindings.length > 0) { + artifactAudit = { + passed: false, + message: `Sensitive values were found in: ${initialFindings.join(', ')}`, + }; + } + const harnessMessage = harnessError instanceof Error + ? harnessError.message + : harnessError === undefined + ? null + : String(harnessError); + const document = resultDocument( + artifacts, + options, + results, + artifactAudit, + harnessMessage, + ); + artifacts.writeJson('results.json', document); + artifacts.write('summary.md', summaryMarkdown(document)); + artifacts.finish(); + + const finalFindings = auditArtifacts(artifacts.runDir, auditValues); + if (finalFindings.length > 0 && artifactAudit.passed) { + artifactAudit = { + passed: false, + message: `Sensitive values were found in: ${finalFindings.join(', ')}`, + }; + const failedAuditDocument = resultDocument( + artifacts, + options, + results, + artifactAudit, + harnessMessage, + ); + artifacts.writeJson('results.json', failedAuditDocument); + artifacts.write('summary.md', summaryMarkdown(failedAuditDocument)); + artifacts.finish(); + } + + for (const result of results) console.log(`${result.status.toUpperCase()} ${result.id}`); + console.log(`Agent acceptance artifacts: ${artifacts.runDir}`); + if (harnessError) console.error(redactor.redact(harnessMessage ?? 'Agent harness failed')); + return computeExitCode(totals(results), artifactAudit, Boolean(harnessError)); +} + +try { + process.exitCode = await runAgentAcceptance(parseArgs(process.argv.slice(2))); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/tests/acceptance/fixtures.ts b/tests/acceptance/fixtures.ts new file mode 100644 index 0000000..96922ea --- /dev/null +++ b/tests/acceptance/fixtures.ts @@ -0,0 +1,246 @@ +import type { ServerResponse } from 'node:http'; +import { + mockAbility, + STANDARD_ABILITIES, +} from '../../src/__tests__/process/fixtures/api-responses.js'; +import type { + MockServer, + RecordedRequest, +} from '../../src/__tests__/process/fixtures/mock-server.js'; + +export const FIXTURE_USERNAME = 'fake-user'; +export const FIXTURE_APP_PASSWORD = 'clearly-fake fixture app password 1234'; +export const FIXTURE_SITE_ID = 101; + +interface FixturePlugin { + slug: string; + name: string; + version: string; + active: boolean; + update_version: string | null; +} + +interface FixtureSite { + id: number; + url: string; + name: string; + status: string; + last_sync: string; + plugins: FixturePlugin[]; +} + +export const FIXTURE_SITES: FixtureSite[] = [ + { + id: FIXTURE_SITE_ID, + url: 'https://alpha.example.invalid', + name: 'Synthetic Alpha Site', + status: 'connected', + last_sync: '2026-01-01T00:00:00.000Z', + plugins: [ + { + slug: 'hello.php', + name: 'Synthetic Hello Dolly', + version: '1.0.0-fake', + active: true, + update_version: null, + }, + { + slug: 'example-cache/example-cache.php', + name: 'Synthetic Example Cache', + version: '2.0.0-fake', + active: false, + update_version: '2.1.0-fake', + }, + ], + }, + { + id: 202, + url: 'https://bravo.example.invalid', + name: 'Synthetic Bravo Site', + status: 'connected', + last_sync: '2026-01-02T00:00:00.000Z', + plugins: [], + }, +]; + +const countSitesAbility = mockAbility({ + name: 'mainwp/count-sites-v1', + readonly: true, + category: 'sites', + input_schema: { + type: 'object', + properties: { + tag_ids: { type: 'array', items: { type: 'integer' } }, + }, + }, +}); + +// Live Dashboards can expose WordPress-core abilities whose names carry no +// -vN version suffix; CLI discovery skips those entries with a warning. +// Mirror one here so the fixture target exercises the same contract. +const unversionedCoreAbility = { + name: 'core/get-site-info', + label: 'get site info', + description: 'Unversioned core ability that discovery must skip.', + category: 'core', + input_schema: { type: 'object', properties: {} }, + meta: { + annotations: { readonly: true, destructive: false, idempotent: false }, + }, +}; + +export const FIXTURE_ABILITIES = [ + ...STANDARD_ABILITIES, + countSitesAbility, + unversionedCoreAbility, +]; + +function json(response: ServerResponse, status: number, body: unknown): void { + const encoded = JSON.stringify(body); + response.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': Buffer.byteLength(encoded), + }); + response.end(encoded); +} + +function requestInput(request: RecordedRequest): Record { + if (request.method === 'GET' || request.method === 'DELETE') { + const input: Record = {}; + for (const [key, value] of Object.entries(request.query)) { + const match = key.match(/^input\[([^\]]+)\](?:\[(?:\d*)\])?$/); + if (!match?.[1]) continue; + const inputKey = match[1]; + const parsed = /^-?\d+$/.test(value) + ? Number(value) + : value === 'true' + ? true + : value === 'false' + ? false + : value; + if (/\[(?:\d*)\]$/.test(key)) { + const current = input[inputKey]; + input[inputKey] = Array.isArray(current) ? [...current, parsed] : [parsed]; + } else { + input[inputKey] = parsed; + } + } + return input; + } + const body = request.body as Record | undefined; + const input = body?.['input']; + return input && typeof input === 'object' && !Array.isArray(input) + ? input as Record + : {}; +} + +function publicSite(site: FixtureSite): Omit { + const { plugins: _plugins, ...publicData } = site; + return publicData; +} + +function findSite(identifier: unknown): FixtureSite | undefined { + const normalized = String(identifier ?? '').replace(/\/+$/, '').toLowerCase(); + return FIXTURE_SITES.find(site => { + const url = site.url.replace(/\/+$/, '').toLowerCase(); + return String(site.id) === normalized || + url === normalized || + new URL(url).hostname === normalized; + }); +} + +export function programFixtureServer( + server: MockServer, + options: { previewFailure?: boolean } = {}, +): void { + server.reset(); + server.setCredentials(FIXTURE_USERNAME, FIXTURE_APP_PASSWORD); + server.setAbilities(FIXTURE_ABILITIES); + + server.addRoute( + 'GET', + '/wp-json/wp-abilities/v1/abilities/mainwp/list-sites-v1/run', + (request, response) => { + const input = requestInput(request); + const page = typeof input['page'] === 'number' ? input['page'] : 1; + const perPage = typeof input['per_page'] === 'number' ? input['per_page'] : 20; + const start = (page - 1) * perPage; + json(response, 200, { + items: FIXTURE_SITES.slice(start, start + perPage).map(publicSite), + page, + per_page: perPage, + total: FIXTURE_SITES.length, + }); + }, + ); + + server.addRoute( + 'GET', + '/wp-json/wp-abilities/v1/abilities/mainwp/count-sites-v1/run', + (_request, response) => json(response, 200, { total: FIXTURE_SITES.length }), + ); + + server.addRoute( + 'GET', + '/wp-json/wp-abilities/v1/abilities/mainwp/get-site-v1/run', + (request, response) => { + const site = findSite(requestInput(request)['site_id_or_domain']); + if (!site) { + json(response, 404, { + code: 'mainwp_site_not_found', + message: 'The requested synthetic site was not found.', + data: { status: 404 }, + }); + return; + } + json(response, 200, publicSite(site)); + }, + ); + + server.addRoute( + 'GET', + '/wp-json/wp-abilities/v1/abilities/mainwp/get-site-plugins-v1/run', + (request, response) => { + const site = findSite(requestInput(request)['site_id_or_domain']); + if (!site) { + json(response, 404, { + code: 'mainwp_site_not_found', + message: 'The requested synthetic site was not found.', + data: { status: 404 }, + }); + return; + } + json(response, 200, { + site_id: site.id, + site_url: site.url, + plugins: site.plugins, + total: site.plugins.length, + }); + }, + ); + + server.addRoute( + 'POST', + '/wp-json/wp-abilities/v1/abilities/mainwp/delete-site-v1/run', + (request, response) => { + const input = requestInput(request); + if (input['dry_run'] === true) { + if (options.previewFailure) { + json(response, 500, { + code: 'fixture_preview_failed', + message: 'Synthetic preview failure.', + }); + return; + } + json(response, 200, { + success: true, + data: { + affected: [{ site_id: FIXTURE_SITE_ID, name: 'Synthetic Alpha Site' }], + }, + }); + return; + } + json(response, 200, { success: true, data: { deleted: true } }); + }, + ); +} diff --git a/tests/acceptance/lib/artifacts.ts b/tests/acceptance/lib/artifacts.ts new file mode 100644 index 0000000..5651a43 --- /dev/null +++ b/tests/acceptance/lib/artifacts.ts @@ -0,0 +1,161 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { CommandRecord, CommandRunner } from './commands.js'; +import type { Redactor } from './redact.js'; + +export const HARNESS_VERSION = '1.0.0'; + +export interface TarballManifest { + filename: string; + sha256: string; + integrity: string; +} + +export interface AcceptanceManifest { + git: { + branch: string; + commit: string; + dirty: boolean; + diffSha256: string; + }; + packageVersion: string; + tarball: TarballManifest | null; + nodeVersion: string; + npmVersion: string; + os: string; + arch: string; + harnessVersion: string; + mode: string; + target: string; + flags: Record; + startTime: string; + endTime: string | null; +} + +export type EventDirection = 'runner-to-cli' | 'cli-to-runner'; + +interface EventRecord { + scenario: string; + direction: EventDirection; + timestamp: string; + monotonicMs: number; + message: unknown; +} + +export class Artifacts { + readonly runDir: string; + readonly manifest: AcceptanceManifest; + private readonly monotonicStart = performance.now(); + + constructor( + readonly repoRoot: string, + readonly runId: string, + private readonly redactor: Redactor, + manifest: AcceptanceManifest + ) { + this.runDir = path.join(repoRoot, 'test-results', 'acceptance', runId); + this.manifest = manifest; + fs.mkdirSync(this.runDir, { recursive: true, mode: 0o700 }); + fs.chmodSync(this.runDir, 0o700); + this.writeJson('manifest.json', manifest); + fs.writeFileSync(path.join(this.runDir, 'events.jsonl'), '', { mode: 0o600 }); + fs.writeFileSync(path.join(this.runDir, 'commands.jsonl'), '', { mode: 0o600 }); + } + + writeJson(filename: string, value: unknown): void { + this.write(filename, `${JSON.stringify(value, null, 2)}\n`); + } + + write(filename: string, value: string): void { + fs.writeFileSync(path.join(this.runDir, filename), this.redactor.redact(value), { + encoding: 'utf8', + mode: 0o600, + }); + } + + appendJsonLine(filename: string, value: unknown): void { + fs.appendFileSync( + path.join(this.runDir, filename), + `${this.redactor.stringify(value)}\n`, + { encoding: 'utf8', mode: 0o600 } + ); + } + + appendEvent(scenario: string, direction: EventDirection, message: unknown): void { + this.appendJsonLine('events.jsonl', { + scenario, + direction, + timestamp: new Date().toISOString(), + monotonicMs: Math.round((performance.now() - this.monotonicStart) * 1000) / 1000, + message, + } satisfies EventRecord); + } + + appendScenarioStderr(scenario: string, value: string): void { + const filename = `scenario-${scenario.replace(/[^a-z0-9_-]/gi, '_')}.stderr.log`; + fs.appendFileSync(path.join(this.runDir, filename), this.redactor.redact(value), 'utf8'); + } + + recordCommand(record: CommandRecord): void { + this.appendJsonLine('commands.jsonl', record); + } + + setTarball(tarball: TarballManifest): void { + this.manifest.tarball = tarball; + this.writeJson('manifest.json', this.manifest); + } + + finish(): void { + this.manifest.endTime = new Date().toISOString(); + this.writeJson('manifest.json', this.manifest); + } +} + +export async function createArtifacts( + repoRoot: string, + redactor: Redactor, + runner: CommandRunner, + mode: string, + target: string, + flags: Record, + suffix = '' +): Promise { + const branch = (await runner.run(['git', 'branch', '--show-current'], repoRoot)).stdout.trim(); + const commit = (await runner.run(['git', 'rev-parse', 'HEAD'], repoRoot)).stdout.trim(); + const status = (await runner.run(['git', 'status', '--porcelain'], repoRoot)).stdout; + const diff = (await runner.run(['git', 'diff', 'HEAD'], repoRoot)).stdout; + const npmVersion = (await runner.run(['npm', '--version'], repoRoot)).stdout.trim(); + const packageJson = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) as { + version: string; + }; + const startTime = new Date().toISOString(); + const timestamp = startTime.replace(/[-:.]/g, ''); + const dirty = status.length > 0; + const runId = `${timestamp}-${commit.slice(0, 8)}${dirty ? '-dirty' : ''}${suffix}`; + const manifest: AcceptanceManifest = { + git: { + branch, + commit, + dirty, + diffSha256: crypto.createHash('sha256').update(diff).digest('hex'), + }, + packageVersion: packageJson.version, + tarball: null, + nodeVersion: process.version, + npmVersion, + os: `${os.platform()} ${os.release()}`, + arch: os.arch(), + harnessVersion: HARNESS_VERSION, + mode, + target, + flags, + startTime, + endTime: null, + }; + const artifacts = new Artifacts(repoRoot, runId, redactor, manifest); + for (const record of runner.records) artifacts.recordCommand(record); + runner.onRecord = record => artifacts.recordCommand(record); + return artifacts; +} diff --git a/tests/acceptance/lib/cli.ts b/tests/acceptance/lib/cli.ts new file mode 100644 index 0000000..a627fee --- /dev/null +++ b/tests/acceptance/lib/cli.ts @@ -0,0 +1,134 @@ +import type { ConfigDir } from '../../../src/__tests__/process/fixtures/config-dir.js'; +import type { AcceptanceCredentials } from './env.js'; +import { + CommandRunner, + type CommandResult, +} from './commands.js'; + +export interface CLIEnvelope { + success: boolean; + data?: T; + error?: { + code: string; + message: string; + details?: unknown; + hint?: string; + }; + meta?: { + command: string; + timestamp: string; + version: string; + }; +} + +export interface CLIInvocationResult extends CommandResult { + json?: CLIEnvelope; +} + +export interface CLIInvocationOptions { + env?: Record; + /** + * Acceptance scenarios normally inspect non-zero exits directly. Set this + * to false when a command failure should throw CommandError immediately. + */ + allowFailure?: boolean; +} + +export interface CLIInvokerOptions { + binaryPath: string; + cwd: string; + runner: CommandRunner; + configDir: ConfigDir; + credentials: AcceptanceCredentials; + env?: Record; + onStderr?: (stderr: string) => void; +} + +/** + * Process-level MainWP Control invoker used by deterministic acceptance tests. + * Each call is one CommandRunner spawn and therefore one recorded invocation. + */ +export class CLIInvoker { + readonly binaryPath: string; + readonly cwd: string; + readonly runner: CommandRunner; + readonly configDir: ConfigDir; + readonly credentials: AcceptanceCredentials; + private readonly extraEnv: Record; + private readonly onStderr: ((stderr: string) => void) | undefined; + + constructor(options: CLIInvokerOptions) { + this.binaryPath = options.binaryPath; + this.cwd = options.cwd; + this.runner = options.runner; + this.configDir = options.configDir; + this.credentials = options.credentials; + this.extraEnv = { ...options.env }; + this.onStderr = options.onStderr; + } + + async run( + args: string[], + options: CLIInvocationOptions = {} + ): Promise> { + if (args.length === 0) { + throw new Error('Acceptance CLI invocations must include an explicit subcommand'); + } + this.assertNoCredentialInArgv(args); + + const result = await this.runner.run( + [this.binaryPath, ...args], + this.cwd, + { + env: this.buildEnv(options.env), + // Error-contract scenarios need the result instead of an exception. + allowFailure: options.allowFailure ?? true, + timeoutMs: 15_000, + } + ); + if (result.stderr) this.onStderr?.(result.stderr); + + let json: CLIEnvelope | undefined; + try { + json = JSON.parse(result.stdout) as CLIEnvelope; + } catch { + // Human-output calls and malformed-output assertions inspect stdout. + } + + return json === undefined ? result : { ...result, json }; + } + + private buildEnv(invocationEnv: Record | undefined): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + PATH: process.env['PATH'] ?? '', + NODE_ENV: 'test', + NODE_NO_WARNINGS: '1', + ...this.extraEnv, + ...invocationEnv, + // These values are invariant per scenario and cannot be overridden by a + // launch or invocation-specific environment. + XDG_CONFIG_HOME: this.configDir.xdgHome, + HOME: this.configDir.xdgHome, + MAINWPCONTROL_NO_KEYTAR: '1', + MAINWP_APP_PASSWORD: this.credentials.appPassword, + }; + + if (new URL(this.credentials.dashboardUrl).protocol === 'http:') { + env['MAINWP_ALLOW_HTTP'] = '1'; + } else { + delete env['MAINWP_ALLOW_HTTP']; + } + + return env; + } + + private assertNoCredentialInArgv(args: string[]): void { + const rawPassword = this.credentials.appPassword; + const compactPassword = rawPassword.replace(/\s/g, ''); + const secrets = [...new Set([rawPassword, compactPassword])].filter(Boolean); + + if (args.some(arg => secrets.some(secret => arg.includes(secret)))) { + throw new Error('Acceptance CLI argv must not contain credentials'); + } + } +} diff --git a/tests/acceptance/lib/commands.ts b/tests/acceptance/lib/commands.ts new file mode 100644 index 0000000..86da89f --- /dev/null +++ b/tests/acceptance/lib/commands.ts @@ -0,0 +1,146 @@ +import { spawn } from 'node:child_process'; + +export interface CommandRecord { + argv: string[]; + cwd: string; + exitCode: number; + durationMs: number; + stdoutTail: string; + stderrTail: string; +} + +export interface CommandResult extends CommandRecord { + stdout: string; + stderr: string; +} + +export class CommandError extends Error { + constructor(readonly result: CommandResult) { + super( + `Command failed with exit code ${result.exitCode}: ${result.argv.join(' ')}\n${result.stderrTail}` + ); + } +} + +function tail(value: string, maxLength = 12_000): string { + return value.length <= maxLength ? value : value.slice(-maxLength); +} + +/** + * Cap per-stream buffering: retain only the trailing bytes once the limit is + * exceeded, so a runaway subprocess cannot balloon harness memory. 10 MiB is + * far beyond any legitimate CLI or npm output in this harness. + */ +const MAX_STREAM_BYTES = 10 * 1024 * 1024; + +export function appendBounded(chunks: Buffer[], chunk: Buffer): void { + chunks.push(chunk); + let total = chunks.reduce((sum, item) => sum + item.length, 0); + while (total > MAX_STREAM_BYTES && chunks.length > 1) { + total -= chunks.shift()!.length; + } +} + +export class CommandRunner { + readonly records: CommandRecord[] = []; + onRecord?: (record: CommandRecord) => void; + + record(record: CommandRecord): void { + this.records.push(record); + this.onRecord?.(record); + } + + async run( + argv: string[], + cwd: string, + options: { env?: NodeJS.ProcessEnv; allowFailure?: boolean; timeoutMs?: number } = {} + ): Promise { + const started = performance.now(); + const command = argv[0]; + if (!command) { + const result: CommandResult = { + argv, + cwd, + exitCode: 1, + durationMs: Math.round(performance.now() - started), + stdout: '', + stderr: 'Command argv must contain an executable.', + stdoutTail: '', + stderrTail: 'Command argv must contain an executable.', + }; + this.record({ + argv: result.argv, + cwd: result.cwd, + exitCode: result.exitCode, + durationMs: result.durationMs, + stdoutTail: result.stdoutTail, + stderrTail: result.stderrTail, + }); + if (!options.allowFailure) throw new CommandError(result); + return result; + } + const child = spawn(command, argv.slice(1), { + cwd, + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + shell: false, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on('data', chunk => appendBounded(stdout, Buffer.from(chunk))); + child.stderr.on('data', chunk => appendBounded(stderr, Buffer.from(chunk))); + + let spawnError: Error | undefined; + let timedOut = false; + const exitCode = await new Promise(resolve => { + let forceKill: NodeJS.Timeout | undefined; + const timeout = options.timeoutMs === undefined + ? undefined + : setTimeout(() => { + timedOut = true; + child.kill('SIGTERM'); + forceKill = setTimeout(() => child.kill('SIGKILL'), 1_000); + }, options.timeoutMs); + child.once('error', error => { + spawnError = error; + if (timeout) clearTimeout(timeout); + if (forceKill) clearTimeout(forceKill); + resolve(1); + }); + child.once('close', code => { + if (timeout) clearTimeout(timeout); + if (forceKill) clearTimeout(forceKill); + resolve(timedOut ? 124 : (code ?? 1)); + }); + }); + const stdoutText = Buffer.concat(stdout).toString('utf8'); + const capturedStderr = Buffer.concat(stderr).toString('utf8'); + const stderrText = timedOut + ? [capturedStderr, `Command timed out after ${options.timeoutMs}ms.`].filter(Boolean).join('\n') + : spawnError + ? [capturedStderr, spawnError.message].filter(Boolean).join('\n') + : capturedStderr; + const result: CommandResult = { + argv, + cwd, + exitCode, + durationMs: Math.round(performance.now() - started), + stdout: stdoutText, + stderr: stderrText, + stdoutTail: tail(stdoutText), + stderrTail: tail(stderrText), + }; + const record: CommandRecord = { + argv: result.argv, + cwd: result.cwd, + exitCode: result.exitCode, + durationMs: result.durationMs, + stdoutTail: result.stdoutTail, + stderrTail: result.stderrTail, + }; + this.record(record); + + if (exitCode !== 0 && !options.allowFailure) throw new CommandError(result); + return result; + } +} diff --git a/tests/acceptance/lib/env.ts b/tests/acceptance/lib/env.ts new file mode 100644 index 0000000..533e651 --- /dev/null +++ b/tests/acceptance/lib/env.ts @@ -0,0 +1,96 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export interface AcceptanceCredentials { + dashboardUrl: string; + username: string; + appPassword: string; +} + +function stripInlineComment(value: string): string { + const match = value.match(/^(.*?)(?:\s+#.*)?$/); + return match?.[1]?.trim() ?? value.trim(); +} + +function unquote(value: string): string { + const first = value[0]; + if (first === '"' || first === "'") { + const closingQuote = value.indexOf(first, 1); + if (closingQuote !== -1) return value.slice(1, closingQuote); + } + return stripInlineComment(value); +} + +export function parseAcceptanceEnv(content: string): AcceptanceCredentials { + const values = new Map(); + for (const rawLine of content.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + const match = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!match) continue; + const key = match[1]; + const rawValue = match[2]; + if (!key || rawValue === undefined) continue; + values.set(key, unquote(rawValue.trim())); + } + + const credentials = { + dashboardUrl: values.get('LLM_DASH_URL') ?? '', + username: values.get('MAINWP_USER') ?? '', + appPassword: values.get('MAINWP_APP_PASSWORD') ?? '', + }; + validateCredentials(credentials, 'acceptance environment file'); + return credentials; +} + +function validateCredentials(credentials: AcceptanceCredentials, source: string): void { + const missing = Object.entries(credentials) + .filter(([, value]) => value.length === 0) + .map(([name]) => name); + if (missing.length > 0) { + throw new Error(`Missing ${missing.join(', ')} in ${source}`); + } +} + +function expandHome(filePath: string): string { + if (filePath === '~') return os.homedir(); + if (filePath.startsWith('~/')) return path.join(os.homedir(), filePath.slice(2)); + return filePath; +} + +export function resolveAcceptanceCredentials( + env: NodeJS.ProcessEnv = process.env +): AcceptanceCredentials { + const fromEnvironment = { + dashboardUrl: env.MAINWP_URL ?? '', + username: env.MAINWP_USER ?? '', + appPassword: env.MAINWP_APP_PASSWORD ?? '', + }; + const setCount = Object.values(fromEnvironment).filter(value => value.length > 0).length; + if (setCount === 3) { + return fromEnvironment; + } + if (setCount > 0) { + // A partial environment must not silently fall back to the file: the + // run would target a different Dashboard than the operator intended. + throw new Error( + 'Partial live acceptance environment: set all of MAINWP_URL, MAINWP_USER, ' + + 'and MAINWP_APP_PASSWORD, or none of them to use the env file.' + ); + } + + const envPath = expandHome( + env.MAINWP_CONTROL_ACCEPTANCE_ENV ?? '~/github/dev-tools/network-testbed/.env' + ); + let content: string; + try { + content = fs.readFileSync(envPath, 'utf8'); + } catch (error) { + throw new Error( + `Live acceptance credentials were not complete in the process environment and ${envPath} could not be read`, + { cause: error } + ); + } + return parseAcceptanceEnv(content); +} diff --git a/tests/acceptance/lib/guards.ts b/tests/acceptance/lib/guards.ts new file mode 100644 index 0000000..6c0cf16 --- /dev/null +++ b/tests/acceptance/lib/guards.ts @@ -0,0 +1,25 @@ +export function isWriteHostAllowed(hostname: string): boolean { + const host = hostname.toLowerCase(); + return ( + host === 'localhost' || + host === '127.0.0.1' || + host.endsWith('.local') + ); +} + +export function getWriteGuardReason( + dashboardUrl: string, + writesEnabled: boolean, + target: 'live' | 'fixture' +): string | null { + if (target === 'fixture') return null; + if (!writesEnabled) return 'Write scenarios require --writes.'; + const hostname = new URL(dashboardUrl).hostname; + if (!isWriteHostAllowed(hostname)) { + return ( + `Dashboard host ${hostname} is not write-allowed. ` + + 'Use localhost, 127.0.0.1, or a .local host.' + ); + } + return null; +} diff --git a/tests/acceptance/lib/local-registry.ts b/tests/acceptance/lib/local-registry.ts new file mode 100644 index 0000000..ee36537 --- /dev/null +++ b/tests/acceptance/lib/local-registry.ts @@ -0,0 +1,149 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; +import type { CommandRunner } from './commands.js'; + +interface PackedDependency { + name: string; + version: string; + filename: string; + shasum: string; + integrity: string; +} + +interface RegistryVersion { + packed: PackedDependency; + packageJson: Record; +} + +export interface LocalRegistry { + url: string; + close(): Promise; +} + +function json(response: http.ServerResponse, status: number, body: unknown): void { + const encoded = JSON.stringify(body); + response.writeHead(status, { + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(encoded), + }); + response.end(encoded); +} + +export async function startLocalDependencyRegistry( + repoRoot: string, + tempRoot: string, + runner: CommandRunner +): Promise { + const lock = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package-lock.json'), 'utf8')) as { + packages: Record; + }; + const packagePaths = Object.entries(lock.packages) + .filter(([packagePath, metadata]) => packagePath.startsWith('node_modules/') && !metadata.dev) + .map(([packagePath]) => path.join(repoRoot, packagePath)) + .filter(packagePath => fs.existsSync(path.join(packagePath, 'package.json'))); + const tarballDir = path.join(tempRoot, 'dependency-tarballs'); + fs.mkdirSync(tarballDir, { recursive: true }); + const packedResult = await runner.run( + [ + 'npm', + 'pack', + '--ignore-scripts', + '--json', + '--pack-destination', + tarballDir, + ...packagePaths, + ], + repoRoot + ); + const packed = JSON.parse(packedResult.stdout) as PackedDependency[]; + const packageMetadata = new Map>(); + for (const packagePath of packagePaths) { + const packageJson = JSON.parse( + fs.readFileSync(path.join(packagePath, 'package.json'), 'utf8') + ) as Record; + const name = packageJson['name']; + const version = packageJson['version']; + if (typeof name === 'string' && typeof version === 'string') { + packageMetadata.set(`${name}@${version}`, packageJson); + } + } + const byName = new Map>(); + for (const dependency of packed) { + const packageJson = packageMetadata.get(`${dependency.name}@${dependency.version}`); + if (!packageJson) { + throw new Error(`Packed dependency metadata was not found for ${dependency.name}@${dependency.version}`); + } + const versions = byName.get(dependency.name) ?? new Map(); + versions.set(dependency.version, { packed: dependency, packageJson }); + byName.set(dependency.name, versions); + } + + const server = http.createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + if (url.pathname.startsWith('/tarballs/')) { + const filename = path.basename(decodeURIComponent(url.pathname.slice('/tarballs/'.length))); + const filePath = path.join(tarballDir, filename); + if (!fs.existsSync(filePath)) return json(response, 404, { error: 'tarball not found' }); + const stat = fs.statSync(filePath); + response.writeHead(200, { + 'content-type': 'application/octet-stream', + 'content-length': stat.size, + }); + fs.createReadStream(filePath).pipe(response); + return; + } + + const name = decodeURIComponent(url.pathname.slice(1)); + const versions = byName.get(name); + if (!versions) return json(response, 404, { error: `package ${name} not found` }); + const address = server.address(); + if (!address || typeof address === 'string') { + return json(response, 500, { error: 'registry is not bound' }); + } + const registryVersions = Object.fromEntries( + [...versions.entries()].map(([version, { packed: dependency, packageJson }]) => { + const tarballUrl = `http://127.0.0.1:${address.port}/tarballs/${encodeURIComponent( + dependency.filename + )}`; + return [version, { + ...packageJson, + dist: { + tarball: tarballUrl, + shasum: dependency.shasum, + integrity: + dependency.integrity || + `sha512-${crypto + .createHash('sha512') + .update(fs.readFileSync(path.join(tarballDir, dependency.filename))) + .digest('base64')}`, + }, + }]; + }) + ); + const latest = [...versions.keys()] + .sort((left, right) => left.localeCompare(right, undefined, { numeric: true })) + .at(-1); + if (!latest) return json(response, 500, { error: `package ${name} has no versions` }); + json(response, 200, { + _id: name, + name, + 'dist-tags': { latest }, + versions: registryVersions, + }); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Local registry failed to bind'); + return { + url: `http://127.0.0.1:${address.port}`, + close: () => + new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())); + }), + }; +} diff --git a/tests/acceptance/lib/pack.ts b/tests/acceptance/lib/pack.ts new file mode 100644 index 0000000..74d663b --- /dev/null +++ b/tests/acceptance/lib/pack.ts @@ -0,0 +1,171 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { Artifacts } from './artifacts.js'; +import type { CommandRunner } from './commands.js'; +import { startLocalDependencyRegistry } from './local-registry.js'; + +interface NpmPackResult { + filename: string; + shasum: string; + integrity: string; +} + +function parseNpmPackOutput(stdout: string): NpmPackResult[] { + const trimmed = stdout.trim(); + const jsonStart = trimmed.lastIndexOf('\n['); + return JSON.parse(jsonStart === -1 ? trimmed : trimmed.slice(jsonStart + 1)) as NpmPackResult[]; +} + +export interface PackChecks { + requiredFilesPresent: boolean; + forbiddenFilesAbsent: boolean; + installedBinPresent: boolean; + installedVersionMatches: boolean; + versionCommandMatches: boolean; +} + +export interface PackedPackage { + tempRoot: string; + consumerDir: string; + tarballPath: string; + filename: string; + sha256: string; + npmShasum: string; + integrity: string; + binPath: string; + version: string; + versionOutput: string; + checks: PackChecks; + cleanup(): void; +} + +function hasForbiddenEntry(entry: string): boolean { + return entry.startsWith('package/src/') || + entry.startsWith('package/tests/') || + entry.startsWith('package/.mwpdev/') || + entry === 'package/src' || + entry === 'package/tests' || + entry === 'package/.mwpdev' || + /^package\/\.env(?:\.|$)/.test(entry) || + /(?:^|\/)[^/]*\.test\.[^/]+$/.test(entry); +} + +export async function packAndInstall( + repoRoot: string, + runner: CommandRunner, + artifacts: Artifacts, + keepConsumer: boolean, +): Promise { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mainwp-control-acceptance-')); + try { + const packCache = path.join(tempRoot, 'pack-npm-cache'); + fs.mkdirSync(packCache); + const packResult = await runner.run( + ['npm', 'pack', '--json', '--pack-destination', tempRoot], + repoRoot, + { env: { ...process.env, npm_config_cache: packCache } }, + ); + const parsed = parseNpmPackOutput(packResult.stdout); + if (parsed.length !== 1 || !parsed[0]) { + throw new Error(`npm pack produced ${parsed.length} package records`); + } + const packed = parsed[0]; + const tarballPath = path.join(tempRoot, packed.filename); + const sha256 = crypto.createHash('sha256').update(fs.readFileSync(tarballPath)).digest('hex'); + const listingResult = await runner.run(['tar', '-tzf', tarballPath], repoRoot); + const entries = listingResult.stdout.split(/\r?\n/).filter(Boolean); + const entrySet = new Set(entries); + const requiredFilesPresent = + entrySet.has('package/bin/run.js') && + entrySet.has('package/oclif.manifest.json') && + entries.some(entry => entry.startsWith('package/dist/')) && + entries.some(entry => entry.startsWith('package/scripts/completions/')); + const forbiddenFilesAbsent = entries.every(entry => !hasForbiddenEntry(entry)); + if (!requiredFilesPresent || !forbiddenFilesAbsent) { + throw new Error( + `Packed tarball content assertions failed: ${JSON.stringify({ + requiredFilesPresent, + forbiddenFilesAbsent, + })}`, + ); + } + + const consumerDir = path.join(tempRoot, 'consumer'); + fs.mkdirSync(consumerDir); + const npmCache = path.join(tempRoot, 'npm-cache'); + fs.mkdirSync(npmCache); + await runner.run(['npm', 'init', '-y'], consumerDir, { + env: { ...process.env, npm_config_cache: npmCache }, + }); + + const registry = await startLocalDependencyRegistry(repoRoot, tempRoot, runner); + try { + await runner.run( + [ + 'npm', + 'install', + tarballPath, + '--ignore-scripts', + '--no-audit', + '--no-fund', + '--registry', + registry.url, + ], + consumerDir, + { env: { ...process.env, npm_config_cache: npmCache } }, + ); + } finally { + await registry.close(); + } + + const packageDir = path.join(consumerDir, 'node_modules', '@mainwp', 'control'); + const binLink = path.join(consumerDir, 'node_modules', '.bin', 'mainwpcontrol'); + const installedBinPresent = fs.existsSync(binLink); + const binPath = binLink; + const installedPackage = JSON.parse( + fs.readFileSync(path.join(packageDir, 'package.json'), 'utf8'), + ) as { version: string }; + const repoPackage = JSON.parse( + fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'), + ) as { version: string }; + const versionResult = await runner.run([binPath, '--version'], consumerDir); + const versionOutput = versionResult.stdout.trim(); + const checks: PackChecks = { + requiredFilesPresent, + forbiddenFilesAbsent, + installedBinPresent, + installedVersionMatches: installedPackage.version === repoPackage.version, + versionCommandMatches: versionOutput.includes(repoPackage.version), + }; + if (Object.values(checks).some(check => !check)) { + throw new Error(`Installed package assertions failed: ${JSON.stringify(checks)}`); + } + artifacts.setTarball({ + filename: packed.filename, + sha256, + integrity: packed.integrity, + }); + + return { + tempRoot, + consumerDir, + tarballPath, + filename: packed.filename, + sha256, + npmShasum: packed.shasum, + integrity: packed.integrity, + binPath, + version: installedPackage.version, + versionOutput, + checks, + cleanup: () => { + if (!keepConsumer) fs.rmSync(tempRoot, { recursive: true, force: true }); + }, + }; + } catch (error) { + fs.rmSync(tempRoot, { recursive: true, force: true }); + throw error; + } +} diff --git a/tests/acceptance/lib/redact.ts b/tests/acceptance/lib/redact.ts new file mode 100644 index 0000000..38a0ee6 --- /dev/null +++ b/tests/acceptance/lib/redact.ts @@ -0,0 +1,58 @@ +export interface RedactorValues { + username?: string; + appPassword?: string; + dashboardUrl?: string; + authorization?: string; +} + +interface Replacement { + value: string; + token: string; +} + +export class Redactor { + private readonly replacements: Replacement[] = []; + + constructor(values: RedactorValues = {}) { + this.add(values); + } + + add(values: RedactorValues): void { + this.addReplacement(values.authorization, ''); + this.addReplacement(values.appPassword, ''); + if (values.appPassword) { + const compact = values.appPassword.replace(/\s+/g, ''); + if (compact !== values.appPassword) { + this.addReplacement(compact, ''); + } + } + this.addReplacement(values.username, ''); + if (values.dashboardUrl) { + try { + this.addReplacement(new URL(values.dashboardUrl).origin, ''); + } catch { + this.addReplacement(values.dashboardUrl, ''); + } + } + this.replacements.sort((a, b) => b.value.length - a.value.length); + } + + private addReplacement(value: string | undefined, token: string): void { + if (!value || this.replacements.some(replacement => replacement.value === value)) return; + this.replacements.push({ value, token }); + } + + redact(value: string): string { + let redacted = value; + for (const replacement of this.replacements) { + redacted = redacted.split(replacement.value).join(replacement.token); + } + return redacted; + } + + stringify(value: unknown, spacing?: number): string { + // JSON.stringify returns undefined for undefined/functions/symbols; + // redact() must always receive a string. + return this.redact(JSON.stringify(value, null, spacing) ?? 'null'); + } +} diff --git a/tests/acceptance/lib/verify.ts b/tests/acceptance/lib/verify.ts new file mode 100644 index 0000000..d1e00e0 --- /dev/null +++ b/tests/acceptance/lib/verify.ts @@ -0,0 +1,221 @@ +import { Agent, request } from 'undici'; +import type { AcceptanceCredentials } from './env.js'; + +export interface AbilityAnnotations { + readonly: boolean; + destructive: boolean; + idempotent: boolean; +} + +export interface AbilityDefinition { + name: string; + label?: string; + description?: string; + category?: string; + input_schema?: { + properties?: Record; + [key: string]: unknown; + }; + output_schema?: Record; + meta?: { annotations?: AbilityAnnotations; [key: string]: unknown }; + [key: string]: unknown; +} + +export interface VerifiedSite { + id: number; + url: string; + name: string; + status?: string; + last_sync?: string | null; + [key: string]: unknown; +} + +export interface VerifiedPlugin { + slug: string; + name: string; + version: string; + active: boolean; + update_version?: string | null; + [key: string]: unknown; +} + +export interface VerifiedPluginResponse { + site_id: number; + site_url: string; + plugins: VerifiedPlugin[]; + total: number; +} + +export function serializeToPhpQueryString(input: Record): string { + const params: string[] = []; + for (const [key, value] of Object.entries(input)) { + if (Array.isArray(value)) { + for (const item of value) { + if (item !== null && typeof item === 'object') { + throw new Error(`Unsupported nested query parameter at "${key}": arrays need scalars`); + } + params.push(`input[${encodeURIComponent(key)}][]=${encodeURIComponent(String(item))}`); + } + } else if (value !== null && typeof value === 'object') { + for (const [subKey, subValue] of Object.entries(value)) { + if (subValue !== null && typeof subValue === 'object') { + throw new Error( + `Unsupported nested query parameter at "${key}": objects may be only one level deep` + ); + } + params.push( + `input[${encodeURIComponent(key)}][${encodeURIComponent(subKey)}]=${encodeURIComponent(String(subValue))}` + ); + } + } else if (value !== undefined && value !== null) { + params.push(`input[${encodeURIComponent(key)}]=${encodeURIComponent(String(value))}`); + } + } + return params.length > 0 ? `?${params.join('&')}` : ''; +} + +export class IndependentVerifier { + private readonly baseUrl: string; + private readonly authorization: string; + private readonly dispatcher?: Agent; + private catalog?: AbilityDefinition[]; + + constructor(credentials: AcceptanceCredentials, skipTlsVerify: boolean) { + this.baseUrl = `${credentials.dashboardUrl.replace(/\/+$/, '')}/wp-json/wp-abilities/v1`; + this.authorization = `Basic ${Buffer.from( + `${credentials.username}:${credentials.appPassword}` + ).toString('base64')}`; + if (skipTlsVerify) { + this.dispatcher = new Agent({ connect: { rejectUnauthorized: false } }); + } + } + + getAuthorizationHeader(): string { + return this.authorization; + } + + async close(): Promise { + await this.dispatcher?.close(); + } + + async fetchCatalog(): Promise { + if (this.catalog) return this.catalog; + const abilities: AbilityDefinition[] = []; + for (let page = 1; page <= 50; page += 1) { + const response = await this.requestJsonResponse( + `${this.baseUrl}/abilities?per_page=100&page=${page}`, + 'GET' + ); + if (!Array.isArray(response.data)) { + throw new Error(`Independent verifier expected an ability array on catalog page ${page}`); + } + abilities.push(...response.data); + const rawTotalPages = response.headers['x-wp-totalpages']; + const totalPages = Number( + Array.isArray(rawTotalPages) ? rawTotalPages[0] : (rawTotalPages ?? 1) + ); + if (page >= totalPages) break; + if (page === 50) throw new Error('Independent verifier catalog exceeded the 50-page cap'); + } + this.catalog = abilities; + return this.catalog; + } + + async getAbilityInputArrayEnum(abilityName: string, argumentName: string): Promise { + const ability = (await this.fetchCatalog()).find(candidate => candidate.name === abilityName); + if (!ability) throw new Error(`Independent verifier could not find ability ${abilityName}`); + const values = ability.input_schema?.properties?.[argumentName]?.items?.enum; + if (!Array.isArray(values) || !values.every(value => typeof value === 'string')) { + throw new Error( + `Independent verifier found no string enum for ${abilityName} argument ${argumentName}` + ); + } + return values as string[]; + } + + async execute(abilityName: string, input: Record = {}): Promise { + const catalog = await this.fetchCatalog(); + const ability = catalog.find(candidate => candidate.name === abilityName); + if (!ability) throw new Error(`Independent verifier could not find ability ${abilityName}`); + + const annotations = ability.meta?.annotations; + const isReadonly = annotations?.readonly ?? false; + const isDestructive = annotations?.destructive ?? true; + const isIdempotent = annotations?.idempotent ?? false; + const endpoint = `${this.baseUrl}/abilities/${abilityName}/run`; + + if (isReadonly || (isDestructive && isIdempotent)) { + const method = isReadonly ? 'GET' : 'DELETE'; + return this.requestJson( + endpoint + (Object.keys(input).length > 0 ? serializeToPhpQueryString(input) : ''), + method + ); + } + return this.requestJson(endpoint, 'POST', JSON.stringify({ input })); + } + + async listSites(): Promise { + const sites: VerifiedSite[] = []; + let page = 1; + for (;;) { + const response = (await this.execute('mainwp/list-sites-v1', { + page, + per_page: 100, + })) as { items: VerifiedSite[]; total: number }; + sites.push(...response.items); + if (sites.length >= response.total || response.items.length === 0) return sites; + page += 1; + } + } + + async countSites(): Promise { + const response = (await this.execute('mainwp/count-sites-v1')) as { total: number }; + return response.total; + } + + async getSite(siteIdOrDomain: number | string): Promise { + return (await this.execute('mainwp/get-site-v1', { + site_id_or_domain: siteIdOrDomain, + })) as VerifiedSite; + } + + async getSitePlugins(siteIdOrDomain: number | string): Promise { + return (await this.execute('mainwp/get-site-plugins-v1', { + site_id_or_domain: siteIdOrDomain, + })) as VerifiedPluginResponse; + } + + private async requestJson( + url: string, + method: 'GET' | 'POST' | 'DELETE', + body?: string + ): Promise { + return (await this.requestJsonResponse(url, method, body)).data; + } + + private async requestJsonResponse( + url: string, + method: 'GET' | 'POST' | 'DELETE', + body?: string + ): Promise<{ data: T; headers: Record }> { + const response = await request(url, { + method, + headers: { + authorization: this.authorization, + 'content-type': 'application/json', + }, + ...(body ? { body } : {}), + ...(this.dispatcher ? { dispatcher: this.dispatcher } : {}), + }); + const responseBody = await response.body.text(); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw new Error( + `Independent verifier request failed with HTTP ${response.statusCode}: ${responseBody}` + ); + } + return { + data: JSON.parse(responseBody) as T, + headers: response.headers, + }; + } +} diff --git a/tests/acceptance/run.ts b/tests/acceptance/run.ts new file mode 100644 index 0000000..fa62eda --- /dev/null +++ b/tests/acceptance/run.ts @@ -0,0 +1,625 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ConfigDir } from '../../src/__tests__/process/fixtures/config-dir.js'; +import { MockServer } from '../../src/__tests__/process/fixtures/mock-server.js'; +import { + FIXTURE_APP_PASSWORD, + FIXTURE_USERNAME, + programFixtureServer, +} from './fixtures.js'; +import { createArtifacts, type Artifacts } from './lib/artifacts.js'; +import { CLIInvoker } from './lib/cli.js'; +import { CommandRunner, type CommandRecord } from './lib/commands.js'; +import { + resolveAcceptanceCredentials, + type AcceptanceCredentials, +} from './lib/env.js'; +import { getWriteGuardReason, isWriteHostAllowed } from './lib/guards.js'; +import { packAndInstall, type PackedPackage } from './lib/pack.js'; +import { Redactor } from './lib/redact.js'; +import { IndependentVerifier } from './lib/verify.js'; +import { scenarios } from './scenarios/index.js'; +import { + AssertionRecorder, + type AcceptanceMode, + type AcceptanceTarget, + type ScenarioContext, + type ScenarioDefinition, + type ScenarioResult, +} from './scenarios/types.js'; + +interface RunnerOptions { + mode: AcceptanceMode; + target: AcceptanceTarget; + scenarioIds: string[]; + writes: boolean; + list: boolean; + keepConsumer: boolean; + help: boolean; +} + +interface ResultDocument { + runId: string; + mode: AcceptanceMode; + target: AcceptanceTarget; + totals: Record<'passed' | 'failed' | 'skipped' | 'unverified', number>; + scenarios: ScenarioResult[]; + artifactAudit: { + passed: boolean; + message: string; + }; + harnessError: string | null; +} + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); + +function requiredValue(argv: string[], index: number, flag: string): string { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${flag} requires a value`); + return value; +} + +function parseArgs(argv: string[]): RunnerOptions { + const options: RunnerOptions = { + mode: 'packed', + target: 'live', + scenarioIds: [], + writes: false, + list: false, + keepConsumer: false, + help: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--mode') { + const value = requiredValue(argv, index, arg); + if (value !== 'packed' && value !== 'source') { + throw new Error(`Invalid --mode value: ${value}`); + } + options.mode = value; + index += 1; + } else if (arg === '--target') { + const value = requiredValue(argv, index, arg); + if (value !== 'live' && value !== 'fixture') { + throw new Error(`Invalid --target value: ${value}`); + } + options.target = value; + index += 1; + } else if (arg === '--scenario') { + options.scenarioIds.push(requiredValue(argv, index, arg)); + index += 1; + } else if (arg === '--writes') { + options.writes = true; + } else if (arg === '--list') { + options.list = true; + } else if (arg === '--keep-consumer') { + options.keepConsumer = true; + } else if (arg === '--help' || arg === '-h') { + options.help = true; + } else { + throw new Error(`Unknown acceptance flag: ${arg}`); + } + } + return options; +} + +function printHelp(): void { + console.log(`Usage: tsx tests/acceptance/run.ts [options] + +Options: + --mode packed|source Run the packed install (default) or repo binary + --target live|fixture Use live credentials (default) or local fixtures + --scenario Run one scenario; repeat to select multiple + --writes Enable guarded live write scenarios + --list List registered scenarios + --keep-consumer Preserve the packed consumer directory + --help Show this help`); +} + +function selectedScenarios(ids: string[]): ScenarioDefinition[] { + if (ids.length === 0) return scenarios; + const byId = new Map(scenarios.map(scenario => [scenario.id, scenario])); + const unknown = ids.filter(id => !byId.has(id)); + if (unknown.length > 0) throw new Error(`Unknown scenario IDs: ${unknown.join(', ')}`); + return ids.map(id => byId.get(id)!); +} + +function summarize(results: ScenarioResult[]): ResultDocument['totals'] { + return { + passed: results.filter(result => result.status === 'passed').length, + failed: results.filter(result => result.status === 'failed').length, + skipped: results.filter(result => result.status === 'skipped').length, + unverified: results.filter(result => result.status === 'unverified').length, + }; +} + +// Unverified scenarios (independent verification could not confirm the +// outcome) must fail the run the same as `failed` ones, or CI reads a run +// with unresolved evidence as green. Skipped scenarios must NOT affect this: +// the live baseline legitimately skips guarded write scenarios. +function computeExitCode( + totals: ResultDocument['totals'], + artifactAudit: ResultDocument['artifactAudit'], + hasHarnessError: boolean, +): number { + return totals.failed > 0 || totals.unverified > 0 || !artifactAudit.passed || hasHarnessError + ? 1 + : 0; +} + +function invocationLabel(record: CommandRecord): string { + return record.argv.slice(1).join(' '); +} + +function summaryMarkdown( + document: ResultDocument, + cliInvocations: CommandRecord[], +): string { + const lines = [ + '# MainWP Control acceptance results', + '', + `- Run: ${document.runId}`, + `- Mode: ${document.mode}`, + `- Target: ${document.target}`, + `- Passed: ${document.totals.passed}`, + `- Failed: ${document.totals.failed}`, + `- Skipped: ${document.totals.skipped}`, + `- Unverified: ${document.totals.unverified}`, + `- Artifact audit: ${document.artifactAudit.passed ? 'passed' : 'failed'} — ${document.artifactAudit.message}`, + `- Exit code: ${computeExitCode(document.totals, document.artifactAudit, document.harnessError !== null)}${document.totals.failed === 0 && document.totals.unverified > 0 ? ' (unverified scenarios present)' : ''}`, + ...(document.harnessError ? [`- Harness error: ${document.harnessError}`] : []), + '', + '| Scenario | Status | Duration (ms) | Purpose |', + '| --- | --- | ---: | --- |', + ...document.scenarios.map(result => + `| ${result.id} | ${result.status} | ${result.durationMs} | ${result.purpose.replace(/\|/g, '\\|')} |` + ), + '', + '## 10 slowest scenarios', + '', + '| Scenario | Duration (ms) |', + '| --- | ---: |', + ...[...document.scenarios] + .sort((left, right) => right.durationMs - left.durationMs) + .slice(0, 10) + .map(result => `| ${result.id} | ${result.durationMs} |`), + '', + '## 10 slowest CLI invocations', + '', + '| Invocation | Duration (ms) |', + '| --- | ---: |', + ...[...cliInvocations] + .sort((left, right) => right.durationMs - left.durationMs) + .slice(0, 10) + .map(record => `| ${invocationLabel(record).replace(/\|/g, '\\|')} | ${record.durationMs} |`), + '', + ]; + return `${lines.join('\n')}\n`; +} + +function resultDocument( + artifacts: Artifacts, + options: RunnerOptions, + results: ScenarioResult[], + artifactAudit: ResultDocument['artifactAudit'], + harnessError: string | null = null, +): ResultDocument { + return { + runId: artifacts.runId, + mode: options.mode, + target: options.target, + totals: summarize(results), + scenarios: results, + artifactAudit, + harnessError, + }; +} + +function recordAuditValues( + values: Set, + credentials: AcceptanceCredentials, +): void { + values.add(credentials.username); + values.add(credentials.appPassword); + values.add(credentials.appPassword.replace(/\s+/g, '')); + values.add(new URL(credentials.dashboardUrl).origin); + // Basic Authorization headers carry the credential base64-encoded; a leaked + // header would otherwise slip past the plaintext checks above. + values.add( + Buffer.from(`${credentials.username}:${credentials.appPassword}`).toString('base64'), + ); + values.add( + Buffer.from( + `${credentials.username}:${credentials.appPassword.replace(/\s+/g, '')}`, + ).toString('base64'), + ); +} + +function auditArtifacts(runDir: string, auditValues: Set): string[] { + const findings: string[] = []; + const values = [...auditValues].filter(Boolean); + const visit = (directory: string): void => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(fullPath); + continue; + } + const content = fs.readFileSync(fullPath, 'utf8'); + if (values.some(value => content.includes(value))) { + findings.push(path.relative(runDir, fullPath)); + } + } + }; + visit(runDir); + return findings; +} + +function combineScenarioErrors(current: unknown, next: unknown): unknown { + if (current === undefined) return next; + const currentMessage = current instanceof Error ? current.message : String(current); + const nextMessage = next instanceof Error ? next.message : String(next); + return new Error(`${currentMessage}\nCleanup error: ${nextMessage}`); +} + +async function runScenario( + definition: ScenarioDefinition, + options: RunnerOptions, + liveCredentials: AcceptanceCredentials | null, + packedPackage: PackedPackage | null, + runner: CommandRunner, + artifacts: Artifacts, + redactor: Redactor, + auditValues: Set, + cliInvocations: CommandRecord[], +): Promise { + const started = performance.now(); + const assert = new AssertionRecorder(); + const skipped = (status: 'skipped' | 'unverified', reason: string): ScenarioResult => ({ + id: definition.id, + purpose: definition.purpose, + kind: definition.kind, + status, + durationMs: Math.round(performance.now() - started), + assertions: assert.results, + reason, + }); + + if (!definition.targets.includes(options.target)) { + return skipped('skipped', `Scenario does not run against the ${options.target} target.`); + } + if (definition.id === 'packed-integrity' && options.mode === 'source') { + return skipped('skipped', 'packed-integrity applies only to --mode packed.'); + } + + const guardUrl = liveCredentials?.dashboardUrl ?? 'http://127.0.0.1'; + if (definition.kind === 'write') { + const guardReason = getWriteGuardReason( + guardUrl, + options.writes, + options.target, + ); + if (guardReason) return skipped('skipped', guardReason); + } + + let mockServer: MockServer | null = null; + let configDir: ConfigDir | null = null; + let verifier: IndependentVerifier | null = null; + let context: ScenarioContext | null = null; + let scenarioError: unknown; + let deferredResult: { status: 'skipped' | 'unverified'; reason: string } | null = null; + const commandStart = runner.records.length; + artifacts.appendScenarioStderr(definition.id, ''); + + try { + let credentials: AcceptanceCredentials; + if (options.target === 'fixture') { + mockServer = new MockServer(); + await mockServer.start(); + programFixtureServer(mockServer); + credentials = { + dashboardUrl: mockServer.baseUrl, + username: FIXTURE_USERNAME, + appPassword: FIXTURE_APP_PASSWORD, + }; + } else { + if (!liveCredentials) throw new Error('Live credentials were not resolved'); + credentials = liveCredentials; + } + + recordAuditValues(auditValues, credentials); + redactor.add({ + username: credentials.username, + appPassword: credentials.appPassword, + dashboardUrl: credentials.dashboardUrl, + authorization: `Basic ${Buffer.from( + `${credentials.username}:${credentials.appPassword}`, + ).toString('base64')}`, + }); + + // TLS verification stays on unless the target is the local self-signed + // testbed (same host classes the write guard trusts). Pointing the + // harness at a non-local Dashboard must never silently disable TLS. + const skipTlsVerify = options.target === 'live' + && isWriteHostAllowed(new URL(credentials.dashboardUrl).hostname); + verifier = new IndependentVerifier(credentials, skipTlsVerify); + let precondition; + try { + precondition = await definition.preconditions?.({ + target: options.target, + mode: options.mode, + credentials, + verifier, + packedPackage, + }); + } catch (error) { + deferredResult = { + status: 'unverified', + reason: `Precondition failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } + if (!deferredResult && precondition?.status) { + deferredResult = { + status: precondition.status, + reason: precondition.reason ?? 'Precondition was not met.', + }; + } + + if (!deferredResult) { + configDir = await ConfigDir.create({ + profiles: [{ + name: 'acceptance', + dashboardUrl: credentials.dashboardUrl, + username: credentials.username, + ...(skipTlsVerify ? { skipSSLVerification: true } : {}), + }], + activeProfile: 'acceptance', + ...(precondition?.launch?.settings + ? { settings: precondition.launch.settings } + : {}), + }); + const binaryPath = options.mode === 'packed' + ? packedPackage?.binPath + : path.join(repoRoot, 'bin', 'run.js'); + if (!binaryPath) throw new Error('Packed mode did not produce an installed binary'); + const cli = new CLIInvoker({ + binaryPath, + cwd: options.mode === 'packed' ? packedPackage!.consumerDir : repoRoot, + runner, + configDir, + credentials, + ...(precondition?.launch?.env ? { env: precondition.launch.env } : {}), + onStderr: stderr => { + artifacts.appendScenarioStderr( + definition.id, + `\n--- CLI invocation stderr ---\n${stderr}`, + ); + }, + }); + context = { + cli, + verifier, + configDir, + mockServer, + config: { + target: options.target, + mode: options.mode, + dashboardUrl: credentials.dashboardUrl, + packageVersion: artifacts.manifest.packageVersion, + }, + packedPackage, + assert, + state: precondition?.state ?? {}, + }; + + artifacts.appendEvent(definition.id, 'runner-to-cli', { event: 'scenario-start' }); + await definition.run(context); + if (process.env['MAINWP_CONTROL_ACCEPTANCE_FORCE_FAILURE'] === definition.id) { + assert.equal('forced harness self-test failure', true, false); + } + } + } catch (error) { + scenarioError = combineScenarioErrors(scenarioError, error); + } finally { + if (context && definition.cleanup) { + try { + await definition.cleanup(context); + } catch (error) { + scenarioError = combineScenarioErrors(scenarioError, error); + } + } + await verifier?.close().catch(error => { + scenarioError = combineScenarioErrors(scenarioError, error); + }); + await configDir?.cleanup().catch(error => { + scenarioError = combineScenarioErrors(scenarioError, error); + }); + await mockServer?.stop().catch(error => { + scenarioError = combineScenarioErrors(scenarioError, error); + }); + + const scenarioCommands = runner.records.slice(commandStart).filter(record => { + const executable = record.argv[0]; + return executable === context?.cli.binaryPath; + }); + cliInvocations.push(...scenarioCommands); + artifacts.appendEvent(definition.id, 'cli-to-runner', { + event: 'scenario-finish', + commandCount: scenarioCommands.length, + }); + } + + if (scenarioError === undefined && deferredResult) { + return skipped(deferredResult.status, deferredResult.reason); + } + const failed = scenarioError !== undefined || assert.results.some(result => !result.pass); + return { + id: definition.id, + purpose: definition.purpose, + kind: definition.kind, + status: failed ? 'failed' : 'passed', + durationMs: Math.round(performance.now() - started), + assertions: assert.results, + ...(scenarioError === undefined + ? {} + : { error: scenarioError instanceof Error ? scenarioError.message : String(scenarioError) }), + }; +} + +async function runAcceptance(options: RunnerOptions): Promise { + if (options.help) { + printHelp(); + return 0; + } + if (options.list) { + for (const scenario of scenarios) { + console.log(`${scenario.id}\t${scenario.kind}\t${scenario.targets.join(',')}\t${scenario.purpose}`); + } + return 0; + } + const definitions = selectedScenarios(options.scenarioIds); + const liveCredentials = options.target === 'live' + ? resolveAcceptanceCredentials() + : null; + const redactor = new Redactor( + liveCredentials + ? { + username: liveCredentials.username, + appPassword: liveCredentials.appPassword, + dashboardUrl: liveCredentials.dashboardUrl, + authorization: `Basic ${Buffer.from( + `${liveCredentials.username}:${liveCredentials.appPassword}`, + ).toString('base64')}`, + } + : { + username: FIXTURE_USERNAME, + appPassword: FIXTURE_APP_PASSWORD, + }, + ); + const auditValues = new Set(); + if (liveCredentials) recordAuditValues(auditValues, liveCredentials); + else { + auditValues.add(FIXTURE_USERNAME); + auditValues.add(FIXTURE_APP_PASSWORD); + auditValues.add(FIXTURE_APP_PASSWORD.replace(/\s+/g, '')); + } + const runner = new CommandRunner(); + const artifacts = await createArtifacts( + repoRoot, + redactor, + runner, + options.mode, + options.target, + { + writes: options.writes, + scenarios: options.scenarioIds, + keepConsumer: options.keepConsumer, + }, + ); + const results: ScenarioResult[] = []; + const cliInvocations: CommandRecord[] = []; + let packedPackage: PackedPackage | null = null; + let harnessError: unknown; + let artifactAudit: ResultDocument['artifactAudit'] = { + passed: true, + message: 'No registered credential or Dashboard-origin values were found.', + }; + + try { + if (options.mode === 'packed') { + packedPackage = await packAndInstall( + repoRoot, + runner, + artifacts, + options.keepConsumer, + ); + } + for (const definition of definitions) { + const result = await runScenario( + definition, + options, + liveCredentials, + packedPackage, + runner, + artifacts, + redactor, + auditValues, + cliInvocations, + ); + results.push(result); + artifacts.writeJson( + 'results.json', + resultDocument(artifacts, options, results, artifactAudit), + ); + } + const document = resultDocument(artifacts, options, results, artifactAudit); + artifacts.writeJson('results.json', document); + artifacts.write('summary.md', summaryMarkdown(document, cliInvocations)); + artifacts.finish(); + } catch (error) { + harnessError = error; + const message = error instanceof Error ? error.message : String(error); + const document = resultDocument(artifacts, options, results, artifactAudit, message); + artifacts.writeJson('results.json', document); + artifacts.write('summary.md', summaryMarkdown(document, cliInvocations)); + artifacts.finish(); + } finally { + packedPackage?.cleanup(); + } + + const findings = auditArtifacts(artifacts.runDir, auditValues); + if (findings.length > 0) { + artifactAudit = { + passed: false, + message: `Sensitive values were found in: ${findings.join(', ')}`, + }; + } + const harnessMessage = harnessError instanceof Error + ? harnessError.message + : harnessError === undefined + ? null + : String(harnessError); + const finalDocument = resultDocument( + artifacts, + options, + results, + artifactAudit, + harnessMessage, + ); + artifacts.writeJson('results.json', finalDocument); + artifacts.write('summary.md', summaryMarkdown(finalDocument, cliInvocations)); + artifacts.finish(); + + const finalFindings = auditArtifacts(artifacts.runDir, auditValues); + if (finalFindings.length > 0 && artifactAudit.passed) { + artifactAudit = { + passed: false, + message: `Sensitive values were found in: ${finalFindings.join(', ')}`, + }; + const failedAuditDocument = resultDocument( + artifacts, + options, + results, + artifactAudit, + harnessMessage, + ); + artifacts.writeJson('results.json', failedAuditDocument); + artifacts.write('summary.md', summaryMarkdown(failedAuditDocument, cliInvocations)); + artifacts.finish(); + } + + console.log(`Acceptance artifacts: ${artifacts.runDir}`); + if (harnessError) { + console.error(redactor.redact(harnessError instanceof Error ? harnessError.message : String(harnessError))); + return 1; + } + return computeExitCode(summarize(results), artifactAudit, harnessError !== undefined); +} + +try { + process.exitCode = await runAcceptance(parseArgs(process.argv.slice(2))); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/tests/acceptance/scenarios/configuration.ts b/tests/acceptance/scenarios/configuration.ts new file mode 100644 index 0000000..25d7f7f --- /dev/null +++ b/tests/acceptance/scenarios/configuration.ts @@ -0,0 +1,27 @@ +import type { ScenarioDefinition } from './types.js'; + +export const packedIntegrity: ScenarioDefinition = { + id: 'packed-integrity', + purpose: 'Surface tarball content, installed-bin, and installed-version assertions.', + kind: 'read', + targets: ['fixture', 'live'], + preconditions: ctx => + ctx.mode === 'packed' + ? {} + : { status: 'skipped', reason: 'packed-integrity applies only to --mode packed.' }, + async run(ctx) { + if (!ctx.packedPackage) throw new Error('Packed package metadata was unavailable'); + for (const [name, value] of Object.entries(ctx.packedPackage.checks)) { + ctx.assert.equal(name, value, true); + } + ctx.assert.equal( + 'installed binary version output matches package version', + ctx.packedPackage.versionOutput.includes(ctx.config.packageVersion), + true + ); + ctx.assert.truthy('tarball sha256 recorded', /^[a-f0-9]{64}$/.test(ctx.packedPackage.sha256)); + ctx.assert.truthy('npm integrity recorded', ctx.packedPackage.integrity.startsWith('sha512-')); + }, +}; + +export const configurationScenarios: ScenarioDefinition[] = [packedIntegrity]; diff --git a/tests/acceptance/scenarios/errors.ts b/tests/acceptance/scenarios/errors.ts new file mode 100644 index 0000000..303fc41 --- /dev/null +++ b/tests/acceptance/scenarios/errors.ts @@ -0,0 +1,151 @@ +import { FIXTURE_SITE_ID } from '../fixtures.js'; +import type { ScenarioDefinition, ScenarioPreconditionContext } from './types.js'; + +interface ErrorEnvelope { + success: boolean; + error?: { + code?: string; + message?: string; + details?: unknown; + }; +} + +function errorEnvelope(value: unknown): ErrorEnvelope { + if (!value || typeof value !== 'object') throw new Error('CLI did not return a JSON envelope'); + return value as ErrorEnvelope; +} + +function requestInput(request: { body: unknown }): Record { + if (!request.body || typeof request.body !== 'object') return {}; + const input = (request.body as Record).input; + return input && typeof input === 'object' ? (input as Record) : {}; +} + +async function destructiveSitePrecondition(ctx: ScenarioPreconditionContext) { + if (ctx.target === 'fixture') return { state: { siteId: FIXTURE_SITE_ID } }; + const sites = await ctx.verifier.listSites(); + if (sites.length === 0) { + return { status: 'skipped' as const, reason: 'No site was available for the safety check.' }; + } + const site = sites[0]; + if (!site) throw new Error('Site discovery returned an inconsistent empty result'); + return { state: { siteId: site.id } }; +} + +export const notFoundInput: ScenarioDefinition = { + id: 'not-found-input', + purpose: 'Require a structured CLI error for a nonexistent site identifier.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const result = await ctx.cli.run([ + 'abilities', + 'run', + 'mainwp/get-site-v1', + '--input', + JSON.stringify({ site_id_or_domain: 99_999_999 }), + '--json', + ]); + const output = errorEnvelope(result.json); + ctx.assert.equal('not-found envelope fails', output.success, false); + if (ctx.config.target === 'live') { + // A live Dashboard answers a missing site id with HTTP 403, which this + // CLI classifies as an authorization failure rather than a lookup miss. + ctx.assert.equal('not-found exits with auth error', result.exitCode, 2); + ctx.assert.equal('not-found classification', output.error?.code, 'AUTH_ERROR'); + } else { + ctx.assert.equal('not-found exits with API error', result.exitCode, 4); + ctx.assert.equal('not-found classification', output.error?.code, 'NOT_FOUND'); + } + }, +}; + +export const invalidArgs: ScenarioDefinition = { + id: 'invalid-args', + purpose: 'Reject schema-invalid ability input with the stable input-error exit code.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const result = await ctx.cli.run([ + 'abilities', + 'run', + 'mainwp/count-sites-v1', + '--input', + JSON.stringify({ tag_ids: 'not-an-array' }), + '--json', + ]); + const output = errorEnvelope(result.json); + ctx.assert.equal('invalid args exit with input error', result.exitCode, 1); + ctx.assert.equal('invalid args envelope fails', output.success, false); + ctx.assert.equal('invalid args classification', output.error?.code, 'SCHEMA_VALIDATION_ERROR'); + }, +}; + +export const unknownAbility: ScenarioDefinition = { + id: 'unknown-ability', + purpose: 'Reject an ability absent from discovery with the documented CLI contract.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const result = await ctx.cli.run([ + 'abilities', + 'run', + 'mainwp/does-not-exist-v1', + '--json', + ]); + const output = errorEnvelope(result.json); + ctx.assert.equal('unknown ability exits with input error', result.exitCode, 1); + ctx.assert.equal('unknown ability envelope fails', output.success, false); + ctx.assert.equal('unknown ability classification', output.error?.code, 'INPUT_ERROR'); + ctx.assert.equal( + 'unknown ability message documents absence', + /ability not found/i.test(output.error?.message ?? ''), + true + ); + }, +}; + +export const nonInteractiveDestructiveWithoutForce: ScenarioDefinition = { + id: 'non-interactive-destructive-without-force', + purpose: 'Refuse destructive confirmation in a non-TTY process unless --force is explicit.', + kind: 'read', + targets: ['fixture', 'live'], + preconditions: destructiveSitePrecondition, + async run(ctx) { + const siteId = ctx.state.siteId; + if (typeof siteId !== 'number') throw new Error('Safety precondition did not provide a site id'); + const result = await ctx.cli.run([ + 'abilities', + 'run', + 'mainwp/delete-site-v1', + '--input', + JSON.stringify({ site_id_or_domain: siteId }), + '--confirm', + '--json', + ]); + const output = errorEnvelope(result.json); + ctx.assert.equal('non-interactive destructive call exits with input error', result.exitCode, 1); + ctx.assert.equal('non-interactive destructive envelope fails', output.success, false); + ctx.assert.equal('non-interactive destructive classification', output.error?.code, 'INPUT_ERROR'); + ctx.assert.equal( + 'non-interactive destructive message requires force', + /interactive|force/i.test(output.error?.message ?? ''), + true + ); + if (ctx.config.target === 'fixture') { + if (!ctx.mockServer) throw new Error('Fixture scenario did not receive a MockServer'); + const confirmRequests = ctx.mockServer + .getRecordedRequests() + .filter(request => request.path.includes('mainwp/delete-site-v1/run')) + .filter(request => requestInput(request).confirm === true); + ctx.assert.equal('no confirm-true execution reached the fixture', confirmRequests.length, 0); + } + }, +}; + +export const errorScenarios: ScenarioDefinition[] = [ + notFoundInput, + invalidArgs, + unknownAbility, + nonInteractiveDestructiveWithoutForce, +]; diff --git a/tests/acceptance/scenarios/index.ts b/tests/acceptance/scenarios/index.ts new file mode 100644 index 0000000..852549e --- /dev/null +++ b/tests/acceptance/scenarios/index.ts @@ -0,0 +1,22 @@ +import { configurationScenarios } from './configuration.js'; +import { errorScenarios } from './errors.js'; +import { readScenarios } from './read.js'; +import { safetyScenarios } from './safety.js'; +import type { ScenarioDefinition } from './types.js'; +import { writeScenarios } from './writes.js'; + +export const scenarios: ScenarioDefinition[] = [ + ...readScenarios, + ...errorScenarios, + ...safetyScenarios, + ...writeScenarios, + ...configurationScenarios, +]; + +const duplicateIds = scenarios + .map(scenario => scenario.id) + .filter((id, index, all) => all.indexOf(id) !== index); + +if (duplicateIds.length > 0) { + throw new Error(`Duplicate acceptance scenario IDs: ${duplicateIds.join(', ')}`); +} diff --git a/tests/acceptance/scenarios/read.ts b/tests/acceptance/scenarios/read.ts new file mode 100644 index 0000000..11238fb --- /dev/null +++ b/tests/acceptance/scenarios/read.ts @@ -0,0 +1,498 @@ +import type { VerifiedPluginResponse, VerifiedSite } from '../lib/verify.js'; +import type { + ScenarioDefinition, + ScenarioPreconditionContext, + ScenarioPreconditionResult, +} from './types.js'; + +interface CLIEnvelope { + success: boolean; + data?: T; + error?: { code?: string; message?: string }; +} + +interface AbilityExecution { + mode: string; + ability: string; + success: boolean; + data: T; +} + +interface Theme { + slug: string; + version: string; + active: boolean; + update_version?: string | null; +} + +interface ThemeResponse { + site_id: number; + site_url: string; + active_theme: string; + themes: Theme[]; + total: number; +} + +interface Update { + site_id: number; + site_url: string; + site_name: string; + type: string; + slug: string; + name: string; + current_version: string; + new_version: string; +} + +interface UpdatesResponse { + updates: Update[]; + total: number; + errors?: unknown[]; +} + +interface UpdatesSnapshot { + updates: Update[]; + errors: unknown[]; +} + +interface Tag { + id: number; + name: string; + sites_count: number; + sites_ids?: number[]; +} + +interface PaginatedResponse { + items: T[]; + total: number; +} + +function sorted(values: string[]): string[] { + return [...values].sort(); +} + +function envelope(value: unknown): CLIEnvelope { + if (!value || typeof value !== 'object') throw new Error('CLI did not return a JSON envelope'); + return value as CLIEnvelope; +} + +async function runAbility( + ctx: Parameters[0], + abilityName: string, + input: Record = {}, + assertionName = abilityName +): Promise { + const args = ['abilities', 'run', abilityName]; + if (Object.keys(input).length > 0) args.push('--input', JSON.stringify(input)); + args.push('--json'); + const result = await ctx.cli.run(args); + const output = envelope>(result.json); + ctx.assert.equal(`${assertionName} exits successfully`, result.exitCode, 0); + ctx.assert.equal(`${assertionName} envelope succeeds`, output.success, true); + ctx.assert.equal(`${assertionName} ability succeeds`, output.data?.success, true); + if (!output.data) throw new Error(`${abilityName} returned no data envelope`); + return output.data.data; +} + +/** + * Safety cap for the pagination loops below: a Dashboard reporting a wrong + * `total` while returning non-empty pages must fail loudly, not hang the run. + */ +const MAX_LIST_PAGES = 100; + +function assertPageWithinCap(page: number, abilityName: string): void { + if (page >= MAX_LIST_PAGES) { + throw new Error(`${abilityName} pagination did not terminate within ${MAX_LIST_PAGES} pages`); + } +} + +async function cliListAll( + ctx: Parameters[0], + abilityName: string +): Promise { + const items: T[] = []; + for (let page = 1; ; page += 1) { + const response = await runAbility>( + ctx, + abilityName, + { page, per_page: 100 }, + `${abilityName} page ${page}` + ); + items.push(...response.items); + if (items.length >= response.total || response.items.length === 0) return items; + assertPageWithinCap(page, abilityName); + } +} + +async function verifierListAll( + ctx: Parameters[0], + abilityName: string +): Promise { + const items: T[] = []; + for (let page = 1; ; page += 1) { + const response = (await ctx.verifier.execute(abilityName, { + page, + per_page: 100, + })) as PaginatedResponse; + items.push(...response.items); + if (items.length >= response.total || response.items.length === 0) return items; + assertPageWithinCap(page, abilityName); + } +} + +async function connectedSitePrecondition( + ctx: ScenarioPreconditionContext +): Promise { + const site = (await ctx.verifier.listSites()).find(candidate => candidate.status === 'connected'); + if (!site) { + return { status: 'skipped', reason: 'No connected site was available for the scenario.' }; + } + return { state: { site } }; +} + +function selectedSite(state: Record): VerifiedSite { + const site = state.site as VerifiedSite | undefined; + if (!site) throw new Error('Connected-site precondition did not provide a site'); + return site; +} + +function themeSignature(theme: Theme): string { + return [theme.slug, theme.version, theme.active, theme.update_version ?? ''].join(':'); +} + +function updateSignature(update: Update): string { + return [ + update.site_id, + update.site_url, + update.site_name, + update.type, + update.slug, + update.name, + update.current_version, + update.new_version, + ].join(':'); +} + +function tagSignature(tag: Tag): string { + return [ + tag.id, + tag.name, + tag.sites_count, + sorted((tag.sites_ids ?? []).map(String)).join(','), + ].join(':'); +} + +async function verifierListUpdates( + ctx: Parameters[0], + siteId: number +): Promise { + const updates: Update[] = []; + const errors: unknown[] = []; + for (let page = 1; ; page += 1) { + const response = (await ctx.verifier.execute('mainwp/list-updates-v1', { + site_ids_or_domains: [siteId], + page, + per_page: 200, + })) as UpdatesResponse; + updates.push(...response.updates); + errors.push(...(response.errors ?? [])); + if (updates.length >= response.total || response.updates.length === 0) { + return { updates, errors }; + } + assertPageWithinCap(page, 'mainwp/list-updates-v1'); + } +} + +async function cliListUpdates( + ctx: Parameters[0], + siteId: number +): Promise { + const updates: Update[] = []; + const errors: unknown[] = []; + for (let page = 1; ; page += 1) { + const response = await runAbility( + ctx, + 'mainwp/list-updates-v1', + { site_ids_or_domains: [siteId], page, per_page: 200 }, + `mainwp/list-updates-v1 page ${page}` + ); + updates.push(...response.updates); + errors.push(...(response.errors ?? [])); + if (updates.length >= response.total || response.updates.length === 0) { + return { updates, errors }; + } + assertPageWithinCap(page, 'mainwp/list-updates-v1'); + } +} + +export const startupDoctor: ScenarioDefinition = { + id: 'startup-doctor', + purpose: 'Prove the CLI starts, reaches the Dashboard, and emits a successful JSON envelope.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const result = await ctx.cli.run(['doctor', '--json']); + const output = envelope(result.json); + ctx.assert.equal('doctor exits successfully', result.exitCode, 0); + ctx.assert.equal('doctor envelope succeeds', output.success, true); + }, +}; + +// Discovery only lists catalog entries whose name is a namespaced, versioned +// identifier (ABILITY_NAME_PATTERN in src/core/abilities-executor.ts); other +// entries — such as WordPress-core abilities without a -vN suffix — are +// skipped with a warning. The independent expectation applies the same rule. +const LISTABLE_ABILITY_NAME = /^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._-]*-v[1-9]\d*$/; + +export const abilitiesList: ScenarioDefinition = { + id: 'abilities-list', + purpose: 'Cross-check the CLI ability catalog count and full-name set against a direct read.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const direct = await ctx.verifier.fetchCatalog(); + const listable = direct.filter( + ability => typeof ability.name === 'string' && LISTABLE_ABILITY_NAME.test(ability.name) + ); + const result = await ctx.cli.run(['abilities', 'list', '--json']); + const output = envelope<{ + abilities: Array<{ name: string }>; + total: number; + }>(result.json); + ctx.assert.equal('abilities list exits successfully', result.exitCode, 0); + ctx.assert.equal('abilities list envelope succeeds', output.success, true); + ctx.assert.equal('ability count matches direct catalog', output.data?.total, listable.length); + ctx.assert.deepEqual( + 'full ability name set matches direct catalog', + sorted((output.data?.abilities ?? []).map(ability => ability.name)), + sorted(listable.map(ability => ability.name)) + ); + }, +}; + +export const abilitiesInfo: ScenarioDefinition = { + id: 'abilities-info', + purpose: 'Cross-check CLI ability details against the independent catalog entry.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const direct = (await ctx.verifier.fetchCatalog()).find( + ability => ability.name === 'mainwp/list-sites-v1' + ); + if (!direct) throw new Error('Independent catalog did not contain mainwp/list-sites-v1'); + const result = await ctx.cli.run([ + 'abilities', + 'info', + 'mainwp/list-sites-v1', + '--json', + ]); + const output = envelope<{ + name: string; + label?: string; + description?: string; + category?: string; + annotations?: Record; + inputSchema?: Record; + outputSchema?: Record; + }>(result.json); + ctx.assert.equal('abilities info exits successfully', result.exitCode, 0); + ctx.assert.equal('abilities info envelope succeeds', output.success, true); + ctx.assert.equal('ability name matches catalog', output.data?.name, direct.name); + ctx.assert.equal('ability label matches catalog', output.data?.label, direct.label); + ctx.assert.equal('ability description matches catalog', output.data?.description, direct.description); + ctx.assert.equal('ability category matches catalog', output.data?.category, direct.category); + ctx.assert.deepEqual( + 'ability annotations match catalog', + output.data?.annotations ?? {}, + direct.meta?.annotations ?? {} + ); + ctx.assert.deepEqual('ability input schema matches catalog', output.data?.inputSchema, direct.input_schema); + ctx.assert.deepEqual('ability output schema matches catalog', output.data?.outputSchema, direct.output_schema); + }, +}; + +export const listSitesCrossCheck: ScenarioDefinition = { + id: 'list-sites-cross-check', + purpose: 'Cross-check CLI site discovery against independent reads before and after it.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const before = await ctx.verifier.listSites(); + const actual = await cliListAll(ctx, 'mainwp/list-sites-v1'); + const after = await ctx.verifier.listSites(); + const actualIds = actual.map(site => site.id).sort((a, b) => a - b); + const actualUrls = sorted(actual.map(site => site.url)); + ctx.assert.equal('site count matches first direct read', actual.length, before.length); + ctx.assert.deepEqual( + 'site id set matches first direct read', + actualIds, + before.map(site => site.id).sort((a, b) => a - b) + ); + ctx.assert.deepEqual('site URL set matches first direct read', actualUrls, sorted(before.map(site => site.url))); + ctx.assert.equal('site count matches second direct read', actual.length, after.length); + ctx.assert.deepEqual( + 'site id set matches second direct read', + actualIds, + after.map(site => site.id).sort((a, b) => a - b) + ); + ctx.assert.deepEqual('site URL set matches second direct read', actualUrls, sorted(after.map(site => site.url))); + }, +}; + +export const countSitesConsistency: ScenarioDefinition = { + id: 'count-sites-consistency', + purpose: 'Verify the CLI count-sites result equals an independent direct count.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const direct = await ctx.verifier.countSites(); + const actual = await runAbility<{ total: number }>(ctx, 'mainwp/count-sites-v1'); + ctx.assert.equal('CLI and independent site counts match', actual.total, direct); + }, +}; + +export const getSite: ScenarioDefinition = { + id: 'get-site', + purpose: 'Verify a discovered site detail response against an independent read.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + const sites = await ctx.verifier.listSites(); + const site = sites[0]; + if (!site) throw new Error('No sites were available'); + const direct = await ctx.verifier.getSite(site.id); + const actual = await runAbility(ctx, 'mainwp/get-site-v1', { + site_id_or_domain: site.id, + }); + ctx.assert.equal('site id matches', actual.id, direct.id); + ctx.assert.equal('site URL matches', actual.url, direct.url); + ctx.assert.equal('site name matches', actual.name, direct.name); + }, +}; + +export const sitePlugins: ScenarioDefinition = { + id: 'site-plugins', + purpose: 'Verify a site plugin inventory and a known slug against an independent read.', + kind: 'read', + targets: ['fixture', 'live'], + async run(ctx) { + let selected: { site: VerifiedSite; plugins: VerifiedPluginResponse } | undefined; + for (const site of await ctx.verifier.listSites()) { + const plugins = await ctx.verifier.getSitePlugins(site.id); + if (plugins.plugins.length > 0) { + selected = { site, plugins }; + break; + } + } + if (!selected) throw new Error('No site with plugins was available for the scenario'); + const actual = await runAbility(ctx, 'mainwp/get-site-plugins-v1', { + site_id_or_domain: selected.site.id, + }); + ctx.assert.deepEqual( + 'plugin inventory matches', + sorted(actual.plugins.map(plugin => `${plugin.slug}:${plugin.active}`)), + sorted(selected.plugins.plugins.map(plugin => `${plugin.slug}:${plugin.active}`)) + ); + const knownPlugin = selected.plugins.plugins[0]; + if (!knownPlugin) throw new Error('Selected plugin inventory was unexpectedly empty'); + const knownSlug = knownPlugin.slug; + ctx.assert.includes('known plugin slug is present', actual.plugins.map(plugin => plugin.slug), knownSlug); + if (ctx.config.target === 'fixture') { + ctx.assert.includes('fixture includes Hello Dolly', actual.plugins.map(plugin => plugin.slug), 'hello.php'); + } + }, +}; + +export const siteThemes: ScenarioDefinition = { + id: 'site-themes', + purpose: 'Cross-check a connected site theme inventory against an independent direct read.', + kind: 'read', + targets: ['live'], + preconditions: connectedSitePrecondition, + async run(ctx) { + const site = selectedSite(ctx.state); + const direct = (await ctx.verifier.execute('mainwp/get-site-themes-v1', { + site_id_or_domain: site.id, + })) as ThemeResponse; + const actual = await runAbility(ctx, 'mainwp/get-site-themes-v1', { + site_id_or_domain: site.id, + }); + ctx.assert.equal('theme site id matches', actual.site_id, direct.site_id); + ctx.assert.equal('theme site URL matches', actual.site_url, direct.site_url); + ctx.assert.equal('active theme matches', actual.active_theme, direct.active_theme); + ctx.assert.equal('theme total matches', actual.total, direct.total); + ctx.assert.deepEqual( + 'theme inventory matches', + sorted(actual.themes.map(themeSignature)), + sorted(direct.themes.map(themeSignature)) + ); + }, +}; + +export const listUpdatesCrossCheck: ScenarioDefinition = { + id: 'list-updates-cross-check', + purpose: 'Cross-check updates against independent direct reads that bracket the CLI snapshot.', + kind: 'read', + targets: ['live'], + preconditions: connectedSitePrecondition, + async run(ctx) { + const site = selectedSite(ctx.state); + const before = await verifierListUpdates(ctx, site.id); + const actual = await cliListUpdates(ctx, site.id); + const after = await verifierListUpdates(ctx, site.id); + const beforeSignatures = sorted(before.updates.map(updateSignature)); + const actualSignatures = sorted(actual.updates.map(updateSignature)); + const afterSignatures = sorted(after.updates.map(updateSignature)); + const directUnion = new Set([...beforeSignatures, ...afterSignatures]); + const directIntersection = beforeSignatures.filter(signature => afterSignatures.includes(signature)); + const actualSet = new Set(actualSignatures); + const oracleUnchanged = JSON.stringify(beforeSignatures) === JSON.stringify(afterSignatures); + ctx.assert.equal('bracketing direct update reads have no site errors', before.errors.length + after.errors.length, 0); + ctx.assert.equal('CLI update read has no site errors', actual.errors.length, 0); + ctx.assert.equal( + 'CLI updates are covered by the bracketing direct reads', + actualSignatures.every(signature => directUnion.has(signature)), + true + ); + ctx.assert.equal( + 'updates stable across direct reads are present through CLI', + directIntersection.every(signature => actualSet.has(signature)), + true + ); + ctx.assert.equal( + 'CLI updates match an unchanged direct snapshot', + oracleUnchanged ? JSON.stringify(actualSignatures) : 'changed', + oracleUnchanged ? JSON.stringify(beforeSignatures) : 'changed' + ); + }, +}; + +export const listTagsCrossCheck: ScenarioDefinition = { + id: 'list-tags-cross-check', + purpose: 'Cross-check the complete CLI tag list against an independent direct list-tags read.', + kind: 'read', + targets: ['live'], + async run(ctx) { + const direct = await verifierListAll(ctx, 'mainwp/list-tags-v1'); + const actual = await cliListAll(ctx, 'mainwp/list-tags-v1'); + ctx.assert.equal('tag count matches the direct list', actual.length, direct.length); + ctx.assert.deepEqual( + 'tag inventory matches the direct list', + sorted(actual.map(tagSignature)), + sorted(direct.map(tagSignature)) + ); + }, +}; + +export const readScenarios: ScenarioDefinition[] = [ + startupDoctor, + abilitiesList, + abilitiesInfo, + listSitesCrossCheck, + countSitesConsistency, + getSite, + sitePlugins, + siteThemes, + listUpdatesCrossCheck, + listTagsCrossCheck, +]; diff --git a/tests/acceptance/scenarios/safety.ts b/tests/acceptance/scenarios/safety.ts new file mode 100644 index 0000000..3ae9265 --- /dev/null +++ b/tests/acceptance/scenarios/safety.ts @@ -0,0 +1,133 @@ +import { FIXTURE_SITE_ID, programFixtureServer } from '../fixtures.js'; +import type { ScenarioDefinition } from './types.js'; + +interface CLIEnvelope { + success: boolean; + data?: T; + error?: { code?: string; message?: string }; +} + +interface SafetyOutput { + mode?: string; + ability?: string; + success?: boolean; + preview?: { + affected?: unknown[]; + summary?: string; + }; +} + +function envelope(value: unknown): CLIEnvelope { + if (!value || typeof value !== 'object') throw new Error('CLI did not return a JSON envelope'); + return value as CLIEnvelope; +} + +function requestInput(request: { body: unknown }): Record { + if (!request.body || typeof request.body !== 'object') return {}; + const input = (request.body as Record).input; + return input && typeof input === 'object' ? (input as Record) : {}; +} + +function deleteRequests(ctx: Parameters[0]) { + if (!ctx.mockServer) throw new Error('Fixture scenario did not receive a MockServer'); + return ctx.mockServer + .getRecordedRequests() + .filter(request => request.path.includes('mainwp/delete-site-v1/run')); +} + +function deleteArgs(...flags: string[]): string[] { + return [ + 'abilities', + 'run', + 'mainwp/delete-site-v1', + '--input', + JSON.stringify({ site_id_or_domain: FIXTURE_SITE_ID }), + ...flags, + '--json', + ]; +} + +export const dryRunPreview: ScenarioDefinition = { + id: 'dry-run-preview', + purpose: 'Prove a destructive dry run previews exactly once and never confirms.', + kind: 'read', + targets: ['fixture'], + async run(ctx) { + const result = await ctx.cli.run(deleteArgs('--dry-run')); + const output = envelope(result.json); + const requests = deleteRequests(ctx); + const dryRuns = requests.filter(request => requestInput(request).dry_run === true); + const confirms = requests.filter(request => requestInput(request).confirm === true); + ctx.assert.equal('dry run exits successfully', result.exitCode, 0); + ctx.assert.equal('dry run envelope succeeds', output.success, true); + ctx.assert.equal('dry run reports preview mode', output.data?.mode, 'preview'); + ctx.assert.equal('dry run ability succeeds', output.data?.success, true); + ctx.assert.truthy('dry run includes a preview summary', output.data?.preview?.summary); + ctx.assert.equal('exactly one delete request reached the fixture in total', requests.length, 1); + ctx.assert.equal('exactly one dry-run request reached the fixture', dryRuns.length, 1); + ctx.assert.equal('no confirm request reached the fixture', confirms.length, 0); + ctx.assert.equal('dry-run request uses POST', dryRuns[0]?.method, 'POST'); + ctx.assert.equal('dry-run request retains the site id', requestInput(dryRuns[0] ?? { body: undefined }).site_id_or_domain, FIXTURE_SITE_ID); + ctx.assert.equal('dry-run request omits user_confirmed', requestInput(dryRuns[0] ?? { body: undefined }).user_confirmed, undefined); + }, +}; + +export const previewThenConfirm: ScenarioDefinition = { + id: 'preview-then-confirm', + purpose: 'Prove force skips only the prompt: preview first, then exactly one confirmation.', + // 'write': the scenario completes a confirmed deletion. It only ever runs + // against the fixture (the write guard passes fixture targets through), + // but if a live target is ever added, the guard must apply. + kind: 'write', + targets: ['fixture'], + async run(ctx) { + const result = await ctx.cli.run(deleteArgs('--confirm', '--force')); + const output = envelope(result.json); + const requests = deleteRequests(ctx); + const dryRuns = requests.filter(request => requestInput(request).dry_run === true); + const confirms = requests.filter(request => requestInput(request).confirm === true); + ctx.assert.equal('confirmed destructive flow exits successfully', result.exitCode, 0); + ctx.assert.equal('confirmed destructive envelope succeeds', output.success, true); + ctx.assert.equal('confirmed destructive flow reports execute mode', output.data?.mode, 'execute'); + ctx.assert.equal('confirmed destructive ability succeeds', output.data?.success, true); + ctx.assert.truthy('confirmed destructive flow includes preview', output.data?.preview); + ctx.assert.equal('exactly two delete requests reached the fixture', requests.length, 2); + ctx.assert.equal('exactly one preview request reached the fixture', dryRuns.length, 1); + ctx.assert.equal('exactly one confirm request reached the fixture', confirms.length, 1); + ctx.assert.equal('preview request is first', requestInput(requests[0] ?? { body: undefined }).dry_run, true); + ctx.assert.equal('confirm request is second', requestInput(requests[1] ?? { body: undefined }).confirm, true); + ctx.assert.equal('confirm request records user confirmation', requestInput(confirms[0] ?? { body: undefined }).user_confirmed, true); + ctx.assert.equal('confirm request retains the site id', requestInput(confirms[0] ?? { body: undefined }).site_id_or_domain, FIXTURE_SITE_ID); + }, +}; + +export const previewFailureFailsClosed: ScenarioDefinition = { + id: 'preview-failure-fails-closed', + purpose: 'Prove a failed destructive preview returns PREVIEW_FAILED and never confirms.', + // 'write': the invocation requests a confirmed deletion; only the fixture's + // programmed preview failure keeps it from executing. + kind: 'write', + targets: ['fixture'], + async run(ctx) { + if (!ctx.mockServer) throw new Error('Fixture scenario did not receive a MockServer'); + await programFixtureServer(ctx.mockServer, { previewFailure: true }); + const result = await ctx.cli.run(deleteArgs('--confirm', '--force')); + const output = envelope(result.json); + const requests = deleteRequests(ctx); + const dryRuns = requests.filter(request => requestInput(request).dry_run === true); + const confirms = requests.filter(request => requestInput(request).confirm === true); + ctx.assert.equal('preview failure exits with API error', result.exitCode, 4); + ctx.assert.equal('preview failure envelope fails', output.success, false); + ctx.assert.equal('preview failure classification', output.error?.code, 'PREVIEW_FAILED'); + ctx.assert.equal('preview failure message identifies preview', /preview/i.test(output.error?.message ?? ''), true); + ctx.assert.equal('exactly one delete request reached the fixture in total', requests.length, 1); + ctx.assert.equal('exactly one failed preview reached the fixture', dryRuns.length, 1); + ctx.assert.equal('failed preview sends no confirm request', confirms.length, 0); + }, +}; + +export const safetyScenarios: ScenarioDefinition[] = [ + dryRunPreview, + previewThenConfirm, + previewFailureFailsClosed, +]; diff --git a/tests/acceptance/scenarios/types.ts b/tests/acceptance/scenarios/types.ts new file mode 100644 index 0000000..0ad6d6a --- /dev/null +++ b/tests/acceptance/scenarios/types.ts @@ -0,0 +1,213 @@ +import { isDeepStrictEqual } from 'node:util'; +import type { ConfigDir } from '../../../src/__tests__/process/fixtures/config-dir.js'; +import type { MockServer } from '../../../src/__tests__/process/fixtures/mock-server.js'; +import type { CLIInvoker } from '../lib/cli.js'; +import type { AcceptanceCredentials } from '../lib/env.js'; +import type { PackedPackage } from '../lib/pack.js'; +import type { + IndependentVerifier, + VerifiedPluginResponse, + VerifiedSite, +} from '../lib/verify.js'; + +export type AcceptanceTarget = 'live' | 'fixture'; +export type AcceptanceMode = 'packed' | 'source'; +export type ScenarioStatus = 'passed' | 'failed' | 'skipped' | 'unverified'; + +export interface AssertionResult { + name: string; + expected: unknown; + actual: unknown; + pass: boolean; +} + +function recordedValue(value: unknown): unknown { + return value === undefined ? '' : value; +} + +export class AssertionRecorder { + readonly results: AssertionResult[] = []; + + equal(name: string, actual: unknown, expected: unknown): void { + this.results.push({ + name, + expected: recordedValue(expected), + actual: recordedValue(actual), + pass: Object.is(actual, expected), + }); + } + + deepEqual(name: string, actual: unknown, expected: unknown): void { + this.results.push({ + name, + expected: recordedValue(expected), + actual: recordedValue(actual), + pass: isDeepStrictEqual(actual, expected), + }); + } + + truthy(name: string, actual: unknown, expected = true): void { + this.results.push({ + name, + expected, + actual: recordedValue(actual), + pass: Boolean(actual), + }); + } + + lessThan(name: string, actual: number, expected: number): void { + this.results.push({ + name, + expected, + actual, + pass: actual < expected, + }); + } + + includes(name: string, values: unknown[], expected: unknown): void { + this.results.push({ + name, + expected: recordedValue(expected), + actual: values, + pass: values.includes(expected), + }); + } +} + +export interface ScenarioLaunch { + env?: Record; + settings?: Record; + omitCredentialEnv?: boolean; +} + +export interface ScenarioPreconditionContext { + target: AcceptanceTarget; + mode: AcceptanceMode; + credentials: AcceptanceCredentials; + verifier: IndependentVerifier; + packedPackage: PackedPackage | null; +} + +export interface ScenarioPreconditionResult { + status?: 'skipped' | 'unverified'; + reason?: string; + launch?: ScenarioLaunch; + state?: Record; +} + +export interface ScenarioContext { + cli: CLIInvoker; + verifier: IndependentVerifier; + configDir: ConfigDir; + /** Present only for fixture-target scenarios. */ + mockServer: MockServer | null; + config: { + target: AcceptanceTarget; + mode: AcceptanceMode; + dashboardUrl: string; + packageVersion: string; + }; + packedPackage: PackedPackage | null; + assert: AssertionRecorder; + state: Record; +} + +export interface ScenarioDefinition { + id: string; + purpose: string; + kind: 'read' | 'write'; + targets: AcceptanceTarget[]; + preconditions?: ( + ctx: ScenarioPreconditionContext + ) => Promise | ScenarioPreconditionResult; + run(ctx: ScenarioContext): Promise; + cleanup?(ctx: ScenarioContext): Promise; +} + +export interface ScenarioResult { + id: string; + purpose: string; + kind: 'read' | 'write'; + status: ScenarioStatus; + durationMs: number; + assertions: AssertionResult[]; + reason?: string; + error?: string; +} + +// Far beyond any testbed (10k sites at per_page 100); a Dashboard paging bug +// must fail the run loudly instead of hanging it. +const MAX_SITE_PAGES = 100; + +export async function cliListAllSites(cli: CLIInvoker): Promise { + const sites: VerifiedSite[] = []; + let page = 1; + for (;;) { + const result = await cli.run<{ + mode: string; + ability: string; + success: boolean; + data: { items: VerifiedSite[]; total: number }; + }>([ + 'abilities', + 'run', + 'mainwp/list-sites-v1', + '--input', + JSON.stringify({ page, per_page: 100 }), + '--json', + ]); + if (result.exitCode !== 0 || result.json?.success !== true) { + throw new Error(`mainwp/list-sites-v1 failed: ${result.stderr || result.stdout}`); + } + const response = result.json.data?.data; + if ( + !response || + !Array.isArray(response.items) || + typeof response.total !== 'number' || + !Number.isFinite(response.total) || + response.total < 0 + ) { + throw new Error('mainwp/list-sites-v1 returned an unexpected CLI envelope'); + } + sites.push(...response.items); + if (sites.length >= response.total || response.items.length === 0) return sites; + if (page >= MAX_SITE_PAGES) { + throw new Error( + `mainwp/list-sites-v1 pagination did not terminate within ${MAX_SITE_PAGES} pages` + ); + } + page += 1; + } +} + +export async function findSiteWithPlugins( + verifier: IndependentVerifier +): Promise<{ site: VerifiedSite; plugins: VerifiedPluginResponse }> { + for (const site of await verifier.listSites()) { + const plugins = await verifier.getSitePlugins(site.id); + if (plugins.plugins.length > 0) return { site, plugins }; + } + throw new Error('No site with plugins was available for the scenario'); +} + +export async function findHelloDolly( + verifier: IndependentVerifier, + preferredSlug?: string +): Promise<{ site: VerifiedSite; slug: string; active: boolean } | null> { + const safeSlugs = [preferredSlug, 'hello.php', 'hello-dolly/hello.php'].filter( + (value): value is string => Boolean(value) + ); + const inventories = await Promise.all( + (await verifier.listSites()).map(async site => ({ + site, + plugins: (await verifier.getSitePlugins(site.id)).plugins, + })) + ); + for (const slug of safeSlugs) { + for (const inventory of inventories) { + const plugin = inventory.plugins.find(candidate => candidate.slug === slug); + if (plugin) return { site: inventory.site, slug: plugin.slug, active: plugin.active }; + } + } + return null; +} diff --git a/tests/acceptance/scenarios/writes.ts b/tests/acceptance/scenarios/writes.ts new file mode 100644 index 0000000..cfb06d6 --- /dev/null +++ b/tests/acceptance/scenarios/writes.ts @@ -0,0 +1,174 @@ +import type { VerifiedSite } from '../lib/verify.js'; +import type { ScenarioDefinition } from './types.js'; + +interface CLIEnvelope { + success: boolean; + data?: T; + error?: { code?: string; message?: string }; +} + +interface AbilityExecution { + success?: boolean; +} + +interface SelectedPlugin { + site: VerifiedSite; + slug: string; + active: boolean; +} + +function envelope(value: unknown): CLIEnvelope { + if (!value || typeof value !== 'object') throw new Error('CLI did not return a JSON envelope'); + return value as CLIEnvelope; +} + +async function waitForSyncAdvance( + ctx: Parameters[0], + siteId: number, + before: string | null | undefined +): Promise { + for (let attempt = 0; attempt < 10; attempt += 1) { + const current = await ctx.verifier.getSite(siteId); + if (current.last_sync && current.last_sync !== before) return current.last_sync; + await new Promise(resolve => setTimeout(resolve, 500)); + } + return (await ctx.verifier.getSite(siteId)).last_sync; +} + +async function findTogglePlugin( + ctx: Parameters>[0] +): Promise { + const preferred = process.env.MAINWP_CONTROL_ACCEPTANCE_TOGGLE_PLUGIN; + const safeSlugs = [preferred, 'hello.php', 'hello-dolly/hello.php'].filter( + (value): value is string => Boolean(value) + ); + const inventories = await Promise.all( + (await ctx.verifier.listSites()).map(async site => ({ + site, + plugins: (await ctx.verifier.getSitePlugins(site.id)).plugins, + })) + ); + for (const slug of safeSlugs) { + for (const inventory of inventories) { + const plugin = inventory.plugins.find(candidate => candidate.slug === slug); + if (plugin) return { site: inventory.site, slug: plugin.slug, active: plugin.active }; + } + } + return null; +} + +async function setPluginActive( + ctx: Parameters[0], + siteId: number, + slug: string, + active: boolean, + recordAssertions = true +): Promise { + const abilityName = active + ? 'mainwp/activate-site-plugins-v1' + : 'mainwp/deactivate-site-plugins-v1'; + const result = await ctx.cli.run([ + 'abilities', + 'run', + abilityName, + '--input', + JSON.stringify({ site_id_or_domain: siteId, plugins: [slug] }), + '--confirm', + '--force', + '--json', + ]); + const output = envelope(result.json); + if (recordAssertions) { + ctx.assert.equal(`${abilityName} exits successfully`, result.exitCode, 0); + ctx.assert.equal(`${abilityName} envelope succeeds`, output.success, true); + ctx.assert.equal(`${abilityName} ability succeeds`, output.data?.success, true); + } + if (result.exitCode !== 0 || !output.success) { + throw new Error(`${abilityName} failed: ${output.error?.message ?? result.stderr}`); + } +} + +export const syncSite: ScenarioDefinition = { + id: 'sync-site', + purpose: 'Sync one discovered site and independently verify its last-sync timestamp advances.', + kind: 'write', + targets: ['live'], + async run(ctx) { + const sites = await ctx.verifier.listSites(); + const site = sites[0]; + if (!site) throw new Error('No site was available to sync'); + const before = await ctx.verifier.getSite(site.id); + const result = await ctx.cli.run([ + 'abilities', + 'run', + 'mainwp/sync-sites-v1', + '--input', + JSON.stringify({ site_ids_or_domains: [site.id] }), + '--json', + ]); + const output = envelope(result.json); + ctx.assert.equal('sync-site exits successfully', result.exitCode, 0); + ctx.assert.equal('sync-site envelope succeeds', output.success, true); + ctx.assert.equal('sync-site ability succeeds', output.data?.success, true); + const after = await waitForSyncAdvance(ctx, site.id, before.last_sync); + ctx.assert.truthy('last_sync advanced', after && after !== before.last_sync); + }, +}; + +export const pluginToggleRoundtrip: ScenarioDefinition = { + id: 'plugin-toggle-roundtrip', + purpose: 'Deactivate and reactivate an allowed plugin with independent state verification.', + kind: 'write', + targets: ['live'], + preconditions: async ctx => { + const catalog = await ctx.verifier.fetchCatalog(); + const requiredAbilities = [ + 'mainwp/deactivate-site-plugins-v1', + 'mainwp/activate-site-plugins-v1', + ]; + const missingAbilities = requiredAbilities.filter( + name => !catalog.some(ability => ability.name === name) + ); + if (missingAbilities.length > 0) { + return { + status: 'unverified', + reason: `Required plugin roundtrip abilities are unavailable: ${missingAbilities.join(', ')}`, + }; + } + const plugin = await findTogglePlugin(ctx); + if (!plugin?.active) { + return { status: 'skipped', reason: 'No active allowed toggle plugin was discovered.' }; + } + return { state: { plugin } }; + }, + async run(ctx) { + const plugin = ctx.state.plugin as SelectedPlugin | undefined; + if (!plugin) throw new Error('Plugin precondition did not provide a plugin'); + + await setPluginActive(ctx, plugin.site.id, plugin.slug, false); + const inactive = await ctx.verifier.getSitePlugins(plugin.site.id); + ctx.assert.equal( + 'plugin is independently inactive', + inactive.plugins.find(candidate => candidate.slug === plugin.slug)?.active, + false + ); + + await setPluginActive(ctx, plugin.site.id, plugin.slug, true); + const restored = await ctx.verifier.getSitePlugins(plugin.site.id); + ctx.assert.equal( + 'plugin is independently active again', + restored.plugins.find(candidate => candidate.slug === plugin.slug)?.active, + true + ); + }, + async cleanup(ctx) { + const plugin = ctx.state.plugin as SelectedPlugin | undefined; + if (!plugin) return; + const current = await ctx.verifier.getSitePlugins(plugin.site.id); + if (current.plugins.find(candidate => candidate.slug === plugin.slug)?.active === false) { + await setPluginActive(ctx, plugin.site.id, plugin.slug, true, false); + } + }, +}; + +export const writeScenarios: ScenarioDefinition[] = [syncSite, pluginToggleRoundtrip]; diff --git a/vitest.config.ts b/vitest.config.ts index ee04806..9463394 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -5,8 +5,13 @@ export default defineConfig({ globals: true, environment: 'node', include: ['src/**/*.test.ts'], + exclude: [ + 'src/__tests__/process/live-api.test.ts', + 'src/__tests__/process/live-workflow-docs.test.ts', + ], // Process tests spawn CLI as child process; Windows CI needs extra time - testTimeout: 30_000, + // (child boot can exceed 15s cold there, see cli-runner DEFAULT_TIMEOUT) + testTimeout: process.platform === 'win32' ? 90_000 : 30_000, coverage: { provider: 'v8', reporter: ['text', 'html'], diff --git a/vitest.process.config.ts b/vitest.process.config.ts index afad23f..4015dc0 100644 --- a/vitest.process.config.ts +++ b/vitest.process.config.ts @@ -5,7 +5,12 @@ export default defineConfig({ globals: true, environment: 'node', include: ['src/__tests__/process/**/*.test.ts'], - testTimeout: 30_000, + exclude: [ + 'src/__tests__/process/live-api.test.ts', + 'src/__tests__/process/live-workflow-docs.test.ts', + ], + // Windows child boot can exceed 15s cold; see cli-runner DEFAULT_TIMEOUT + testTimeout: process.platform === 'win32' ? 90_000 : 30_000, hookTimeout: 30_000, }, });