From 3a1d9d1de4cc64693ea148cfb5b1645c51fd1922 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 5 Aug 2026 22:46:17 +0100 Subject: [PATCH] fix: normalize Copilot reasoning effort --- .../development/agent_parameter_comparison.md | 41 +++++++++------ .../adapters/copilot_cli_adapter.py | 16 ++++++ tests/unit/test_copilot_cli_stream.py | 52 +++++++++++++++++++ 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/docs/development/agent_parameter_comparison.md b/docs/development/agent_parameter_comparison.md index 2dae9c5..899facd 100644 --- a/docs/development/agent_parameter_comparison.md +++ b/docs/development/agent_parameter_comparison.md @@ -1,7 +1,7 @@ # Agent CLI Parameter Comparison Comparison of headless/non-interactive configuration across supported CLI coding agents. -Last updated: 2026-08-02 +Last updated: 2026-08-05 > The summary matrix below has no Pi or Cursor column (it predates both adapters); see the > per-agent detail sections, the model-discovery section, and the `disallowed_tools` table. @@ -20,9 +20,9 @@ Last updated: 2026-08-02 | **Allowed tools** | `--allowed-tools` | No direct flag | `--allow-tool`, `--available-tools` | `tools` in config | | **Disallowed tools** | `--disallowed-tools` | `web_search` config only | `--deny-tool` | `OPENCODE_PERMISSION` env / `permission` config | | **Stream output** | `--output-format stream-json` | `--json` (NDJSON) | `--output-format=json` (JSONL) | `--format json` | -| **Working dir** | cwd + `--add-dir` | `--cd` / `-C` | cwd (no flag) | cwd (no flag) | +| **Working dir** | cwd + `--add-dir` | `--cd` / `-C` | cwd / `-C` | cwd | | **System prompt** | `--system-prompt` / `--append-system-prompt` | No flag (files only) | No flag (files only) | `instructions` in config | -| **Budget** | `--max-budget-usd` | No direct flag | No direct flag | No direct flag | +| **Budget** | `--max-budget-usd` | No direct flag | `--max-ai-credits` | No direct flag | | **Auto-approve** | `--dangerously-skip-permissions` | `--yolo` | `--yolo` / `--allow-all` | Auto in `run` mode | | **Session resume** | `--resume` | `exec resume ` | `--resume` | `-s ` | @@ -123,23 +123,32 @@ real discovery commands. ## Copilot CLI (GitHub) -- **Headless mode**: `-p` / `--prompt` for one-shot, `--acp --stdio` for programmatic JSON-RPC -- **Model**: `--model` (default `claude-sonnet-4.5`) -- **Effort**: `--effort low|medium|high` or `--reasoning-effort low|medium|high` -- **Allowed tools**: `--allow-tool`, `--deny-tool`, `--available-tools`, `--excluded-tools`, `--allow-all-tools` -- **Auto-approve**: `--allow-all` / `--yolo`, `--autopilot` +- **Headless mode**: `-p` / `--prompt` for one-shot; `--acp` for Agent Client Protocol +- **Model**: `--model `; use `auto` to let Copilot choose +- **Effort**: `--effort` / `--reasoning-effort` with choices + `none|minimal|low|medium|high|xhigh|max`. AgentShell accepts effort values + case-insensitively, validates them against these choices, and passes the normalized lowercase + value to Copilot via `--effort`. +- **Allowed tools**: `--allow-tool`, `--deny-tool`, `--available-tools`, + `--excluded-tools`, `--allow-all-tools` +- **Auto-approve**: `--allow-all-tools`; `--allow-all` / `--yolo` also grant path and URL + permissions +- **Agent mode**: `--mode interactive|plan|autopilot`; `--autopilot` is the shortcut - **Output format**: `--output-format=json` (JSONL) - **Silent mode**: `--silent` suppresses stats, prints only response -- **Working directory**: Uses cwd (no flag), ACP mode uses `newSession` parameter -- **System prompt**: No CLI flag, uses `.github/copilot-instructions.md` and `AGENTS.md` files -- **Budget**: No per-run flag, auto-compacts at 95% token limit +- **Working directory**: CLI supports `-C `; AgentShell sets subprocess `cwd` +- **System prompt**: No CLI flag; uses `.github/copilot-instructions.md` and `AGENTS.md` files +- **AI credit budget**: `--max-ai-credits ` limits credits for the session +- **Token budget**: No per-run flag; auto-compacts at 95% of the token limit - **Path permissions**: `--allow-all-paths`, `--disallow-temp-dir` - **URL permissions**: `--allow-all-urls`, `--allow-url`, `--deny-url` -- **Session**: `--resume`, `--continue`, plus `--session-id=` to start a NEW session under - a chosen id. The adapter resumes with `--resume `. The resumed run reports the SAME - `sessionId`, and an unknown id is rejected ("No session, task, or name matched"), so id - identity is real evidence the CLI continued that session (measured 2026-07-26) -- **ACP mode**: `--acp --stdio` or `--acp --port 3000` for JSON-RPC integration +- **Session**: `--resume`, `--continue`, and `--session-id `. The latter resumes an existing + session or task, or assigns a UUID to a new session. AgentShell resumes with `--resume `. + The resumed run reports the SAME `sessionId`, and an unknown id is rejected + ("No session, task, or name matched"), so id identity is real evidence the CLI continued that + session (measured 2026-07-26). +- **ACP mode**: `--acp` uses stdio by default. Copilot CLI 1.0.78 also accepts hidden + `--stdio` and `--port ` transport options; they are not shown by `copilot --help`. ## OpenCode diff --git a/src/agent_shell/adapters/copilot_cli_adapter.py b/src/agent_shell/adapters/copilot_cli_adapter.py index 49557b7..99ded86 100644 --- a/src/agent_shell/adapters/copilot_cli_adapter.py +++ b/src/agent_shell/adapters/copilot_cli_adapter.py @@ -44,6 +44,21 @@ "edit": ["write"], } +_COPILOT_EFFORTS = ("none", "minimal", "low", "medium", "high", "xhigh", "max") + + +def _normalize_effort(effort: str | None) -> str | None: + if effort is None: + return None + + normalized = effort.lower() if isinstance(effort, str) else None + if normalized not in _COPILOT_EFFORTS: + choices = ", ".join(_COPILOT_EFFORTS) + raise ValueError( + f"Unsupported Copilot effort {effort!r}; accepted choices: {choices}" + ) + return normalized + def _json_rpc_frame(message: dict) -> bytes: payload = json.dumps(message, separators=(",", ":")).encode("utf-8") @@ -160,6 +175,7 @@ async def stream( session_id: str | None = None, disallowed_tools: list[str] | None = None, ) -> AsyncIterator[StreamEvent]: + effort = _normalize_effort(effort) cmd = [ "copilot", "-p", prompt, "--output-format", "json", diff --git a/tests/unit/test_copilot_cli_stream.py b/tests/unit/test_copilot_cli_stream.py index 14f7381..05b4782 100644 --- a/tests/unit/test_copilot_cli_stream.py +++ b/tests/unit/test_copilot_cli_stream.py @@ -115,6 +115,58 @@ async def test_includes_effort_flag_in_command(self): assert "--effort" in cmd_args assert cmd_args[cmd_args.index("--effort") + 1] == "high" + @pytest.mark.parametrize( + ("effort", "expected"), + [ + ("none", "none"), + ("minimal", "minimal"), + ("low", "low"), + ("medium", "medium"), + ("high", "high"), + ("xhigh", "xhigh"), + ("max", "max"), + ("NONE", "none"), + ("Minimal", "minimal"), + ("HIGH", "high"), + ("xHiGh", "xhigh"), + ("MAX", "max"), + ("Max", "max"), + ], + ) + async def test_accepts_and_normalizes_supported_effort_values( + self, effort: str, expected: str + ): + # Arrange + adapter = CopilotCLIAdapter() + ndjson = [MESSAGE_DELTA_EVENT, RESULT_EVENT_SUCCESS] + mock_process = _make_mock_process(ndjson) + + # Act + with patch("asyncio.create_subprocess_exec", return_value=mock_process) as mock_exec: + async for _ in adapter.stream(cwd="/tmp", prompt="test", effort=effort): + pass + + # Assert + cmd_args = mock_exec.call_args[0] + assert cmd_args[cmd_args.index("--effort") + 1] == expected + + async def test_rejects_unsupported_effort_before_spawning(self): + # Arrange + adapter = CopilotCLIAdapter() + + # Act + with patch("asyncio.create_subprocess_exec") as mock_exec: + with pytest.raises(ValueError) as error: + async for _ in adapter.stream(cwd="/tmp", prompt="test", effort="turbo"): + pass + + # Assert + assert str(error.value) == ( + "Unsupported Copilot effort 'turbo'; accepted choices: " + "none, minimal, low, medium, high, xhigh, max" + ) + mock_exec.assert_not_called() + async def test_omits_effort_flag_when_none(self): # Arrange adapter = CopilotCLIAdapter()