diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json new file mode 100644 index 0000000..987947d --- /dev/null +++ b/.github/plugin/marketplace.json @@ -0,0 +1,32 @@ +{ + "name": "agent-trace", + "owner": { + "name": "Siddhant Khare", + "email": "siddhantkhare2694@gmail.com" + }, + "metadata": { + "description": "GitHub Copilot CLI plugins for agent observability.", + "version": "0.94.0" + }, + "plugins": [ + { + "name": "agent-strace", + "description": "Capture, replay, and analyze GitHub Copilot CLI agent sessions.", + "version": "0.94.0", + "source": ".", + "author": { + "name": "Siddhant Khare", + "email": "siddhantkhare2694@gmail.com" + }, + "homepage": "https://github.com/Siddhant-K-code/agent-trace", + "repository": "https://github.com/Siddhant-K-code/agent-trace", + "license": "MIT", + "keywords": [ + "agents", + "copilot-cli", + "observability", + "tracing" + ] + } + ] +} diff --git a/README.md b/README.md index 482576d..2d1f913 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,17 @@ agent-strace replay # replay the latest Full config and JSON: [docs/setup.md](docs/setup.md) +### GitHub Copilot CLI plugin + +After installing `agent-strace`, install the repository as a Copilot CLI plugin to enable tracing hooks plus the trace analyst agent and skill: + +```bash +copilot plugin marketplace add Siddhant-K-code/agent-trace +copilot plugin install agent-strace@agent-trace +``` + +See the [Copilot CLI plugin guide](docs/copilot-plugin.md) for verification and usage. + **Option 2: MCP proxy** — wraps any MCP server, works with Cursor, Windsurf, and Copilot Desktop MCP servers ```bash diff --git a/agents/trace-analyst.agent.md b/agents/trace-analyst.agent.md new file mode 100644 index 0000000..f525c63 --- /dev/null +++ b/agents/trace-analyst.agent.md @@ -0,0 +1,23 @@ +--- +name: trace-analyst +description: Investigates agent-strace sessions to explain behavior, failures, cost, and tool usage. +tools: ["bash"] +--- + +You are an agent-session forensic analyst. Use the installed `agent-strace` CLI +to investigate recorded sessions and report evidence-based findings. + +1. Confirm `agent-strace` is available before analysis. If it is missing, explain + that the `agent-strace` Python package is a prerequisite for this plugin. +2. Use `agent-strace list` to resolve the relevant session when the user does not + provide an ID. +3. Start with `inspect`, `explain`, and `timeline`. Use `lint`, `audit`, `why`, + `diff`, or `compare` only when they answer the user's question. +4. Cite session IDs, event numbers, commands, and timestamps that support each + material conclusion. Distinguish observed facts from inferences. +5. Treat `.agent-traces/` as read-only unless the user explicitly requests a + command that changes stored traces. +6. Never expose redacted values or infer secrets from placeholders. + +Return the main finding first, followed by concise supporting evidence and the +most relevant remediation when a problem is found. diff --git a/docs/copilot-plugin.md b/docs/copilot-plugin.md new file mode 100644 index 0000000..d01c63a --- /dev/null +++ b/docs/copilot-plugin.md @@ -0,0 +1,75 @@ +# GitHub Copilot CLI plugin + +The `agent-strace` Copilot CLI plugin records lifecycle events and adds a trace +analyst agent and skill for investigating sessions. + +## Install + +Install the zero-dependency Python CLI first so the plugin hooks can invoke it: + +```bash +uv tool install agent-strace +# or: pip install agent-strace +``` + +Add this repository as a marketplace, then install its plugin: + +```bash +copilot plugin marketplace add Siddhant-K-code/agent-trace +copilot plugin install agent-strace@agent-trace +``` + +Restart Copilot CLI after installation. Do not also run +`agent-strace setup --cli copilot`; both approaches register the same hooks and +would record duplicate events. + +## Verify + +```bash +copilot plugin list +agent-strace --version +``` + +In an interactive Copilot CLI session, use `/agent` to select `trace-analyst` or +run `/skills list` to confirm the `agent-strace` skill loaded. New sessions are +written to `.agent-traces/` in the working directory. + +## Use + +Ask Copilot to analyze the latest agent trace, or run commands directly: + +```bash +agent-strace list +agent-strace replay +agent-strace explain +agent-strace timeline +agent-strace lint +``` + +The hooks capture session starts and ends, prompts, successful and failed tool +calls, and agent-stop events exposed by Copilot CLI. Secret redaction remains +enabled by default. + +## Develop locally + +From a clone of this repository: + +```bash +pip install -e . +copilot plugin marketplace add "$PWD" +copilot plugin install agent-strace@agent-trace +``` + +Copilot caches installed plugin components. Run +`copilot plugin marketplace update agent-trace` followed by +`copilot plugin update agent-strace` after changing the manifest, hooks, agent, +or skill. + +## Uninstall + +```bash +copilot plugin uninstall agent-strace +``` + +Locally installed plugins run only in Copilot CLI. Copilot cloud agent does not +load plugins installed on a developer machine. diff --git a/docs/setup.md b/docs/setup.md index 1b012f3..357b280 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -141,7 +141,14 @@ Cursor hook coverage depends on the events Cursor emits. Native hooks capture pr ### GitHub Copilot CLI hooks -`agent-strace setup --cli copilot` writes user-level Copilot hooks: +The recommended setup is the Copilot CLI plugin, which bundles the hooks plus a trace analyst agent and skill: + +```bash +copilot plugin marketplace add Siddhant-K-code/agent-trace +copilot plugin install agent-strace@agent-trace +``` + +See [Copilot CLI plugin](copilot-plugin.md) for verification and usage. Alternatively, `agent-strace setup --cli copilot` writes only the user-level Copilot hooks: ``` ~/.copilot/ @@ -149,19 +156,19 @@ Cursor hook coverage depends on the events Cursor emits. Native hooks capture pr └── agent-strace.json ``` -Set `COPILOT_HOME` to install into a different Copilot config directory. The generated config registers Copilot lifecycle hooks using the VS Code-compatible event names: +Set `COPILOT_HOME` to install into a different Copilot config directory. The generated config registers Copilot lifecycle hooks using the native event names: ```json { "version": 1, "hooks": { - "SessionStart": [{ "hooks": [{ "type": "command", "command": "agent-strace hook --provider copilot session-start" }] }], - "UserPromptSubmit": [{ "hooks": [{ "type": "command", "command": "agent-strace hook --provider copilot user-prompt" }] }], - "PreToolUse": [{ "matcher": ".*", "hooks": [{ "type": "command", "command": "agent-strace hook --provider copilot pre-tool" }] }], - "PostToolUse": [{ "matcher": ".*", "hooks": [{ "type": "command", "command": "agent-strace hook --provider copilot post-tool" }] }], - "PostToolUseFailure": [{ "matcher": ".*", "hooks": [{ "type": "command", "command": "agent-strace hook --provider copilot post-tool-failure" }] }], - "Stop": [{ "hooks": [{ "type": "command", "command": "agent-strace hook --provider copilot stop" }] }], - "SessionEnd": [{ "hooks": [{ "type": "command", "command": "agent-strace hook --provider copilot session-end" }] }] + "sessionStart": [{ "type": "command", "command": "agent-strace hook --provider copilot session-start", "timeoutSec": 10 }], + "userPromptSubmitted": [{ "type": "command", "command": "agent-strace hook --provider copilot user-prompt", "timeoutSec": 10 }], + "preToolUse": [{ "type": "command", "command": "agent-strace hook --provider copilot pre-tool", "timeoutSec": 10 }], + "postToolUse": [{ "type": "command", "command": "agent-strace hook --provider copilot post-tool", "timeoutSec": 10 }], + "postToolUseFailure": [{ "type": "command", "command": "agent-strace hook --provider copilot post-tool-failure", "timeoutSec": 10 }], + "agentStop": [{ "type": "command", "command": "agent-strace hook --provider copilot stop", "timeoutSec": 10 }], + "sessionEnd": [{ "type": "command", "command": "agent-strace hook --provider copilot session-end", "timeoutSec": 10 }] } } ``` diff --git a/hooks.json b/hooks.json new file mode 100644 index 0000000..81945f7 --- /dev/null +++ b/hooks.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "hooks": { + "sessionStart": [ + { + "type": "command", + "command": "agent-strace hook --provider copilot session-start", + "timeoutSec": 10 + } + ], + "userPromptSubmitted": [ + { + "type": "command", + "command": "agent-strace hook --provider copilot user-prompt", + "timeoutSec": 10 + } + ], + "preToolUse": [ + { + "type": "command", + "command": "agent-strace hook --provider copilot pre-tool", + "timeoutSec": 10 + } + ], + "postToolUse": [ + { + "type": "command", + "command": "agent-strace hook --provider copilot post-tool", + "timeoutSec": 10 + } + ], + "postToolUseFailure": [ + { + "type": "command", + "command": "agent-strace hook --provider copilot post-tool-failure", + "timeoutSec": 10 + } + ], + "agentStop": [ + { + "type": "command", + "command": "agent-strace hook --provider copilot stop", + "timeoutSec": 10 + } + ], + "sessionEnd": [ + { + "type": "command", + "command": "agent-strace hook --provider copilot session-end", + "timeoutSec": 10 + } + ] + } +} diff --git a/plugin.json b/plugin.json new file mode 100644 index 0000000..4c17591 --- /dev/null +++ b/plugin.json @@ -0,0 +1,21 @@ +{ + "name": "agent-strace", + "description": "Capture, replay, and analyze GitHub Copilot CLI agent sessions.", + "version": "0.94.0", + "author": { + "name": "Siddhant Khare", + "email": "siddhantkhare2694@gmail.com" + }, + "homepage": "https://github.com/Siddhant-K-code/agent-trace", + "repository": "https://github.com/Siddhant-K-code/agent-trace", + "license": "MIT", + "keywords": [ + "agents", + "copilot-cli", + "observability", + "tracing" + ], + "agents": "agents/", + "skills": "skills/", + "hooks": "hooks.json" +} diff --git a/skills/agent-strace/SKILL.md b/skills/agent-strace/SKILL.md new file mode 100644 index 0000000..8fd7e3e --- /dev/null +++ b/skills/agent-strace/SKILL.md @@ -0,0 +1,56 @@ +--- +name: agent-strace +description: Trace and investigate GitHub Copilot CLI sessions with agent-strace. Use when the user wants to capture a Copilot session, replay agent activity, explain a decision, diagnose repeated tool calls or failures, inspect cost, or audit a recorded session. +--- + +# agent-strace + +Use the local `agent-strace` executable. The plugin contributes lifecycle hooks, +but the Python package must also be installed and available on `PATH`. + +## Prerequisite check + +Run `agent-strace --version`. If it is unavailable, stop and provide one of: + +```bash +uv tool install agent-strace +pip install agent-strace +``` + +Do not silently substitute another tracing tool. + +## Capture + +Installed plugin hooks automatically record new Copilot CLI sessions to +`.agent-traces/`. Do not also run `agent-strace setup --cli copilot`; that would +register duplicate hooks. + +## Investigate + +Choose the smallest command set that answers the request: + +```bash +agent-strace list +agent-strace inspect +agent-strace replay +agent-strace explain +agent-strace timeline +agent-strace why +agent-strace lint +agent-strace audit +agent-strace cost +agent-strace diff +agent-strace compare +``` + +Resolve an omitted session ID with `agent-strace list`. Prefer text output for +interactive analysis and JSON only when structured processing is necessary. +Report exact session IDs and event numbers for material findings. + +## Safety + +- Treat trace storage as read-only unless the user requests retention, import, + anonymization, or another mutating operation. +- Preserve secret redaction. +- Do not claim causal intent from proximity alone; use `why` or clearly label the + conclusion as an inference. diff --git a/src/agent_trace/__init__.py b/src/agent_trace/__init__.py index 1a3fba9..38e217c 100644 --- a/src/agent_trace/__init__.py +++ b/src/agent_trace/__init__.py @@ -1,3 +1,3 @@ """agent-trace: strace for AI agents.""" -__version__ = "0.93.1" +__version__ = "0.94.0" diff --git a/src/agent_trace/cli.py b/src/agent_trace/cli.py index aee1630..2741a4c 100644 --- a/src/agent_trace/cli.py +++ b/src/agent_trace/cli.py @@ -662,50 +662,40 @@ def _copilot_hooks_config(args: argparse.Namespace) -> dict: return { "version": 1, "hooks": { - "SessionStart": [{ - "hooks": [{ - "type": "command", - "command": f"{cmd_prefix} session-start", - }], + "sessionStart": [{ + "type": "command", + "command": f"{cmd_prefix} session-start", + "timeoutSec": 10, }], - "UserPromptSubmit": [{ - "hooks": [{ - "type": "command", - "command": f"{cmd_prefix} user-prompt", - }], + "userPromptSubmitted": [{ + "type": "command", + "command": f"{cmd_prefix} user-prompt", + "timeoutSec": 10, }], - "PreToolUse": [{ - "matcher": ".*", - "hooks": [{ - "type": "command", - "command": f"{cmd_prefix} pre-tool", - }], + "preToolUse": [{ + "type": "command", + "command": f"{cmd_prefix} pre-tool", + "timeoutSec": 10, }], - "PostToolUse": [{ - "matcher": ".*", - "hooks": [{ - "type": "command", - "command": f"{cmd_prefix} post-tool", - }], + "postToolUse": [{ + "type": "command", + "command": f"{cmd_prefix} post-tool", + "timeoutSec": 10, }], - "PostToolUseFailure": [{ - "matcher": ".*", - "hooks": [{ - "type": "command", - "command": f"{cmd_prefix} post-tool-failure", - }], + "postToolUseFailure": [{ + "type": "command", + "command": f"{cmd_prefix} post-tool-failure", + "timeoutSec": 10, }], - "Stop": [{ - "hooks": [{ - "type": "command", - "command": f"{cmd_prefix} stop", - }], + "agentStop": [{ + "type": "command", + "command": f"{cmd_prefix} stop", + "timeoutSec": 10, }], - "SessionEnd": [{ - "hooks": [{ - "type": "command", - "command": f"{cmd_prefix} session-end", - }], + "sessionEnd": [{ + "type": "command", + "command": f"{cmd_prefix} session-end", + "timeoutSec": 10, }], } } diff --git a/src/agent_trace/hooks.py b/src/agent_trace/hooks.py index 88128b8..371a5ba 100644 --- a/src/agent_trace/hooks.py +++ b/src/agent_trace/hooks.py @@ -288,8 +288,21 @@ def _normalise_payload(input_data: dict, provider: str, event: str) -> dict: data["tool_name"] = data.get("toolName") if data.get("toolArgs") is not None and "tool_input" not in data: data["tool_input"] = data.get("toolArgs") - if (data.get("toolResult") is not None or data.get("textResultForLlm") is not None) and "tool_output" not in data: - data["tool_output"] = data.get("toolResult", data.get("textResultForLlm", "")) + result = data.get("toolResult", data.get("tool_result")) + if isinstance(result, dict) and "tool_output" not in data: + for key in ("textResultForLlm", "text_result_for_llm", "output"): + if key in result: + data["tool_output"] = result[key] + break + else: + data["tool_output"] = result + elif result is not None and "tool_output" not in data: + data["tool_output"] = result + if data.get("textResultForLlm") is not None and "tool_output" not in data: + data["tool_output"] = data.get("textResultForLlm") + if event == "post-tool-failure" and data.get("error") and "tool_output" not in data: + error = data["error"] + data["tool_output"] = error.get("message", error) if isinstance(error, dict) else error command = data.get("command") if command and not data.get("tool_name"): data.setdefault("tool_name", "shell") @@ -341,7 +354,18 @@ def handle_session_start(input_data: dict, provider: str = "claude") -> None: if session_id: meta.session_id = session_id[:16] - store.create_session(meta) + source = input_data.get("source", "startup") + if source == "resume" and session_id: + try: + existing_meta = store.load_meta(meta.session_id) + except FileNotFoundError: + store.create_session(meta) + else: + existing_meta.ended_at = None + meta = existing_meta + store.update_meta(meta) + else: + store.create_session(meta) _write_session_meta(meta) if session_id: @@ -353,7 +377,7 @@ def handle_session_start(input_data: dict, provider: str = "claude") -> None: event_data = { "mode": f"{agent_name}-hooks", "provider": provider, - "source": input_data.get("source", "startup"), + "source": source, "model": input_data.get("model", ""), } for key in ("cwd", "transcript_path", "permission_mode"): diff --git a/tests/test_copilot_hooks.py b/tests/test_copilot_hooks.py index 7d97259..6ed1b5e 100644 --- a/tests/test_copilot_hooks.py +++ b/tests/test_copilot_hooks.py @@ -93,6 +93,99 @@ def test_copilot_hook_main_normalizes_camel_case_payloads(self): self.assertEqual(stops[0].data["stop_reason"], "end_turn") self.assertEqual(stops[0].data["transcript_path"], "/tmp/copilot-transcript.jsonl") + def test_copilot_hook_main_records_official_tool_result_payloads(self): + with patch.object(sys, "stdin", io.StringIO(json.dumps({ + "sessionId": "copilotofficial123", + "source": "startup", + }))): + hook_main(["--provider", "copilot", "session-start"]) + + with patch.object(sys, "stdin", io.StringIO(json.dumps({ + "sessionId": "copilotofficial123", + "toolName": "bash", + "toolArgs": {"command": "echo ok"}, + }))): + hook_main(["--provider", "copilot", "pre-tool"]) + + with patch.object(sys, "stdin", io.StringIO(json.dumps({ + "sessionId": "copilotofficial123", + "toolName": "bash", + "toolArgs": {"command": "echo ok"}, + "toolResult": { + "resultType": "success", + "textResultForLlm": "ok", + }, + }))): + hook_main(["--provider", "copilot", "post-tool"]) + + session_id = _read_active_session(provider="copilot") + events = TraceStore(self.tmpdir).load_events(session_id) + results = [event for event in events if event.event_type == EventType.TOOL_RESULT] + + self.assertEqual(results[0].data["result"], "ok") + + def test_copilot_hook_main_records_official_failure_payload(self): + with patch.object(sys, "stdin", io.StringIO(json.dumps({ + "sessionId": "copilotfailure123", + "source": "startup", + }))): + hook_main(["--provider", "copilot", "session-start"]) + + with patch.object(sys, "stdin", io.StringIO(json.dumps({ + "sessionId": "copilotfailure123", + "toolName": "bash", + "toolArgs": {"command": "false"}, + }))): + hook_main(["--provider", "copilot", "pre-tool"]) + + with patch.object(sys, "stdin", io.StringIO(json.dumps({ + "sessionId": "copilotfailure123", + "toolName": "bash", + "toolArgs": {"command": "false"}, + "error": "Command exited with status 1", + }))): + hook_main(["--provider", "copilot", "post-tool-failure"]) + + session_id = _read_active_session(provider="copilot") + events = TraceStore(self.tmpdir).load_events(session_id) + errors = [event for event in events if event.event_type == EventType.ERROR] + + self.assertEqual(errors[0].data["error"], "Command exited with status 1") + + def test_copilot_resume_preserves_existing_session_metadata(self): + session_payload = { + "sessionId": "copilotresume1234", + "source": "startup", + } + with patch.object(sys, "stdin", io.StringIO(json.dumps(session_payload))): + hook_main(["--provider", "copilot", "session-start"]) + + store = TraceStore(self.tmpdir) + session_id = _read_active_session(provider="copilot") + meta = store.load_meta(session_id) + started_at = meta.started_at + meta.ended_at = started_at + 4 + meta.total_duration_ms = 4000 + meta.tool_calls = 3 + meta.errors = 1 + store.update_meta(meta) + + session_payload["source"] = "resume" + with patch.object(sys, "stdin", io.StringIO(json.dumps(session_payload))): + hook_main(["--provider", "copilot", "session-start"]) + + resumed = store.load_meta(session_id) + events = store.load_events(session_id) + starts = [event for event in events if event.event_type == EventType.SESSION_START] + + self.assertEqual(resumed.started_at, started_at) + self.assertIsNone(resumed.ended_at) + self.assertEqual(resumed.total_duration_ms, 4000) + self.assertEqual(resumed.tool_calls, 3) + self.assertEqual(resumed.errors, 1) + self.assertEqual(len(starts), 2) + self.assertEqual(starts[-1].data["source"], "resume") + class TestCopilotSetup(unittest.TestCase): def setUp(self): @@ -121,16 +214,16 @@ def test_setup_cli_copilot_writes_user_hooks_file(self): self.assertEqual(printed_hooks, hooks) self.assertEqual(hooks["version"], 1) - self.assertIn("SessionStart", hooks["hooks"]) - self.assertIn("PostToolUseFailure", hooks["hooks"]) - self.assertIn("Stop", hooks["hooks"]) - self.assertIn("SessionEnd", hooks["hooks"]) + self.assertIn("sessionStart", hooks["hooks"]) + self.assertIn("postToolUseFailure", hooks["hooks"]) + self.assertIn("agentStop", hooks["hooks"]) + self.assertIn("sessionEnd", hooks["hooks"]) self.assertEqual( - hooks["hooks"]["UserPromptSubmit"][0]["hooks"][0]["command"], + hooks["hooks"]["userPromptSubmitted"][0]["command"], "agent-strace hook --provider copilot user-prompt", ) self.assertEqual( - hooks["hooks"]["Stop"][0]["hooks"][0]["command"], + hooks["hooks"]["agentStop"][0]["command"], "agent-strace hook --provider copilot stop", ) self.assertIn("GitHub Copilot hooks config", err.getvalue()) diff --git a/tests/test_plugin.py b/tests/test_plugin.py new file mode 100644 index 0000000..0e2811e --- /dev/null +++ b/tests/test_plugin.py @@ -0,0 +1,53 @@ +"""Tests for the GitHub Copilot CLI plugin package.""" + +import argparse +import json +import unittest +from pathlib import Path + +from agent_trace import __version__ +from agent_trace.cli import _copilot_hooks_config + + +ROOT = Path(__file__).resolve().parents[1] + + +class TestCopilotPlugin(unittest.TestCase): + def test_manifest_references_existing_components(self): + manifest = json.loads((ROOT / "plugin.json").read_text()) + + self.assertEqual(manifest["name"], "agent-strace") + self.assertEqual(manifest["version"], __version__) + for field in ("agents", "skills", "hooks"): + self.assertTrue((ROOT / manifest[field]).exists()) + + def test_marketplace_references_root_plugin(self): + marketplace = json.loads( + (ROOT / ".github" / "plugin" / "marketplace.json").read_text() + ) + + self.assertEqual(marketplace["name"], "agent-trace") + self.assertEqual(marketplace["metadata"]["version"], __version__) + self.assertEqual(len(marketplace["plugins"]), 1) + self.assertEqual(marketplace["plugins"][0]["name"], "agent-strace") + self.assertEqual(marketplace["plugins"][0]["version"], __version__) + self.assertEqual(marketplace["plugins"][0]["source"], ".") + + def test_plugin_hooks_match_generated_copilot_hooks(self): + plugin_hooks = json.loads((ROOT / "hooks.json").read_text()) + args = argparse.Namespace(redact=False, no_redact=False) + + self.assertEqual(plugin_hooks, _copilot_hooks_config(args)) + + def test_agent_and_skill_have_required_frontmatter(self): + agent = (ROOT / "agents" / "trace-analyst.agent.md").read_text() + skill = (ROOT / "skills" / "agent-strace" / "SKILL.md").read_text() + + self.assertTrue(agent.startswith("---\nname: trace-analyst\n")) + self.assertIn("\ndescription:", agent) + self.assertTrue(skill.startswith("---\nname: agent-strace\n")) + self.assertIn("\ndescription:", skill) + + +if __name__ == "__main__": + unittest.main()