diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 000000000..ce7406e54 --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,66 @@ +{ + "hooks": { + "PostToolUse": [ + { + "hooks": [ + { + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex post-tool-use'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex post-tool-use)\"", + "type": "command", + "statusMessage": "Chainloop Trace: record tool changes" + } + ], + "matcher": "^(apply_patch|Bash)$" + } + ], + "PreToolUse": [ + { + "hooks": [ + { + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex pre-tool-use'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex pre-tool-use)\"", + "type": "command", + "statusMessage": "Chainloop Trace: snapshot files before changes" + } + ], + "matcher": "^(apply_patch|Bash)$" + } + ], + "SessionEnd": [ + { + "hooks": [ + { + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-end'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex session-end)\"", + "type": "command", + "statusMessage": "Chainloop Trace: finish session" + } + ] + } + ], + "SessionStart": [ + { + "hooks": [ + { + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-start'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex session-start)\"", + "type": "command", + "statusMessage": "Chainloop Trace: start session" + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex user-prompt-submit'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex user-prompt-submit)\"", + "type": "command", + "statusMessage": "Chainloop Trace: capture prompt context" + } + ] + } + ] + } +} diff --git a/README.md b/README.md index 745248368..7cc44a352 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,9 @@ chainloop trace init # installs the git and agent hooks, writes .chainloo `trace init` writes `.chainloop.yml` and git hooks (`pre-push`, `post-commit`, `commit-msg`, `post-rewrite`) in the current repo, and `trace uninstall` removes them. Session data stays on your machine until `git push`, and the CLI redacts secrets before upload. The [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=try-it-in-60-seconds) lists what it sends. -Now code with Claude Code, OpenCode, or Cursor, commit, and `git push`. The push hook attests the session and prints where to see it: +For Codex, run `chainloop trace init --codex`. Review and trust the installed hooks in Codex before starting work. See the [agent support matrix](./app/cli/internal/trace/README.md) for verified features and limits. + +Now code with Claude Code, OpenCode, Codex, or Cursor, commit, and `git push`. The push hook attests the session and prints where to see it: ```text Coding Session Available at https://app.chainloop.dev/u//sessions/ @@ -184,7 +186,7 @@ A workflow contract is the API between the people who set the rules and the peop | Component | What it does | |---|---| -| [AI session collector](./app/cli/cmd/trace.go) (`chainloop trace`) | Records Claude Code, OpenCode, and Cursor sessions, redacts secrets, attributes AI and human lines, and attests the session on `git push`. `requireTrace` in `.chainloop.yml` blocks a push that has no record. | +| [AI session collector](./app/cli/cmd/trace.go) (`chainloop trace`) | Records Claude Code, OpenCode, Codex, and Cursor sessions, redacts secrets, attributes AI and human lines, and attests the session on `git push`. `requireTrace` in `.chainloop.yml` blocks a push that has no record. | | [CLI](./app/cli) (`chainloop`) | One integration point for every major CI runner and 45 evidence types. It crafts, signs, and pushes attestations, and runs policies. | | [Trusted store](./app/artifact-cas) | Content-addressable storage for evidence, backed by an OCI registry, S3, Azure Blob Storage, or inline. Your storage, your keys. Called the Artifact CAS in the code. | | [Signing](./pkg/attestation/signer) | in-toto attestations, keyless by default, or your own cosign / KMS keys, or SignServer. Timestamped. | @@ -231,7 +233,7 @@ During an attestation you can attach these evidence types. The CLI uploads files ## Use cases -**Put guardrails on coding agents.** Record every Claude Code, OpenCode, or Cursor session as a signed record, and run your policies on every session. Catch an unapproved model, a secret in the transcript, or more AI lines than the rule allows. Chainloop signs violations into the record, and they fail the pipeline that builds the PR. `requireTrace: true` rejects a push with no recorded session. → [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [Policies](https://docs.chainloop.dev/concepts/policies?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [AI coding governance](https://chainloop.dev/solutions/ai-coding-governance/?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) +**Put guardrails on coding agents.** Record every Claude Code, OpenCode, Codex, or Cursor session as a signed record, and run your policies on every session. Catch an unapproved model, a secret in the transcript, or more AI lines than the rule allows. Chainloop signs violations into the record, and they fail the pipeline that builds the PR. `requireTrace: true` rejects a push with no recorded session. → [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [Policies](https://docs.chainloop.dev/concepts/policies?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [AI coding governance](https://chainloop.dev/solutions/ai-coding-governance/?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) **Trace a release back to the prompt.** Follow any release back through the build and the commit to the AI session that started it. `chainloop referrer discover --digest ` returns every attestation that references an artifact, SBOM, or commit, and what those reference in turn. One record, from prompt to production. → [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) @@ -259,7 +261,8 @@ Git tells you what changed. It no longer tells you how. `chainloop trace` record | Claude Code | Supported, with token usage and cost | | OpenCode | Supported, with token usage and cost | | Cursor | Experimental, no token usage or cost yet | -| Codex, GitHub Copilot, Gemini, Windsurf, Amp | Coming. [Contributions welcome](#community-and-contributing) | +| Codex | Supported, with token usage and estimated cost | +| GitHub Copilot, Gemini, Windsurf, Amp | Coming. [Contributions welcome](#community-and-contributing) | Wanted: new collector providers for the agents above, and new evidence types. The provider interface is in [app/cli/internal/trace](./app/cli/internal/trace). @@ -351,7 +354,7 @@ You can also download the CLI from the [releases page](https://github.com/chainl ## For AI coding agents > [!NOTE] -> **If you are an AI coding agent setting up Chainloop in this repository:** Install the CLI with `curl -sfL https://dl.chainloop.dev/cli/install.sh | bash -s -- --oss`. Ask the user to run `chainloop auth login`, which opens a browser. Then run `chainloop trace init --project ` from the repository root. Add `--claude`, `--cursor`, or `--opencode` for the agents in use. The command writes `.chainloop.yml` and git hooks. Commit `.chainloop.yml`. The docs index for agents is https://docs.chainloop.dev/llms.txt. Any docs page is available as Markdown when you add `.md` to its URL. [Documentation](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=for-ai-coding-agents) +> **If you are an AI coding agent setting up Chainloop in this repository:** Install the CLI with `curl -sfL https://dl.chainloop.dev/cli/install.sh | bash -s -- --oss`. Ask the user to run `chainloop auth login`, which opens a browser. Then run `chainloop trace init --project ` from the repository root. Add `--claude`, `--cursor`, `--opencode`, or `--codex` for the agents in use. The command writes `.chainloop.yml` and Git hooks, plus `.codex/hooks.json` when using Codex. Commit `.chainloop.yml` and the generated `.codex/hooks.json`. Each clone must still run `chainloop trace init` to install its local Git hooks. The docs index for agents is https://docs.chainloop.dev/llms.txt. Any docs page is available as Markdown when you add `.md` to its URL. [Documentation](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=for-ai-coding-agents) Or paste this into Claude Code, Codex, or Cursor: diff --git a/app/cli/cmd/trace_hook.go b/app/cli/cmd/trace_hook.go index 26da0805e..442b68369 100644 --- a/app/cli/cmd/trace_hook.go +++ b/app/cli/cmd/trace_hook.go @@ -20,6 +20,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/config" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" @@ -40,6 +41,7 @@ func newTraceHookCmd() *cobra.Command { newTraceHookClaudeCmd(), newTraceHookCursorCmd(), newTraceHookOpenCodeCmd(), + newTraceHookCodexCmd(), ) return cmd @@ -451,3 +453,21 @@ func newTraceHookOpenCodePostToolUseCmd() *cobra.Command { }, } } + +func newTraceHookCodexCmd() *cobra.Command { + cmd := &cobra.Command{Use: "codex", Short: "Handle Codex hook invocations"} + handlers := []struct { + name, description string + run func(*cobra.Command, []string) error + }{ + {"session-start", "Capture the Codex session", func(cmd *cobra.Command, _ []string) error { return runAgentSessionStart(cmd, codex.New()) }}, + {"user-prompt-submit", "Remind Codex to capture specs", func(_ *cobra.Command, _ []string) error { return action.HandleAgentPromptSubmit(codex.New(), logger) }}, + {"pre-tool-use", "Snapshot files before a Codex tool", func(_ *cobra.Command, _ []string) error { return action.HandleAgentPreToolUse(codex.New(), logger) }}, + {"post-tool-use", "Record changes after a Codex tool", func(_ *cobra.Command, _ []string) error { return action.HandleAgentPostToolUse(codex.New(), logger) }}, + {"session-end", "Copy the completed Codex session", func(_ *cobra.Command, _ []string) error { return action.HandleAgentSessionEnd(codex.New(), logger) }}, + } + for _, h := range handlers { + cmd.AddCommand(&cobra.Command{Use: h.name, Short: h.description, Args: cobra.NoArgs, Annotations: map[string]string{skipActionOptsInit: trueString}, RunE: func(cmd *cobra.Command, args []string) error { InitHookLogger(); return h.run(cmd, args) }}) + } + return cmd +} diff --git a/app/cli/cmd/trace_init.go b/app/cli/cmd/trace_init.go index e75a18947..3057c342d 100644 --- a/app/cli/cmd/trace_init.go +++ b/app/cli/cmd/trace_init.go @@ -27,6 +27,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/repositoryconfig" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/config" tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/hooks" @@ -47,6 +48,7 @@ func newTraceInitCmd() *cobra.Command { claudeFlag bool cursorFlag bool opencodeFlag bool + codexFlag bool ) cmd := &cobra.Command{ @@ -72,7 +74,7 @@ named freely; the name is normalized to the lowercase, dash-separated form Chainloop stores. It then asks which harnesses to trace, with the ones the repository is already set up for ticked, or Claude Code on a first run; use space to tick more. Passing --org, --project or a harness flag -(--claude, --cursor, --opencode) skips the matching question. +(--claude, --cursor, --opencode, --codex) skips the matching question. Nothing is asked in CI or when the output is redirected: there --project is required unless .chainloop.yml carries projectName. Set CHAINLOOP_NO_PROMPT to @@ -155,7 +157,7 @@ cannot use them with --local. You can use --require-trace with --local.`, // so every question is answered up front and an abort leaves the // repository untouched. selected, err := resolveTraceProviders(newNamingPrompter(os.LookupEnv), - traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag}, + traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag, codex: codexFlag}, traceInitCanPrompt(), func() []string { return configuredTraceProviders(repoRoot) }) if err != nil { return stopIfAborted(err) @@ -293,6 +295,7 @@ cannot use them with --local. You can use --require-trace with --local.`, cmd.Flags().Bool("require-trace", false, "block pushes when a required trace is missing") cmd.Flags().BoolVar(&claudeFlag, "claude", false, "install Claude Code hooks (default when no harness flag is set)") cmd.Flags().BoolVar(&cursorFlag, "cursor", false, "install Cursor hooks") + cmd.Flags().BoolVar(&codexFlag, "codex", false, "install Codex hooks") cmd.Flags().BoolVar(&opencodeFlag, "opencode", false, "install opencode hooks") return cmd @@ -380,6 +383,11 @@ What's next Learn more: %s `, strings.Join(files, " "), joinWithOr(providerNames(installed)), traceDocsURL) + for _, p := range installed { + if p.Name() == codex.Name { + fmt.Fprintln(w, "For Codex, review and trust the installed hooks in Codex before starting work.") + } + } } // writeTraceInitLocalSummary reports what a local init set up. It names no @@ -437,6 +445,11 @@ What's next Learn more: %s `, strings.Join(files, " "), joinWithOr(providerNames(installed)), traceDocsURL) + for _, p := range installed { + if p.Name() == codex.Name { + fmt.Fprintln(w, "For Codex, review and trust the installed hooks in Codex before starting work.") + } + } } // providerNames names the harnesses, in the order they were installed. diff --git a/app/cli/cmd/trace_init_providers.go b/app/cli/cmd/trace_init_providers.go index e88d1f735..db98dc92e 100644 --- a/app/cli/cmd/trace_init_providers.go +++ b/app/cli/cmd/trace_init_providers.go @@ -20,6 +20,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/providers" @@ -27,21 +28,20 @@ import ( const providersPromptTitle = "Harnesses to trace in this repository" -// traceProviderFlags records which --claude / --cursor / --opencode flags the -// user set. +// traceProviderFlags records the harness flags that the user set. type traceProviderFlags struct { - claude, cursor, opencode bool + claude, cursor, opencode, codex bool } // any reports whether the user named at least one provider. func (f traceProviderFlags) any() bool { - return f.claude || f.cursor || f.opencode + return f.claude || f.cursor || f.opencode || f.codex } // names returns the providers the flags select, in the registry's order rather // than the order the flags happen to appear in. func (f traceProviderFlags) names() []string { - out := make([]string, 0, 3) + out := make([]string, 0, 4) if f.claude { out = append(out, claude.Name) } @@ -52,6 +52,9 @@ func (f traceProviderFlags) names() []string { out = append(out, opencode.Name) } + if f.codex { + out = append(out, codex.Name) + } return out } diff --git a/app/cli/cmd/trace_init_providers_test.go b/app/cli/cmd/trace_init_providers_test.go index 83170a267..80e5021b0 100644 --- a/app/cli/cmd/trace_init_providers_test.go +++ b/app/cli/cmd/trace_init_providers_test.go @@ -194,6 +194,11 @@ func TestConfiguredTraceProvidersSkipsUnreadableSettings(t *testing.T) { } func TestTraceProviderFlags(t *testing.T) { + t.Run("codex flag", func(t *testing.T) { + f := traceProviderFlags{codex: true} + assert.True(t, f.any()) + assert.Equal(t, []string{"codex"}, f.namesOrDefault()) + }) t.Run("no flag set", func(t *testing.T) { assert.False(t, traceProviderFlags{}.any()) assert.Empty(t, traceProviderFlags{}.names()) diff --git a/app/cli/cmd/trace_init_test.go b/app/cli/cmd/trace_init_test.go index 2b16b373f..665fca8c2 100644 --- a/app/cli/cmd/trace_init_test.go +++ b/app/cli/cmd/trace_init_test.go @@ -22,6 +22,7 @@ import ( "slices" "testing" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/providers" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -326,14 +327,14 @@ func TestWriteTraceNextSteps(t *testing.T) { }, { name: "every harness", - providers: []string{providerClaudeCode, "cursor", "opencode"}, + providers: []string{providerClaudeCode, "cursor", opencode.Name, "codex"}, // Named in full: ".opencode" alone would pass on any file under that // directory, so it would not catch the plugin being renamed. wantFiles: []string{ chainloopYMLName, claudeSettings, ".cursor/hooks.json", - ".opencode/plugins/chainloop-trace.ts", + ".opencode/plugins/chainloop-trace.ts", ".codex/hooks.json", }, - wantHarnesses: "claude-code, cursor or opencode", + wantHarnesses: "claude-code, cursor, opencode or codex", }, } diff --git a/app/cli/cmd/trace_run.go b/app/cli/cmd/trace_run.go index 02bed925e..ecb451a40 100644 --- a/app/cli/cmd/trace_run.go +++ b/app/cli/cmd/trace_run.go @@ -64,6 +64,7 @@ func newTraceRunCmd() *cobra.Command { claudeFlag bool cursorFlag bool opencodeFlag bool + codexFlag bool ) cmd := &cobra.Command{ @@ -165,7 +166,7 @@ func newTraceRunCmd() *cobra.Command { return action.TraceRun(cmd.Context(), logger, action.TraceRunOpts{ Store: store, RepoRoot: repoRoot, - Providers: traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag}.namesOrDefault(), + Providers: traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag, codex: codexFlag}.namesOrDefault(), Command: args, ProjectName: projectFlag, Organization: organization, @@ -187,6 +188,7 @@ func newTraceRunCmd() *cobra.Command { cmd.Flags().BoolVar(&localFlag, "local", false, "store the session on disk, with no control plane, login or workflow") cmd.Flags().BoolVar(&claudeFlag, "claude", false, "install Claude Code hooks (default when no provider flag is set)") cmd.Flags().BoolVar(&cursorFlag, "cursor", false, "install Cursor hooks") + cmd.Flags().BoolVar(&codexFlag, "codex", false, "install Codex hooks") cmd.Flags().BoolVar(&opencodeFlag, "opencode", false, "install opencode hooks") // project and workflow are not required in local mode, so the check is done diff --git a/app/cli/documentation/cli-reference.md b/app/cli/documentation/cli-reference.md index ba01605fb..471e73bcf 100644 --- a/app/cli/documentation/cli-reference.md +++ b/app/cli/documentation/cli-reference.md @@ -3314,7 +3314,7 @@ named freely; the name is normalized to the lowercase, dash-separated form Chainloop stores. It then asks which harnesses to trace, with the ones the repository is already set up for ticked, or Claude Code on a first run; use space to tick more. Passing --org, --project or a harness flag -(--claude, --cursor, --opencode) skips the matching question. +(--claude, --cursor, --opencode, --codex) skips the matching question. Nothing is asked in CI or when the output is redirected: there --project is required unless .chainloop.yml carries projectName. Set CHAINLOOP_NO_PROMPT to @@ -3345,6 +3345,7 @@ Options ``` --claude install Claude Code hooks (default when no harness flag is set) +--codex install Codex hooks --contract string workflow contract to attach when creating the workflow; it must already exist and be usable by the project (default: "chainloop-ai-coding-session" when the organization has it, an empty contract otherwise) --cursor install Cursor hooks -h, --help help for init @@ -3413,6 +3414,7 @@ Options ``` --claude install Claude Code hooks (default when no provider flag is set) +--codex install Codex hooks --contract string workflow contract to attach when creating the workflow; it must already exist and be usable by the project (default: "chainloop-ai-coding-session" when the organization has it, an empty contract otherwise) --cursor install Cursor hooks -h, --help help for run diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 9f66dddae..a9fee17de 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -1,6 +1,6 @@ # Chainloop Trace: agent support -Chainloop Trace records the sessions of AI coding agents as `CHAINLOOP_AI_CODING_SESSION` evidence. Each agent is a provider in this package: `claude/`, `cursor/` and `opencode/`. The interfaces are in `provider.go`, and the registry is in `providers/`. To add a provider, follow [PROVIDERS.md](PROVIDERS.md). One provider covers OpenCode 1.x and OpenCode 2: it detects the version at runtime. +Chainloop Trace records the sessions of AI coding agents as `CHAINLOOP_AI_CODING_SESSION` evidence. Each agent is a provider in this package: `claude/`, `cursor/`, `opencode/` and `codex/`. The interfaces are in `provider.go`, and the registry is in `providers/`. To add a provider, follow [PROVIDERS.md](PROVIDERS.md). One provider covers OpenCode 1.x and OpenCode 2: it detects the version at runtime. This page lists the features of Trace, by area, and tells which agents support each one. **Keep it current.** When you add or change a provider, or add or change a feature of Trace, update this page in the same pull request. @@ -10,7 +10,15 @@ Claude Code has the most complete support. Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and `trace run --pi` do not exist, so no Pi session is recorded. Each Pi cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Pi support is added, set the Pi cells to **Unknown** and run the end-to-end suite to find the values. -Trace does not support Codex yet: `chainloop trace init --codex` and `trace run --codex` do not exist, so no Codex session is recorded. Each Codex cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Codex support is added, set the Codex cells to **Unknown** and run the end-to-end suite to find the values. +Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --local --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Installed hooks have a `Chainloop Trace` status label in Codex. Re-running init adds labels to existing Chainloop hooks. Changing the hooks requires a new trust review. + +Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 22 feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts, skills tracking, spec source pointers, and pasted image pointers are not implemented. + +Codex token usage comes from cumulative rollout counters. Cost is an estimate from the embedded standard API prices, with a warning for unknown models. Subscription billing and third-party provider prices can differ. Trace reads rollouts from `$CODEX_HOME/sessions` (default `~/.codex/sessions`) or `archived_sessions`. Their JSONL format is not a stable Codex interface. + +In a linked Git worktree, Codex 0.163 reads project hooks from the primary checkout. Install the Codex hooks in that checkout too, then review and trust them in Codex. + +Codex evidence converts native messages, tool calls, and tool results to the message shape used by the existing session viewer. Text uses `message.content`, calls use `tool_use` blocks, and results use `tool_result` blocks with matching IDs. Each converted entry retains its original event in `source_event`; native metadata stays unchanged. Codex regression and E2E checks require renderable messages. When tools are used, they also require a completed exchange with readable tool output. A nonempty native transcript alone does not pass. These checks apply to Codex; `raw_session` remains open in the shared evidence schema. ## Setup @@ -18,9 +26,9 @@ How Trace is added to a repository and to an agent. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Yes | Unknown | Yes | No | No | -| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Yes | Unknown | Yes | No | No | -| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | No | +| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Yes | Unknown | Yes | No | Yes | +| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Yes | Unknown | Yes | No | Yes | +| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | Unknown | Why not Yes: @@ -33,17 +41,19 @@ What the session evidence holds about the conversation. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | No | No | -| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | No | No | -| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | No | No | +| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | No | Yes | +| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | No | Yes | +| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | No | Yes | | Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | No | No | -| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | No | No | +| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | No | Unknown | Why not Yes: +- **Missed session start, Codex:** the scenario produces evidence, but its check also needs hook messages that Codex does not report in headless mode. - **Transcript, OpenCode:** OpenCode has no transcript file. The provider runs `opencode export` (1.x) or `opencode session export` (2) and rebuilds the transcript from the export. It keeps the text and the inputs of the tool calls. It drops the tool outputs, the reasoning and the file parts. OpenCode 2 exports also have no agent version and no session slug. - **Token usage and cost, Cursor:** the Cursor transcript does not hold token usage. Usage and cost are zero, and the evidence has a warning that tells so. (For OpenCode, the cost is the value that OpenCode reports. For Claude Code, Chainloop computes it from its price table.) - **Model detection, Cursor:** the primary model comes from the session-start hook. The transcript does not tell the models, so the evidence does not list the other models. In auto mode, the hook names the model `default`, not the model that ran. +- **Subagents, Codex:** the initial provider records only the root session transcript. Child-session transcripts are not collected. - **Subagents, Cursor:** the provider does not read subagent conversations. - **Subagents, OpenCode:** a subagent runs in a child session with its own export. It is not part of the evidence of its parent session. - **Missed session start, Cursor and OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). @@ -54,14 +64,14 @@ Which lines of a commit the AI wrote. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence reports the totals of the session in `ai_generated_lines_added` and `ai_generated_lines_removed`. A session with no commit sends no evidence, so it reports no totals (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | No | -| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the removals of the session's commits, less the lines that a commit of the session wrote. The line counts of each file are the sum of its AI and human lines, and a file that the session created stays created when a later commit of the session changes it (spec issue-3639, issue 3674). | Partial | Partial | Partial | Partial | No | No | -| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | -| Line ranges | The line ranges of each file are positions in the file at the commit that the push sends. They cover the lines of the session that are still there, and each range says if its lines are AI or human. The AI ranges hold as many lines as the net AI lines added, and the human ranges as many lines as the net human lines added. A file that the pushed commit no longer holds has no ranges. A range with no attribution is an AI range at edit time. Older CLIs write only these ranges. This CLI writes them for a file that it cannot blame at the pushed commit, and for a file with edits from before line fingerprints (spec issue-3639, line ranges). | Partial | Partial | Partial | Partial | No | No | -| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | +| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | Yes | +| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | Yes | +| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence reports the totals of the session in `ai_generated_lines_added` and `ai_generated_lines_removed`. A session with no commit sends no evidence, so it reports no totals (spec issue-3639). | Yes | Partial | Yes | Yes | No | Yes | +| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | Partial | +| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the removals of the session's commits, less the lines that a commit of the session wrote. The line counts of each file are the sum of its AI and human lines, and a file that the session created stays created when a later commit of the session changes it (spec issue-3639, issue 3674). | Partial | Partial | Partial | Partial | No | Partial | +| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | Partial | +| Line ranges | The line ranges of each file are positions in the file at the commit that the push sends. They cover the lines of the session that are still there, and each range says if its lines are AI or human. The AI ranges hold as many lines as the net AI lines added, and the human ranges as many lines as the net human lines added. A file that the pushed commit no longer holds has no ranges. A range with no attribution is an AI range at edit time. Older CLIs write only these ranges. This CLI writes them for a file that it cannot blame at the pushed commit, and for a file with edits from before line fingerprints (spec issue-3639, line ranges). | Partial | Partial | Partial | Partial | No | Partial | +| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | No | Partial | Why not Yes: @@ -75,14 +85,17 @@ Why not Yes: What the user sees in the agent. +The Codex welcome message and session link after push were verified manually in the interactive CLI 0.163.0. Codex runs its session-start hook after the first user message, so the welcome message appears then. The local-mode E2E suite does not verify these interactive messages. + | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | No | -| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | No | -| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | No | +| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | Yes | +| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | Yes | +| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | Unknown | Why not Yes: +- **Credentials warning, Codex:** this check needs a control plane. The suite uses local storage only, and the warning has not been verified manually. - **Welcome message, Cursor and OpenCode:** the provider gives the session-start context to the model only. It has no channel that shows a message to the user at session start, so the welcome message is dropped. - **Credentials warning, Cursor and OpenCode:** the provider has no channel to the user at session start. The model is told to tell the user, so the warning depends on the model. - **Session link after push, Cursor:** the `postToolUse` hook runs after a shell command, but its response has no field that Cursor shows to the user. The link stays pending until it expires. @@ -94,18 +107,20 @@ What the session was asked to build, and which skills it used. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | No | No | -| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | No | -| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | No | -| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Yes | Unknown | Yes | No | No | +| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | No | Yes | +| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | Unknown | +| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | Yes | +| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Yes | Unknown | Yes | No | Yes | | Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | No | | Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | No | | Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | No | Why not Yes: +- **Spec capture reminder, Codex:** Codex does not report hook messages in its headless output, so the suite cannot inspect the reminder. - **Spec capture reminder, Cursor:** Cursor has no hook at each user prompt. The agent gets the session-start instruction only. - **Spec source role, title, and description, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Skills tracking, spec source pointers, and pasted image pointers, Codex:** these optional integrations are not implemented. - **Skills tracking, Cursor:** the provider does not track skills. The evidence has no skill entries, which means "not recorded", not "no skill used". - **Skills tracking, OpenCode:** only the skills that the model starts with the `skill` tool are counted. Skills that the user starts, and skills used in subagents, are not. A plugin installed by an earlier version does not send the skill folder for OpenCode 2: install the hooks again to update it. - **Spec source pointers, Cursor and OpenCode:** the provider has no finder for the copies in its transcript yet. For OpenCode, the rebuilt transcript holds no tool outputs, so only the write copies would apply. @@ -117,9 +132,9 @@ How the evidence is protected before it leaves the machine. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | No | No | -| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | No | No | -| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | No | No | +| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | No | Yes | +| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | No | Yes | +| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | No | Yes | Why not Yes: @@ -132,10 +147,10 @@ How the sessions are linked to the commits and attested. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | No | No | -| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | No | No | -| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | No | No | -| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | No | No | +| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | No | Yes | +| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | No | Unknown | +| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | No | Yes | +| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | No | Yes | ### Switching storage modes @@ -150,20 +165,32 @@ Run `chainloop trace init` one time for each repository. Git does not clone the Why not Yes: +- **Attestation at push, Codex:** the suite uses local storage only and cannot verify control-plane uploads. - **Shell commands in another checkout, Cursor:** a relative path in the command resolves against the first workspace root, not against the folder where the shell runs. + ## How each agent is hooked in | Agent | Hook installation | Hooks | |-------|-------------------|-------| | Claude Code | `.claude/settings.json` | Session start and end, user prompt, and before and after each file and shell tool and the Skill tool. | | Cursor | `.cursor/hooks.json` | Session start and end, `afterFileEdit`, and `preToolUse` and `postToolUse` for the `Shell` tool. The `preToolUse` hook always allows the command. The provider installs no prompt hook. | +| Codex | `.codex/hooks.json` | `SessionStart`, `SessionEnd`, `UserPromptSubmit`, `PreToolUse` and `PostToolUse` for `Bash` and `apply_patch`. Patches use one snapshot per affected file, including both paths of a move. | | OpenCode | `.opencode/plugins/chainloop-trace.ts` | Session start and end, user prompt, and before and after each tool. One plugin covers both versions: `server()` for 1.x (1.3.4 or later) and `setup()` for 2. The provider picks the export command from `opencode --version`. | All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. ## End-to-end tests -The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code, Cursor, OpenCode 2, Pi, and Codex columns. The Pi and Codex cells are **No**, so the suite skips them until Pi and Codex support is added. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. Pi runs with `pi -p --mode json --approve`, with its configuration and session folders next to the workspace. Codex runs with `codex exec --json`, with its `CODEX_HOME` folder next to the workspace, no sandbox, and no hook trust prompt. Cursor runs with `cursor-agent -p --output-format stream-json --force --trust`, with its `HOME` folder next to the workspace, so the Cursor configuration and hooks of the user do not load. Cursor runs its shell tool in a login shell, and on macOS `path_helper` puts the system folders first in `PATH`. So the harness writes shell startup files in that `HOME` that set `PATH` back. Otherwise a `chainloop` in `/usr/local/bin` runs the git hooks instead of the CLI under test. The Cursor CLI then has no stored login, so the run needs `CURSOR_API_KEY`. The headless Cursor CLI cannot attach a pasted image, and its output does not show the hook messages. The OpenCode 2, Pi, and Codex models in `e2e/config.yaml` are OpenRouter models, so the run needs `OPENROUTER_API_KEY`. A Codex model with no `openrouter/` prefix runs on the default Codex provider, which needs `CODEX_API_KEY`. +The `e2e/` package checks the tables against real agents and models (spec issue-3635). It creates throwaway repositories and uses Trace in local storage mode. It currently runs the Claude Code, Cursor, OpenCode 2, Pi, and Codex columns. Pi cells are **No**, so the suite skips them. + +The harnesses run as follows: + +- OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace. +- Cursor runs with `cursor-agent -p --output-format stream-json --force --trust` and an isolated `HOME`. Shell startup files restore the test CLI to the front of `PATH`, including after macOS `path_helper` runs. The isolated home has no stored login, so the run needs `CURSOR_API_KEY`. The headless CLI cannot attach a pasted image or report hook messages. +- Pi runs with `pi -p --mode json --approve`. +- Codex runs with `codex exec --json`, an isolated `CODEX_HOME`, no sandbox, and no hook trust prompt. The isolated configuration disables personal and bundled skills while keeping project skills available. For bundled OpenAI models on OpenRouter, the harness preserves native instructions and patch tools by aliasing the model metadata to OpenRouter's prefixed model ID. + +The configured OpenCode 2, Pi, and Codex models use OpenRouter and need `OPENROUTER_API_KEY`. A Codex model without the `openrouter/` prefix uses the default Codex provider and needs `CODEX_API_KEY`. The pinned Codex model is `openrouter/openai/gpt-6-luna`. GPT-5 Mini frequently ended the scripted task before committing and pushing, so it is not the default test model. Some scenarios add the git commands of a human after the session, for example a commit that replaces an AI line, an amend, or a rename, and then push again. They work on a feature branch, because the push reads only the commits that are not on the default branch of the remote. @@ -179,4 +206,20 @@ A feature row maps to its checks by the feature name. When you rename a feature make -C app/cli test-trace-e2e ``` +To run Codex only, set `OPENROUTER_API_KEY` in your environment, then run: + +```bash +CHAINLOOP_E2E_HARNESS=Codex make -C app/cli test-trace-e2e +``` + +To retain the workspace and agent output for a failure: + +```bash +CHAINLOOP_E2E_HARNESS=Codex \ +CHAINLOOP_E2E_WORK_DIR="$(mktemp -d)" \ +make -C app/cli test-trace-e2e +``` + +Each scenario saves `harness-output.jsonl` next to its repository. The process fails when a **Yes** or **Partial** cell fails. Failures in **Unknown** cells appear in the result matrix without failing the process. + `CHAINLOOP_E2E_HARNESS`, `CHAINLOOP_E2E_MODEL`, and `CHAINLOOP_E2E_FEATURE` limit a run to some columns, models, or features. Separate the values with `;`, because feature names can hold commas. Set `CHAINLOOP_E2E_WORK_DIR` to keep the workspaces and the agent output. The results go to `e2e/results/`, or to `CHAINLOOP_E2E_RESULTS_DIR`. Only a run with no filter becomes the previous result of the next comparison. diff --git a/app/cli/internal/trace/codex/announce_test.go b/app/cli/internal/trace/codex/announce_test.go new file mode 100644 index 000000000..4ddfab568 --- /dev/null +++ b/app/cli/internal/trace/codex/announce_test.go @@ -0,0 +1,224 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "encoding/json" + "io" + "os" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAnnounceToUser pins the wire shape Codex expects. systemMessage +// must stay top-level: nested inside hookSpecificOutput it is silently ignored. +const emptyMessageCase = "nothing to say emits nothing" + +func TestAnnounceToUser(t *testing.T) { + const msg = "Coding Session Available at https://app.chainloop.dev/u/chainloop/sessions/ses_1" + + testCases := []struct { + name string + msg string + wantEmitted bool + }{ + { + name: "a message goes out on both channels", + msg: msg, + wantEmitted: true, + }, + { + name: emptyMessageCase, + msg: "", + wantEmitted: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := captureStdout(t, func() { + require.NoError(t, New().AnnounceToUser(tc.msg)) + }) + + if !tc.wantEmitted { + assert.Empty(t, out, "no message means no stdout, so the hook stays a no-op") + return + } + + var got map[string]any + require.NoError(t, json.Unmarshal([]byte(out), &got)) + + assert.Equal(t, tc.msg, got["systemMessage"], "systemMessage must be top-level") + + hookOut, ok := got["hookSpecificOutput"].(map[string]any) + require.True(t, ok, "hookSpecificOutput must be present") + assert.Equal(t, "PostToolUse", hookOut["hookEventName"]) + + // The model needs the message verbatim to be able to repeat it. + assert.Contains(t, hookOut["additionalContext"], tc.msg) + }) + } +} + +// TestAnnounceSessionStart pins the session-start wire shape. The two channels +// travel in one document because Codex parses a hook's stdout as a single +// JSON value: a second write would be discarded together with whatever it +// carried, which is why they are one call rather than two. +func TestAnnounceSessionStart(t *testing.T) { + const ( + banner = "Chainloop Trace is recording this session." + instruction = "Write the specification this session is working from into /repo/.chainloop/specs/abc-123 now." + ) + + testCases := []struct { + name string + msg trace.SessionStartMessage + wantBanner string + wantInstruction string + wantEmitted bool + }{ + { + name: "both channels in one document", + msg: trace.SessionStartMessage{Banner: banner, Instruction: instruction}, + wantBanner: banner, + wantInstruction: instruction, + wantEmitted: true, + }, + { + // What the hook emits once a spec is already captured: the user + // still gets the banner, the model is told nothing. + name: "a banner with nothing to instruct", + msg: trace.SessionStartMessage{Banner: banner}, + wantBanner: banner, + wantEmitted: true, + }, + { + name: "an instruction with no banner", + msg: trace.SessionStartMessage{Instruction: instruction}, + wantInstruction: instruction, + wantEmitted: true, + }, + { + name: emptyMessageCase, + msg: trace.SessionStartMessage{}, + wantEmitted: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := captureStdout(t, func() { + require.NoError(t, New().AnnounceSessionStart(tc.msg)) + }) + + if !tc.wantEmitted { + assert.Empty(t, out, "no message means no stdout, so the hook stays a no-op") + return + } + + dec := json.NewDecoder(strings.NewReader(out)) + + var got map[string]any + require.NoError(t, dec.Decode(&got)) + + // The constraint this whole shape exists to respect: one document, + // never two. + require.ErrorIs(t, dec.Decode(&map[string]any{}), io.EOF, "stdout must carry exactly one JSON document") + + if tc.wantBanner == "" { + assert.NotContains(t, got, "systemMessage", "nothing to show means no empty banner to render") + } else { + assert.Equal(t, tc.wantBanner, got["systemMessage"], "systemMessage must be top-level") + } + + hookOut, ok := got["hookSpecificOutput"].(map[string]any) + require.True(t, ok, "hookSpecificOutput must be present") + assert.Equal(t, "SessionStart", hookOut["hookEventName"], "the event must be the one that fired") + + if tc.wantInstruction == "" { + assert.NotContains(t, hookOut, "additionalContext") + return + } + assert.Equal(t, tc.wantInstruction, hookOut["additionalContext"], "the model needs the instruction verbatim") + }) + } +} + +// TestAnnouncePromptSubmit pins the prompt-submit wire shape: Codex adds +// additionalContext of a UserPromptSubmit response to the context of the turn. +func TestAnnouncePromptSubmit(t *testing.T) { + const reminder = "Chainloop spec capture. Folder: /repo/.chainloop/specs/abc-123" + + testCases := []struct { + name string + reminder string + wantEmitted bool + }{ + {name: "a reminder reaches the model", reminder: reminder, wantEmitted: true}, + {name: emptyMessageCase, reminder: "", wantEmitted: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := captureStdout(t, func() { + require.NoError(t, New().AnnouncePromptSubmit(tc.reminder)) + }) + + if !tc.wantEmitted { + assert.Empty(t, out) + return + } + + var got map[string]any + require.NoError(t, json.Unmarshal([]byte(out), &got)) + + // The reminder is for the model only: a banner at every turn + // would distract the user. + assert.NotContains(t, got, "systemMessage") + + hookOut, ok := got["hookSpecificOutput"].(map[string]any) + require.True(t, ok, "hookSpecificOutput must be present") + assert.Equal(t, "UserPromptSubmit", hookOut["hookEventName"], "the event must be the one that fired") + assert.Equal(t, tc.reminder, hookOut["additionalContext"]) + }) + } +} + +// captureStdout runs fn with os.Stdout redirected to a pipe and returns +// everything written to it. Reads to EOF rather than into a fixed buffer: a +// truncated read would corrupt the payload these tests parse as JSON. +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + + orig := os.Stdout + r, w, err := os.Pipe() + require.NoError(t, err) + os.Stdout = w + t.Cleanup(func() { os.Stdout = orig }) + + fn() + require.NoError(t, w.Close()) + + out, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + + return string(out) +} diff --git a/app/cli/internal/trace/codex/copy.go b/app/cli/internal/trace/codex/copy.go new file mode 100644 index 000000000..9840fdcb2 --- /dev/null +++ b/app/cli/internal/trace/codex/copy.go @@ -0,0 +1,219 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +func codexHome() string { + if home := os.Getenv("CODEX_HOME"); home != "" { + return home + } + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, ".codex") +} + +// CopySessionData refreshes a private transcript copy atomically. A missing +// transcript leaves the previous copy intact. Only Codex session storage is read. +func (p *Provider) CopySessionData(store *state.Store, loc trace.SessionLocation) error { + if store == nil || !state.ValidSessionID(loc.SessionID) { + return errors.New("copy Codex session: invalid session") + } + rec, err := store.LoadSessionRecord(loc.SessionID) + if err != nil { + // The record is an optional storage hint. A damaged hint must not + // prevent copying a transcript validated against configured storage. + rec = nil + } + home := codexHome() + if rec != nil && rec.CodexStorageRoot != "" { + home = rec.CodexStorageRoot + } + if home == "" { + return nil + } + roots := []string{filepath.Join(home, "sessions"), filepath.Join(home, "archived_sessions")} + source := loc.TranscriptPath + if source != "" { + if !filepath.IsAbs(source) { + return errors.New("copy Codex session: transcript path must be absolute") + } + if !insideStorage(source, roots) { + return errors.New("copy Codex session: transcript outside Codex storage") + } + if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) { + source = "" + } + } + if source == "" { + var err error + source, err = findRollout(roots, loc.SessionID) + if err != nil { + return err + } + } + + if source == "" { + return nil + } + if !insideStorage(source, roots) { + return errors.New("copy Codex session: transcript outside Codex storage") + } + src, err := os.Open(source) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + defer src.Close() + info, err := src.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("copy Codex session: transcript is not a regular file") + } + // The rollout header, rather than the file name, owns the session identity. + scanner := bufio.NewScanner(src) + scanner.Buffer(make([]byte, 4096), maxHookPayloadBytes) + if !scanner.Scan() { + return errors.New("copy Codex session: missing rollout header") + } + var header struct { + Type string `json:"type"` + Payload struct { + ID string `json:"id"` + } `json:"payload"` + } + if json.Unmarshal(scanner.Bytes(), &header) != nil || header.Type != "session_meta" || header.Payload.ID != loc.SessionID { + return errors.New("copy Codex session: rollout session ID mismatch") + } + // Persist only after validating both storage containment and session identity. + // Do not infer a trusted root from the hook's transcript path. + if err := saveStorageRoot(store, rec, home); err != nil { + return fmt.Errorf("copy Codex session: save storage root: %w", err) + } + if _, err := src.Seek(0, io.SeekStart); err != nil { + return err + } + dir := store.RawSessionDir() + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".codex-session-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + if _, err := io.Copy(tmp, src); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), state.RawSessionPath(dir, loc.SessionID)) +} + +func saveStorageRoot(store *state.Store, rec *state.SessionRecord, home string) error { + if rec == nil || rec.CodexStorageRoot != "" { + return nil + } + root, err := filepath.EvalSymlinks(home) + if err != nil { + return err + } + root, err = filepath.Abs(root) + if err != nil { + return err + } + // Reload after discovery and validation so an overlapping hook's checkout + // registrations and active state are not replaced by the earlier snapshot. + current, err := store.LoadSessionRecord(rec.SessionID) + if err != nil || current == nil { + // This optional hint must not recreate a deleted or damaged record. + return nil + } + if current.CodexStorageRoot != "" { + return nil + } + current.CodexStorageRoot = root + return store.SaveSessionRecord(current) +} + +// Resolve symlinks as well as lexical traversal before opening a hook path. +func insideStorage(path string, roots []string) bool { + resolved, err := filepath.EvalSymlinks(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return false + } + if errors.Is(err, os.ErrNotExist) { + resolved = filepath.Clean(path) + } + for _, root := range roots { + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + resolvedRoot = filepath.Clean(root) + } + rel, err := filepath.Rel(resolvedRoot, resolved) + if err == nil && filepath.IsLocal(rel) && rel != "." { + return true + } + } + return false +} + +func findRollout(roots []string, sessionID string) (string, error) { + source := "" + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if entry.Type().IsRegular() && strings.HasSuffix(entry.Name(), "-"+sessionID+".jsonl") { + source = path + return fs.SkipAll + } + return nil + }) + if err != nil { + return "", fmt.Errorf("find Codex session: %w", err) + } + if source != "" { + break + } + } + return source, nil +} diff --git a/app/cli/internal/trace/codex/copy_metadata_test.go b/app/cli/internal/trace/codex/copy_metadata_test.go new file mode 100644 index 000000000..48a5f092e --- /dev/null +++ b/app/cli/internal/trace/codex/copy_metadata_test.go @@ -0,0 +1,68 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "os" + "path/filepath" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStorageRootPreservesNewerMetadata(t *testing.T) { + store := state.NewOutOfTreeStore(t.TempDir()) + require.NoError(t, store.SaveSessionRecord(&state.SessionRecord{SessionID: testSessionID, Provider: Name, Active: true})) + stale, err := store.LoadSessionRecord(testSessionID) + require.NoError(t, err) + newer := *stale + newer.Active = false + newer.Checkouts = []string{t.TempDir()} + require.NoError(t, store.SaveSessionRecord(&newer)) + home := t.TempDir() + require.NoError(t, saveStorageRoot(store, stale, home)) + got, err := store.LoadSessionRecord(testSessionID) + require.NoError(t, err) + assert.False(t, got.Active) + assert.Equal(t, newer.Checkouts, got.Checkouts) + canonicalHome, err := filepath.EvalSymlinks(home) + require.NoError(t, err) + assert.Equal(t, canonicalHome, got.CodexStorageRoot) +} + +func TestCopyWithDamagedOptionalMetadata(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + source := filepath.Join(home, "sessions", "session.jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(source), 0700)) + require.NoError(t, os.WriteFile(source, raw, 0600)) //nolint:gosec // fixture copied into a private temporary Codex home + store := state.NewOutOfTreeStore(t.TempDir()) + require.NoError(t, store.SaveSessionRecord(&state.SessionRecord{SessionID: testSessionID, Provider: Name})) + record := filepath.Join(store.TraceDir(), "sessions", testSessionID+".json") + require.NoError(t, os.WriteFile(record, []byte("{"), 0600)) + require.NoError(t, New().CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: source})) + got, err := os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, raw, got) + unchanged, err := os.ReadFile(record) + require.NoError(t, err) + assert.Equal(t, "{", string(unchanged)) +} diff --git a/app/cli/internal/trace/codex/hooks.go b/app/cli/internal/trace/codex/hooks.go new file mode 100644 index 000000000..40b2a0099 --- /dev/null +++ b/app/cli/internal/trace/codex/hooks.go @@ -0,0 +1,390 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" +) + +const ( + hookMarker = "chainloop trace hook codex" + settingsFile = ".codex/hooks.json" + // Skip tracing when the CLI is missing, before any Go handler can run. + hookCommandPrefix = "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec " + windowsHookPrefix = `cmd.exe /d /s /c "where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (` + + // Codex hook event names. + eventSessionStart = "SessionStart" + eventUserPromptSubmit = "UserPromptSubmit" + eventPreToolUse = "PreToolUse" + eventPostToolUse = "PostToolUse" + eventSessionEnd = "SessionEnd" +) + +// fileWritingTools is the single source of truth for Codex hook tool names that modify files. +var fileWritingTools = []string{"apply_patch"} + +// commandTools are Codex hook tool names that run shell commands. They can create +// or modify arbitrary files, so their changes are captured via a before/after +// working-tree snapshot rather than a per-file snapshot. +var commandTools = []string{"Bash"} + +// hookToolMatcher is the pipe-separated matcher for the pre/post-tool-use hooks. +// It covers both file-writing and command tools; the handler branches on which +// kind fired. +var hookToolMatcher = "^(" + strings.Join(append(append([]string{}, fileWritingTools...), commandTools...), "|") + ")$" + +type hookEvent struct { + event string + command string + matcher string // optional: tool name matcher + statusMessage string +} + +var hookEvents = []hookEvent{ + {eventSessionStart, "chainloop trace hook codex session-start", "", "Chainloop Trace: start session"}, + {eventUserPromptSubmit, "chainloop trace hook codex user-prompt-submit", "", "Chainloop Trace: capture prompt context"}, + {eventPreToolUse, "chainloop trace hook codex pre-tool-use", hookToolMatcher, "Chainloop Trace: snapshot files before changes"}, + {eventPostToolUse, "chainloop trace hook codex post-tool-use", hookToolMatcher, "Chainloop Trace: record tool changes"}, + {eventSessionEnd, "chainloop trace hook codex session-end", "", "Chainloop Trace: finish session"}, +} + +// SettingsFile returns the absolute path to .codex/hooks.json for the given repo. +func (p *Provider) SettingsFile(repoRoot string) string { + return filepath.Join(repoRoot, settingsFile) +} + +// InstallHooks adds Chainloop hooks to .codex/hooks.json. +func (p *Provider) InstallHooks(repoRoot string) error { + return p.installHookEvents(repoRoot, hookEvents) +} + +// InstallHooksForTraceRun installs every hook except SessionEnd; trace +// run drives end-of-session itself. +func (p *Provider) InstallHooksForTraceRun(repoRoot string) error { + events := make([]hookEvent, 0, len(hookEvents)) + for _, h := range hookEvents { + if h.event == eventSessionEnd { + continue + } + events = append(events, h) + } + + return p.installHookEvents(repoRoot, events) +} + +func (p *Provider) installHookEvents(repoRoot string, events []hookEvent) error { + settingsPath := filepath.Join(repoRoot, settingsFile) + + settings, err := readJSONFile(settingsPath) + if err != nil { + return err + } + + hooks, _ := settings["hooks"].(map[string]any) + if hooks == nil { + hooks = make(map[string]any) + } + + changed := false + if len(events) < len(hookEvents) { + changed = removeHookEntry(hooks, eventSessionEnd) + } + for _, h := range events { + if addHookEntry(hooks, h.event, h.command, h.matcher, h.statusMessage) { + changed = true + } + } + + if !changed { + return nil + } + + settings["hooks"] = hooks + + return writeJSONFile(settingsPath, settings) +} + +// UninstallHooks removes Chainloop hooks from .codex/hooks.json. +func (p *Provider) UninstallHooks(repoRoot string) error { + settingsPath := filepath.Join(repoRoot, settingsFile) + + settings, err := readJSONFile(settingsPath) + if err != nil { + return err + } + + hooks, _ := settings["hooks"].(map[string]any) + if hooks == nil { + return nil + } + + changed := false + for _, h := range hookEvents { + if removeHookEntry(hooks, h.event) { + changed = true + } + } + + if !changed { + return nil + } + + // Clean up empty hooks map + if len(hooks) == 0 { + delete(settings, "hooks") + } + + // Delete file if empty + if len(settings) == 0 { + return os.Remove(settingsPath) + } + + return writeJSONFile(settingsPath, settings) +} + +// HooksInstalled reports whether .codex/hooks.json holds any Chainloop +// hook. User-authored hooks are ignored. +func (p *Provider) HooksInstalled(repoRoot string) (bool, error) { + settings, err := readJSONFile(filepath.Join(repoRoot, settingsFile)) + if err != nil { + return false, err + } + + hooks, _ := settings["hooks"].(map[string]any) + for _, h := range hookEvents { + entries, _ := hooks[h.event].([]any) + if slices.ContainsFunc(entries, entryContainsChainloopHook) { + return true, nil + } + } + + return false, nil +} + +// maxHookPayloadBytes caps hook payload reads to defend against runaway or +// malformed payloads. +const maxHookPayloadBytes = 16 * 1024 * 1024 + +// ReadHookInput reads and parses the Codex hook JSON from the given reader. +func (p *Provider) ReadHookInput(r io.Reader) (*trace.HookInput, error) { + data, err := io.ReadAll(io.LimitReader(r, maxHookPayloadBytes+1)) + if err != nil { + return nil, err + } + + if len(data) > maxHookPayloadBytes { + return nil, errors.New("codex hook payload exceeds size limit") + } + var raw struct { + trace.HookInput + Model string `json:"model"` + ToolInput struct { + Command string `json:"command"` + } `json:"tool_input"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + input := raw.HookInput + input.Model = raw.Model + if p.IsCommandTool(input.ToolName) { + input.Command = raw.ToolInput.Command + } + if p.IsFileWritingTool(input.ToolName) { + input.FilePaths = patchPaths(raw.ToolInput.Command, input.Cwd) + } + return &input, nil +} + +// patchPaths lists both sides of moves so deletions and new paths are recorded. +func patchPaths(patch, cwd string) []string { + var paths []string + if !filepath.IsAbs(cwd) { + return nil + } + for _, line := range strings.Split(patch, "\n") { + for _, prefix := range []string{"*** Add File: ", "*** Update File: ", "*** Delete File: ", "*** Move to: "} { + path, ok := strings.CutPrefix(line, prefix) + if !ok || path == "" { + continue + } + if !filepath.IsAbs(path) { + path = filepath.Join(cwd, path) + } + path = filepath.Clean(path) + if !slices.Contains(paths, path) { + paths = append(paths, path) + } + } + } + return paths +} + +func addHookEntry(hooks map[string]any, event, command, matcher, statusMessage string) bool { + entries, _ := hooks[event].([]any) + for _, entry := range entries { + group, ok := entry.(map[string]any) + if !ok { + continue + } + handlers, _ := group["hooks"].([]any) + existingMatcher, _ := group["matcher"].(string) + if len(handlers) == 1 && existingMatcher == matcher { + handler, _ := handlers[0].(map[string]any) + if handler["command"] == hookCommandPrefix+command+"'" && handler["commandWindows"] == windowsHookPrefix+command+`)"` && handler["statusMessage"] == statusMessage { + return false + } + } + } + removeHookEntry(hooks, event) + entries, _ = hooks[event].([]any) + hooks[event] = append(entries, buildHookEntry(command, matcher, statusMessage)) + return true +} + +func buildHookEntry(command, matcher, statusMessage string) map[string]any { + entry := map[string]any{ + "hooks": []any{ + map[string]any{ + "type": "command", + "statusMessage": statusMessage, + "command": hookCommandPrefix + command + "'", + "commandWindows": windowsHookPrefix + command + `)"`, + }, + }, + } + if matcher != "" { + entry["matcher"] = matcher + } + return entry +} + +func removeHookEntry(hooks map[string]any, event string) bool { + entries, _ := hooks[event].([]any) + var kept []any + changed := false + for _, entry := range entries { + group, ok := entry.(map[string]any) + if !ok { + kept = append(kept, entry) + continue + } + handlers, _ := group["hooks"].([]any) + var userHooks []any + for _, handler := range handlers { + if isChainloopHandler(handler) { + changed = true + } else { + userHooks = append(userHooks, handler) + } + } + if len(userHooks) == len(handlers) { + kept = append(kept, entry) + } else if len(userHooks) > 0 { + group["hooks"] = userHooks + kept = append(kept, group) + } + } + if !changed { + return false + } + if len(kept) == 0 { + delete(hooks, event) + } else { + hooks[event] = kept + } + return true +} + +func isChainloopHandler(handler any) bool { + h, ok := handler.(map[string]any) + if !ok { + return false + } + command, _ := h["command"].(string) + if strings.HasPrefix(command, hookCommandPrefix) && strings.HasSuffix(command, "'") { + command = strings.TrimSuffix(strings.TrimPrefix(command, hookCommandPrefix), "'") + } + return strings.HasPrefix(command, hookMarker+" ") +} + +func entryContainsChainloopHook(entry any) bool { + group, ok := entry.(map[string]any) + if !ok { + return false + } + handlers, _ := group["hooks"].([]any) + return slices.ContainsFunc(handlers, isChainloopHandler) +} + +func readJSONFile(path string) (map[string]any, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return make(map[string]any), nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + + var result map[string]any + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + + // A file containing `null` unmarshals successfully into a nil map, which + // panics on assignment. + if result == nil { + result = make(map[string]any) + } + + if hooks, ok := result["hooks"]; ok && hooks != nil { + groups, ok := hooks.(map[string]any) + if !ok { + return nil, fmt.Errorf("parse %s: hooks must be an object", path) + } + for event, entries := range groups { + if _, ok := entries.([]any); !ok { + return nil, fmt.Errorf("parse %s: %s hooks must be an array", path, event) + } + } + } + return result, nil +} + +func writeJSONFile(path string, data map[string]any) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create directory: %w", err) + } + + content, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("marshal JSON: %w", err) + } + + content = append(content, '\n') + + return os.WriteFile(path, content, 0600) +} diff --git a/app/cli/internal/trace/codex/hooks_test.go b/app/cli/internal/trace/codex/hooks_test.go new file mode 100644 index 000000000..3b8243287 --- /dev/null +++ b/app/cli/internal/trace/codex/hooks_test.go @@ -0,0 +1,188 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const windowsOS = "windows" + +func TestInstalledHooksWithoutCLI(t *testing.T) { + if runtime.GOOS == windowsOS { + t.Skip("covered by TestWindowsHookExecution") + } + shell, err := exec.LookPath("sh") + require.NoError(t, err) + bin := t.TempDir() + require.NoError(t, os.Symlink(shell, filepath.Join(bin, "sh"))) + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + settings, err := readJSONFile(p.SettingsFile(root)) + require.NoError(t, err) + + for _, event := range hookEvents { + t.Run(event.event, func(t *testing.T) { + command := installedHookCommand(t, settings, event.event) + cmd := exec.CommandContext(t.Context(), shell, "-c", command) //nolint:gosec // execute the installed hook to verify its shell behavior + cmd.Env = append(os.Environ(), "PATH="+bin) + cmd.Stdin = strings.NewReader(`{"session_id":"test"}`) + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + assert.Empty(t, out) + }) + } +} + +func TestInstalledHookPreservesIOAndExitStatus(t *testing.T) { + if runtime.GOOS == windowsOS { + t.Skip("covered by TestWindowsHookExecution") + } + shell, err := exec.LookPath("sh") + require.NoError(t, err) + bin := t.TempDir() + require.NoError(t, os.Symlink(shell, filepath.Join(bin, "sh"))) + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + settings, err := readJSONFile(p.SettingsFile(root)) + require.NoError(t, err) + command := installedHookCommand(t, settings, eventPostToolUse) + stub := "#!/bin/sh\nIFS= read -r payload\nprintf '%s\\n%s\\n' \"$*\" \"$payload\"\nexit \"$HOOK_TEST_EXIT_CODE\"\n" + require.NoError(t, os.WriteFile(filepath.Join(bin, "chainloop"), []byte(stub), 0700)) //nolint:gosec // executable stub CLI in a temporary test directory + + for _, code := range []string{"0", "37"} { + t.Run("exit_"+code, func(t *testing.T) { + cmd := exec.CommandContext(t.Context(), shell, "-c", command) //nolint:gosec // execute the installed hook against a local stub CLI + cmd.Env = append(os.Environ(), "PATH="+bin, "HOOK_TEST_EXIT_CODE="+code) + cmd.Stdin = strings.NewReader("{\"session_id\":\"test\"}\n") + out, err := cmd.Output() + if code == "0" { + require.NoError(t, err) + } else { + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, 37, exitErr.ExitCode()) + } + assert.Equal(t, "trace hook codex post-tool-use\n{\"session_id\":\"test\"}\n", string(out)) + }) + } +} + +func TestInstallUpgradesUnguardedHooks(t *testing.T) { + root := t.TempDir() + p := New() + path := p.SettingsFile(root) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + legacy := `{"hooks":{"PostToolUse":[{"matcher":"^(apply_patch|Bash)$","hooks":[{"type":"command","command":"chainloop trace hook codex post-tool-use"},{"type":"command","command":"echo user-hook"}]}]}}` + require.NoError(t, os.WriteFile(path, []byte(legacy), 0600)) + require.NoError(t, p.InstallHooks(root)) + first, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(string(first), "exec chainloop trace hook codex post-tool-use")) + assert.NotContains(t, string(first), `"command": "chainloop trace hook codex post-tool-use"`) + assert.Contains(t, string(first), "command -v chainloop") + assert.Contains(t, string(first), "echo user-hook") + // Existing guarded hooks also gain labels on init, without duplicating them. + settings, err := readJSONFile(path) + require.NoError(t, err) + for _, groups := range settings["hooks"].(map[string]any) { + for _, group := range groups.([]any) { + for _, handler := range group.(map[string]any)["hooks"].([]any) { + if isChainloopHandler(handler) { + delete(handler.(map[string]any), "statusMessage") + } + } + } + } + require.NoError(t, writeJSONFile(path, settings)) + require.NoError(t, p.InstallHooks(root)) + upgraded, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, first, upgraded) + assert.Contains(t, string(upgraded), "Chainloop Trace: snapshot files before changes") + + require.NoError(t, p.InstallHooks(root)) + second, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.Equal(t, first, second) + require.NoError(t, p.UninstallHooks(root)) + remaining, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.NotContains(t, string(remaining), hookMarker) + assert.Contains(t, string(remaining), "echo user-hook") +} + +func installedHookCommand(t *testing.T, settings map[string]any, event string) string { + t.Helper() + hooks := settings["hooks"].(map[string]any) + groups := hooks[event].([]any) + require.Len(t, groups, 1) + handlers := groups[0].(map[string]any)["hooks"].([]any) + require.Len(t, handlers, 1) + return handlers[0].(map[string]any)["command"].(string) +} + +func TestInstalledHooksWindowsOverride(t *testing.T) { + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + settings, err := readJSONFile(p.SettingsFile(root)) + require.NoError(t, err) + hooks := settings["hooks"].(map[string]any) + for _, event := range hookEvents { + groups := hooks[event.event].([]any) + handlers := groups[0].(map[string]any)["hooks"].([]any) + command := handlers[0].(map[string]any)["commandWindows"].(string) + assert.Contains(t, command, "where chainloop") + assert.Contains(t, command, event.command) + assert.NotContains(t, command, "sh -c") + } +} + +func TestWindowsHookExecution(t *testing.T) { + if runtime.GOOS != windowsOS { + t.Skip("requires the native Windows command interpreter") + } + shell, err := exec.LookPath("cmd.exe") + require.NoError(t, err) + bin := t.TempDir() + t.Setenv("PATH", bin+string(os.PathListSeparator)+filepath.Join(os.Getenv("SystemRoot"), "System32")) + command := windowsHookPrefix + "chainloop trace hook codex post-tool-use" + `)"` + missing := exec.CommandContext(t.Context(), shell, "/d", "/s", "/c", command) + out, err := missing.CombinedOutput() + require.NoError(t, err, string(out)) + assert.Empty(t, out) + stub := "@echo off\r\nset /p payload=\r\necho %*\r\necho %payload%\r\nexit /b 37\r\n" + require.NoError(t, os.WriteFile(filepath.Join(bin, "chainloop.cmd"), []byte(stub), 0600)) + available := exec.CommandContext(t.Context(), shell, "/d", "/s", "/c", command) + available.Stdin = strings.NewReader("{\"session_id\":\"test\"}\n") + out, err = available.Output() + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, 37, exitErr.ExitCode()) + assert.Contains(t, string(out), "trace hook codex post-tool-use") + assert.Contains(t, string(out), `{"session_id":"test"}`) +} diff --git a/app/cli/internal/trace/codex/parse.go b/app/cli/internal/trace/codex/parse.go new file mode 100644 index 000000000..a91e3750d --- /dev/null +++ b/app/cli/internal/trace/codex/parse.go @@ -0,0 +1,285 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "time" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +type tokenUsage struct { + Input int `json:"input_tokens"` + Cached int `json:"cached_input_tokens"` + CacheWrite int `json:"cache_write_input_tokens"` + Output int `json:"output_tokens"` + Total int `json:"total_tokens"` +} + +type rolloutEntry struct { + Timestamp string `json:"timestamp"` + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` +} + +type rolloutPayload struct { + ID string `json:"id"` + Type string `json:"type"` + Version string `json:"cli_version"` + ModelProvider string `json:"model_provider"` + Model string `json:"model"` + Role string `json:"role"` + Name string `json:"name"` + CallID string `json:"call_id"` + ThreadUsage *tokenUsage `json:"thread_token_usage"` + Info *struct { + Total *tokenUsage `json:"total_token_usage"` + } `json:"info"` +} + +// ParseSession reads only the private rollout copy. Unknown record types stay +// in the raw transcript, because the Codex rollout format can evolve. +func (p *Provider) ParseSession(ctx context.Context, opts *trace.ParseOpts) (*aicodingsession.Evidence, error) { + if opts == nil || !state.ValidSessionID(opts.SessionID) { + return nil, errors.New("parse Codex session: invalid options") + } + path, err := state.FindRawSessionFile(opts.SessionDir, opts.SessionID) + if err != nil { + return nil, err + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + d := aicodingsession.Data{ + SchemaVersion: "v1", + Agent: aicodingsession.Agent{Name: Name, Version: opts.AgentVersion}, + Session: aicodingsession.Session{ID: opts.SessionID}, + Model: &aicodingsession.Model{Primary: opts.Model}, + ToolsUsed: &aicodingsession.ToolsUsed{}, + Conversation: &aicodingsession.Conversation{}, + RawSession: map[string][]json.RawMessage{"main": {}}, + } + parser := rolloutParser{ + data: &d, + models: map[string]int{}, + tools: map[string]int{}, + calls: map[string]bool{}, + usageByModel: map[string]tokenUsage{}, + model: opts.Model, + } + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 16*1024*1024) + line := 0 + for scanner.Scan() { + if err := ctx.Err(); err != nil { + return nil, err + } + line++ + raw := scanner.Bytes() + if len(raw) == 0 { + continue + } + var e rolloutEntry + var payload rolloutPayload + if json.Unmarshal(raw, &e) != nil || e.Type == "" || len(e.Payload) == 0 || json.Unmarshal(e.Payload, &payload) != nil { + d.Warnings = append(d.Warnings, fmt.Sprintf("invalid Codex rollout record on line %d", line)) + continue + } + entry, err := transcriptEntry(e, payload, append(json.RawMessage(nil), raw...), parser.model, line) + if err != nil { + return nil, err + } + d.RawSession["main"] = append(d.RawSession["main"], entry) + if err := parser.consume(e, payload, line); err != nil { + return nil, err + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read Codex rollout: %w", err) + } + if err := parser.finish(); err != nil { + return nil, err + } + + return aicodingsession.NewEvidence(d), nil +} + +type rolloutParser struct { + data *aicodingsession.Data + models map[string]int + tools map[string]int + calls map[string]bool + usageByModel map[string]tokenUsage + model string + first, last time.Time + previous tokenUsage + haveUsage, haveHeader bool +} + +func (p *rolloutParser) consume(e rolloutEntry, payload rolloutPayload, line int) error { + if at, err := time.Parse(time.RFC3339Nano, e.Timestamp); err == nil { + if p.first.IsZero() || at.Before(p.first) { + p.first = at + } + if p.last.IsZero() || at.After(p.last) { + p.last = at + } + } else { + p.data.Warnings = append(p.data.Warnings, fmt.Sprintf("invalid Codex timestamp on line %d", line)) + } + switch e.Type { + case "session_meta": + if payload.ID != p.data.Session.ID { + return errors.New("parse Codex session: rollout session ID mismatch") + } + p.haveHeader = true + p.data.Agent.Version = payload.Version + p.data.Model.Provider = payload.ModelProvider + case "turn_context": + if payload.Model != "" { + p.model = payload.Model + if _, ok := p.models[p.model]; !ok { + p.models[p.model] = 0 + } + } + case "response_item": + switch payload.Type { + case "message": + switch payload.Role { + case "user": + p.data.Conversation.UserMessages++ + case "assistant": + p.data.Conversation.AssistantMessages++ + if p.model != "" { + p.models[p.model]++ + } + } + case "function_call", "custom_tool_call": + if payload.Name != "" && (payload.CallID == "" || !p.calls[payload.CallID]) { + p.tools[payload.Name]++ + p.calls[payload.CallID] = true + } + } + case "event_msg": + if payload.Type == "token_count" && payload.Info != nil && payload.Info.Total != nil { + p.consumeUsage(*payload.Info.Total, line) + } + case "token_usage_record": + if payload.ThreadUsage != nil { + p.consumeUsage(*payload.ThreadUsage, line) + } + } + + return nil +} + +func (p *rolloutParser) consumeUsage(total tokenUsage, line int) { + if total.Input < 0 || total.Output < 0 || total.Cached < 0 || total.CacheWrite < 0 || total.Total < 0 { + p.data.Warnings = append(p.data.Warnings, fmt.Sprintf("negative Codex token count on line %d", line)) + return + } + // Counters are session totals, not increments. Repeated snapshots add zero. + delta := tokenUsage{ + Input: total.Input - p.previous.Input, + Cached: total.Cached - p.previous.Cached, + CacheWrite: total.CacheWrite - p.previous.CacheWrite, + Output: total.Output - p.previous.Output, + Total: total.Total - p.previous.Total, + } + if delta.Input < 0 || delta.Output < 0 || delta.Cached < 0 || delta.CacheWrite < 0 || delta.Total < 0 { + p.data.Warnings = append(p.data.Warnings, fmt.Sprintf("Codex token counters reset on line %d", line)) + delta = total + } + u := p.usageByModel[p.model] + u.Input += delta.Input + u.Cached += delta.Cached + u.CacheWrite += delta.CacheWrite + u.Output += delta.Output + u.Total += delta.Total + p.usageByModel[p.model] = u + p.previous = total + p.haveUsage = true +} + +func (p *rolloutParser) finish() error { + if !p.haveHeader { + return errors.New("parse Codex session: missing session metadata") + } + if !p.first.IsZero() { + p.data.Session.StartedAt = p.first.UTC().Format(time.RFC3339) + p.data.Session.EndedAt = p.last.UTC().Format(time.RFC3339) + p.data.Session.DurationSeconds = int(p.last.Sub(p.first).Seconds()) + } + best := -1 + for name := range p.models { + p.data.Model.ModelsUsed = append(p.data.Model.ModelsUsed, name) + } + sort.Strings(p.data.Model.ModelsUsed) + for _, name := range p.data.Model.ModelsUsed { + if p.models[name] > best { + p.data.Model.Primary = name + best = p.models[name] + } + } + if p.data.Model.Primary == "" { + p.data.Warnings = append(p.data.Warnings, "Codex rollout has no model information") + } + for name, count := range p.tools { + p.data.ToolsUsed.Summary = append(p.data.ToolsUsed.Summary, aicodingsession.ToolSummary{ToolName: name, InvocationCount: count}) + p.data.ToolsUsed.TotalInvocations += count + } + sort.Slice(p.data.ToolsUsed.Summary, func(i, j int) bool { + a, b := p.data.ToolsUsed.Summary[i], p.data.ToolsUsed.Summary[j] + if a.InvocationCount != b.InvocationCount { + return a.InvocationCount > b.InvocationCount + } + return a.ToolName < b.ToolName + }) + p.data.Conversation.TotalMessages = p.data.Conversation.UserMessages + p.data.Conversation.AssistantMessages + if p.haveUsage { + p.data.Usage = &aicodingsession.Usage{} + // Stable ordering also keeps warnings deterministic for sessions using several models. + names := make([]string, 0, len(p.usageByModel)) + for name := range p.usageByModel { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + u := p.usageByModel[name] + p.data.Usage.InputTokens += u.Input + p.data.Usage.OutputTokens += u.Output + p.data.Usage.TotalTokens += u.Total + p.data.Usage.CacheReadInputTokens += u.Cached + p.data.Usage.CacheCreationInputTokens += u.CacheWrite + p.data.Usage.EstimatedCostUSD += estimateCost(name, u, &p.data.Warnings) + } + p.data.Usage.EstimatedCostUSD = roundCost(p.data.Usage.EstimatedCostUSD) + } else { + p.data.Warnings = append(p.data.Warnings, "Codex rollout has no token usage; cost is unavailable") + } + return nil +} diff --git a/app/cli/internal/trace/codex/pricing.go b/app/cli/internal/trace/codex/pricing.go new file mode 100644 index 000000000..a52865df0 --- /dev/null +++ b/app/cli/internal/trace/codex/pricing.go @@ -0,0 +1,75 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + _ "embed" + "encoding/json" + "fmt" + "math" + "strings" + "time" +) + +type modelPricing struct { + Input float64 `json:"input"` + Cached float64 `json:"cached"` + Output float64 `json:"output"` + CacheWrite float64 `json:"cache_write"` +} + +//go:embed pricing.json +var pricingJSON []byte +var prices struct { + Default string `json:"default_model"` + Models map[string]modelPricing `json:"models"` +} + +func init() { + if err := json.Unmarshal(pricingJSON, &prices); err != nil { + panic(fmt.Sprintf("codex trace: invalid embedded pricing: %v", err)) + } + if _, ok := prices.Models[prices.Default]; !ok { + panic("codex trace: missing default model price") + } +} + +func estimateCost(model string, u tokenUsage, warnings *[]string) float64 { + name := strings.TrimPrefix(model, "openai/") + price, ok := prices.Models[name] + if !ok { + // Date-pinned model IDs use the same prices as their family alias. + if len(name) > 11 && name[len(name)-11] == '-' { + if _, err := time.Parse(time.DateOnly, name[len(name)-10:]); err == nil { + price, ok = prices.Models[name[:len(name)-11]] + } + } + } + if !ok { + price = prices.Models[prices.Default] + *warnings = append(*warnings, fmt.Sprintf("unknown Codex model %q; cost estimated using %s pricing", model, prices.Default)) + } + // Codex input_tokens already includes cached tokens. Cache writes are part + // of input, and reasoning tokens are already part of output_tokens. + uncached := max(0, u.Input-u.Cached) + written := 0 + if price.CacheWrite > 0 { + written = min(u.CacheWrite, uncached) + } + return (float64(uncached-written)*price.Input + float64(written)*price.CacheWrite + float64(u.Cached)*price.Cached + float64(u.Output)*price.Output) / 1e6 +} + +func roundCost(cost float64) float64 { return math.Round(cost*1e4) / 1e4 } diff --git a/app/cli/internal/trace/codex/pricing.json b/app/cli/internal/trace/codex/pricing.json new file mode 100644 index 000000000..2a9dd638f --- /dev/null +++ b/app/cli/internal/trace/codex/pricing.json @@ -0,0 +1,145 @@ +{ + "version": "2026-10-21", + "source": "https://developers.openai.com/api/docs/pricing", + "notes": "USD per million tokens, standard short-context API rates. Estimates exclude priority tiers, long-context surcharges, and subscription or third-party billing. Unknown models use the default with a warning.", + "default_model": "gpt-5-mini", + "models": { + "gpt-5-mini": { + "input": 0.25, + "cached": 0.025, + "output": 2.0 + }, + "gpt-6-astra": { + "input": 10.0, + "cached": 1.0, + "output": 50.0, + "cache_write": 12.5 + }, + "gpt-6.1-sol": { + "input": 2.0, + "cached": 0.1, + "output": 10.0, + "cache_write": 2.5 + }, + "gpt-6-luna": { + "input": 0.1, + "cached": 0.01, + "output": 0.5, + "cache_write": 0.125 + }, + "gpt-6-sol": { + "input": 2.0, + "cached": 0.2, + "output": 10.0, + "cache_write": 2.5 + }, + "gpt-5.6-sol": { + "input": 4.0, + "cached": 0.4, + "output": 20.0, + "cache_write": 5.0 + }, + "gpt-5.6-terra": { + "input": 2.0, + "cached": 0.2, + "output": 12.0, + "cache_write": 2.5 + }, + "gpt-5.6-luna": { + "input": 0.2, + "cached": 0.02, + "output": 1.2, + "cache_write": 0.25 + }, + "gpt-5.5": { + "input": 5.0, + "cached": 0.5, + "output": 30.0 + }, + "gpt-5.4": { + "input": 2.5, + "cached": 0.25, + "output": 15.0 + }, + "gpt-5.4-mini": { + "input": 0.75, + "cached": 0.075, + "output": 4.5 + }, + "gpt-5.4-nano": { + "input": 0.2, + "cached": 0.02, + "output": 1.25 + }, + "gpt-5.2": { + "input": 1.75, + "cached": 0.175, + "output": 14.0 + }, + "gpt-5.1": { + "input": 1.25, + "cached": 0.125, + "output": 10.0 + }, + "gpt-5": { + "input": 1.25, + "cached": 0.125, + "output": 10.0 + }, + "gpt-5-nano": { + "input": 0.05, + "cached": 0.005, + "output": 0.4 + }, + "gpt-4.1": { + "input": 2.0, + "cached": 0.5, + "output": 8.0 + }, + "gpt-4.1-mini": { + "input": 0.4, + "cached": 0.1, + "output": 1.6 + }, + "gpt-4.1-nano": { + "input": 0.1, + "cached": 0.025, + "output": 0.4 + }, + "gpt-4o": { + "input": 2.5, + "cached": 1.25, + "output": 10.0 + }, + "gpt-4o-mini": { + "input": 0.15, + "cached": 0.075, + "output": 0.6 + }, + "o1": { + "input": 15.0, + "cached": 7.5, + "output": 60.0 + }, + "o3": { + "input": 2.0, + "cached": 0.5, + "output": 8.0 + }, + "o4-mini": { + "input": 1.1, + "cached": 0.275, + "output": 4.4 + }, + "o3-mini": { + "input": 1.1, + "cached": 0.55, + "output": 4.4 + }, + "gpt-5.3-codex": { + "input": 1.75, + "output": 14, + "cached": 0.175 + } + } +} diff --git a/app/cli/internal/trace/codex/provider.go b/app/cli/internal/trace/codex/provider.go new file mode 100644 index 000000000..b7bccc751 --- /dev/null +++ b/app/cli/internal/trace/codex/provider.go @@ -0,0 +1,149 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package codex captures sessions from the Codex CLI's native hooks and JSONL rollouts. +package codex + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "slices" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +// Name is the identifier stored in session records and evidence. +const Name = "codex" + +var _ trace.Provider = (*Provider)(nil) + +// Provider connects Codex to Trace. It holds no session state. +type Provider struct{} + +// New creates a Codex provider. +func New() *Provider { return &Provider{} } + +// Name returns the provider identifier. +func (p *Provider) Name() string { return Name } + +// CaptureFileSnapshot reads the file at input.FilePath and stores its +// content under the store's snapshots/ directory so the post-tool-use +// handler can compute line-range diffs after the edit. A missing file +// (a patch creating a new file) is treated as a non-error; +// other read errors (permission denied, I/O failures) are surfaced. +func (p *Provider) CaptureFileSnapshot(store *state.Store, input *trace.HookInput) error { + if input == nil || input.FilePath == "" { + return nil + } + + content, err := os.ReadFile(input.FilePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + + return fmt.Errorf("read %q: %w", input.FilePath, err) + } + + return store.SaveFileSnapshot(input.SessionID, input.FilePath, content) +} + +// ResolveBeforeContent returns the snapshot saved by CaptureFileSnapshot. +// The "after" argument is unused for Codex — we always rely on the pre-edit +// snapshot — but is part of the interface for providers that reconstruct +// from edits instead of snapshots. +func (p *Provider) ResolveBeforeContent(store *state.Store, input *trace.HookInput, _ []byte) []byte { + if input == nil || input.FilePath == "" { + return nil + } + + snap, err := store.LoadFileSnapshot(input.SessionID, input.FilePath) + if err != nil { + return nil + } + + return snap +} + +// CleanupAfterEdit removes the per-edit snapshot. Safe to call when no +// snapshot was ever taken. +func (p *Provider) CleanupAfterEdit(store *state.Store, input *trace.HookInput) { + if input == nil || input.FilePath == "" { + return + } + + store.DeleteFileSnapshot(input.SessionID, input.FilePath) +} + +// IsFileWritingTool returns true if the named tool modifies files on disk. +func (p *Provider) IsFileWritingTool(toolName string) bool { + return slices.Contains(fileWritingTools, toolName) +} + +// IsCommandTool returns true if the named tool runs a shell command. +func (p *Provider) IsCommandTool(toolName string) bool { + return slices.Contains(commandTools, toolName) +} + +// SupportsSessionStartBanner reports whether Codex shows systemMessage to the user. +func (p *Provider) SupportsSessionStartBanner() bool { return true } + +// SupportsSessionStartInstruction reports whether Codex accepts startup context. +func (p *Provider) SupportsSessionStartInstruction() bool { return true } + +// SupportsPromptReminder reports whether Codex accepts context at each prompt. +func (p *Provider) SupportsPromptReminder() bool { return true } + +// AnnounceSessionStart sends the banner and instruction in one hook response. +func (p *Provider) AnnounceSessionStart(msg trace.SessionStartMessage) error { + return writeHookResponse(eventSessionStart, msg.Banner, msg.Instruction) +} + +// AnnouncePromptSubmit adds the reminder to the model's context. +func (p *Provider) AnnouncePromptSubmit(reminder string) error { + return writeHookResponse(eventUserPromptSubmit, "", reminder) +} + +// AnnounceToUser shows the session link and gives the model context to repeat it. +func (p *Provider) AnnounceToUser(msg string) error { + if msg == "" { + return nil + } + return writeHookResponse(eventPostToolUse, msg, "Tell the user the following, including any link verbatim: "+msg) +} + +type hookSpecificOutput struct { + HookEventName string `json:"hookEventName"` + AdditionalContext string `json:"additionalContext,omitempty"` +} + +type hookResponse struct { + SystemMessage string `json:"systemMessage,omitempty"` + HookSpecificOutput hookSpecificOutput `json:"hookSpecificOutput"` +} + +// Codex reads one JSON document per hook. Empty messages emit nothing. +func writeHookResponse(event, banner, instruction string) error { + if banner == "" && instruction == "" { + return nil + } + return json.NewEncoder(os.Stdout).Encode(hookResponse{ + SystemMessage: banner, + HookSpecificOutput: hookSpecificOutput{HookEventName: event, AdditionalContext: instruction}, + }) +} diff --git a/app/cli/internal/trace/codex/provider_test.go b/app/cli/internal/trace/codex/provider_test.go new file mode 100644 index 000000000..ebb858f2d --- /dev/null +++ b/app/cli/internal/trace/codex/provider_test.go @@ -0,0 +1,345 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/internal/schemavalidators" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testSessionID = "sess-1" +const testModel = "gpt-5-mini" + +func TestHooks(t *testing.T) { + for _, run := range []bool{false, true} { + t.Run(map[bool]string{false: "init", true: "run"}[run], func(t *testing.T) { + p := New() + root := t.TempDir() + path := p.SettingsFile(root) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte(`{"description":"user settings","hooks":{"PostToolUse":[{"hooks":[{"type":"command","command":"echo user-hook"}]}]}}`), 0600)) + installed, err := p.HooksInstalled(root) + require.NoError(t, err) + require.False(t, installed) + install := p.InstallHooks + if run { + install = p.InstallHooksForTraceRun + } + require.NoError(t, install(root)) + first, err := os.ReadFile(path) + require.NoError(t, err) + require.NoError(t, install(root)) + second, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, first, second) + installed, err = p.HooksInstalled(root) + require.NoError(t, err) + assert.True(t, installed) + assert.Contains(t, string(first), "echo user-hook") + assert.Contains(t, string(first), "apply_patch") + assert.Equal(t, !run, strings.Contains(string(first), `"SessionEnd"`)) + require.NoError(t, p.UninstallHooks(root)) + left, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(left), "echo user-hook") + assert.Contains(t, string(left), "user settings") + assert.NotContains(t, string(left), hookMarker) + require.NoError(t, p.UninstallHooks(root)) + }) + } +} + +func TestReadHookInput(t *testing.T) { + for _, tc := range []struct { + name, tool, command string + paths []string + }{ + {"shell", "Bash", "git status", nil}, + {"patch", "apply_patch", "*** Begin Patch\n*** Update File: one.go\n@@\n-old\n+new\n*** Add File: two.go\n+new\n*** Delete File: gone.go\n*** Update File: old.go\n*** Move to: new.go\n@@\n-old\n+new\n*** End Patch", []string{"one.go", "two.go", "gone.go", "old.go", "new.go"}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + raw, _ := json.Marshal(map[string]any{"session_id": testSessionID, "cwd": root, "transcript_path": "/sessions/rollout.jsonl", "tool_name": tc.tool, "tool_use_id": "call-1", "agent_id": "child-1", "model": testModel, "tool_input": map[string]string{"command": tc.command}}) + input, err := New().ReadHookInput(strings.NewReader(string(raw))) + require.NoError(t, err) + assert.Equal(t, testSessionID, input.SessionID) + assert.Equal(t, "call-1", input.ToolUseID) + assert.Equal(t, "child-1", input.AgentID) + assert.Equal(t, testModel, input.Model) + if tc.tool == "Bash" { + assert.Equal(t, tc.command, input.Command) + } else { + var paths []string + for _, path := range tc.paths { + paths = append(paths, filepath.Join(root, path)) + } + assert.Equal(t, paths, input.FilePaths) + } + }) + } + _, err := New().ReadHookInput(strings.NewReader(`{`)) + assert.Error(t, err) +} + +func TestParseSession(t *testing.T) { + dir := t.TempDir() + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // temporary fixture path and a fixed session ID + evidence, err := New().ParseSession(context.Background(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + d := evidence.Data + assert.Equal(t, Name, d.Agent.Name) + assert.Equal(t, "0.163.0", d.Agent.Version) + assert.Equal(t, 60, d.Session.DurationSeconds) + assert.Equal(t, testModel, d.Model.Primary) + assert.Equal(t, "openai", d.Model.Provider) + assert.Equal(t, 2000, d.Usage.InputTokens) + assert.Equal(t, 400, d.Usage.OutputTokens) + assert.Equal(t, 2400, d.Usage.TotalTokens) + assert.Equal(t, 1000, d.Usage.CacheReadInputTokens) + assert.Equal(t, 100, d.Usage.CacheCreationInputTokens) + assert.InDelta(t, 0.0011, d.Usage.EstimatedCostUSD, 0.00001) + assert.Equal(t, 2, d.Conversation.TotalMessages) + assert.Equal(t, 2, d.ToolsUsed.TotalInvocations) + assert.Equal(t, 1, d.Conversation.UserMessages) + assert.Equal(t, 1, d.Conversation.AssistantMessages) + assert.Len(t, d.RawSession["main"], 12) + assert.Empty(t, d.Warnings) + doc, err := json.Marshal(d) + require.NoError(t, err) + var data map[string]any + require.NoError(t, json.Unmarshal(doc, &data)) + require.NoError(t, schemavalidators.ValidateAICodingSession(data, schemavalidators.AICodingSessionVersion0_1)) +} + +func TestCopySessionData(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + p := New() + store := state.NewOutOfTreeStore(t.TempDir()) + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + path := filepath.Join(home, "sessions", "2026", "10", "21", "rollout-time-sess-1.jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, raw, 0600)) //nolint:gosec // temporary Codex session fixture + for _, reported := range []string{path, "", filepath.Join(home, "sessions", "moved.jsonl")} { + require.NoError(t, p.CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: reported})) + got, err := os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, raw, got) + } + outside := filepath.Join(t.TempDir(), "rollout-time-sess-1.jsonl") + require.NoError(t, os.WriteFile(outside, raw, 0600)) //nolint:gosec // deliberately outside storage to verify rejection + assert.Error(t, p.CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: outside})) + link := filepath.Join(home, "sessions", "rollout-link-sess-1.jsonl") + require.NoError(t, os.Symlink(outside, link)) + assert.Error(t, p.CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: link})) + require.NoError(t, p.CopySessionData(store, trace.SessionLocation{SessionID: "missing"})) + assert.Error(t, p.CopySessionData(store, trace.SessionLocation{SessionID: "../escape"})) +} + +func TestUninstallKeepsMixedUserHookGroup(t *testing.T) { + p := New() + root := t.TempDir() + path := p.SettingsFile(root) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte(`{"hooks":{"PostToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":"chainloop trace hook codex post-tool-use"},{"type":"command","command":"echo keep-me"}]}]}}`), 0600)) + require.NoError(t, p.InstallHooks(root)) + require.NoError(t, p.UninstallHooks(root)) + raw, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(raw), "echo keep-me") + assert.NotContains(t, string(raw), hookMarker) +} + +func TestParseSessionWarnings(t *testing.T) { + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + for _, tc := range []struct { + name string + data string + wantErr bool + warning string + }{ + {name: "truncated tail", data: string(raw) + `{"type":`, warning: "invalid Codex rollout record"}, + {name: "wrong session", data: strings.ReplaceAll(string(raw), testSessionID, "other"), wantErr: true}, + {name: "no header", data: `{"type":"event_msg","timestamp":"2026-10-21T10:00:00Z","payload":{"type":"other"}}`, wantErr: true}, + {name: "unknown model", data: strings.ReplaceAll(string(raw), testModel, "future-model"), warning: "unknown Codex model"}, + {name: "no usage", data: strings.ReplaceAll(string(raw), "token_count", "other"), warning: "no token usage"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), []byte(tc.data), 0600)) + e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Contains(t, strings.Join(e.Data.Warnings, "\n"), tc.warning) + }) + } +} + +func TestTraceRunOmitsPreviouslyInstalledEndHook(t *testing.T) { + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + require.NoError(t, p.InstallHooksForTraceRun(root)) + raw, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.NotContains(t, string(raw), `"SessionEnd"`) +} + +func TestUsageAcrossModels(t *testing.T) { + dir := t.TempDir() + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + // The second model owns only the increment after the first cumulative snapshot. + extra := `{"timestamp":"2026-10-21T10:01:01Z","type":"turn_context","payload":{"model":"gpt-5"}} +{"timestamp":"2026-10-21T10:01:02Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[]}} +{"timestamp":"2026-10-21T10:01:03Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[]}} +{"timestamp":"2026-10-21T10:01:04Z","type":"token_usage_record","payload":{"usage":{"input_tokens":2000,"output_tokens":404,"total_tokens":2404},"thread_token_usage":{"input_tokens":4000,"cached_input_tokens":2000,"cache_write_input_tokens":200,"output_tokens":804,"total_tokens":4804}}} +{"timestamp":"2026-10-21T10:01:04Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":4000,"cached_input_tokens":2000,"cache_write_input_tokens":200,"output_tokens":804,"total_tokens":4804}}}} +` + raw = append(raw, extra...) + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // temporary fixture and a fixed session ID + e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + assert.Equal(t, []string{"gpt-5", testModel}, e.Data.Model.ModelsUsed) + assert.Equal(t, "gpt-5", e.Data.Model.Primary) + assert.Equal(t, 4000, e.Data.Usage.InputTokens) + assert.Equal(t, 4804, e.Data.Usage.TotalTokens) + assert.InDelta(t, 0.0065, e.Data.Usage.EstimatedCostUSD, 0.00001) +} + +func TestNativeRollout(t *testing.T) { + raw, err := os.ReadFile("testdata/native-session.jsonl") + require.NoError(t, err) + lines := strings.SplitAfter(string(raw), "\n") + for _, tc := range []struct { + name, data string + }{ + {"before mirrored event", strings.Join(lines[:6], "")}, + {"with mirrored event", string(raw)}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), []byte(tc.data), 0600)) + e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + require.NotNil(t, e.Data.Usage) + assert.Equal(t, 19064, e.Data.Usage.InputTokens) + assert.Equal(t, 128, e.Data.Usage.OutputTokens) + assert.Equal(t, 19192, e.Data.Usage.TotalTokens) + assert.Equal(t, 19061, e.Data.Usage.CacheCreationInputTokens) + assert.InDelta(t, 0.0489, e.Data.Usage.EstimatedCostUSD, 0.00001) + assert.NotEmpty(t, e.Data.Model.Primary) + assert.NotEmpty(t, e.Data.ToolsUsed.Summary) + assert.Empty(t, e.Data.Warnings) + }) + } +} + +func TestCopySessionDataPreservesStorageRoot(t *testing.T) { + for _, env := range []string{"unset", "different home"} { + t.Run(env, func(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + store := state.NewOutOfTreeStore(t.TempDir()) + require.NoError(t, store.SaveSessionRecord(&state.SessionRecord{SessionID: testSessionID, Provider: Name, Active: true})) + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + path := filepath.Join(home, "sessions", "rollout-time-"+testSessionID+".jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, raw, 0600)) //nolint:gosec // temporary Codex storage fixture and a fixed session ID + loc := trace.SessionLocation{SessionID: testSessionID, TranscriptPath: path} + require.NoError(t, New().CopySessionData(store, loc)) + rec, err := store.LoadSessionRecord(testSessionID) + require.NoError(t, err) + canonicalHome, err := filepath.EvalSymlinks(home) + require.NoError(t, err) + assert.Equal(t, canonicalHome, rec.CodexStorageRoot) + assert.True(t, rec.Active) + + updated := append(append([]byte(nil), raw...), []byte("\n{\"new_record\":true}\n")...) + require.NoError(t, os.WriteFile(path, updated, 0600)) //nolint:gosec // temporary Codex storage fixture and a fixed session ID + if env == "unset" { + require.NoError(t, os.Unsetenv("CODEX_HOME")) + } else { + t.Setenv("CODEX_HOME", t.TempDir()) + } + // A new provider instance represents a later Git hook process. + require.NoError(t, New().CopySessionData(store, loc)) + got, err := os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, updated, got) + + archived := filepath.Join(home, "archived_sessions", filepath.Base(path)) + require.NoError(t, os.MkdirAll(filepath.Dir(archived), 0700)) + require.NoError(t, os.Rename(path, archived)) + require.NoError(t, New().CopySessionData(store, loc)) + got, err = os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, updated, got) + + outside := filepath.Join(t.TempDir(), "outside.jsonl") + require.NoError(t, os.WriteFile(outside, raw, 0600)) //nolint:gosec // deliberately outside storage to verify rejection + assert.Error(t, New().CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: outside})) + link := filepath.Join(home, "sessions", "escaped.jsonl") + require.NoError(t, os.Symlink(outside, link)) + assert.Error(t, New().CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: link})) + }) + } +} + +func TestEstimateCostDateSuffix(t *testing.T) { + for _, tc := range []struct { + model string + cost float64 + warning bool + }{ + {"gpt-6-luna", 0.1, false}, + {"gpt-6-luna-2026-10-21", 0.1, false}, + {"openai/gpt-6-luna-2024-02-29", 0.1, false}, + {"gpt-6-luna-foo-123456", 0.25, true}, + {"gpt-6-luna-2026-02-29", 0.25, true}, + {"gpt-6-luna-2026-13-21", 0.25, true}, + {"gpt-6-luna-2026-1-21", 0.25, true}, + } { + t.Run(tc.model, func(t *testing.T) { + var warnings []string + assert.InDelta(t, tc.cost, estimateCost(tc.model, tokenUsage{Input: 1_000_000}, &warnings), 0.00001) + if tc.warning { + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0], "unknown Codex model") + } else { + assert.Empty(t, warnings) + } + }) + } +} diff --git a/app/cli/internal/trace/codex/testdata/README.md b/app/cli/internal/trace/codex/testdata/README.md new file mode 100644 index 000000000..bdee0632b --- /dev/null +++ b/app/cli/internal/trace/codex/testdata/README.md @@ -0,0 +1,8 @@ +`native-session.jsonl` contains selected records from a Codex CLI 0.163.0 +rollout. Instructions, prompts, tool inputs and outputs, paths, IDs, and times +were removed or replaced. The native usage counters and record shapes remain. +It includes both `token_usage_record` and `event_msg/token_count` records for +the same usage, which must count once. + +`session.jsonl` is a small constructed fixture for predictable totals, repeated +cumulative counters, and duplicate conversation events. diff --git a/app/cli/internal/trace/codex/testdata/native-session.jsonl b/app/cli/internal/trace/codex/testdata/native-session.jsonl new file mode 100644 index 000000000..ab9a8ac43 --- /dev/null +++ b/app/cli/internal/trace/codex/testdata/native-session.jsonl @@ -0,0 +1,10 @@ +{"timestamp":"2026-10-21T10:00:00Z","type":"session_meta","payload":{"id":"sess-1","cli_version":"0.163.0","model_provider":"openai","cwd":"/workspace","timestamp":"2026-10-21T10:00:00Z"}} +{"timestamp":"2026-10-21T10:00:01Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Sanitized example message"}]}} +{"timestamp":"2026-10-21T10:00:02Z","type":"turn_context","payload":{"model":"gpt-6.1-sol","cwd":"/workspace"}} +{"timestamp":"2026-10-21T10:00:03Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Sanitized example message"}]}} +{"timestamp":"2026-10-21T10:00:04Z","type":"response_item","payload":{"type":"custom_tool_call","name":"exec","call_id":"call-4","input":"Sanitized tool input"}} +{"timestamp":"2026-10-21T10:00:05Z","type":"token_usage_record","payload":{"usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"thread_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"turn_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192}}} +{"timestamp":"2026-10-21T10:00:06Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"sanitized-call","output":"Sanitized tool output"}} +{"timestamp":"2026-10-21T10:00:07Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"last_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"model_context_window":475000}}} +{"timestamp":"2026-10-21T10:00:08Z","type":"response_item","payload":{"type":"function_call","name":"wait","call_id":"call-8","arguments":"Sanitized tool input"}} +{"timestamp":"2026-10-21T10:00:09Z","type":"response_item","payload":{"type":"function_call_output","call_id":"sanitized-call","output":"Sanitized tool output"}} diff --git a/app/cli/internal/trace/codex/testdata/session.jsonl b/app/cli/internal/trace/codex/testdata/session.jsonl new file mode 100644 index 000000000..ad714752f --- /dev/null +++ b/app/cli/internal/trace/codex/testdata/session.jsonl @@ -0,0 +1,12 @@ +{"timestamp":"2026-10-21T10:00:00Z","type":"session_meta","payload":{"id":"sess-1","timestamp":"2026-10-21T10:00:00Z","cwd":"/workspace","cli_version":"0.163.0","source":"cli","model_provider":"openai"}} +{"timestamp":"2026-10-21T10:00:01Z","type":"turn_context","payload":{"model":"gpt-5-mini","cwd":"/workspace"}} +{"timestamp":"2026-10-21T10:00:02Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Create hello.txt"}]}} +{"timestamp":"2026-10-21T10:00:02Z","type":"event_msg","payload":{"type":"user_message","message":"Create hello.txt"}} +{"timestamp":"2026-10-21T10:00:03Z","type":"response_item","payload":{"type":"custom_tool_call","name":"apply_patch","call_id":"call-1","input":"*** Begin Patch\n*** Add File: hello.txt\n+hello\n*** End Patch"}} +{"timestamp":"2026-10-21T10:00:04Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call-1","output":"Success"}} +{"timestamp":"2026-10-21T10:00:05Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":1000,"cached_input_tokens":500,"cache_write_input_tokens":50,"output_tokens":200,"reasoning_output_tokens":100,"total_tokens":1200}}}} +{"timestamp":"2026-10-21T10:00:06Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call-2","arguments":"{\"cmd\":\"cat hello.txt\"}"}} +{"timestamp":"2026-10-21T10:00:07Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call-2","output":"hello"}} +{"timestamp":"2026-10-21T10:00:58Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Created hello.txt"}]}} +{"timestamp":"2026-10-21T10:00:59Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":1000,"cache_write_input_tokens":100,"output_tokens":400,"reasoning_output_tokens":200,"total_tokens":2400}}}} +{"timestamp":"2026-10-21T10:01:00Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":1000,"cache_write_input_tokens":100,"output_tokens":400,"total_tokens":2400}}}} diff --git a/app/cli/internal/trace/codex/transcript.go b/app/cli/internal/trace/codex/transcript.go new file mode 100644 index 000000000..25ef182ac --- /dev/null +++ b/app/cli/internal/trace/codex/transcript.go @@ -0,0 +1,147 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" +) + +const ( + responseItem = "response_item" + customToolCall = "custom_tool_call" + functionCall = "function_call" + messageItem = "message" +) + +// transcriptEntry normalizes messages and tool exchanges for every consumer, +// while retaining their original event and all unknown metadata records. +func transcriptEntry(e rolloutEntry, payload rolloutPayload, raw json.RawMessage, model string, line int) (json.RawMessage, error) { + if e.Type != responseItem { + return raw, nil + } + switch payload.Type { + case messageItem, functionCall, customToolCall, "function_call_output", "custom_tool_call_output", "reasoning": + default: + return raw, nil + } + var item struct { + Content json.RawMessage `json:"content"` + Arguments string `json:"arguments"` + Input string `json:"input"` + Output json.RawMessage `json:"output"` + Summary []struct { + Text string `json:"text"` + } `json:"summary"` + } + if err := json.Unmarshal(e.Payload, &item); err != nil { + return nil, fmt.Errorf("read Codex transcript item on line %d: %w", line, err) + } + entry := trace.RawSessionEntry{ + UUID: payload.ID, + Timestamp: e.Timestamp, + SourceEvent: raw, + Message: trace.RawSessionMessage{Model: model}, + } + if entry.UUID == "" { + entry.UUID = fmt.Sprintf("codex-event-%d", line) + } + var blocks []any + switch payload.Type { + case messageItem: + if payload.Role != trace.RoleUser && payload.Role != trace.RoleAssistant { + return raw, nil + } + entry.Type = payload.Role + content, err := transcriptContent(item.Content) + if err != nil { + return nil, fmt.Errorf("read Codex message on line %d: %w", line, err) + } + entry.Message.Content = content + case functionCall, customToolCall: + entry.Type = trace.RoleAssistant + input := json.RawMessage(item.Arguments) + var object map[string]json.RawMessage + if payload.Type == customToolCall { + input, _ = json.Marshal(map[string]string{"input": item.Input}) + } else if json.Unmarshal(input, &object) != nil || object == nil { + input, _ = json.Marshal(map[string]string{"arguments": item.Arguments}) + } + blocks = append(blocks, trace.RawSessionToolUseBlock{ + Type: "tool_use", ID: payload.CallID, Name: payload.Name, Input: input, + }) + case "function_call_output", "custom_tool_call_output": + entry.Type = trace.RoleUser + output := item.Output + if len(output) == 0 { + output = json.RawMessage(`""`) + } + if bytes.HasPrefix(bytes.TrimSpace(output), []byte("[")) { + var err error + output, err = transcriptContent(output) + if err != nil { + return nil, fmt.Errorf("read Codex tool result on line %d: %w", line, err) + } + } + blocks = append(blocks, trace.RawSessionToolResultBlock{ + Type: "tool_result", ToolUseID: payload.CallID, Content: output, + }) + case "reasoning": + var summary []string + for _, part := range item.Summary { + if part.Text != "" { + summary = append(summary, part.Text) + } + } + if len(summary) == 0 { + return raw, nil + } + entry.Type = trace.RoleAssistant + blocks = append(blocks, map[string]string{"type": "thinking", "thinking": strings.Join(summary, "\n")}) + default: + return raw, nil + } + entry.Message.Role = entry.Type + if blocks != nil { + entry.Message.Content, _ = json.Marshal(blocks) + } + return json.Marshal(entry) +} + +func transcriptContent(content json.RawMessage) (json.RawMessage, error) { + if len(content) == 0 || bytes.Equal(bytes.TrimSpace(content), []byte("null")) { + return json.RawMessage(`[]`), nil + } + var text string + if json.Unmarshal(content, &text) == nil { + return content, nil + } + var blocks []map[string]any + if err := json.Unmarshal(content, &blocks); err != nil { + return nil, err + } + for _, block := range blocks { + switch block["type"] { + case "input_text", "output_text": + block["type"] = "text" + } + } + return json.Marshal(blocks) +} diff --git a/app/cli/internal/trace/codex/transcript_test.go b/app/cli/internal/trace/codex/transcript_test.go new file mode 100644 index 000000000..59838726e --- /dev/null +++ b/app/cli/internal/trace/codex/transcript_test.go @@ -0,0 +1,115 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranscriptContract(t *testing.T) { + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + dir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // temporary fixture path and a fixed session ID + evidence, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + entries := evidence.Data.RawSession["main"] + require.Len(t, entries, len(lines)) + var texts []string + calls := map[string]trace.RawSessionToolUseBlock{} + results := map[string]string{} + for i, rawEntry := range entries { + var entry trace.RawSessionEntry + require.NoError(t, json.Unmarshal(rawEntry, &entry)) + if entry.Type != trace.RoleUser && entry.Type != trace.RoleAssistant { + assert.JSONEq(t, lines[i], string(rawEntry)) + continue + } + assert.NotEmpty(t, entry.UUID) + assert.Equal(t, entry.Type, entry.Message.Role) + assert.JSONEq(t, lines[i], string(entry.SourceEvent)) + var blocks []json.RawMessage + require.NoError(t, json.Unmarshal(entry.Message.Content, &blocks)) + for _, rawBlock := range blocks { + var block struct{ Type string } + require.NoError(t, json.Unmarshal(rawBlock, &block)) + switch block.Type { + case "text": + var text trace.RawSessionTextBlock + require.NoError(t, json.Unmarshal(rawBlock, &text)) + texts = append(texts, text.Text) + case "tool_use": + var call trace.RawSessionToolUseBlock + require.NoError(t, json.Unmarshal(rawBlock, &call)) + calls[call.ID] = call + case "tool_result": + var result trace.RawSessionToolResultBlock + require.NoError(t, json.Unmarshal(rawBlock, &result)) + var output string + require.NoError(t, json.Unmarshal(result.Content, &output)) + results[result.ToolUseID] = output + } + } + } + assert.Equal(t, []string{"Create hello.txt", "Created hello.txt"}, texts) + require.Len(t, calls, 2) + assert.Equal(t, "apply_patch", calls["call-1"].Name) + assert.Contains(t, string(calls["call-1"].Input), "hello.txt") + assert.Equal(t, "exec_command", calls["call-2"].Name) + assert.JSONEq(t, `{"cmd":"cat hello.txt"}`, string(calls["call-2"].Input)) + assert.Equal(t, map[string]string{"call-1": "Success", "call-2": "hello"}, results) +} + +func TestTranscriptReasoningAndUnknownEvents(t *testing.T) { + for _, tc := range []struct { + name, payload string + wantType string + }{ + {name: "reasoning summary", payload: `{"type":"reasoning","summary":[{"text":"Check the tests"}]}`, wantType: trace.RoleAssistant}, + {name: "encrypted reasoning", payload: `{"type":"reasoning","encrypted_content":"opaque"}`, wantType: responseItem}, + {name: "future event", payload: `{"type":"future_item","arguments":{"new":"shape"}}`, wantType: responseItem}, + {name: "non-JSON arguments", payload: `{"type":"function_call","name":"Bash","call_id":"c1","arguments":"not JSON"}`, wantType: trace.RoleAssistant}, + {name: "array tool output", payload: `{"type":"custom_tool_call_output","call_id":"c1","output":[{"type":"input_text","text":"Success"}]}`, wantType: trace.RoleUser}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := json.RawMessage(`{"type":"response_item","timestamp":"2026-10-21T10:00:00Z","payload":` + tc.payload + `}`) + var event rolloutEntry + var payload rolloutPayload + require.NoError(t, json.Unmarshal(raw, &event)) + require.NoError(t, json.Unmarshal(event.Payload, &payload)) + got, err := transcriptEntry(event, payload, raw, testModel, 1) + require.NoError(t, err) + var entry trace.RawSessionEntry + require.NoError(t, json.Unmarshal(got, &entry)) + assert.Equal(t, tc.wantType, entry.Type) + if tc.wantType == responseItem { + assert.JSONEq(t, string(raw), string(got)) + } else { + assert.JSONEq(t, string(raw), string(entry.SourceEvent)) + assert.Contains(t, string(entry.Message.Content), map[string]string{"reasoning summary": "Check the tests", "non-JSON arguments": "not JSON", "array tool output": `"type":"text"`}[tc.name]) + } + }) + } +} diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index d368e5a99..a8a5c75a2 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -17,6 +17,7 @@ package e2e import ( "bytes" + "encoding/json" "errors" "fmt" "io/fs" @@ -26,6 +27,7 @@ import ( "strings" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" @@ -146,16 +148,93 @@ func evidence(s *Session) *aicodingsession.Data { } func checkTranscript(s *Session) error { - main := evidence(s).RawSession["main"] + d := evidence(s) + main := d.RawSession["main"] if len(main) == 0 { return errors.New("the evidence has no main transcript") } + // Other providers retain their existing native transcript checks. Codex + // converts its rollout to the message shape used by the session viewer. + if d.Agent.Name != codex.Name { + for _, msg := range main { + if bytes.Contains(msg, []byte("greeting.txt")) { + return nil + } + } + return errors.New("the transcript does not show the work of the session") + } + + var user, assistant, showsWork bool + calls, results := map[string]bool{}, map[string]bool{} for _, msg := range main { - if bytes.Contains(msg, []byte("greeting.txt")) { - return nil + var entry trace.RawSessionEntry + if err := json.Unmarshal(msg, &entry); err != nil { + return err + } + role := entry.Type + if role == "" { + role = entry.Role + } + if role != trace.RoleUser && role != trace.RoleAssistant { + continue + } + readable := transcriptHasText(entry.Message.Content) + var blocks []struct { + Type, Text, ID, Name string + ToolUseID string `json:"tool_use_id"` + Content json.RawMessage `json:"content"` + Input json.RawMessage `json:"input"` + } + if json.Unmarshal(entry.Message.Content, &blocks) == nil { + for _, block := range blocks { + switch block.Type { + case "tool_use": + var input map[string]json.RawMessage + if block.ID == "" || block.Name == "" || json.Unmarshal(block.Input, &input) != nil || input == nil { + return errors.New("the Codex transcript has a malformed tool call") + } + calls[block.ID] = true + readable = true + case "tool_result": + results[block.ToolUseID] = transcriptHasText(block.Content) + } + } + } + user = user || role == trace.RoleUser && readable + assistant = assistant || role == trace.RoleAssistant && readable + showsWork = showsWork || bytes.Contains(entry.Message.Content, []byte("greeting.txt")) + } + if !user || !assistant { + return errors.New("the transcript has no renderable user and assistant messages") + } + if !showsWork { + return errors.New("the normalized transcript does not show the work of the session") + } + if d.ToolsUsed != nil && d.ToolsUsed.TotalInvocations > 0 { + for id := range calls { + if results[id] { + return nil + } + } + return errors.New("the Codex transcript has no matching tool call and result") + } + return nil +} + +func transcriptHasText(content json.RawMessage) bool { + var text string + if json.Unmarshal(content, &text) == nil { + return strings.TrimSpace(text) != "" + } + var blocks []trace.RawSessionTextBlock + if json.Unmarshal(content, &blocks) == nil { + for _, block := range blocks { + if block.Type == "text" && strings.TrimSpace(block.Text) != "" { + return true + } } } - return errors.New("the transcript does not show the work of the session") + return false } func checkTokenUsage(s *Session) error { diff --git a/app/cli/internal/trace/e2e/codex.go b/app/cli/internal/trace/e2e/codex.go index 727f2da75..565decb75 100644 --- a/app/cli/internal/trace/e2e/codex.go +++ b/app/cli/internal/trace/e2e/codex.go @@ -16,21 +16,24 @@ package e2e import ( + "bufio" "bytes" "context" - "errors" + "encoding/json" "fmt" "os" "os/exec" "path/filepath" "regexp" "slices" + "strconv" "strings" + + "github.com/pelletier/go-toml/v2" ) const ( - // codexHooksPath is the project file where Codex reads its hooks. Trace - // has no Codex provider yet, so no Chainloop hook is in it. + // codexHooksPath is the project file where Codex reads its hooks. codexHooksPath = ".codex/hooks.json" // codexOpenRouterPrefix selects the OpenRouter provider for a model of // the suite config. @@ -105,13 +108,24 @@ func codexModelArgs(model string) []string { // --ignore-user-config is not set, because it also turns off the project // hooks. // -// Trace has no Codex provider yet, so no hook message is reported. +// Hook messages are not extracted from the headless event stream yet. func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { base := filepath.Dir(req.Dir) codexHome := filepath.Join(base, "codex-home") if err := os.MkdirAll(codexHome, 0o700); err != nil { return nil, err } + userHome, err := os.UserHomeDir() + if err != nil { + return nil, err + } + config, err := codexIsolatedConfig(userHome) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(codexHome, "config.toml"), config, 0600); err != nil { + return nil, err + } args := []string{ "exec", "--json", @@ -120,6 +134,27 @@ func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { "-C", req.Dir, } args = append(args, codexModelArgs(req.Model)...) + if strings.HasPrefix(req.Model, codexOpenRouterPrefix+"openai/") { + export := exec.CommandContext(ctx, c.bin(), "debug", "models", "--bundled") //nolint:gosec // the harness binary under test // nosemgrep + export.Env = append(slices.Clone(req.Env), "CODEX_HOME="+codexHome) + raw, err := export.Output() + if err != nil { + return nil, fmt.Errorf("codex bundled model catalog: %w", err) + } + catalog, err := codexModelCatalog(raw, req.Model) + if err != nil { + return nil, err + } + if catalog != nil { + path := filepath.Join(codexHome, "models.json") + if err := os.WriteFile(path, catalog, 0600); err != nil { + return nil, err + } + // OpenRouter prefixes model names with openai/. Keep the native Codex + // model metadata so that file tools and instructions match that model. + args = append(args, "-c", "model_catalog_json="+strconv.Quote(path)) + } + } // A pasted image goes in as an attached file, from outside the // repository, so that the agent cannot commit it. @@ -138,7 +173,7 @@ func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { } } // The prompt follows "--", so that the list of images ends before it. - args = append(args, "--", req.Prompt) + args = append(args, "--", "-") name := c.bin() if len(req.Wrapper) > 0 { @@ -148,13 +183,14 @@ func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario // nosemgrep cmd.Dir = req.Dir + cmd.Stdin = strings.NewReader(req.Prompt) cmd.Env = append(slices.Clone(req.Env), "CODEX_HOME="+codexHome) var stderr bytes.Buffer cmd.Stderr = &stderr raw, err := cmd.Output() if err != nil { - return &RunOutput{Raw: raw}, fmt.Errorf("codex exec: %w: %s", err, strings.TrimSpace(stderr.String())) + return &RunOutput{Raw: raw}, fmt.Errorf("codex exec: %w: %s", err, strings.TrimSpace(stderr.String()+"\n"+codexStreamErrors(raw))) } return &RunOutput{Raw: raw}, nil @@ -178,14 +214,34 @@ func (c *Codex) HookConfig() HookConfig { } } -// errCodexSessionStart is the error of DisableSessionStart until Trace has a -// Codex provider. -var errCodexSessionStart = errors.New("codex: the suite cannot remove the session-start hook of Trace, because Trace has no Codex provider yet") - -// DisableSessionStart implements Harness. Trace installs no Codex hooks yet, -// so there is no hook to remove. -func (c *Codex) DisableSessionStart(string) error { - return errCodexSessionStart +// DisableSessionStart removes only the session-start hook for recovery tests. +func (c *Codex) DisableSessionStart(repo string) error { + path := filepath.Join(repo, codexHooksPath) + data, err := os.ReadFile(path) + if err != nil { + return err + } + var settings map[string]json.RawMessage + if err := json.Unmarshal(data, &settings); err != nil { + return err + } + var hooks map[string]json.RawMessage + if err := json.Unmarshal(settings["hooks"], &hooks); err != nil { + return err + } + if _, ok := hooks["SessionStart"]; !ok { + return fmt.Errorf("%s has no SessionStart hook", codexHooksPath) + } + delete(hooks, "SessionStart") + settings["hooks"], err = json.Marshal(hooks) + if err != nil { + return err + } + data, err = json.MarshalIndent(settings, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, data, 0600) } // SkillPath implements Harness. Codex finds project skills in @@ -193,3 +249,86 @@ func (c *Codex) DisableSessionStart(string) error { func (c *Codex) SkillPath(name string) string { return ".agents/skills/" + name + "/SKILL.md" } + +// codexStreamErrors extracts diagnostics that codex exec writes to JSON stdout. +func codexStreamErrors(raw []byte) string { + var messages []string + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 4096), 16*1024*1024) + for scanner.Scan() { + var event struct { + Type string `json:"type"` + Message string `json:"message"` + Error struct { + Message string `json:"message"` + } `json:"error"` + } + if json.Unmarshal(scanner.Bytes(), &event) != nil { + continue + } + if event.Type == "error" && event.Message != "" { + messages = append(messages, event.Message) + } + if event.Type == "turn.failed" && event.Error.Message != "" { + messages = append(messages, event.Error.Message) + } + } + return strings.Join(messages, "\n") +} + +// codexIsolatedConfig disables personal skills that Codex discovers outside +// CODEX_HOME. Project skills remain available for the skill scenario. +func codexIsolatedConfig(userHome string) ([]byte, error) { + type skillConfig struct { + Path string `toml:"path"` + Enabled bool `toml:"enabled"` + } + config := struct { + Skills struct { + Config []skillConfig `toml:"config"` + Bundled struct { + Enabled bool `toml:"enabled"` + } `toml:"bundled"` + } `toml:"skills"` + }{} + for _, dir := range []string{filepath.Join(userHome, ".agents", "skills"), filepath.Join(userHome, ".codex", "skills"), filepath.Join(userHome, ".codex", "skills", ".system")} { + paths, err := filepath.Glob(filepath.Join(dir, "*", "SKILL.md")) + if err != nil { + return nil, err + } + for _, path := range paths { + config.Skills.Config = append(config.Skills.Config, skillConfig{Path: path, Enabled: false}) + } + } + return toml.Marshal(config) +} + +// codexModelCatalog aliases bundled OpenAI metadata to OpenRouter's model ID. +// Without it, Codex treats the prefixed name as unknown and uses generic tools. +func codexModelCatalog(raw []byte, model string) ([]byte, error) { + var catalog struct { + Models []map[string]json.RawMessage `json:"models"` + } + if err := json.Unmarshal(raw, &catalog); err != nil { + return nil, fmt.Errorf("decode Codex model catalog: %w", err) + } + target := strings.TrimPrefix(model, codexOpenRouterPrefix) + canonical := strings.TrimPrefix(target, "openai/") + for _, entry := range catalog.Models { + var slug string + if err := json.Unmarshal(entry["slug"], &slug); err != nil { + return nil, err + } + if slug != canonical { + continue + } + alias := make(map[string]json.RawMessage, len(entry)) + for k, v := range entry { + alias[k] = v + } + alias["slug"], _ = json.Marshal(target) + catalog.Models = append(catalog.Models, alias) + return json.Marshal(catalog) + } + return nil, nil +} diff --git a/app/cli/internal/trace/e2e/codex_test.go b/app/cli/internal/trace/e2e/codex_test.go index f0e79e989..cbc8738b3 100644 --- a/app/cli/internal/trace/e2e/codex_test.go +++ b/app/cli/internal/trace/e2e/codex_test.go @@ -16,8 +16,12 @@ package e2e import ( + "encoding/json" + "os" + "path/filepath" "testing" + "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -72,3 +76,52 @@ func TestCodexModelArgs(t *testing.T) { }) } } + +func TestCodexStreamErrors(t *testing.T) { + raw := []byte("{\"type\":\"thread.started\"}\n{\"type\":\"error\",\"message\":\"missing key\"}\n{\"type\":\"turn.failed\",\"error\":{\"message\":\"turn failed\"}}\n") + assert.Equal(t, "missing key\nturn failed", codexStreamErrors(raw)) +} + +func TestCodexIsolatedConfig(t *testing.T) { + home := t.TempDir() + for _, dir := range []string{".agents/skills/personal", ".codex/skills/legacy", ".codex/skills/.system/builtin"} { + path := filepath.Join(home, dir, "SKILL.md") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("personal instructions"), 0600)) + } + raw, err := codexIsolatedConfig(home) + require.NoError(t, err) + var config struct { + Skills struct { + Config []struct { + Path string + Enabled bool + } + Bundled struct{ Enabled bool } + } + } + require.NoError(t, toml.Unmarshal(raw, &config)) + require.Len(t, config.Skills.Config, 3) + for _, skill := range config.Skills.Config { + assert.False(t, skill.Enabled) + assert.Contains(t, skill.Path, home) + } + assert.False(t, config.Skills.Bundled.Enabled) +} + +func TestCodexModelCatalog(t *testing.T) { + raw := []byte(`{"models":[{"slug":"gpt-6-luna","apply_patch_tool_type":"freeform","base_instructions":"native instructions"}]}`) + catalog, err := codexModelCatalog(raw, "openrouter/openai/gpt-6-luna") + require.NoError(t, err) + var got struct { + Models []map[string]any `json:"models"` + } + require.NoError(t, json.Unmarshal(catalog, &got)) + require.Len(t, got.Models, 2) + assert.Equal(t, "openai/gpt-6-luna", got.Models[1]["slug"]) + assert.Equal(t, "freeform", got.Models[1]["apply_patch_tool_type"]) + assert.Equal(t, "native instructions", got.Models[1]["base_instructions"]) + unknown, err := codexModelCatalog(raw, "openrouter/openai/unknown") + require.NoError(t, err) + assert.Nil(t, unknown) +} diff --git a/app/cli/internal/trace/e2e/config.yaml b/app/cli/internal/trace/e2e/config.yaml index 829cb7154..d696dd2dc 100644 --- a/app/cli/internal/trace/e2e/config.yaml +++ b/app/cli/internal/trace/e2e/config.yaml @@ -32,7 +32,7 @@ harnesses: Codex: version: "0.163.0" models: - - openrouter/openai/gpt-5-mini + - openrouter/openai/gpt-6-luna # Runs and pass threshold of a cell whose feature depends on the model # following an instruction (R-006, D-004). A cell of any other feature runs diff --git a/app/cli/internal/trace/e2e/runner.go b/app/cli/internal/trace/e2e/runner.go index c54fff4fa..64d9056d6 100644 --- a/app/cli/internal/trace/e2e/runner.go +++ b/app/cli/internal/trace/e2e/runner.go @@ -287,7 +287,7 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc // errNoStoredSession is the failure of a check when the push stored no // session, for example because the agent did not commit or push. -var errNoStoredSession = errors.New("the session stored nothing: the agent did not commit and push") +var errNoStoredSession = errors.New("the push stored no session evidence; inspect harness-output.jsonl and the workspace git history") // evaluate runs the checks of a planned cell on its sessions and sets its // status and pass rate. diff --git a/app/cli/internal/trace/e2e/transcript_test.go b/app/cli/internal/trace/e2e/transcript_test.go new file mode 100644 index 000000000..f7c498ffc --- /dev/null +++ b/app/cli/internal/trace/e2e/transcript_test.go @@ -0,0 +1,83 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "testing" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/stretchr/testify/require" +) + +func TestTranscriptRequiresRenderableMessages(t *testing.T) { + const prompt = `{"type":"user","message":{"content":[{"type":"text","text":"Create greeting.txt"}]}}` + const answer = `{"type":"assistant","message":{"content":[{"type":"text","text":"Created greeting.txt"}]}}` + const call = `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"call-1","name":"apply_patch","input":{"input":"Create greeting.txt"}}]}}` + const result = `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","content":"Success"}]}}` + for _, tc := range []struct { + name string + entries []string + tools int + wantErr bool + }{ + {name: "messages", entries: []string{prompt, answer}}, + {name: "completed tool exchange", entries: []string{prompt, call, result}, tools: 1}, + {name: "native messages alone", entries: []string{`{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Create greeting.txt"}]}}`}, wantErr: true}, + {name: "filename only in metadata", entries: []string{`{"type":"user","message":{"content":"Hello"},"source_event":{"file":"greeting.txt"}}`, `{"type":"assistant","message":{"content":"Hello"}}`}, wantErr: true}, + {name: "no assistant", entries: []string{prompt}, wantErr: true}, + {name: "missing tool result", entries: []string{prompt, call}, tools: 1, wantErr: true}, + {name: "empty tool result", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","content":""}]}}`}, tools: 1, wantErr: true}, + {name: "missing tool name", entries: []string{prompt, `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"call-1","input":{}}]}}`, result}, tools: 1, wantErr: true}, + {name: "missing tool ID", entries: []string{prompt, `{"type":"assistant","message":{"content":[{"type":"tool_use","name":"apply_patch","input":{}}]}}`, result}, tools: 1, wantErr: true}, + {name: "invalid tool input", entries: []string{prompt, `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"call-1","name":"apply_patch","input":"native arguments"}]}}`, result}, tools: 1, wantErr: true}, + {name: "native tool output", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","content":[{"type":"output_text","text":"Success"}]}]}}`}, tools: 1, wantErr: true}, + {name: "unrelated tool result", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"other","content":"Success"}]}}`}, tools: 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var entries []json.RawMessage + for _, entry := range tc.entries { + entries = append(entries, json.RawMessage(entry)) + } + d := aicodingsession.Data{ + SchemaVersion: "v1", + Agent: aicodingsession.Agent{Name: "codex"}, + Session: aicodingsession.Session{ID: "test", StartedAt: "2026-10-21T10:00:00Z"}, + Conversation: &aicodingsession.Conversation{TotalMessages: 2}, + ToolsUsed: &aicodingsession.ToolsUsed{TotalInvocations: tc.tools}, + RawSession: map[string][]json.RawMessage{"main": entries}, + } + s := &Session{Stored: &StoredSession{Evidence: *aicodingsession.NewEvidence(d)}} + if tc.wantErr { + require.Error(t, checkTranscript(s)) + } else { + require.NoError(t, checkTranscript(s)) + } + }) + } +} + +// The Codex renderer checks must not impose a format on other providers. +func TestTranscriptPreservesOtherProviderFormats(t *testing.T) { + d := aicodingsession.Data{ + Agent: aicodingsession.Agent{Name: "pi"}, + RawSession: map[string][]json.RawMessage{"main": { + json.RawMessage(`{"type":"message","message":{"role":"user","content":"Create greeting.txt"}}`), + }}, + } + s := &Session{Stored: &StoredSession{Evidence: *aicodingsession.NewEvidence(d)}} + require.NoError(t, checkTranscript(s)) +} diff --git a/app/cli/internal/trace/provider.go b/app/cli/internal/trace/provider.go index fba5c355b..490af29a5 100644 --- a/app/cli/internal/trace/provider.go +++ b/app/cli/internal/trace/provider.go @@ -307,6 +307,9 @@ type HookInput struct { ToolName string `json:"tool_name,omitempty"` // FilePath is the absolute path of the file being edited, set by provider's ReadHookInput. FilePath string `json:"-"` + // FilePaths holds all paths of a file tool that edits several files. + // Handlers process each path with its own snapshot and checkout. + FilePaths []string `json:"-"` // Command is the command line of a shell command tool, set by provider's // ReadHookInput. Empty when the provider does not report it. Command string `json:"-"` diff --git a/app/cli/internal/trace/providers/capabilities_test.go b/app/cli/internal/trace/providers/capabilities_test.go index 8ee20a644..ed630dcff 100644 --- a/app/cli/internal/trace/providers/capabilities_test.go +++ b/app/cli/internal/trace/providers/capabilities_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" "github.com/stretchr/testify/assert" @@ -59,6 +60,7 @@ func TestSessionStartChannels(t *testing.T) { wantReminder: true, why: "the opencode plugin posts the instruction and the reminder as context-only messages and has no banner", }, + {provider: codex.Name, wantBanner: true, wantInstruction: true, wantReminder: true, why: "Codex surfaces systemMessage and supports additionalContext at session start and user prompt"}, } for _, tc := range testCases { @@ -77,7 +79,7 @@ func TestSessionStartChannels(t *testing.T) { // without a decision recorded above, since the default zero value would // silently claim no support. func TestSessionStartChannelsCoverEveryProvider(t *testing.T) { - known := map[string]bool{claude.Name: true, cursor.Name: true, opencode.Name: true} + known := map[string]bool{claude.Name: true, cursor.Name: true, opencode.Name: true, codex.Name: true} for _, p := range All() { assert.True(t, known[p.Name()], "provider %q has no session-start channel expectation", p.Name()) diff --git a/app/cli/internal/trace/providers/registry.go b/app/cli/internal/trace/providers/registry.go index 183192db7..f112fcae2 100644 --- a/app/cli/internal/trace/providers/registry.go +++ b/app/cli/internal/trace/providers/registry.go @@ -22,6 +22,7 @@ package providers import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" ) @@ -37,6 +38,7 @@ var all = []trace.Provider{ claude.New(), cursor.New(), opencode.New(), + codex.New(), } // byName is a lookup-optimized view of all, built once at init. diff --git a/app/cli/internal/trace/state/session.go b/app/cli/internal/trace/state/session.go index 02bbd0dd3..11de110bf 100644 --- a/app/cli/internal/trace/state/session.go +++ b/app/cli/internal/trace/state/session.go @@ -44,6 +44,9 @@ type SessionRecord struct { // over Cwd to find the transcripts. Filled in by a later hook when the // first one did not carry it. Empty when the agent does not report it. TranscriptPath string `json:"transcript_path,omitempty"` + // CodexStorageRoot is the canonical Codex home validated while copying + // this session. Later pushes can refresh it without the original CODEX_HOME. + CodexStorageRoot string `json:"codex_storage_root,omitempty"` // Checkouts lists the roots of the other checkouts that this session // edited with a file tool, sorted. Shell hooks run in the session's own // checkout, and they also snapshot these checkouts, so that a command diff --git a/app/cli/internal/trace/types.go b/app/cli/internal/trace/types.go index f776959ee..c74595334 100644 --- a/app/cli/internal/trace/types.go +++ b/app/cli/internal/trace/types.go @@ -24,6 +24,9 @@ const ( AttributionAI = "ai" // AttributionHuman marks file changes produced by a human. AttributionHuman = "human" + + RoleUser = "user" + RoleAssistant = "assistant" ) // RawSessionEntry is the normalized wire format most providers use for a @@ -33,12 +36,13 @@ const ( // This shape mirrors Claude's JSONL transcript format, which the frontend was // originally built against. type RawSessionEntry struct { - Type string `json:"type,omitempty"` - Role string `json:"role,omitempty"` - UUID string `json:"uuid,omitempty"` - Timestamp string `json:"timestamp,omitempty"` - IsMeta bool `json:"isMeta,omitempty"` - Message RawSessionMessage `json:"message"` + Type string `json:"type,omitempty"` + Role string `json:"role,omitempty"` + UUID string `json:"uuid,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + IsMeta bool `json:"isMeta,omitempty"` + Message RawSessionMessage `json:"message"` + SourceEvent json.RawMessage `json:"source_event,omitempty"` } // RawSessionMessage is the message body inside a RawSessionEntry. @@ -62,6 +66,14 @@ type RawSessionToolUseBlock struct { Input json.RawMessage `json:"input,omitempty"` } +// RawSessionToolResultBlock carries the result of a tool_use with the same ID. +// It belongs to a user-shaped entry, not to a new user prompt. +type RawSessionToolResultBlock struct { + Type string `json:"type"` + ToolUseID string `json:"tool_use_id"` + Content json.RawMessage `json:"content"` +} + // MustRawSessionEntry marshals a RawSessionEntry to json.RawMessage. // Panics only on impossible encoding failures (struct field bugs). func MustRawSessionEntry(e RawSessionEntry) json.RawMessage { diff --git a/app/cli/pkg/action/trace_agent_hook.go b/app/cli/pkg/action/trace_agent_hook.go index e2a7b0ee0..c4c071fb2 100644 --- a/app/cli/pkg/action/trace_agent_hook.go +++ b/app/cli/pkg/action/trace_agent_hook.go @@ -380,6 +380,15 @@ func HandleAgentPreToolUse(provider trace.Provider, log zerolog.Logger) error { return nil } + for _, fileInput := range fileHookInputs(input) { + if err := handleAgentPreToolUse(provider, fileInput, log); err != nil { + return err + } + } + return nil +} + +func handleAgentPreToolUse(provider trace.Provider, input *trace.HookInput, log zerolog.Logger) error { log.Debug().Str("session_id", input.SessionID).Msg("pre-tool-use hook invoked") store, repoRoot, err := locateForHook(provider, input) @@ -747,6 +756,15 @@ func HandleAgentPostToolUse(provider trace.Provider, log zerolog.Logger) error { return nil } + for _, fileInput := range fileHookInputs(input) { + if err := handleAgentPostToolUse(provider, fileInput, log); err != nil { + return err + } + } + return nil +} + +func handleAgentPostToolUse(provider trace.Provider, input *trace.HookInput, log zerolog.Logger) error { sessionID := input.SessionID isCommand := provider.IsCommandTool(input.ToolName) @@ -1000,3 +1018,18 @@ func autoInstallGitHooks(store *state.Store, repoRoot string, log zerolog.Logger log.Debug().Err(err).Msg("mark trace initialized failed") } } + +// fileHookInputs expands a multi-file edit into the existing per-file path. +func fileHookInputs(input *trace.HookInput) []*trace.HookInput { + if len(input.FilePaths) == 0 { + return []*trace.HookInput{input} + } + inputs := make([]*trace.HookInput, 0, len(input.FilePaths)) + for _, path := range input.FilePaths { + one := *input + one.FilePath = path + one.FilePaths = nil + inputs = append(inputs, &one) + } + return inputs +} diff --git a/app/cli/pkg/action/trace_codex_hook_test.go b/app/cli/pkg/action/trace_codex_hook_test.go new file mode 100644 index 000000000..0c212db7b --- /dev/null +++ b/app/cli/pkg/action/trace_codex_hook_test.go @@ -0,0 +1,63 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCodexMultiFilePatchAttribution(t *testing.T) { + root := initTempGitRepo(t) + t.Chdir(root) + t.Setenv("CODEX_HOME", t.TempDir()) + require.NoError(t, os.WriteFile(filepath.Join(root, ".chainloop.yml"), []byte("traceStorage: LOCAL\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "one.txt"), []byte("human\nold\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "gone.txt"), []byte("gone\n"), 0600)) + payload, _ := json.Marshal(map[string]any{"session_id": "codex-patch", "cwd": root, "tool_name": "apply_patch", "tool_use_id": "call-1", "tool_input": map[string]string{"command": "*** Begin Patch\n*** Update File: one.txt\n@@\n human\n-old\n+new\n*** Add File: two.txt\n+two\n*** Delete File: gone.txt\n*** End Patch"}}) + withStdin(t, string(payload)) + require.NoError(t, HandleAgentPreToolUse(codex.New(), zerolog.Nop())) + require.NoError(t, os.WriteFile(filepath.Join(root, "one.txt"), []byte("human\nnew\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "two.txt"), []byte("two\n"), 0600)) + require.NoError(t, os.Remove(filepath.Join(root, "gone.txt"))) + withStdin(t, string(payload)) + require.NoError(t, HandleAgentPostToolUse(codex.New(), zerolog.Nop())) + store := state.NewGitStore(filepath.Join(root, ".git")) + ledger := store.LoadAILineAttribution("codex-patch") + assert.Contains(t, ledger.Open, "one.txt") + assert.Contains(t, ledger.Open, "two.txt") + assert.Contains(t, ledger.Open, "gone.txt") + assert.Len(t, ledger.Open["gone.txt"].Removed, 1) + assert.Equal(t, 2, ledger.GeneratedAdded) + assert.Equal(t, 2, ledger.GeneratedRemoved) + // An unchanged human line must not become an AI addition. + assert.Len(t, ledger.Open["one.txt"].Added, 1) + session, err := store.LoadSessionRecord("codex-patch") + require.NoError(t, err) + assert.Equal(t, codex.Name, session.Provider) + for _, name := range []string{"one.txt", "two.txt", "gone.txt"} { + _, err := store.LoadFileSnapshot("codex-patch", filepath.Join(root, name)) + assert.ErrorIs(t, err, os.ErrNotExist) + } +} diff --git a/docs/specs/issue-3657-codex-trace-provider.md b/docs/specs/issue-3657-codex-trace-provider.md new file mode 100644 index 000000000..75bb51022 --- /dev/null +++ b/docs/specs/issue-3657-codex-trace-provider.md @@ -0,0 +1,122 @@ +--- +owner: jiparis +ticket: https://github.com/chainloop-dev/chainloop/issues/3657 +prd: +--- + + + +# Spec issue-3657: Codex support in Chainloop Trace + +## Summary + +Chainloop Trace supports developers who use Codex. Developers can select Codex during initialization or trace one command with temporary hooks. Trace records Codex sessions as AI coding-session evidence. The integration uses native Codex hooks and persisted session transcripts. + +## Problem + +Developers who use Codex cannot attach native Trace evidence to their commits. Codex already records conversations and exposes lifecycle hooks. Trace needs to connect those records to its existing evidence pipeline. + +## Goals and Non-Goals + +- Goal: Developers can record Codex sessions in local storage or the control plane. +- Goal: Evidence includes conversations, models, tokens, estimated cost, tools, and code attribution. +- Non-goal: Capture child-session transcripts, track skills, or replace spec and pasted-image copies with pointers. These features need separate transcript analysis. +- Non-goal: Attribute arbitrary MCP tools or reconstruct sessions that produced no Trace hook. +- Non-goal: Change the evidence schema, control-plane APIs, or default provider. + +## Requirements + +### R-001: Provider selection + +Trace MUST offer Codex during initialization. Initialization and the command wrapper MUST accept the Codex provider flag. Developers MUST be able to select Codex with other providers. + +### R-002: Hook lifecycle + +Installation MUST preserve user hooks and produce identical content on repeated runs. Uninstall MUST delete only Trace hooks. The command wrapper MUST restore the previous hook configuration after execution. Its temporary configuration MUST omit the Trace session-end hook. + +### R-003: Session evidence + +Trace MUST capture session identity, timing, agent version, conversations, tool calls, and model information from persisted Codex transcripts. Missing or unreadable data MUST produce a warning or parsing error. Repeated cumulative usage records MUST count tokens once. Cost MUST use embedded API prices and identify an unknown-model fallback. + +### R-004: Code attribution + +Trace MUST attribute edits from native patch and shell tools. A patch that changes several files MUST preserve separate before-and-after content for each file. Attribution MUST include additions, deletions, and both paths of a move. Tool hooks MUST recover a missed session-start event. + +### R-005: Private session copies + +Trace MUST parse its private copy of the session transcript. Copying MUST check session identity and reject paths outside Codex session storage, including symlink escapes. A failed copy MUST preserve the previous copy. Local capture and parsing MUST require no network or login. + +### R-006: Session instructions + +Trace MUST send spec capture instructions at session start and reminders at each user prompt. Each hook response MUST contain at most one JSON document. Empty announcements MUST emit no output. Initialization MUST tell Codex users to review and trust the installed hooks. + +## Constraints + +Native hook behavior follows Codex CLI 0.163.0. Codex requires users to trust project configuration and hook definitions. Trace does not bypass that review during normal use. The test harness bypasses hook trust only in isolated test repositories. + +## Proposal + +The developer selects Codex during Trace initialization, reviews the installed hooks in Codex, and starts a session. Codex sends session and tool events to Trace. Trace records each patch path separately and uses working-tree snapshots for shell changes. + +Trace copies the Codex rollout into private storage at session boundaries and before evidence assembly. The parser retains native JSONL records and derives summaries from metadata, response items, turn context, and cumulative token counters. Git hooks use the existing evidence pipeline. + +```mermaid +flowchart LR + A[Initialize Trace for Codex] --> B[Review and trust hooks] + B --> C[Codex session and tool events] + C --> D[Private transcript and attribution] + D --> E[Commit and push] + E --> F[Local evidence or control-plane attestation] +``` + +## Decision Record + +### D-001: Use native Codex hooks + +Native hooks connect Codex to the existing session lifecycle. + +- Why: They cover session boundaries, prompts, patches, and shell tools without a separate extension. +- Alternatives considered: A notification-only integration misses pre-edit state. A separate extension duplicates lifecycle support. +- Source: drafting, based on the [official hook contract](https://learn.chatgpt.com/docs/hooks). + +### D-002: Expand patch events per file + +Trace snapshots each affected file before a patch and records its changes after the patch. + +- Why: Native patches can change several files, while attribution needs the original content of each file. +- Alternatives considered: Whole-worktree shell attribution loses uncommitted human content when it substitutes the last commit for the original content. +- Source: drafting. + +### D-003: Parse copied rollouts + +The provider reads copied native rollouts and estimates cost from embedded prices. + +- Why: Evidence remains available after Codex moves or deletes its original transcript. +- Alternatives considered: Live transcript parsing depends on agent storage during assembly. Notification payloads omit conversation and usage details. +- Source: drafting. + +## Open Questions + +None for the initial provider scope. + +## Risks + +| Risk | Mitigation | +|------|------------| +| Codex changes its rollout format. | Preserve unknown records, warn about unreadable records, and test sanitized rollout fixtures. | +| Users have not trusted the hooks. | Explain the trust step during initialization and in provider documentation. | +| Estimated cost differs from billing. | Document standard API estimates and warn when the model price is unknown. | diff --git a/go.mod b/go.mod index 2df6be5df..32aa47669 100644 --- a/go.mod +++ b/go.mod @@ -76,6 +76,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/openvex/go-vex v0.2.9 github.com/owenrumney/go-sarif/v3 v3.3.1 + github.com/pelletier/go-toml/v2 v2.4.3 github.com/posthog/posthog-go v1.34.0 github.com/prometheus/client_golang v1.25.0 github.com/prometheus/common v0.72.1 @@ -376,7 +377,6 @@ require ( github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pjbgf/sha1cd v0.7.0 // indirect