From f2912a38ceb7ce1581c178ef15df16ea6204268b Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 13:49:27 +0200 Subject: [PATCH 1/8] feat(trace): add Codex provider Support native Codex hooks, rollout evidence, and multi-file attribution. Add an isolated Codex E2E harness and document verified compatibility. Refs #3657 Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino --- .codex/hooks.json | 56 +++ README.md | 13 +- app/cli/cmd/trace_hook.go | 20 + app/cli/cmd/trace_init.go | 17 +- app/cli/cmd/trace_init_providers.go | 13 +- app/cli/cmd/trace_init_providers_test.go | 5 + app/cli/cmd/trace_init_test.go | 7 +- app/cli/cmd/trace_run.go | 4 +- app/cli/documentation/cli-reference.md | 4 +- app/cli/internal/trace/README.md | 173 +------- app/cli/internal/trace/codex/announce_test.go | 224 ++++++++++ app/cli/internal/trace/codex/copy.go | 179 ++++++++ app/cli/internal/trace/codex/hooks.go | 381 ++++++++++++++++++ app/cli/internal/trace/codex/parse.go | 278 +++++++++++++ app/cli/internal/trace/codex/pricing.go | 72 ++++ app/cli/internal/trace/codex/pricing.json | 145 +++++++ app/cli/internal/trace/codex/provider.go | 149 +++++++ app/cli/internal/trace/codex/provider_test.go | 247 ++++++++++++ .../internal/trace/codex/testdata/README.md | 8 + .../trace/codex/testdata/native-session.jsonl | 10 + .../trace/codex/testdata/session.jsonl | 12 + app/cli/internal/trace/e2e/codex.go | 167 +++++++- app/cli/internal/trace/e2e/codex_test.go | 53 +++ app/cli/internal/trace/e2e/config.yaml | 2 +- app/cli/internal/trace/e2e/runner.go | 2 +- app/cli/internal/trace/provider.go | 3 + .../trace/providers/capabilities_test.go | 4 +- app/cli/internal/trace/providers/registry.go | 2 + app/cli/pkg/action/trace_agent_hook.go | 33 ++ app/cli/pkg/action/trace_codex_hook_test.go | 63 +++ docs/specs/issue-3657-codex-trace-provider.md | 122 ++++++ 31 files changed, 2270 insertions(+), 198 deletions(-) create mode 100644 .codex/hooks.json create mode 100644 app/cli/internal/trace/codex/announce_test.go create mode 100644 app/cli/internal/trace/codex/copy.go create mode 100644 app/cli/internal/trace/codex/hooks.go create mode 100644 app/cli/internal/trace/codex/parse.go create mode 100644 app/cli/internal/trace/codex/pricing.go create mode 100644 app/cli/internal/trace/codex/pricing.json create mode 100644 app/cli/internal/trace/codex/provider.go create mode 100644 app/cli/internal/trace/codex/provider_test.go create mode 100644 app/cli/internal/trace/codex/testdata/README.md create mode 100644 app/cli/internal/trace/codex/testdata/native-session.jsonl create mode 100644 app/cli/internal/trace/codex/testdata/session.jsonl create mode 100644 app/cli/pkg/action/trace_codex_hook_test.go create mode 100644 docs/specs/issue-3657-codex-trace-provider.md diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 000000000..0f1e67470 --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,56 @@ +{ + "hooks": { + "PostToolUse": [ + { + "hooks": [ + { + "command": "chainloop trace hook codex post-tool-use", + "type": "command" + } + ], + "matcher": "^(apply_patch|Bash)$" + } + ], + "PreToolUse": [ + { + "hooks": [ + { + "command": "chainloop trace hook codex pre-tool-use", + "type": "command" + } + ], + "matcher": "^(apply_patch|Bash)$" + } + ], + "SessionEnd": [ + { + "hooks": [ + { + "command": "chainloop trace hook codex session-end", + "type": "command" + } + ] + } + ], + "SessionStart": [ + { + "hooks": [ + { + "command": "chainloop trace hook codex session-start", + "type": "command" + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "command": "chainloop trace hook codex user-prompt-submit", + "type": "command" + } + ] + } + ] + } +} diff --git a/README.md b/README.md index 745248368..69866e110 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,9 @@ chainloop trace init # installs the git and agent hooks, writes .chainloo `trace init` writes `.chainloop.yml` and git hooks (`pre-push`, `post-commit`, `commit-msg`, `post-rewrite`) in the current repo, and `trace uninstall` removes them. Session data stays on your machine until `git push`, and the CLI redacts secrets before upload. The [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=try-it-in-60-seconds) lists what it sends. -Now code with Claude Code, OpenCode, or Cursor, commit, and `git push`. The push hook attests the session and prints where to see it: +For Codex, run `chainloop trace init --codex`. Review and trust the installed hooks in Codex before starting work. See the [agent support matrix](./app/cli/internal/trace/README.md) for verified features and limits. + +Now code with Claude Code, OpenCode, Codex, or Cursor, commit, and `git push`. The push hook attests the session and prints where to see it: ```text Coding Session Available at https://app.chainloop.dev/u//sessions/ @@ -184,7 +186,7 @@ A workflow contract is the API between the people who set the rules and the peop | Component | What it does | |---|---| -| [AI session collector](./app/cli/cmd/trace.go) (`chainloop trace`) | Records Claude Code, OpenCode, and Cursor sessions, redacts secrets, attributes AI and human lines, and attests the session on `git push`. `requireTrace` in `.chainloop.yml` blocks a push that has no record. | +| [AI session collector](./app/cli/cmd/trace.go) (`chainloop trace`) | Records Claude Code, OpenCode, Codex, and Cursor sessions, redacts secrets, attributes AI and human lines, and attests the session on `git push`. `requireTrace` in `.chainloop.yml` blocks a push that has no record. | | [CLI](./app/cli) (`chainloop`) | One integration point for every major CI runner and 45 evidence types. It crafts, signs, and pushes attestations, and runs policies. | | [Trusted store](./app/artifact-cas) | Content-addressable storage for evidence, backed by an OCI registry, S3, Azure Blob Storage, or inline. Your storage, your keys. Called the Artifact CAS in the code. | | [Signing](./pkg/attestation/signer) | in-toto attestations, keyless by default, or your own cosign / KMS keys, or SignServer. Timestamped. | @@ -231,7 +233,7 @@ During an attestation you can attach these evidence types. The CLI uploads files ## Use cases -**Put guardrails on coding agents.** Record every Claude Code, OpenCode, or Cursor session as a signed record, and run your policies on every session. Catch an unapproved model, a secret in the transcript, or more AI lines than the rule allows. Chainloop signs violations into the record, and they fail the pipeline that builds the PR. `requireTrace: true` rejects a push with no recorded session. → [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [Policies](https://docs.chainloop.dev/concepts/policies?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [AI coding governance](https://chainloop.dev/solutions/ai-coding-governance/?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) +**Put guardrails on coding agents.** Record every Claude Code, OpenCode, Codex, or Cursor session as a signed record, and run your policies on every session. Catch an unapproved model, a secret in the transcript, or more AI lines than the rule allows. Chainloop signs violations into the record, and they fail the pipeline that builds the PR. `requireTrace: true` rejects a push with no recorded session. → [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [Policies](https://docs.chainloop.dev/concepts/policies?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) · [AI coding governance](https://chainloop.dev/solutions/ai-coding-governance/?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) **Trace a release back to the prompt.** Follow any release back through the build and the commit to the AI session that started it. `chainloop referrer discover --digest ` returns every attestation that references an artifact, SBOM, or commit, and what those reference in turn. One record, from prompt to production. → [AI Sessions Quickstart](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=use-cases) @@ -259,7 +261,8 @@ Git tells you what changed. It no longer tells you how. `chainloop trace` record | Claude Code | Supported, with token usage and cost | | OpenCode | Supported, with token usage and cost | | Cursor | Experimental, no token usage or cost yet | -| Codex, GitHub Copilot, Gemini, Windsurf, Amp | Coming. [Contributions welcome](#community-and-contributing) | +| Codex | Supported, with token usage and estimated cost | +| GitHub Copilot, Gemini, Windsurf, Amp | Coming. [Contributions welcome](#community-and-contributing) | Wanted: new collector providers for the agents above, and new evidence types. The provider interface is in [app/cli/internal/trace](./app/cli/internal/trace). @@ -351,7 +354,7 @@ You can also download the CLI from the [releases page](https://github.com/chainl ## For AI coding agents > [!NOTE] -> **If you are an AI coding agent setting up Chainloop in this repository:** Install the CLI with `curl -sfL https://dl.chainloop.dev/cli/install.sh | bash -s -- --oss`. Ask the user to run `chainloop auth login`, which opens a browser. Then run `chainloop trace init --project ` from the repository root. Add `--claude`, `--cursor`, or `--opencode` for the agents in use. The command writes `.chainloop.yml` and git hooks. Commit `.chainloop.yml`. The docs index for agents is https://docs.chainloop.dev/llms.txt. Any docs page is available as Markdown when you add `.md` to its URL. [Documentation](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=for-ai-coding-agents) +> **If you are an AI coding agent setting up Chainloop in this repository:** Install the CLI with `curl -sfL https://dl.chainloop.dev/cli/install.sh | bash -s -- --oss`. Ask the user to run `chainloop auth login`, which opens a browser. Then run `chainloop trace init --project ` from the repository root. Add `--claude`, `--cursor`, `--opencode`, or `--codex` for the agents in use. The command writes `.chainloop.yml` and git hooks. Commit `.chainloop.yml`. The docs index for agents is https://docs.chainloop.dev/llms.txt. Any docs page is available as Markdown when you add `.md` to its URL. [Documentation](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=for-ai-coding-agents) Or paste this into Claude Code, Codex, or Cursor: diff --git a/app/cli/cmd/trace_hook.go b/app/cli/cmd/trace_hook.go index 26da0805e..442b68369 100644 --- a/app/cli/cmd/trace_hook.go +++ b/app/cli/cmd/trace_hook.go @@ -20,6 +20,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/config" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" @@ -40,6 +41,7 @@ func newTraceHookCmd() *cobra.Command { newTraceHookClaudeCmd(), newTraceHookCursorCmd(), newTraceHookOpenCodeCmd(), + newTraceHookCodexCmd(), ) return cmd @@ -451,3 +453,21 @@ func newTraceHookOpenCodePostToolUseCmd() *cobra.Command { }, } } + +func newTraceHookCodexCmd() *cobra.Command { + cmd := &cobra.Command{Use: "codex", Short: "Handle Codex hook invocations"} + handlers := []struct { + name, description string + run func(*cobra.Command, []string) error + }{ + {"session-start", "Capture the Codex session", func(cmd *cobra.Command, _ []string) error { return runAgentSessionStart(cmd, codex.New()) }}, + {"user-prompt-submit", "Remind Codex to capture specs", func(_ *cobra.Command, _ []string) error { return action.HandleAgentPromptSubmit(codex.New(), logger) }}, + {"pre-tool-use", "Snapshot files before a Codex tool", func(_ *cobra.Command, _ []string) error { return action.HandleAgentPreToolUse(codex.New(), logger) }}, + {"post-tool-use", "Record changes after a Codex tool", func(_ *cobra.Command, _ []string) error { return action.HandleAgentPostToolUse(codex.New(), logger) }}, + {"session-end", "Copy the completed Codex session", func(_ *cobra.Command, _ []string) error { return action.HandleAgentSessionEnd(codex.New(), logger) }}, + } + for _, h := range handlers { + cmd.AddCommand(&cobra.Command{Use: h.name, Short: h.description, Args: cobra.NoArgs, Annotations: map[string]string{skipActionOptsInit: trueString}, RunE: func(cmd *cobra.Command, args []string) error { InitHookLogger(); return h.run(cmd, args) }}) + } + return cmd +} diff --git a/app/cli/cmd/trace_init.go b/app/cli/cmd/trace_init.go index e75a18947..3057c342d 100644 --- a/app/cli/cmd/trace_init.go +++ b/app/cli/cmd/trace_init.go @@ -27,6 +27,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/repositoryconfig" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/config" tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/hooks" @@ -47,6 +48,7 @@ func newTraceInitCmd() *cobra.Command { claudeFlag bool cursorFlag bool opencodeFlag bool + codexFlag bool ) cmd := &cobra.Command{ @@ -72,7 +74,7 @@ named freely; the name is normalized to the lowercase, dash-separated form Chainloop stores. It then asks which harnesses to trace, with the ones the repository is already set up for ticked, or Claude Code on a first run; use space to tick more. Passing --org, --project or a harness flag -(--claude, --cursor, --opencode) skips the matching question. +(--claude, --cursor, --opencode, --codex) skips the matching question. Nothing is asked in CI or when the output is redirected: there --project is required unless .chainloop.yml carries projectName. Set CHAINLOOP_NO_PROMPT to @@ -155,7 +157,7 @@ cannot use them with --local. You can use --require-trace with --local.`, // so every question is answered up front and an abort leaves the // repository untouched. selected, err := resolveTraceProviders(newNamingPrompter(os.LookupEnv), - traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag}, + traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag, codex: codexFlag}, traceInitCanPrompt(), func() []string { return configuredTraceProviders(repoRoot) }) if err != nil { return stopIfAborted(err) @@ -293,6 +295,7 @@ cannot use them with --local. You can use --require-trace with --local.`, cmd.Flags().Bool("require-trace", false, "block pushes when a required trace is missing") cmd.Flags().BoolVar(&claudeFlag, "claude", false, "install Claude Code hooks (default when no harness flag is set)") cmd.Flags().BoolVar(&cursorFlag, "cursor", false, "install Cursor hooks") + cmd.Flags().BoolVar(&codexFlag, "codex", false, "install Codex hooks") cmd.Flags().BoolVar(&opencodeFlag, "opencode", false, "install opencode hooks") return cmd @@ -380,6 +383,11 @@ What's next Learn more: %s `, strings.Join(files, " "), joinWithOr(providerNames(installed)), traceDocsURL) + for _, p := range installed { + if p.Name() == codex.Name { + fmt.Fprintln(w, "For Codex, review and trust the installed hooks in Codex before starting work.") + } + } } // writeTraceInitLocalSummary reports what a local init set up. It names no @@ -437,6 +445,11 @@ What's next Learn more: %s `, strings.Join(files, " "), joinWithOr(providerNames(installed)), traceDocsURL) + for _, p := range installed { + if p.Name() == codex.Name { + fmt.Fprintln(w, "For Codex, review and trust the installed hooks in Codex before starting work.") + } + } } // providerNames names the harnesses, in the order they were installed. diff --git a/app/cli/cmd/trace_init_providers.go b/app/cli/cmd/trace_init_providers.go index e88d1f735..db98dc92e 100644 --- a/app/cli/cmd/trace_init_providers.go +++ b/app/cli/cmd/trace_init_providers.go @@ -20,6 +20,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/providers" @@ -27,21 +28,20 @@ import ( const providersPromptTitle = "Harnesses to trace in this repository" -// traceProviderFlags records which --claude / --cursor / --opencode flags the -// user set. +// traceProviderFlags records the harness flags that the user set. type traceProviderFlags struct { - claude, cursor, opencode bool + claude, cursor, opencode, codex bool } // any reports whether the user named at least one provider. func (f traceProviderFlags) any() bool { - return f.claude || f.cursor || f.opencode + return f.claude || f.cursor || f.opencode || f.codex } // names returns the providers the flags select, in the registry's order rather // than the order the flags happen to appear in. func (f traceProviderFlags) names() []string { - out := make([]string, 0, 3) + out := make([]string, 0, 4) if f.claude { out = append(out, claude.Name) } @@ -52,6 +52,9 @@ func (f traceProviderFlags) names() []string { out = append(out, opencode.Name) } + if f.codex { + out = append(out, codex.Name) + } return out } diff --git a/app/cli/cmd/trace_init_providers_test.go b/app/cli/cmd/trace_init_providers_test.go index 83170a267..80e5021b0 100644 --- a/app/cli/cmd/trace_init_providers_test.go +++ b/app/cli/cmd/trace_init_providers_test.go @@ -194,6 +194,11 @@ func TestConfiguredTraceProvidersSkipsUnreadableSettings(t *testing.T) { } func TestTraceProviderFlags(t *testing.T) { + t.Run("codex flag", func(t *testing.T) { + f := traceProviderFlags{codex: true} + assert.True(t, f.any()) + assert.Equal(t, []string{"codex"}, f.namesOrDefault()) + }) t.Run("no flag set", func(t *testing.T) { assert.False(t, traceProviderFlags{}.any()) assert.Empty(t, traceProviderFlags{}.names()) diff --git a/app/cli/cmd/trace_init_test.go b/app/cli/cmd/trace_init_test.go index 2b16b373f..665fca8c2 100644 --- a/app/cli/cmd/trace_init_test.go +++ b/app/cli/cmd/trace_init_test.go @@ -22,6 +22,7 @@ import ( "slices" "testing" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/providers" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -326,14 +327,14 @@ func TestWriteTraceNextSteps(t *testing.T) { }, { name: "every harness", - providers: []string{providerClaudeCode, "cursor", "opencode"}, + providers: []string{providerClaudeCode, "cursor", opencode.Name, "codex"}, // Named in full: ".opencode" alone would pass on any file under that // directory, so it would not catch the plugin being renamed. wantFiles: []string{ chainloopYMLName, claudeSettings, ".cursor/hooks.json", - ".opencode/plugins/chainloop-trace.ts", + ".opencode/plugins/chainloop-trace.ts", ".codex/hooks.json", }, - wantHarnesses: "claude-code, cursor or opencode", + wantHarnesses: "claude-code, cursor, opencode or codex", }, } diff --git a/app/cli/cmd/trace_run.go b/app/cli/cmd/trace_run.go index 02bed925e..ecb451a40 100644 --- a/app/cli/cmd/trace_run.go +++ b/app/cli/cmd/trace_run.go @@ -64,6 +64,7 @@ func newTraceRunCmd() *cobra.Command { claudeFlag bool cursorFlag bool opencodeFlag bool + codexFlag bool ) cmd := &cobra.Command{ @@ -165,7 +166,7 @@ func newTraceRunCmd() *cobra.Command { return action.TraceRun(cmd.Context(), logger, action.TraceRunOpts{ Store: store, RepoRoot: repoRoot, - Providers: traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag}.namesOrDefault(), + Providers: traceProviderFlags{claude: claudeFlag, cursor: cursorFlag, opencode: opencodeFlag, codex: codexFlag}.namesOrDefault(), Command: args, ProjectName: projectFlag, Organization: organization, @@ -187,6 +188,7 @@ func newTraceRunCmd() *cobra.Command { cmd.Flags().BoolVar(&localFlag, "local", false, "store the session on disk, with no control plane, login or workflow") cmd.Flags().BoolVar(&claudeFlag, "claude", false, "install Claude Code hooks (default when no provider flag is set)") cmd.Flags().BoolVar(&cursorFlag, "cursor", false, "install Cursor hooks") + cmd.Flags().BoolVar(&codexFlag, "codex", false, "install Codex hooks") cmd.Flags().BoolVar(&opencodeFlag, "opencode", false, "install opencode hooks") // project and workflow are not required in local mode, so the check is done diff --git a/app/cli/documentation/cli-reference.md b/app/cli/documentation/cli-reference.md index ba01605fb..471e73bcf 100644 --- a/app/cli/documentation/cli-reference.md +++ b/app/cli/documentation/cli-reference.md @@ -3314,7 +3314,7 @@ named freely; the name is normalized to the lowercase, dash-separated form Chainloop stores. It then asks which harnesses to trace, with the ones the repository is already set up for ticked, or Claude Code on a first run; use space to tick more. Passing --org, --project or a harness flag -(--claude, --cursor, --opencode) skips the matching question. +(--claude, --cursor, --opencode, --codex) skips the matching question. Nothing is asked in CI or when the output is redirected: there --project is required unless .chainloop.yml carries projectName. Set CHAINLOOP_NO_PROMPT to @@ -3345,6 +3345,7 @@ Options ``` --claude install Claude Code hooks (default when no harness flag is set) +--codex install Codex hooks --contract string workflow contract to attach when creating the workflow; it must already exist and be usable by the project (default: "chainloop-ai-coding-session" when the organization has it, an empty contract otherwise) --cursor install Cursor hooks -h, --help help for init @@ -3413,6 +3414,7 @@ Options ``` --claude install Claude Code hooks (default when no provider flag is set) +--codex install Codex hooks --contract string workflow contract to attach when creating the workflow; it must already exist and be usable by the project (default: "chainloop-ai-coding-session" when the organization has it, an empty contract otherwise) --cursor install Cursor hooks -h, --help help for run diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 9f66dddae..18222f050 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -1,6 +1,6 @@ # Chainloop Trace: agent support -Chainloop Trace records the sessions of AI coding agents as `CHAINLOOP_AI_CODING_SESSION` evidence. Each agent is a provider in this package: `claude/`, `cursor/` and `opencode/`. The interfaces are in `provider.go`, and the registry is in `providers/`. To add a provider, follow [PROVIDERS.md](PROVIDERS.md). One provider covers OpenCode 1.x and OpenCode 2: it detects the version at runtime. +Chainloop Trace records the sessions of AI coding agents as `CHAINLOOP_AI_CODING_SESSION` evidence. Each agent is a provider in this package: `claude/`, `cursor/`, `opencode/` and `codex/`. The interfaces are in `provider.go`, and the registry is in `providers/`. To add a provider, follow [PROVIDERS.md](PROVIDERS.md). One provider covers OpenCode 1.x and OpenCode 2: it detects the version at runtime. This page lists the features of Trace, by area, and tells which agents support each one. **Keep it current.** When you add or change a provider, or add or change a feature of Trace, update this page in the same pull request. @@ -10,173 +10,18 @@ Claude Code has the most complete support. Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and `trace run --pi` do not exist, so no Pi session is recorded. Each Pi cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Pi support is added, set the Pi cells to **Unknown** and run the end-to-end suite to find the values. -Trace does not support Codex yet: `chainloop trace init --codex` and `trace run --codex` do not exist, so no Codex session is recorded. Each Codex cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Codex support is added, set the Codex cells to **Unknown** and run the end-to-end suite to find the values. +Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Changing the hooks requires a new trust review. -## Setup - -How Trace is added to a repository and to an agent. - -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | -|---------|---------------|-------------|--------|--------------|------------|---|---| -| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Yes | Unknown | Yes | No | No | -| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Yes | Unknown | Yes | No | No | -| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | No | - -Why not Yes: - -- **Hook installation and deletion, and Trace run wrapper, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). -- **More than one agent in one repository, all agents:** not verified yet. The end-to-end suite needs a second agent to test it (spec issue-3635). - -## Session collection - -What the session evidence holds about the conversation. - -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | -|---------|---------------|-------------|--------|--------------|------------|---|---| -| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | No | No | -| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | No | No | -| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | No | No | -| Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | No | No | -| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | No | No | - -Why not Yes: - -- **Transcript, OpenCode:** OpenCode has no transcript file. The provider runs `opencode export` (1.x) or `opencode session export` (2) and rebuilds the transcript from the export. It keeps the text and the inputs of the tool calls. It drops the tool outputs, the reasoning and the file parts. OpenCode 2 exports also have no agent version and no session slug. -- **Token usage and cost, Cursor:** the Cursor transcript does not hold token usage. Usage and cost are zero, and the evidence has a warning that tells so. (For OpenCode, the cost is the value that OpenCode reports. For Claude Code, Chainloop computes it from its price table.) -- **Model detection, Cursor:** the primary model comes from the session-start hook. The transcript does not tell the models, so the evidence does not list the other models. In auto mode, the hook names the model `default`, not the model that ran. -- **Subagents, Cursor:** the provider does not read subagent conversations. -- **Subagents, OpenCode:** a subagent runs in a child session with its own export. It is not part of the evidence of its parent session. -- **Missed session start, Cursor and OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). - -## Code attribution +Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 19 feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts, skills tracking, spec source pointers, and pasted image pointers are not implemented. -Which lines of a commit the AI wrote. +Codex token usage comes from cumulative rollout counters. Cost is an estimate from the embedded standard API prices, with a warning for unknown models. Subscription billing and third-party provider prices can differ. Trace reads rollouts from `$CODEX_HOME/sessions` (default `~/.codex/sessions`) or `archived_sessions`. Their JSONL format is not a stable Codex interface. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | -|---------|---------------|-------------|--------|--------------|------------|---|---| -| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence reports the totals of the session in `ai_generated_lines_added` and `ai_generated_lines_removed`. A session with no commit sends no evidence, so it reports no totals (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | No | -| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the removals of the session's commits, less the lines that a commit of the session wrote. The line counts of each file are the sum of its AI and human lines, and a file that the session created stays created when a later commit of the session changes it (spec issue-3639, issue 3674). | Partial | Partial | Partial | Partial | No | No | -| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | -| Line ranges | The line ranges of each file are positions in the file at the commit that the push sends. They cover the lines of the session that are still there, and each range says if its lines are AI or human. The AI ranges hold as many lines as the net AI lines added, and the human ranges as many lines as the net human lines added. A file that the pushed commit no longer holds has no ranges. A range with no attribution is an AI range at edit time. Older CLIs write only these ranges. This CLI writes them for a file that it cannot blame at the pushed commit, and for a file with edits from before line fingerprints (spec issue-3639, line ranges). | Partial | Partial | Partial | Partial | No | No | -| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | - -Why not Yes: - -- **Line attribution, partial commits and generated line counts, Cursor:** Cursor has only the `afterFileEdit` hook, which gives the edits but no snapshot before the edit. The provider rebuilds the content before the edit from the edits. -- **Net contribution and line ranges, all agents:** the push finds the last writer of each line with `git blame`. A later commit that only re-indents a line takes the line, because blame sees a change. For Cursor, the limits of line attribution apply too. -- **Session updates, all agents:** the update works only in the clone where the work happened, while the CLI keeps the trace state of the session (30 days), and only when the commits of the session are on the branch that the push sends. The push reads only the commits that are not on the default branch of the remote (`origin/HEAD`, or else `origin/main` or `origin/master`). After the commits of a session are pushed to that branch, a later commit does not update the session. -- **Renames and rewrites, all agents:** this works only in the clone where the work happened. Git finds a rename only when the file keeps at least half of its content. Two pushed commits can rename a file one after the other. A later change to the file under its middle name does not send the session again. A rename inside an amend or a squash of the session commit loses the AI lines: the new commit creates the file under its new name, so no line of it matches the trace state (issue 3686). The limits of session updates apply too. For Cursor, the limits of line attribution apply too. -- **Shell changes attribution, all agents:** the hook before a shell command records only a hash of each file, not its content. The file at the `HEAD` commit from before the command stands in for the content before the command. Uncommitted lines that an earlier agent edit recorded are not recorded again, but uncommitted human lines of a file that the command changes count as AI. - -## User experience - -What the user sees in the agent. - -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | -|---------|---------------|-------------|--------|--------------|------------|---|---| -| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | No | -| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | No | -| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | No | - -Why not Yes: - -- **Welcome message, Cursor and OpenCode:** the provider gives the session-start context to the model only. It has no channel that shows a message to the user at session start, so the welcome message is dropped. -- **Credentials warning, Cursor and OpenCode:** the provider has no channel to the user at session start. The model is told to tell the user, so the warning depends on the model. -- **Session link after push, Cursor:** the `postToolUse` hook runs after a shell command, but its response has no field that Cursor shows to the user. The link stays pending until it expires. -- **Session link after push, OpenCode:** the plugin hook fires after a shell command, but the response that shows a message to the user is not verified yet. The link stays pending until it expires. - -## Specs and skills - -What the session was asked to build, and which skills it used. - -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | -|---------|---------------|-------------|--------|--------------|------------|---|---| -| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | No | No | -| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | No | -| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | No | -| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Yes | Unknown | Yes | No | No | -| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | No | -| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | No | -| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | No | - -Why not Yes: - -- **Spec capture reminder, Cursor:** Cursor has no hook at each user prompt. The agent gets the session-start instruction only. -- **Spec source role, title, and description, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). -- **Skills tracking, Cursor:** the provider does not track skills. The evidence has no skill entries, which means "not recorded", not "no skill used". -- **Skills tracking, OpenCode:** only the skills that the model starts with the `skill` tool are counted. Skills that the user starts, and skills used in subagents, are not. A plugin installed by an earlier version does not send the skill folder for OpenCode 2: install the hooks again to update it. -- **Spec source pointers, Cursor and OpenCode:** the provider has no finder for the copies in its transcript yet. For OpenCode, the rebuilt transcript holds no tool outputs, so only the write copies would apply. -- **Pasted image pointers, Cursor and OpenCode:** the provider has no finder for pasted images yet. Each agent puts pasted images in a different block shape. The pasted images stay inline in the transcript. The agent is told not to capture them in the spec folder. - -## Security - -How the evidence is protected before it leaves the machine. - -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | -|---------|---------------|-------------|--------|--------------|------------|---|---| -| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | No | No | -| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | No | No | -| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | No | No | - -Why not Yes: - -- **Media skipped by redaction, Cursor:** the Cursor transcript holds no tool outputs, so a file that the agent reads leaves no media in it. A pasted image stays inline in the transcript, but it is not verified that it is in the base64 media shape that the redaction skips. The headless Cursor CLI cannot attach an image, so the end-to-end suite cannot check it. -- **Media skipped by redaction, OpenCode:** the rebuilt transcript holds no media. - -## Git integration +## Setup -How the sessions are linked to the commits and attested. +How Trace is added to a repository and to an agent. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | No | No | -| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | No | No | -| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | No | No | -| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | No | No | - -### Switching storage modes - -`traceStorage` in `.chainloop.yml` is a committed repository policy. It is the same for every clone. When a repository uses `LOCAL`, Trace stores every session of every user on disk. To change the mode: - -- **To local:** run `chainloop trace init --local`. It writes `traceStorage: LOCAL`, installs the hooks, and does not connect to a control plane. Commit `.chainloop.yml` to share the change. -- **Back to the control plane:** run `chainloop trace init` without `--local`. It writes `traceStorage: CONTROL_PLANE` and sets up the workflow. The command tells you about the change. Commit `.chainloop.yml`. - -`tracePath` in `.chainloop.yml` sets the trace directory, where Chainloop Trace keeps its data in every mode. The default is `.git/chainloop-trace`. A relative path starts at the repository root. `chainloop trace init --trace-path ` writes it. A directory in the working tree must be in `.gitignore`. - -Run `chainloop trace init` one time for each repository. Git does not clone the git hooks in `.git`. In a clone, the first traced session installs them, so a teammate does not run init. - -Why not Yes: - -- **Shell commands in another checkout, Cursor:** a relative path in the command resolves against the first workspace root, not against the folder where the shell runs. -## How each agent is hooked in - -| Agent | Hook installation | Hooks | -|-------|-------------------|-------| -| Claude Code | `.claude/settings.json` | Session start and end, user prompt, and before and after each file and shell tool and the Skill tool. | -| Cursor | `.cursor/hooks.json` | Session start and end, `afterFileEdit`, and `preToolUse` and `postToolUse` for the `Shell` tool. The `preToolUse` hook always allows the command. The provider installs no prompt hook. | -| OpenCode | `.opencode/plugins/chainloop-trace.ts` | Session start and end, user prompt, and before and after each tool. One plugin covers both versions: `server()` for 1.x (1.3.4 or later) and `setup()` for 2. The provider picks the export command from `opencode --version`. | - -All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. - -## End-to-end tests - -The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code, Cursor, OpenCode 2, Pi, and Codex columns. The Pi and Codex cells are **No**, so the suite skips them until Pi and Codex support is added. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. Pi runs with `pi -p --mode json --approve`, with its configuration and session folders next to the workspace. Codex runs with `codex exec --json`, with its `CODEX_HOME` folder next to the workspace, no sandbox, and no hook trust prompt. Cursor runs with `cursor-agent -p --output-format stream-json --force --trust`, with its `HOME` folder next to the workspace, so the Cursor configuration and hooks of the user do not load. Cursor runs its shell tool in a login shell, and on macOS `path_helper` puts the system folders first in `PATH`. So the harness writes shell startup files in that `HOME` that set `PATH` back. Otherwise a `chainloop` in `/usr/local/bin` runs the git hooks instead of the CLI under test. The Cursor CLI then has no stored login, so the run needs `CURSOR_API_KEY`. The headless Cursor CLI cannot attach a pasted image, and its output does not show the hook messages. The OpenCode 2, Pi, and Codex models in `e2e/config.yaml` are OpenRouter models, so the run needs `OPENROUTER_API_KEY`. A Codex model with no `openrouter/` prefix runs on the default Codex provider, which needs `CODEX_API_KEY`. - -Some scenarios add the git commands of a human after the session, for example a commit that replaces an AI line, an amend, or a rename, and then push again. They work on a feature branch, because the push reads only the commits that are not on the default branch of the remote. - -A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows: - -- A cell marked **No** or **Not applicable** is `skipped`. -- A feature that needs a control plane is `not covered`, because the suite uses local storage mode only. -- A row with no registered check is `no check`. - -`e2e/config.yaml` sets the version of each agent that the suite expects, and the models. The suite does not install the agent. It runs the installed version, and it logs a warning when that version is not the one in the configuration. The suite uses model tokens. To run it: - -```bash -make -C app/cli test-trace-e2e -``` - -`CHAINLOOP_E2E_HARNESS`, `CHAINLOOP_E2E_MODEL`, and `CHAINLOOP_E2E_FEATURE` limit a run to some columns, models, or features. Separate the values with `;`, because feature names can hold commas. Set `CHAINLOOP_E2E_WORK_DIR` to keep the workspaces and the agent output. The results go to `e2e/results/`, or to `CHAINLOOP_E2E_RESULTS_DIR`. Only a run with no filter becomes the previous result of the next comparison. +| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Yes | Unknown | Yes | No | Yes | +| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Yes | Unknown | Yes | No | Yes | +| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | Unknown | diff --git a/app/cli/internal/trace/codex/announce_test.go b/app/cli/internal/trace/codex/announce_test.go new file mode 100644 index 000000000..4ddfab568 --- /dev/null +++ b/app/cli/internal/trace/codex/announce_test.go @@ -0,0 +1,224 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "encoding/json" + "io" + "os" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAnnounceToUser pins the wire shape Codex expects. systemMessage +// must stay top-level: nested inside hookSpecificOutput it is silently ignored. +const emptyMessageCase = "nothing to say emits nothing" + +func TestAnnounceToUser(t *testing.T) { + const msg = "Coding Session Available at https://app.chainloop.dev/u/chainloop/sessions/ses_1" + + testCases := []struct { + name string + msg string + wantEmitted bool + }{ + { + name: "a message goes out on both channels", + msg: msg, + wantEmitted: true, + }, + { + name: emptyMessageCase, + msg: "", + wantEmitted: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := captureStdout(t, func() { + require.NoError(t, New().AnnounceToUser(tc.msg)) + }) + + if !tc.wantEmitted { + assert.Empty(t, out, "no message means no stdout, so the hook stays a no-op") + return + } + + var got map[string]any + require.NoError(t, json.Unmarshal([]byte(out), &got)) + + assert.Equal(t, tc.msg, got["systemMessage"], "systemMessage must be top-level") + + hookOut, ok := got["hookSpecificOutput"].(map[string]any) + require.True(t, ok, "hookSpecificOutput must be present") + assert.Equal(t, "PostToolUse", hookOut["hookEventName"]) + + // The model needs the message verbatim to be able to repeat it. + assert.Contains(t, hookOut["additionalContext"], tc.msg) + }) + } +} + +// TestAnnounceSessionStart pins the session-start wire shape. The two channels +// travel in one document because Codex parses a hook's stdout as a single +// JSON value: a second write would be discarded together with whatever it +// carried, which is why they are one call rather than two. +func TestAnnounceSessionStart(t *testing.T) { + const ( + banner = "Chainloop Trace is recording this session." + instruction = "Write the specification this session is working from into /repo/.chainloop/specs/abc-123 now." + ) + + testCases := []struct { + name string + msg trace.SessionStartMessage + wantBanner string + wantInstruction string + wantEmitted bool + }{ + { + name: "both channels in one document", + msg: trace.SessionStartMessage{Banner: banner, Instruction: instruction}, + wantBanner: banner, + wantInstruction: instruction, + wantEmitted: true, + }, + { + // What the hook emits once a spec is already captured: the user + // still gets the banner, the model is told nothing. + name: "a banner with nothing to instruct", + msg: trace.SessionStartMessage{Banner: banner}, + wantBanner: banner, + wantEmitted: true, + }, + { + name: "an instruction with no banner", + msg: trace.SessionStartMessage{Instruction: instruction}, + wantInstruction: instruction, + wantEmitted: true, + }, + { + name: emptyMessageCase, + msg: trace.SessionStartMessage{}, + wantEmitted: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := captureStdout(t, func() { + require.NoError(t, New().AnnounceSessionStart(tc.msg)) + }) + + if !tc.wantEmitted { + assert.Empty(t, out, "no message means no stdout, so the hook stays a no-op") + return + } + + dec := json.NewDecoder(strings.NewReader(out)) + + var got map[string]any + require.NoError(t, dec.Decode(&got)) + + // The constraint this whole shape exists to respect: one document, + // never two. + require.ErrorIs(t, dec.Decode(&map[string]any{}), io.EOF, "stdout must carry exactly one JSON document") + + if tc.wantBanner == "" { + assert.NotContains(t, got, "systemMessage", "nothing to show means no empty banner to render") + } else { + assert.Equal(t, tc.wantBanner, got["systemMessage"], "systemMessage must be top-level") + } + + hookOut, ok := got["hookSpecificOutput"].(map[string]any) + require.True(t, ok, "hookSpecificOutput must be present") + assert.Equal(t, "SessionStart", hookOut["hookEventName"], "the event must be the one that fired") + + if tc.wantInstruction == "" { + assert.NotContains(t, hookOut, "additionalContext") + return + } + assert.Equal(t, tc.wantInstruction, hookOut["additionalContext"], "the model needs the instruction verbatim") + }) + } +} + +// TestAnnouncePromptSubmit pins the prompt-submit wire shape: Codex adds +// additionalContext of a UserPromptSubmit response to the context of the turn. +func TestAnnouncePromptSubmit(t *testing.T) { + const reminder = "Chainloop spec capture. Folder: /repo/.chainloop/specs/abc-123" + + testCases := []struct { + name string + reminder string + wantEmitted bool + }{ + {name: "a reminder reaches the model", reminder: reminder, wantEmitted: true}, + {name: emptyMessageCase, reminder: "", wantEmitted: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := captureStdout(t, func() { + require.NoError(t, New().AnnouncePromptSubmit(tc.reminder)) + }) + + if !tc.wantEmitted { + assert.Empty(t, out) + return + } + + var got map[string]any + require.NoError(t, json.Unmarshal([]byte(out), &got)) + + // The reminder is for the model only: a banner at every turn + // would distract the user. + assert.NotContains(t, got, "systemMessage") + + hookOut, ok := got["hookSpecificOutput"].(map[string]any) + require.True(t, ok, "hookSpecificOutput must be present") + assert.Equal(t, "UserPromptSubmit", hookOut["hookEventName"], "the event must be the one that fired") + assert.Equal(t, tc.reminder, hookOut["additionalContext"]) + }) + } +} + +// captureStdout runs fn with os.Stdout redirected to a pipe and returns +// everything written to it. Reads to EOF rather than into a fixed buffer: a +// truncated read would corrupt the payload these tests parse as JSON. +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + + orig := os.Stdout + r, w, err := os.Pipe() + require.NoError(t, err) + os.Stdout = w + t.Cleanup(func() { os.Stdout = orig }) + + fn() + require.NoError(t, w.Close()) + + out, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + + return string(out) +} diff --git a/app/cli/internal/trace/codex/copy.go b/app/cli/internal/trace/codex/copy.go new file mode 100644 index 000000000..0aca8e56e --- /dev/null +++ b/app/cli/internal/trace/codex/copy.go @@ -0,0 +1,179 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +func codexHome() string { + if home := os.Getenv("CODEX_HOME"); home != "" { + return home + } + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, ".codex") +} + +// CopySessionData refreshes a private transcript copy atomically. A missing +// transcript leaves the previous copy intact. Only Codex session storage is read. +func (p *Provider) CopySessionData(store *state.Store, loc trace.SessionLocation) error { + if store == nil || !state.ValidSessionID(loc.SessionID) { + return errors.New("copy Codex session: invalid session") + } + home := codexHome() + if home == "" { + return nil + } + roots := []string{filepath.Join(home, "sessions"), filepath.Join(home, "archived_sessions")} + source := loc.TranscriptPath + if source != "" { + if !filepath.IsAbs(source) { + return errors.New("copy Codex session: transcript path must be absolute") + } + if !insideStorage(source, roots) { + return errors.New("copy Codex session: transcript outside Codex storage") + } + if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) { + source = "" + } + } + if source == "" { + var err error + source, err = findRollout(roots, loc.SessionID) + if err != nil { + return err + } + } + + if source == "" { + return nil + } + if !insideStorage(source, roots) { + return errors.New("copy Codex session: transcript outside Codex storage") + } + src, err := os.Open(source) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + defer src.Close() + info, err := src.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("copy Codex session: transcript is not a regular file") + } + // The rollout header, rather than the file name, owns the session identity. + scanner := bufio.NewScanner(src) + scanner.Buffer(make([]byte, 4096), maxHookPayloadBytes) + if !scanner.Scan() { + return errors.New("copy Codex session: missing rollout header") + } + var header struct { + Type string `json:"type"` + Payload struct { + ID string `json:"id"` + } `json:"payload"` + } + if json.Unmarshal(scanner.Bytes(), &header) != nil || header.Type != "session_meta" || header.Payload.ID != loc.SessionID { + return errors.New("copy Codex session: rollout session ID mismatch") + } + if _, err := src.Seek(0, io.SeekStart); err != nil { + return err + } + dir := store.RawSessionDir() + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".codex-session-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + if _, err := io.Copy(tmp, src); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), state.RawSessionPath(dir, loc.SessionID)) +} + +// Resolve symlinks as well as lexical traversal before opening a hook path. +func insideStorage(path string, roots []string) bool { + resolved, err := filepath.EvalSymlinks(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return false + } + if errors.Is(err, os.ErrNotExist) { + resolved = filepath.Clean(path) + } + for _, root := range roots { + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + resolvedRoot = filepath.Clean(root) + } + rel, err := filepath.Rel(resolvedRoot, resolved) + if err == nil && filepath.IsLocal(rel) && rel != "." { + return true + } + } + return false +} + +func findRollout(roots []string, sessionID string) (string, error) { + source := "" + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if entry.Type().IsRegular() && strings.HasSuffix(entry.Name(), "-"+sessionID+".jsonl") { + source = path + return fs.SkipAll + } + return nil + }) + if err != nil { + return "", fmt.Errorf("find Codex session: %w", err) + } + if source != "" { + break + } + } + return source, nil +} diff --git a/app/cli/internal/trace/codex/hooks.go b/app/cli/internal/trace/codex/hooks.go new file mode 100644 index 000000000..a7070bd22 --- /dev/null +++ b/app/cli/internal/trace/codex/hooks.go @@ -0,0 +1,381 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" +) + +const ( + hookMarker = "chainloop trace hook codex" + settingsFile = ".codex/hooks.json" + + // Codex hook event names. + eventSessionStart = "SessionStart" + eventUserPromptSubmit = "UserPromptSubmit" + eventPreToolUse = "PreToolUse" + eventPostToolUse = "PostToolUse" + eventSessionEnd = "SessionEnd" +) + +// fileWritingTools is the single source of truth for Codex hook tool names that modify files. +var fileWritingTools = []string{"apply_patch"} + +// commandTools are Codex hook tool names that run shell commands. They can create +// or modify arbitrary files, so their changes are captured via a before/after +// working-tree snapshot rather than a per-file snapshot. +var commandTools = []string{"Bash"} + +// hookToolMatcher is the pipe-separated matcher for the pre/post-tool-use hooks. +// It covers both file-writing and command tools; the handler branches on which +// kind fired. +var hookToolMatcher = "^(" + strings.Join(append(append([]string{}, fileWritingTools...), commandTools...), "|") + ")$" + +type hookEvent struct { + event string + command string + matcher string // optional: tool name matcher +} + +var hookEvents = []hookEvent{ + {eventSessionStart, "chainloop trace hook codex session-start", ""}, + {eventUserPromptSubmit, "chainloop trace hook codex user-prompt-submit", ""}, + {eventPreToolUse, "chainloop trace hook codex pre-tool-use", hookToolMatcher}, + {eventPostToolUse, "chainloop trace hook codex post-tool-use", hookToolMatcher}, + {eventSessionEnd, "chainloop trace hook codex session-end", ""}, +} + +// SettingsFile returns the absolute path to .codex/hooks.json for the given repo. +func (p *Provider) SettingsFile(repoRoot string) string { + return filepath.Join(repoRoot, settingsFile) +} + +// InstallHooks adds Chainloop hooks to .codex/hooks.json. +func (p *Provider) InstallHooks(repoRoot string) error { + return p.installHookEvents(repoRoot, hookEvents) +} + +// InstallHooksForTraceRun installs every hook except SessionEnd; trace +// run drives end-of-session itself. +func (p *Provider) InstallHooksForTraceRun(repoRoot string) error { + events := make([]hookEvent, 0, len(hookEvents)) + for _, h := range hookEvents { + if h.event == eventSessionEnd { + continue + } + events = append(events, h) + } + + return p.installHookEvents(repoRoot, events) +} + +func (p *Provider) installHookEvents(repoRoot string, events []hookEvent) error { + settingsPath := filepath.Join(repoRoot, settingsFile) + + settings, err := readJSONFile(settingsPath) + if err != nil { + return err + } + + hooks, _ := settings["hooks"].(map[string]any) + if hooks == nil { + hooks = make(map[string]any) + } + + changed := false + if len(events) < len(hookEvents) { + changed = removeHookEntry(hooks, eventSessionEnd) + } + for _, h := range events { + if addHookEntry(hooks, h.event, h.command, h.matcher) { + changed = true + } + } + + if !changed { + return nil + } + + settings["hooks"] = hooks + + return writeJSONFile(settingsPath, settings) +} + +// UninstallHooks removes Chainloop hooks from .codex/hooks.json. +func (p *Provider) UninstallHooks(repoRoot string) error { + settingsPath := filepath.Join(repoRoot, settingsFile) + + settings, err := readJSONFile(settingsPath) + if err != nil { + return err + } + + hooks, _ := settings["hooks"].(map[string]any) + if hooks == nil { + return nil + } + + changed := false + for _, h := range hookEvents { + if removeHookEntry(hooks, h.event) { + changed = true + } + } + + if !changed { + return nil + } + + // Clean up empty hooks map + if len(hooks) == 0 { + delete(settings, "hooks") + } + + // Delete file if empty + if len(settings) == 0 { + return os.Remove(settingsPath) + } + + return writeJSONFile(settingsPath, settings) +} + +// HooksInstalled reports whether .codex/hooks.json holds any Chainloop +// hook. User-authored hooks are ignored. +func (p *Provider) HooksInstalled(repoRoot string) (bool, error) { + settings, err := readJSONFile(filepath.Join(repoRoot, settingsFile)) + if err != nil { + return false, err + } + + hooks, _ := settings["hooks"].(map[string]any) + for _, h := range hookEvents { + entries, _ := hooks[h.event].([]any) + if slices.ContainsFunc(entries, entryContainsChainloopHook) { + return true, nil + } + } + + return false, nil +} + +// maxHookPayloadBytes caps hook payload reads to defend against runaway or +// malformed payloads. +const maxHookPayloadBytes = 16 * 1024 * 1024 + +// ReadHookInput reads and parses the Codex hook JSON from the given reader. +func (p *Provider) ReadHookInput(r io.Reader) (*trace.HookInput, error) { + data, err := io.ReadAll(io.LimitReader(r, maxHookPayloadBytes+1)) + if err != nil { + return nil, err + } + + if len(data) > maxHookPayloadBytes { + return nil, errors.New("codex hook payload exceeds size limit") + } + var raw struct { + trace.HookInput + Model string `json:"model"` + ToolInput struct { + Command string `json:"command"` + } `json:"tool_input"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + input := raw.HookInput + input.Model = raw.Model + if p.IsCommandTool(input.ToolName) { + input.Command = raw.ToolInput.Command + } + if p.IsFileWritingTool(input.ToolName) { + input.FilePaths = patchPaths(raw.ToolInput.Command, input.Cwd) + } + return &input, nil +} + +// patchPaths lists both sides of moves so deletions and new paths are recorded. +func patchPaths(patch, cwd string) []string { + var paths []string + if !filepath.IsAbs(cwd) { + return nil + } + for _, line := range strings.Split(patch, "\n") { + for _, prefix := range []string{"*** Add File: ", "*** Update File: ", "*** Delete File: ", "*** Move to: "} { + path, ok := strings.CutPrefix(line, prefix) + if !ok || path == "" { + continue + } + if !filepath.IsAbs(path) { + path = filepath.Join(cwd, path) + } + path = filepath.Clean(path) + if !slices.Contains(paths, path) { + paths = append(paths, path) + } + } + } + return paths +} + +func addHookEntry(hooks map[string]any, event, command, matcher string) bool { + entries, _ := hooks[event].([]any) + for _, entry := range entries { + group, ok := entry.(map[string]any) + if !ok { + continue + } + handlers, _ := group["hooks"].([]any) + existingMatcher, _ := group["matcher"].(string) + if len(handlers) == 1 && existingMatcher == matcher { + handler, _ := handlers[0].(map[string]any) + if handler["command"] == command { + return false + } + } + } + removeHookEntry(hooks, event) + entries, _ = hooks[event].([]any) + hooks[event] = append(entries, buildHookEntry(command, matcher)) + return true +} + +func buildHookEntry(command, matcher string) map[string]any { + entry := map[string]any{ + "hooks": []any{ + map[string]any{ + "type": "command", + "command": command, + }, + }, + } + if matcher != "" { + entry["matcher"] = matcher + } + return entry +} + +func removeHookEntry(hooks map[string]any, event string) bool { + entries, _ := hooks[event].([]any) + var kept []any + changed := false + for _, entry := range entries { + group, ok := entry.(map[string]any) + if !ok { + kept = append(kept, entry) + continue + } + handlers, _ := group["hooks"].([]any) + var userHooks []any + for _, handler := range handlers { + if isChainloopHandler(handler) { + changed = true + } else { + userHooks = append(userHooks, handler) + } + } + if len(userHooks) == len(handlers) { + kept = append(kept, entry) + } else if len(userHooks) > 0 { + group["hooks"] = userHooks + kept = append(kept, group) + } + } + if !changed { + return false + } + if len(kept) == 0 { + delete(hooks, event) + } else { + hooks[event] = kept + } + return true +} + +func isChainloopHandler(handler any) bool { + h, ok := handler.(map[string]any) + if !ok { + return false + } + command, _ := h["command"].(string) + return strings.HasPrefix(command, hookMarker+" ") +} + +func entryContainsChainloopHook(entry any) bool { + group, ok := entry.(map[string]any) + if !ok { + return false + } + handlers, _ := group["hooks"].([]any) + return slices.ContainsFunc(handlers, isChainloopHandler) +} + +func readJSONFile(path string) (map[string]any, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return make(map[string]any), nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + + var result map[string]any + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + + // A file containing `null` unmarshals successfully into a nil map, which + // panics on assignment. + if result == nil { + result = make(map[string]any) + } + + if hooks, ok := result["hooks"]; ok && hooks != nil { + groups, ok := hooks.(map[string]any) + if !ok { + return nil, fmt.Errorf("parse %s: hooks must be an object", path) + } + for event, entries := range groups { + if _, ok := entries.([]any); !ok { + return nil, fmt.Errorf("parse %s: %s hooks must be an array", path, event) + } + } + } + return result, nil +} + +func writeJSONFile(path string, data map[string]any) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create directory: %w", err) + } + + content, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("marshal JSON: %w", err) + } + + content = append(content, '\n') + + return os.WriteFile(path, content, 0600) +} diff --git a/app/cli/internal/trace/codex/parse.go b/app/cli/internal/trace/codex/parse.go new file mode 100644 index 000000000..f7629fcce --- /dev/null +++ b/app/cli/internal/trace/codex/parse.go @@ -0,0 +1,278 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "time" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +type tokenUsage struct { + Input int `json:"input_tokens"` + Cached int `json:"cached_input_tokens"` + CacheWrite int `json:"cache_write_input_tokens"` + Output int `json:"output_tokens"` + Total int `json:"total_tokens"` +} + +type rolloutEntry struct { + Timestamp string `json:"timestamp"` + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` +} + +type rolloutPayload struct { + ID string `json:"id"` + Type string `json:"type"` + Version string `json:"cli_version"` + ModelProvider string `json:"model_provider"` + Model string `json:"model"` + Role string `json:"role"` + Name string `json:"name"` + CallID string `json:"call_id"` + Info *struct { + Total *tokenUsage `json:"total_token_usage"` + } `json:"info"` +} + +// ParseSession reads only the private rollout copy. Unknown record types stay +// in the raw transcript, because the Codex rollout format can evolve. +func (p *Provider) ParseSession(ctx context.Context, opts *trace.ParseOpts) (*aicodingsession.Evidence, error) { + if opts == nil || !state.ValidSessionID(opts.SessionID) { + return nil, errors.New("parse Codex session: invalid options") + } + path, err := state.FindRawSessionFile(opts.SessionDir, opts.SessionID) + if err != nil { + return nil, err + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + d := aicodingsession.Data{ + SchemaVersion: "0.1", + Agent: aicodingsession.Agent{Name: Name, Version: opts.AgentVersion}, + Session: aicodingsession.Session{ID: opts.SessionID}, + Model: &aicodingsession.Model{Primary: opts.Model}, + ToolsUsed: &aicodingsession.ToolsUsed{}, + Conversation: &aicodingsession.Conversation{}, + RawSession: map[string][]json.RawMessage{"main": {}}, + } + parser := rolloutParser{ + data: &d, + models: map[string]int{}, + tools: map[string]int{}, + calls: map[string]bool{}, + usageByModel: map[string]tokenUsage{}, + model: opts.Model, + } + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 16*1024*1024) + line := 0 + for scanner.Scan() { + if err := ctx.Err(); err != nil { + return nil, err + } + line++ + raw := scanner.Bytes() + if len(raw) == 0 { + continue + } + var e rolloutEntry + var payload rolloutPayload + if json.Unmarshal(raw, &e) != nil || e.Type == "" || len(e.Payload) == 0 || json.Unmarshal(e.Payload, &payload) != nil { + d.Warnings = append(d.Warnings, fmt.Sprintf("invalid Codex rollout record on line %d", line)) + continue + } + d.RawSession["main"] = append(d.RawSession["main"], append(json.RawMessage(nil), raw...)) + if err := parser.consume(e, payload, line); err != nil { + return nil, err + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read Codex rollout: %w", err) + } + if err := parser.finish(); err != nil { + return nil, err + } + + return aicodingsession.NewEvidence(d), nil +} + +type rolloutParser struct { + data *aicodingsession.Data + models map[string]int + tools map[string]int + calls map[string]bool + usageByModel map[string]tokenUsage + model string + first, last time.Time + previous tokenUsage + haveUsage, haveHeader bool +} + +func (p *rolloutParser) consume(e rolloutEntry, payload rolloutPayload, line int) error { + if at, err := time.Parse(time.RFC3339Nano, e.Timestamp); err == nil { + if p.first.IsZero() || at.Before(p.first) { + p.first = at + } + if p.last.IsZero() || at.After(p.last) { + p.last = at + } + } else { + p.data.Warnings = append(p.data.Warnings, fmt.Sprintf("invalid Codex timestamp on line %d", line)) + } + switch e.Type { + case "session_meta": + if payload.ID != p.data.Session.ID { + return errors.New("parse Codex session: rollout session ID mismatch") + } + p.haveHeader = true + p.data.Agent.Version = payload.Version + p.data.Model.Provider = payload.ModelProvider + case "turn_context": + if payload.Model != "" { + p.model = payload.Model + if _, ok := p.models[p.model]; !ok { + p.models[p.model] = 0 + } + } + case "response_item": + switch payload.Type { + case "message": + switch payload.Role { + case "user": + p.data.Conversation.UserMessages++ + case "assistant": + p.data.Conversation.AssistantMessages++ + if p.model != "" { + p.models[p.model]++ + } + } + case "function_call", "custom_tool_call": + if payload.Name != "" && (payload.CallID == "" || !p.calls[payload.CallID]) { + p.tools[payload.Name]++ + p.calls[payload.CallID] = true + } + } + case "event_msg": + p.consumeUsage(payload, line) + } + + return nil +} + +func (p *rolloutParser) consumeUsage(payload rolloutPayload, line int) { + if payload.Type != "token_count" || payload.Info == nil || payload.Info.Total == nil { + return + } + total := *payload.Info.Total + if total.Input < 0 || total.Output < 0 || total.Cached < 0 || total.CacheWrite < 0 || total.Total < 0 { + p.data.Warnings = append(p.data.Warnings, fmt.Sprintf("negative Codex token count on line %d", line)) + return + } + // Counters are session totals, not increments. Repeated snapshots add zero. + delta := tokenUsage{ + Input: total.Input - p.previous.Input, + Cached: total.Cached - p.previous.Cached, + CacheWrite: total.CacheWrite - p.previous.CacheWrite, + Output: total.Output - p.previous.Output, + Total: total.Total - p.previous.Total, + } + if delta.Input < 0 || delta.Output < 0 || delta.Cached < 0 || delta.CacheWrite < 0 || delta.Total < 0 { + p.data.Warnings = append(p.data.Warnings, fmt.Sprintf("Codex token counters reset on line %d", line)) + delta = total + } + u := p.usageByModel[p.model] + u.Input += delta.Input + u.Cached += delta.Cached + u.CacheWrite += delta.CacheWrite + u.Output += delta.Output + u.Total += delta.Total + p.usageByModel[p.model] = u + p.previous = total + p.haveUsage = true +} + +func (p *rolloutParser) finish() error { + if !p.haveHeader { + return errors.New("parse Codex session: missing session metadata") + } + if !p.first.IsZero() { + p.data.Session.StartedAt = p.first.UTC().Format(time.RFC3339) + p.data.Session.EndedAt = p.last.UTC().Format(time.RFC3339) + p.data.Session.DurationSeconds = int(p.last.Sub(p.first).Seconds()) + } + best := -1 + for name := range p.models { + p.data.Model.ModelsUsed = append(p.data.Model.ModelsUsed, name) + } + sort.Strings(p.data.Model.ModelsUsed) + for _, name := range p.data.Model.ModelsUsed { + if p.models[name] > best { + p.data.Model.Primary = name + best = p.models[name] + } + } + if p.data.Model.Primary == "" { + p.data.Warnings = append(p.data.Warnings, "Codex rollout has no model information") + } + for name, count := range p.tools { + p.data.ToolsUsed.Summary = append(p.data.ToolsUsed.Summary, aicodingsession.ToolSummary{ToolName: name, InvocationCount: count}) + p.data.ToolsUsed.TotalInvocations += count + } + sort.Slice(p.data.ToolsUsed.Summary, func(i, j int) bool { + a, b := p.data.ToolsUsed.Summary[i], p.data.ToolsUsed.Summary[j] + if a.InvocationCount != b.InvocationCount { + return a.InvocationCount > b.InvocationCount + } + return a.ToolName < b.ToolName + }) + p.data.Conversation.TotalMessages = p.data.Conversation.UserMessages + p.data.Conversation.AssistantMessages + if p.haveUsage { + p.data.Usage = &aicodingsession.Usage{} + // Stable ordering also keeps warnings deterministic for sessions using several models. + names := make([]string, 0, len(p.usageByModel)) + for name := range p.usageByModel { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + u := p.usageByModel[name] + p.data.Usage.InputTokens += u.Input + p.data.Usage.OutputTokens += u.Output + p.data.Usage.TotalTokens += u.Total + p.data.Usage.CacheReadInputTokens += u.Cached + p.data.Usage.CacheCreationInputTokens += u.CacheWrite + p.data.Usage.EstimatedCostUSD += estimateCost(name, u, &p.data.Warnings) + } + p.data.Usage.EstimatedCostUSD = roundCost(p.data.Usage.EstimatedCostUSD) + } else { + p.data.Warnings = append(p.data.Warnings, "Codex rollout has no token usage; cost is unavailable") + } + return nil +} diff --git a/app/cli/internal/trace/codex/pricing.go b/app/cli/internal/trace/codex/pricing.go new file mode 100644 index 000000000..52e3d5eba --- /dev/null +++ b/app/cli/internal/trace/codex/pricing.go @@ -0,0 +1,72 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + _ "embed" + "encoding/json" + "fmt" + "math" + "strings" +) + +type modelPricing struct { + Input float64 `json:"input"` + Cached float64 `json:"cached"` + Output float64 `json:"output"` + CacheWrite float64 `json:"cache_write"` +} + +//go:embed pricing.json +var pricingJSON []byte +var prices struct { + Default string `json:"default_model"` + Models map[string]modelPricing `json:"models"` +} + +func init() { + if err := json.Unmarshal(pricingJSON, &prices); err != nil { + panic(fmt.Sprintf("codex trace: invalid embedded pricing: %v", err)) + } + if _, ok := prices.Models[prices.Default]; !ok { + panic("codex trace: missing default model price") + } +} + +func estimateCost(model string, u tokenUsage, warnings *[]string) float64 { + name := strings.TrimPrefix(model, "openai/") + price, ok := prices.Models[name] + if !ok { + // Date-pinned model IDs use the same prices as their family alias. + if len(name) > 11 && name[len(name)-11] == '-' { + price, ok = prices.Models[name[:len(name)-11]] + } + } + if !ok { + price = prices.Models[prices.Default] + *warnings = append(*warnings, fmt.Sprintf("unknown Codex model %q; cost estimated using %s pricing", model, prices.Default)) + } + // Codex input_tokens already includes cached tokens. Cache writes are part + // of input, and reasoning tokens are already part of output_tokens. + uncached := max(0, u.Input-u.Cached) + written := 0 + if price.CacheWrite > 0 { + written = min(u.CacheWrite, uncached) + } + return (float64(uncached-written)*price.Input + float64(written)*price.CacheWrite + float64(u.Cached)*price.Cached + float64(u.Output)*price.Output) / 1e6 +} + +func roundCost(cost float64) float64 { return math.Round(cost*1e4) / 1e4 } diff --git a/app/cli/internal/trace/codex/pricing.json b/app/cli/internal/trace/codex/pricing.json new file mode 100644 index 000000000..2a9dd638f --- /dev/null +++ b/app/cli/internal/trace/codex/pricing.json @@ -0,0 +1,145 @@ +{ + "version": "2026-10-21", + "source": "https://developers.openai.com/api/docs/pricing", + "notes": "USD per million tokens, standard short-context API rates. Estimates exclude priority tiers, long-context surcharges, and subscription or third-party billing. Unknown models use the default with a warning.", + "default_model": "gpt-5-mini", + "models": { + "gpt-5-mini": { + "input": 0.25, + "cached": 0.025, + "output": 2.0 + }, + "gpt-6-astra": { + "input": 10.0, + "cached": 1.0, + "output": 50.0, + "cache_write": 12.5 + }, + "gpt-6.1-sol": { + "input": 2.0, + "cached": 0.1, + "output": 10.0, + "cache_write": 2.5 + }, + "gpt-6-luna": { + "input": 0.1, + "cached": 0.01, + "output": 0.5, + "cache_write": 0.125 + }, + "gpt-6-sol": { + "input": 2.0, + "cached": 0.2, + "output": 10.0, + "cache_write": 2.5 + }, + "gpt-5.6-sol": { + "input": 4.0, + "cached": 0.4, + "output": 20.0, + "cache_write": 5.0 + }, + "gpt-5.6-terra": { + "input": 2.0, + "cached": 0.2, + "output": 12.0, + "cache_write": 2.5 + }, + "gpt-5.6-luna": { + "input": 0.2, + "cached": 0.02, + "output": 1.2, + "cache_write": 0.25 + }, + "gpt-5.5": { + "input": 5.0, + "cached": 0.5, + "output": 30.0 + }, + "gpt-5.4": { + "input": 2.5, + "cached": 0.25, + "output": 15.0 + }, + "gpt-5.4-mini": { + "input": 0.75, + "cached": 0.075, + "output": 4.5 + }, + "gpt-5.4-nano": { + "input": 0.2, + "cached": 0.02, + "output": 1.25 + }, + "gpt-5.2": { + "input": 1.75, + "cached": 0.175, + "output": 14.0 + }, + "gpt-5.1": { + "input": 1.25, + "cached": 0.125, + "output": 10.0 + }, + "gpt-5": { + "input": 1.25, + "cached": 0.125, + "output": 10.0 + }, + "gpt-5-nano": { + "input": 0.05, + "cached": 0.005, + "output": 0.4 + }, + "gpt-4.1": { + "input": 2.0, + "cached": 0.5, + "output": 8.0 + }, + "gpt-4.1-mini": { + "input": 0.4, + "cached": 0.1, + "output": 1.6 + }, + "gpt-4.1-nano": { + "input": 0.1, + "cached": 0.025, + "output": 0.4 + }, + "gpt-4o": { + "input": 2.5, + "cached": 1.25, + "output": 10.0 + }, + "gpt-4o-mini": { + "input": 0.15, + "cached": 0.075, + "output": 0.6 + }, + "o1": { + "input": 15.0, + "cached": 7.5, + "output": 60.0 + }, + "o3": { + "input": 2.0, + "cached": 0.5, + "output": 8.0 + }, + "o4-mini": { + "input": 1.1, + "cached": 0.275, + "output": 4.4 + }, + "o3-mini": { + "input": 1.1, + "cached": 0.55, + "output": 4.4 + }, + "gpt-5.3-codex": { + "input": 1.75, + "output": 14, + "cached": 0.175 + } + } +} diff --git a/app/cli/internal/trace/codex/provider.go b/app/cli/internal/trace/codex/provider.go new file mode 100644 index 000000000..b7bccc751 --- /dev/null +++ b/app/cli/internal/trace/codex/provider.go @@ -0,0 +1,149 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package codex captures sessions from the Codex CLI's native hooks and JSONL rollouts. +package codex + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "slices" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +// Name is the identifier stored in session records and evidence. +const Name = "codex" + +var _ trace.Provider = (*Provider)(nil) + +// Provider connects Codex to Trace. It holds no session state. +type Provider struct{} + +// New creates a Codex provider. +func New() *Provider { return &Provider{} } + +// Name returns the provider identifier. +func (p *Provider) Name() string { return Name } + +// CaptureFileSnapshot reads the file at input.FilePath and stores its +// content under the store's snapshots/ directory so the post-tool-use +// handler can compute line-range diffs after the edit. A missing file +// (a patch creating a new file) is treated as a non-error; +// other read errors (permission denied, I/O failures) are surfaced. +func (p *Provider) CaptureFileSnapshot(store *state.Store, input *trace.HookInput) error { + if input == nil || input.FilePath == "" { + return nil + } + + content, err := os.ReadFile(input.FilePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + + return fmt.Errorf("read %q: %w", input.FilePath, err) + } + + return store.SaveFileSnapshot(input.SessionID, input.FilePath, content) +} + +// ResolveBeforeContent returns the snapshot saved by CaptureFileSnapshot. +// The "after" argument is unused for Codex — we always rely on the pre-edit +// snapshot — but is part of the interface for providers that reconstruct +// from edits instead of snapshots. +func (p *Provider) ResolveBeforeContent(store *state.Store, input *trace.HookInput, _ []byte) []byte { + if input == nil || input.FilePath == "" { + return nil + } + + snap, err := store.LoadFileSnapshot(input.SessionID, input.FilePath) + if err != nil { + return nil + } + + return snap +} + +// CleanupAfterEdit removes the per-edit snapshot. Safe to call when no +// snapshot was ever taken. +func (p *Provider) CleanupAfterEdit(store *state.Store, input *trace.HookInput) { + if input == nil || input.FilePath == "" { + return + } + + store.DeleteFileSnapshot(input.SessionID, input.FilePath) +} + +// IsFileWritingTool returns true if the named tool modifies files on disk. +func (p *Provider) IsFileWritingTool(toolName string) bool { + return slices.Contains(fileWritingTools, toolName) +} + +// IsCommandTool returns true if the named tool runs a shell command. +func (p *Provider) IsCommandTool(toolName string) bool { + return slices.Contains(commandTools, toolName) +} + +// SupportsSessionStartBanner reports whether Codex shows systemMessage to the user. +func (p *Provider) SupportsSessionStartBanner() bool { return true } + +// SupportsSessionStartInstruction reports whether Codex accepts startup context. +func (p *Provider) SupportsSessionStartInstruction() bool { return true } + +// SupportsPromptReminder reports whether Codex accepts context at each prompt. +func (p *Provider) SupportsPromptReminder() bool { return true } + +// AnnounceSessionStart sends the banner and instruction in one hook response. +func (p *Provider) AnnounceSessionStart(msg trace.SessionStartMessage) error { + return writeHookResponse(eventSessionStart, msg.Banner, msg.Instruction) +} + +// AnnouncePromptSubmit adds the reminder to the model's context. +func (p *Provider) AnnouncePromptSubmit(reminder string) error { + return writeHookResponse(eventUserPromptSubmit, "", reminder) +} + +// AnnounceToUser shows the session link and gives the model context to repeat it. +func (p *Provider) AnnounceToUser(msg string) error { + if msg == "" { + return nil + } + return writeHookResponse(eventPostToolUse, msg, "Tell the user the following, including any link verbatim: "+msg) +} + +type hookSpecificOutput struct { + HookEventName string `json:"hookEventName"` + AdditionalContext string `json:"additionalContext,omitempty"` +} + +type hookResponse struct { + SystemMessage string `json:"systemMessage,omitempty"` + HookSpecificOutput hookSpecificOutput `json:"hookSpecificOutput"` +} + +// Codex reads one JSON document per hook. Empty messages emit nothing. +func writeHookResponse(event, banner, instruction string) error { + if banner == "" && instruction == "" { + return nil + } + return json.NewEncoder(os.Stdout).Encode(hookResponse{ + SystemMessage: banner, + HookSpecificOutput: hookSpecificOutput{HookEventName: event, AdditionalContext: instruction}, + }) +} diff --git a/app/cli/internal/trace/codex/provider_test.go b/app/cli/internal/trace/codex/provider_test.go new file mode 100644 index 000000000..60c8dfe7f --- /dev/null +++ b/app/cli/internal/trace/codex/provider_test.go @@ -0,0 +1,247 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testSessionID = "sess-1" +const testModel = "gpt-5-mini" + +func TestHooks(t *testing.T) { + for _, run := range []bool{false, true} { + t.Run(map[bool]string{false: "init", true: "run"}[run], func(t *testing.T) { + p := New() + root := t.TempDir() + path := p.SettingsFile(root) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte(`{"description":"user settings","hooks":{"PostToolUse":[{"hooks":[{"type":"command","command":"echo user-hook"}]}]}}`), 0600)) + installed, err := p.HooksInstalled(root) + require.NoError(t, err) + require.False(t, installed) + install := p.InstallHooks + if run { + install = p.InstallHooksForTraceRun + } + require.NoError(t, install(root)) + first, err := os.ReadFile(path) + require.NoError(t, err) + require.NoError(t, install(root)) + second, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, first, second) + installed, err = p.HooksInstalled(root) + require.NoError(t, err) + assert.True(t, installed) + assert.Contains(t, string(first), "echo user-hook") + assert.Contains(t, string(first), "apply_patch") + assert.Equal(t, !run, strings.Contains(string(first), `"SessionEnd"`)) + require.NoError(t, p.UninstallHooks(root)) + left, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(left), "echo user-hook") + assert.Contains(t, string(left), "user settings") + assert.NotContains(t, string(left), hookMarker) + require.NoError(t, p.UninstallHooks(root)) + }) + } +} + +func TestReadHookInput(t *testing.T) { + for _, tc := range []struct { + name, tool, command string + paths []string + }{ + {"shell", "Bash", "git status", nil}, + {"patch", "apply_patch", "*** Begin Patch\n*** Update File: one.go\n@@\n-old\n+new\n*** Add File: two.go\n+new\n*** Delete File: gone.go\n*** Update File: old.go\n*** Move to: new.go\n@@\n-old\n+new\n*** End Patch", []string{"one.go", "two.go", "gone.go", "old.go", "new.go"}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + raw, _ := json.Marshal(map[string]any{"session_id": testSessionID, "cwd": root, "transcript_path": "/sessions/rollout.jsonl", "tool_name": tc.tool, "tool_use_id": "call-1", "agent_id": "child-1", "model": testModel, "tool_input": map[string]string{"command": tc.command}}) + input, err := New().ReadHookInput(strings.NewReader(string(raw))) + require.NoError(t, err) + assert.Equal(t, testSessionID, input.SessionID) + assert.Equal(t, "call-1", input.ToolUseID) + assert.Equal(t, "child-1", input.AgentID) + assert.Equal(t, testModel, input.Model) + if tc.tool == "Bash" { + assert.Equal(t, tc.command, input.Command) + } else { + var paths []string + for _, path := range tc.paths { + paths = append(paths, filepath.Join(root, path)) + } + assert.Equal(t, paths, input.FilePaths) + } + }) + } + _, err := New().ReadHookInput(strings.NewReader(`{`)) + assert.Error(t, err) +} + +func TestParseSession(t *testing.T) { + dir := t.TempDir() + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // temporary fixture path and a fixed session ID + evidence, err := New().ParseSession(context.Background(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + d := evidence.Data + assert.Equal(t, Name, d.Agent.Name) + assert.Equal(t, "0.163.0", d.Agent.Version) + assert.Equal(t, 60, d.Session.DurationSeconds) + assert.Equal(t, testModel, d.Model.Primary) + assert.Equal(t, "openai", d.Model.Provider) + assert.Equal(t, 2000, d.Usage.InputTokens) + assert.Equal(t, 400, d.Usage.OutputTokens) + assert.Equal(t, 2400, d.Usage.TotalTokens) + assert.Equal(t, 1000, d.Usage.CacheReadInputTokens) + assert.Equal(t, 100, d.Usage.CacheCreationInputTokens) + assert.InDelta(t, 0.0011, d.Usage.EstimatedCostUSD, 0.00001) + assert.Equal(t, 2, d.Conversation.TotalMessages) + assert.Equal(t, 2, d.ToolsUsed.TotalInvocations) + assert.Equal(t, 1, d.Conversation.UserMessages) + assert.Equal(t, 1, d.Conversation.AssistantMessages) + assert.Len(t, d.RawSession["main"], 12) + assert.Empty(t, d.Warnings) +} + +func TestCopySessionData(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + p := New() + store := state.NewOutOfTreeStore(t.TempDir()) + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + path := filepath.Join(home, "sessions", "2026", "10", "21", "rollout-time-sess-1.jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, raw, 0600)) //nolint:gosec // temporary Codex session fixture + for _, reported := range []string{path, "", filepath.Join(home, "sessions", "moved.jsonl")} { + require.NoError(t, p.CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: reported})) + got, err := os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, raw, got) + } + outside := filepath.Join(t.TempDir(), "rollout-time-sess-1.jsonl") + require.NoError(t, os.WriteFile(outside, raw, 0600)) //nolint:gosec // deliberately outside storage to verify rejection + assert.Error(t, p.CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: outside})) + link := filepath.Join(home, "sessions", "rollout-link-sess-1.jsonl") + require.NoError(t, os.Symlink(outside, link)) + assert.Error(t, p.CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: link})) + require.NoError(t, p.CopySessionData(store, trace.SessionLocation{SessionID: "missing"})) + assert.Error(t, p.CopySessionData(store, trace.SessionLocation{SessionID: "../escape"})) +} + +func TestUninstallKeepsMixedUserHookGroup(t *testing.T) { + p := New() + root := t.TempDir() + path := p.SettingsFile(root) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte(`{"hooks":{"PostToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":"chainloop trace hook codex post-tool-use"},{"type":"command","command":"echo keep-me"}]}]}}`), 0600)) + require.NoError(t, p.InstallHooks(root)) + require.NoError(t, p.UninstallHooks(root)) + raw, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(raw), "echo keep-me") + assert.NotContains(t, string(raw), hookMarker) +} + +func TestParseSessionWarnings(t *testing.T) { + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + for _, tc := range []struct { + name string + data string + wantErr bool + warning string + }{ + {name: "truncated tail", data: string(raw) + `{"type":`, warning: "invalid Codex rollout record"}, + {name: "wrong session", data: strings.ReplaceAll(string(raw), testSessionID, "other"), wantErr: true}, + {name: "no header", data: `{"type":"event_msg","timestamp":"2026-10-21T10:00:00Z","payload":{"type":"other"}}`, wantErr: true}, + {name: "unknown model", data: strings.ReplaceAll(string(raw), testModel, "future-model"), warning: "unknown Codex model"}, + {name: "no usage", data: strings.ReplaceAll(string(raw), "token_count", "other"), warning: "no token usage"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), []byte(tc.data), 0600)) + e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Contains(t, strings.Join(e.Data.Warnings, "\n"), tc.warning) + }) + } +} + +func TestTraceRunOmitsPreviouslyInstalledEndHook(t *testing.T) { + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + require.NoError(t, p.InstallHooksForTraceRun(root)) + raw, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.NotContains(t, string(raw), `"SessionEnd"`) +} + +func TestUsageAcrossModels(t *testing.T) { + dir := t.TempDir() + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + // The second model owns only the increment after the first cumulative snapshot. + extra := `{"timestamp":"2026-10-21T10:01:01Z","type":"turn_context","payload":{"model":"gpt-5"}} +{"timestamp":"2026-10-21T10:01:02Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[]}} +{"timestamp":"2026-10-21T10:01:03Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[]}} +{"timestamp":"2026-10-21T10:01:04Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":4000,"cached_input_tokens":2000,"cache_write_input_tokens":200,"output_tokens":804,"total_tokens":4804}}}} +` + raw = append(raw, extra...) + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // temporary fixture and a fixed session ID + e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + assert.Equal(t, []string{"gpt-5", testModel}, e.Data.Model.ModelsUsed) + assert.Equal(t, "gpt-5", e.Data.Model.Primary) + assert.Equal(t, 4000, e.Data.Usage.InputTokens) + assert.Equal(t, 4804, e.Data.Usage.TotalTokens) + assert.InDelta(t, 0.0065, e.Data.Usage.EstimatedCostUSD, 0.00001) +} + +func TestNativeRollout(t *testing.T) { + dir := t.TempDir() + raw, err := os.ReadFile("testdata/native-session.jsonl") + require.NoError(t, err) + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // sanitized fixture and a fixed session ID + e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + assert.Equal(t, 19064, e.Data.Usage.InputTokens) + assert.Equal(t, 128, e.Data.Usage.OutputTokens) + assert.Equal(t, 19192, e.Data.Usage.TotalTokens) + assert.Equal(t, 19061, e.Data.Usage.CacheCreationInputTokens) + assert.NotEmpty(t, e.Data.Model.Primary) + assert.NotEmpty(t, e.Data.ToolsUsed.Summary) + assert.Empty(t, e.Data.Warnings) +} diff --git a/app/cli/internal/trace/codex/testdata/README.md b/app/cli/internal/trace/codex/testdata/README.md new file mode 100644 index 000000000..bdee0632b --- /dev/null +++ b/app/cli/internal/trace/codex/testdata/README.md @@ -0,0 +1,8 @@ +`native-session.jsonl` contains selected records from a Codex CLI 0.163.0 +rollout. Instructions, prompts, tool inputs and outputs, paths, IDs, and times +were removed or replaced. The native usage counters and record shapes remain. +It includes both `token_usage_record` and `event_msg/token_count` records for +the same usage, which must count once. + +`session.jsonl` is a small constructed fixture for predictable totals, repeated +cumulative counters, and duplicate conversation events. diff --git a/app/cli/internal/trace/codex/testdata/native-session.jsonl b/app/cli/internal/trace/codex/testdata/native-session.jsonl new file mode 100644 index 000000000..ab9a8ac43 --- /dev/null +++ b/app/cli/internal/trace/codex/testdata/native-session.jsonl @@ -0,0 +1,10 @@ +{"timestamp":"2026-10-21T10:00:00Z","type":"session_meta","payload":{"id":"sess-1","cli_version":"0.163.0","model_provider":"openai","cwd":"/workspace","timestamp":"2026-10-21T10:00:00Z"}} +{"timestamp":"2026-10-21T10:00:01Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Sanitized example message"}]}} +{"timestamp":"2026-10-21T10:00:02Z","type":"turn_context","payload":{"model":"gpt-6.1-sol","cwd":"/workspace"}} +{"timestamp":"2026-10-21T10:00:03Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Sanitized example message"}]}} +{"timestamp":"2026-10-21T10:00:04Z","type":"response_item","payload":{"type":"custom_tool_call","name":"exec","call_id":"call-4","input":"Sanitized tool input"}} +{"timestamp":"2026-10-21T10:00:05Z","type":"token_usage_record","payload":{"usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"thread_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"turn_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192}}} +{"timestamp":"2026-10-21T10:00:06Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"sanitized-call","output":"Sanitized tool output"}} +{"timestamp":"2026-10-21T10:00:07Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"last_token_usage":{"input_tokens":19064,"cached_input_tokens":0,"cache_write_input_tokens":19061,"output_tokens":128,"reasoning_output_tokens":0,"total_tokens":19192},"model_context_window":475000}}} +{"timestamp":"2026-10-21T10:00:08Z","type":"response_item","payload":{"type":"function_call","name":"wait","call_id":"call-8","arguments":"Sanitized tool input"}} +{"timestamp":"2026-10-21T10:00:09Z","type":"response_item","payload":{"type":"function_call_output","call_id":"sanitized-call","output":"Sanitized tool output"}} diff --git a/app/cli/internal/trace/codex/testdata/session.jsonl b/app/cli/internal/trace/codex/testdata/session.jsonl new file mode 100644 index 000000000..ad714752f --- /dev/null +++ b/app/cli/internal/trace/codex/testdata/session.jsonl @@ -0,0 +1,12 @@ +{"timestamp":"2026-10-21T10:00:00Z","type":"session_meta","payload":{"id":"sess-1","timestamp":"2026-10-21T10:00:00Z","cwd":"/workspace","cli_version":"0.163.0","source":"cli","model_provider":"openai"}} +{"timestamp":"2026-10-21T10:00:01Z","type":"turn_context","payload":{"model":"gpt-5-mini","cwd":"/workspace"}} +{"timestamp":"2026-10-21T10:00:02Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Create hello.txt"}]}} +{"timestamp":"2026-10-21T10:00:02Z","type":"event_msg","payload":{"type":"user_message","message":"Create hello.txt"}} +{"timestamp":"2026-10-21T10:00:03Z","type":"response_item","payload":{"type":"custom_tool_call","name":"apply_patch","call_id":"call-1","input":"*** Begin Patch\n*** Add File: hello.txt\n+hello\n*** End Patch"}} +{"timestamp":"2026-10-21T10:00:04Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call-1","output":"Success"}} +{"timestamp":"2026-10-21T10:00:05Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":1000,"cached_input_tokens":500,"cache_write_input_tokens":50,"output_tokens":200,"reasoning_output_tokens":100,"total_tokens":1200}}}} +{"timestamp":"2026-10-21T10:00:06Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call-2","arguments":"{\"cmd\":\"cat hello.txt\"}"}} +{"timestamp":"2026-10-21T10:00:07Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call-2","output":"hello"}} +{"timestamp":"2026-10-21T10:00:58Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Created hello.txt"}]}} +{"timestamp":"2026-10-21T10:00:59Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":1000,"cache_write_input_tokens":100,"output_tokens":400,"reasoning_output_tokens":200,"total_tokens":2400}}}} +{"timestamp":"2026-10-21T10:01:00Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":2000,"cached_input_tokens":1000,"cache_write_input_tokens":100,"output_tokens":400,"total_tokens":2400}}}} diff --git a/app/cli/internal/trace/e2e/codex.go b/app/cli/internal/trace/e2e/codex.go index 727f2da75..98e5c74c5 100644 --- a/app/cli/internal/trace/e2e/codex.go +++ b/app/cli/internal/trace/e2e/codex.go @@ -16,21 +16,24 @@ package e2e import ( + "bufio" "bytes" "context" - "errors" + "encoding/json" "fmt" "os" "os/exec" "path/filepath" "regexp" "slices" + "strconv" "strings" + + "github.com/pelletier/go-toml/v2" ) const ( - // codexHooksPath is the project file where Codex reads its hooks. Trace - // has no Codex provider yet, so no Chainloop hook is in it. + // codexHooksPath is the project file where Codex reads its hooks. codexHooksPath = ".codex/hooks.json" // codexOpenRouterPrefix selects the OpenRouter provider for a model of // the suite config. @@ -105,13 +108,24 @@ func codexModelArgs(model string) []string { // --ignore-user-config is not set, because it also turns off the project // hooks. // -// Trace has no Codex provider yet, so no hook message is reported. +// Hook messages are not extracted from the headless event stream yet. func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { base := filepath.Dir(req.Dir) codexHome := filepath.Join(base, "codex-home") if err := os.MkdirAll(codexHome, 0o700); err != nil { return nil, err } + userHome, err := os.UserHomeDir() + if err != nil { + return nil, err + } + config, err := codexIsolatedConfig(userHome) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(codexHome, "config.toml"), config, 0600); err != nil { + return nil, err + } args := []string{ "exec", "--json", @@ -120,6 +134,27 @@ func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { "-C", req.Dir, } args = append(args, codexModelArgs(req.Model)...) + if strings.HasPrefix(req.Model, codexOpenRouterPrefix+"openai/") { + export := exec.CommandContext(ctx, c.bin(), "debug", "models", "--bundled") //nolint:gosec // the harness binary under test + export.Env = append(slices.Clone(req.Env), "CODEX_HOME="+codexHome) + raw, err := export.Output() + if err != nil { + return nil, fmt.Errorf("codex bundled model catalog: %w", err) + } + catalog, err := codexModelCatalog(raw, req.Model) + if err != nil { + return nil, err + } + if catalog != nil { + path := filepath.Join(codexHome, "models.json") + if err := os.WriteFile(path, catalog, 0600); err != nil { + return nil, err + } + // OpenRouter prefixes model names with openai/. Keep the native Codex + // model metadata so that file tools and instructions match that model. + args = append(args, "-c", "model_catalog_json="+strconv.Quote(path)) + } + } // A pasted image goes in as an attached file, from outside the // repository, so that the agent cannot commit it. @@ -138,7 +173,7 @@ func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { } } // The prompt follows "--", so that the list of images ends before it. - args = append(args, "--", req.Prompt) + args = append(args, "--", "-") name := c.bin() if len(req.Wrapper) > 0 { @@ -148,13 +183,14 @@ func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario // nosemgrep cmd.Dir = req.Dir + cmd.Stdin = strings.NewReader(req.Prompt) cmd.Env = append(slices.Clone(req.Env), "CODEX_HOME="+codexHome) var stderr bytes.Buffer cmd.Stderr = &stderr raw, err := cmd.Output() if err != nil { - return &RunOutput{Raw: raw}, fmt.Errorf("codex exec: %w: %s", err, strings.TrimSpace(stderr.String())) + return &RunOutput{Raw: raw}, fmt.Errorf("codex exec: %w: %s", err, strings.TrimSpace(stderr.String()+"\n"+codexStreamErrors(raw))) } return &RunOutput{Raw: raw}, nil @@ -178,14 +214,34 @@ func (c *Codex) HookConfig() HookConfig { } } -// errCodexSessionStart is the error of DisableSessionStart until Trace has a -// Codex provider. -var errCodexSessionStart = errors.New("codex: the suite cannot remove the session-start hook of Trace, because Trace has no Codex provider yet") - -// DisableSessionStart implements Harness. Trace installs no Codex hooks yet, -// so there is no hook to remove. -func (c *Codex) DisableSessionStart(string) error { - return errCodexSessionStart +// DisableSessionStart removes only the session-start hook for recovery tests. +func (c *Codex) DisableSessionStart(repo string) error { + path := filepath.Join(repo, codexHooksPath) + data, err := os.ReadFile(path) + if err != nil { + return err + } + var settings map[string]json.RawMessage + if err := json.Unmarshal(data, &settings); err != nil { + return err + } + var hooks map[string]json.RawMessage + if err := json.Unmarshal(settings["hooks"], &hooks); err != nil { + return err + } + if _, ok := hooks["SessionStart"]; !ok { + return fmt.Errorf("%s has no SessionStart hook", codexHooksPath) + } + delete(hooks, "SessionStart") + settings["hooks"], err = json.Marshal(hooks) + if err != nil { + return err + } + data, err = json.MarshalIndent(settings, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, data, 0600) } // SkillPath implements Harness. Codex finds project skills in @@ -193,3 +249,86 @@ func (c *Codex) DisableSessionStart(string) error { func (c *Codex) SkillPath(name string) string { return ".agents/skills/" + name + "/SKILL.md" } + +// codexStreamErrors extracts diagnostics that codex exec writes to JSON stdout. +func codexStreamErrors(raw []byte) string { + var messages []string + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 4096), 16*1024*1024) + for scanner.Scan() { + var event struct { + Type string `json:"type"` + Message string `json:"message"` + Error struct { + Message string `json:"message"` + } `json:"error"` + } + if json.Unmarshal(scanner.Bytes(), &event) != nil { + continue + } + if event.Type == "error" && event.Message != "" { + messages = append(messages, event.Message) + } + if event.Type == "turn.failed" && event.Error.Message != "" { + messages = append(messages, event.Error.Message) + } + } + return strings.Join(messages, "\n") +} + +// codexIsolatedConfig disables personal skills that Codex discovers outside +// CODEX_HOME. Project skills remain available for the skill scenario. +func codexIsolatedConfig(userHome string) ([]byte, error) { + type skillConfig struct { + Path string `toml:"path"` + Enabled bool `toml:"enabled"` + } + config := struct { + Skills struct { + Config []skillConfig `toml:"config"` + Bundled struct { + Enabled bool `toml:"enabled"` + } `toml:"bundled"` + } `toml:"skills"` + }{} + for _, dir := range []string{filepath.Join(userHome, ".agents", "skills"), filepath.Join(userHome, ".codex", "skills"), filepath.Join(userHome, ".codex", "skills", ".system")} { + paths, err := filepath.Glob(filepath.Join(dir, "*", "SKILL.md")) + if err != nil { + return nil, err + } + for _, path := range paths { + config.Skills.Config = append(config.Skills.Config, skillConfig{Path: path, Enabled: false}) + } + } + return toml.Marshal(config) +} + +// codexModelCatalog aliases bundled OpenAI metadata to OpenRouter's model ID. +// Without it, Codex treats the prefixed name as unknown and uses generic tools. +func codexModelCatalog(raw []byte, model string) ([]byte, error) { + var catalog struct { + Models []map[string]json.RawMessage `json:"models"` + } + if err := json.Unmarshal(raw, &catalog); err != nil { + return nil, fmt.Errorf("decode Codex model catalog: %w", err) + } + target := strings.TrimPrefix(model, codexOpenRouterPrefix) + canonical := strings.TrimPrefix(target, "openai/") + for _, entry := range catalog.Models { + var slug string + if err := json.Unmarshal(entry["slug"], &slug); err != nil { + return nil, err + } + if slug != canonical { + continue + } + alias := make(map[string]json.RawMessage, len(entry)) + for k, v := range entry { + alias[k] = v + } + alias["slug"], _ = json.Marshal(target) + catalog.Models = append(catalog.Models, alias) + return json.Marshal(catalog) + } + return nil, nil +} diff --git a/app/cli/internal/trace/e2e/codex_test.go b/app/cli/internal/trace/e2e/codex_test.go index f0e79e989..cbc8738b3 100644 --- a/app/cli/internal/trace/e2e/codex_test.go +++ b/app/cli/internal/trace/e2e/codex_test.go @@ -16,8 +16,12 @@ package e2e import ( + "encoding/json" + "os" + "path/filepath" "testing" + "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -72,3 +76,52 @@ func TestCodexModelArgs(t *testing.T) { }) } } + +func TestCodexStreamErrors(t *testing.T) { + raw := []byte("{\"type\":\"thread.started\"}\n{\"type\":\"error\",\"message\":\"missing key\"}\n{\"type\":\"turn.failed\",\"error\":{\"message\":\"turn failed\"}}\n") + assert.Equal(t, "missing key\nturn failed", codexStreamErrors(raw)) +} + +func TestCodexIsolatedConfig(t *testing.T) { + home := t.TempDir() + for _, dir := range []string{".agents/skills/personal", ".codex/skills/legacy", ".codex/skills/.system/builtin"} { + path := filepath.Join(home, dir, "SKILL.md") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("personal instructions"), 0600)) + } + raw, err := codexIsolatedConfig(home) + require.NoError(t, err) + var config struct { + Skills struct { + Config []struct { + Path string + Enabled bool + } + Bundled struct{ Enabled bool } + } + } + require.NoError(t, toml.Unmarshal(raw, &config)) + require.Len(t, config.Skills.Config, 3) + for _, skill := range config.Skills.Config { + assert.False(t, skill.Enabled) + assert.Contains(t, skill.Path, home) + } + assert.False(t, config.Skills.Bundled.Enabled) +} + +func TestCodexModelCatalog(t *testing.T) { + raw := []byte(`{"models":[{"slug":"gpt-6-luna","apply_patch_tool_type":"freeform","base_instructions":"native instructions"}]}`) + catalog, err := codexModelCatalog(raw, "openrouter/openai/gpt-6-luna") + require.NoError(t, err) + var got struct { + Models []map[string]any `json:"models"` + } + require.NoError(t, json.Unmarshal(catalog, &got)) + require.Len(t, got.Models, 2) + assert.Equal(t, "openai/gpt-6-luna", got.Models[1]["slug"]) + assert.Equal(t, "freeform", got.Models[1]["apply_patch_tool_type"]) + assert.Equal(t, "native instructions", got.Models[1]["base_instructions"]) + unknown, err := codexModelCatalog(raw, "openrouter/openai/unknown") + require.NoError(t, err) + assert.Nil(t, unknown) +} diff --git a/app/cli/internal/trace/e2e/config.yaml b/app/cli/internal/trace/e2e/config.yaml index 829cb7154..d696dd2dc 100644 --- a/app/cli/internal/trace/e2e/config.yaml +++ b/app/cli/internal/trace/e2e/config.yaml @@ -32,7 +32,7 @@ harnesses: Codex: version: "0.163.0" models: - - openrouter/openai/gpt-5-mini + - openrouter/openai/gpt-6-luna # Runs and pass threshold of a cell whose feature depends on the model # following an instruction (R-006, D-004). A cell of any other feature runs diff --git a/app/cli/internal/trace/e2e/runner.go b/app/cli/internal/trace/e2e/runner.go index c54fff4fa..64d9056d6 100644 --- a/app/cli/internal/trace/e2e/runner.go +++ b/app/cli/internal/trace/e2e/runner.go @@ -287,7 +287,7 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc // errNoStoredSession is the failure of a check when the push stored no // session, for example because the agent did not commit or push. -var errNoStoredSession = errors.New("the session stored nothing: the agent did not commit and push") +var errNoStoredSession = errors.New("the push stored no session evidence; inspect harness-output.jsonl and the workspace git history") // evaluate runs the checks of a planned cell on its sessions and sets its // status and pass rate. diff --git a/app/cli/internal/trace/provider.go b/app/cli/internal/trace/provider.go index fba5c355b..490af29a5 100644 --- a/app/cli/internal/trace/provider.go +++ b/app/cli/internal/trace/provider.go @@ -307,6 +307,9 @@ type HookInput struct { ToolName string `json:"tool_name,omitempty"` // FilePath is the absolute path of the file being edited, set by provider's ReadHookInput. FilePath string `json:"-"` + // FilePaths holds all paths of a file tool that edits several files. + // Handlers process each path with its own snapshot and checkout. + FilePaths []string `json:"-"` // Command is the command line of a shell command tool, set by provider's // ReadHookInput. Empty when the provider does not report it. Command string `json:"-"` diff --git a/app/cli/internal/trace/providers/capabilities_test.go b/app/cli/internal/trace/providers/capabilities_test.go index 8ee20a644..ed630dcff 100644 --- a/app/cli/internal/trace/providers/capabilities_test.go +++ b/app/cli/internal/trace/providers/capabilities_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" "github.com/stretchr/testify/assert" @@ -59,6 +60,7 @@ func TestSessionStartChannels(t *testing.T) { wantReminder: true, why: "the opencode plugin posts the instruction and the reminder as context-only messages and has no banner", }, + {provider: codex.Name, wantBanner: true, wantInstruction: true, wantReminder: true, why: "Codex surfaces systemMessage and supports additionalContext at session start and user prompt"}, } for _, tc := range testCases { @@ -77,7 +79,7 @@ func TestSessionStartChannels(t *testing.T) { // without a decision recorded above, since the default zero value would // silently claim no support. func TestSessionStartChannelsCoverEveryProvider(t *testing.T) { - known := map[string]bool{claude.Name: true, cursor.Name: true, opencode.Name: true} + known := map[string]bool{claude.Name: true, cursor.Name: true, opencode.Name: true, codex.Name: true} for _, p := range All() { assert.True(t, known[p.Name()], "provider %q has no session-start channel expectation", p.Name()) diff --git a/app/cli/internal/trace/providers/registry.go b/app/cli/internal/trace/providers/registry.go index 183192db7..f112fcae2 100644 --- a/app/cli/internal/trace/providers/registry.go +++ b/app/cli/internal/trace/providers/registry.go @@ -22,6 +22,7 @@ package providers import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/cursor" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/opencode" ) @@ -37,6 +38,7 @@ var all = []trace.Provider{ claude.New(), cursor.New(), opencode.New(), + codex.New(), } // byName is a lookup-optimized view of all, built once at init. diff --git a/app/cli/pkg/action/trace_agent_hook.go b/app/cli/pkg/action/trace_agent_hook.go index e2a7b0ee0..c4c071fb2 100644 --- a/app/cli/pkg/action/trace_agent_hook.go +++ b/app/cli/pkg/action/trace_agent_hook.go @@ -380,6 +380,15 @@ func HandleAgentPreToolUse(provider trace.Provider, log zerolog.Logger) error { return nil } + for _, fileInput := range fileHookInputs(input) { + if err := handleAgentPreToolUse(provider, fileInput, log); err != nil { + return err + } + } + return nil +} + +func handleAgentPreToolUse(provider trace.Provider, input *trace.HookInput, log zerolog.Logger) error { log.Debug().Str("session_id", input.SessionID).Msg("pre-tool-use hook invoked") store, repoRoot, err := locateForHook(provider, input) @@ -747,6 +756,15 @@ func HandleAgentPostToolUse(provider trace.Provider, log zerolog.Logger) error { return nil } + for _, fileInput := range fileHookInputs(input) { + if err := handleAgentPostToolUse(provider, fileInput, log); err != nil { + return err + } + } + return nil +} + +func handleAgentPostToolUse(provider trace.Provider, input *trace.HookInput, log zerolog.Logger) error { sessionID := input.SessionID isCommand := provider.IsCommandTool(input.ToolName) @@ -1000,3 +1018,18 @@ func autoInstallGitHooks(store *state.Store, repoRoot string, log zerolog.Logger log.Debug().Err(err).Msg("mark trace initialized failed") } } + +// fileHookInputs expands a multi-file edit into the existing per-file path. +func fileHookInputs(input *trace.HookInput) []*trace.HookInput { + if len(input.FilePaths) == 0 { + return []*trace.HookInput{input} + } + inputs := make([]*trace.HookInput, 0, len(input.FilePaths)) + for _, path := range input.FilePaths { + one := *input + one.FilePath = path + one.FilePaths = nil + inputs = append(inputs, &one) + } + return inputs +} diff --git a/app/cli/pkg/action/trace_codex_hook_test.go b/app/cli/pkg/action/trace_codex_hook_test.go new file mode 100644 index 000000000..0c212db7b --- /dev/null +++ b/app/cli/pkg/action/trace_codex_hook_test.go @@ -0,0 +1,63 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCodexMultiFilePatchAttribution(t *testing.T) { + root := initTempGitRepo(t) + t.Chdir(root) + t.Setenv("CODEX_HOME", t.TempDir()) + require.NoError(t, os.WriteFile(filepath.Join(root, ".chainloop.yml"), []byte("traceStorage: LOCAL\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "one.txt"), []byte("human\nold\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "gone.txt"), []byte("gone\n"), 0600)) + payload, _ := json.Marshal(map[string]any{"session_id": "codex-patch", "cwd": root, "tool_name": "apply_patch", "tool_use_id": "call-1", "tool_input": map[string]string{"command": "*** Begin Patch\n*** Update File: one.txt\n@@\n human\n-old\n+new\n*** Add File: two.txt\n+two\n*** Delete File: gone.txt\n*** End Patch"}}) + withStdin(t, string(payload)) + require.NoError(t, HandleAgentPreToolUse(codex.New(), zerolog.Nop())) + require.NoError(t, os.WriteFile(filepath.Join(root, "one.txt"), []byte("human\nnew\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "two.txt"), []byte("two\n"), 0600)) + require.NoError(t, os.Remove(filepath.Join(root, "gone.txt"))) + withStdin(t, string(payload)) + require.NoError(t, HandleAgentPostToolUse(codex.New(), zerolog.Nop())) + store := state.NewGitStore(filepath.Join(root, ".git")) + ledger := store.LoadAILineAttribution("codex-patch") + assert.Contains(t, ledger.Open, "one.txt") + assert.Contains(t, ledger.Open, "two.txt") + assert.Contains(t, ledger.Open, "gone.txt") + assert.Len(t, ledger.Open["gone.txt"].Removed, 1) + assert.Equal(t, 2, ledger.GeneratedAdded) + assert.Equal(t, 2, ledger.GeneratedRemoved) + // An unchanged human line must not become an AI addition. + assert.Len(t, ledger.Open["one.txt"].Added, 1) + session, err := store.LoadSessionRecord("codex-patch") + require.NoError(t, err) + assert.Equal(t, codex.Name, session.Provider) + for _, name := range []string{"one.txt", "two.txt", "gone.txt"} { + _, err := store.LoadFileSnapshot("codex-patch", filepath.Join(root, name)) + assert.ErrorIs(t, err, os.ErrNotExist) + } +} diff --git a/docs/specs/issue-3657-codex-trace-provider.md b/docs/specs/issue-3657-codex-trace-provider.md new file mode 100644 index 000000000..75bb51022 --- /dev/null +++ b/docs/specs/issue-3657-codex-trace-provider.md @@ -0,0 +1,122 @@ +--- +owner: jiparis +ticket: https://github.com/chainloop-dev/chainloop/issues/3657 +prd: +--- + + + +# Spec issue-3657: Codex support in Chainloop Trace + +## Summary + +Chainloop Trace supports developers who use Codex. Developers can select Codex during initialization or trace one command with temporary hooks. Trace records Codex sessions as AI coding-session evidence. The integration uses native Codex hooks and persisted session transcripts. + +## Problem + +Developers who use Codex cannot attach native Trace evidence to their commits. Codex already records conversations and exposes lifecycle hooks. Trace needs to connect those records to its existing evidence pipeline. + +## Goals and Non-Goals + +- Goal: Developers can record Codex sessions in local storage or the control plane. +- Goal: Evidence includes conversations, models, tokens, estimated cost, tools, and code attribution. +- Non-goal: Capture child-session transcripts, track skills, or replace spec and pasted-image copies with pointers. These features need separate transcript analysis. +- Non-goal: Attribute arbitrary MCP tools or reconstruct sessions that produced no Trace hook. +- Non-goal: Change the evidence schema, control-plane APIs, or default provider. + +## Requirements + +### R-001: Provider selection + +Trace MUST offer Codex during initialization. Initialization and the command wrapper MUST accept the Codex provider flag. Developers MUST be able to select Codex with other providers. + +### R-002: Hook lifecycle + +Installation MUST preserve user hooks and produce identical content on repeated runs. Uninstall MUST delete only Trace hooks. The command wrapper MUST restore the previous hook configuration after execution. Its temporary configuration MUST omit the Trace session-end hook. + +### R-003: Session evidence + +Trace MUST capture session identity, timing, agent version, conversations, tool calls, and model information from persisted Codex transcripts. Missing or unreadable data MUST produce a warning or parsing error. Repeated cumulative usage records MUST count tokens once. Cost MUST use embedded API prices and identify an unknown-model fallback. + +### R-004: Code attribution + +Trace MUST attribute edits from native patch and shell tools. A patch that changes several files MUST preserve separate before-and-after content for each file. Attribution MUST include additions, deletions, and both paths of a move. Tool hooks MUST recover a missed session-start event. + +### R-005: Private session copies + +Trace MUST parse its private copy of the session transcript. Copying MUST check session identity and reject paths outside Codex session storage, including symlink escapes. A failed copy MUST preserve the previous copy. Local capture and parsing MUST require no network or login. + +### R-006: Session instructions + +Trace MUST send spec capture instructions at session start and reminders at each user prompt. Each hook response MUST contain at most one JSON document. Empty announcements MUST emit no output. Initialization MUST tell Codex users to review and trust the installed hooks. + +## Constraints + +Native hook behavior follows Codex CLI 0.163.0. Codex requires users to trust project configuration and hook definitions. Trace does not bypass that review during normal use. The test harness bypasses hook trust only in isolated test repositories. + +## Proposal + +The developer selects Codex during Trace initialization, reviews the installed hooks in Codex, and starts a session. Codex sends session and tool events to Trace. Trace records each patch path separately and uses working-tree snapshots for shell changes. + +Trace copies the Codex rollout into private storage at session boundaries and before evidence assembly. The parser retains native JSONL records and derives summaries from metadata, response items, turn context, and cumulative token counters. Git hooks use the existing evidence pipeline. + +```mermaid +flowchart LR + A[Initialize Trace for Codex] --> B[Review and trust hooks] + B --> C[Codex session and tool events] + C --> D[Private transcript and attribution] + D --> E[Commit and push] + E --> F[Local evidence or control-plane attestation] +``` + +## Decision Record + +### D-001: Use native Codex hooks + +Native hooks connect Codex to the existing session lifecycle. + +- Why: They cover session boundaries, prompts, patches, and shell tools without a separate extension. +- Alternatives considered: A notification-only integration misses pre-edit state. A separate extension duplicates lifecycle support. +- Source: drafting, based on the [official hook contract](https://learn.chatgpt.com/docs/hooks). + +### D-002: Expand patch events per file + +Trace snapshots each affected file before a patch and records its changes after the patch. + +- Why: Native patches can change several files, while attribution needs the original content of each file. +- Alternatives considered: Whole-worktree shell attribution loses uncommitted human content when it substitutes the last commit for the original content. +- Source: drafting. + +### D-003: Parse copied rollouts + +The provider reads copied native rollouts and estimates cost from embedded prices. + +- Why: Evidence remains available after Codex moves or deletes its original transcript. +- Alternatives considered: Live transcript parsing depends on agent storage during assembly. Notification payloads omit conversation and usage details. +- Source: drafting. + +## Open Questions + +None for the initial provider scope. + +## Risks + +| Risk | Mitigation | +|------|------------| +| Codex changes its rollout format. | Preserve unknown records, warn about unreadable records, and test sanitized rollout fixtures. | +| Users have not trusted the hooks. | Explain the trust step during initialization and in provider documentation. | +| Estimated cost differs from billing. | Document standard API estimates and warn when the model price is unknown. | From 3e37ae6027a8140dfb25ad9756408acb77b73bd6 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 14:13:06 +0200 Subject: [PATCH 2/8] fix(trace): skip Codex hooks when the CLI is unavailable Guard generated Codex commands with a PATH availability check while preserving hook input, output, and exit status when the CLI is installed. Upgrade existing entries and retain unrelated user hooks. Refs #3687 Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino --- .codex/hooks.json | 10 +- app/cli/internal/trace/codex/hooks.go | 7 +- app/cli/internal/trace/codex/hooks_test.go | 117 +++++++++++++++++++++ 3 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 app/cli/internal/trace/codex/hooks_test.go diff --git a/.codex/hooks.json b/.codex/hooks.json index 0f1e67470..198931de4 100644 --- a/.codex/hooks.json +++ b/.codex/hooks.json @@ -4,7 +4,7 @@ { "hooks": [ { - "command": "chainloop trace hook codex post-tool-use", + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex post-tool-use'", "type": "command" } ], @@ -15,7 +15,7 @@ { "hooks": [ { - "command": "chainloop trace hook codex pre-tool-use", + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex pre-tool-use'", "type": "command" } ], @@ -26,7 +26,7 @@ { "hooks": [ { - "command": "chainloop trace hook codex session-end", + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-end'", "type": "command" } ] @@ -36,7 +36,7 @@ { "hooks": [ { - "command": "chainloop trace hook codex session-start", + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-start'", "type": "command" } ] @@ -46,7 +46,7 @@ { "hooks": [ { - "command": "chainloop trace hook codex user-prompt-submit", + "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex user-prompt-submit'", "type": "command" } ] diff --git a/app/cli/internal/trace/codex/hooks.go b/app/cli/internal/trace/codex/hooks.go index a7070bd22..f2a10b216 100644 --- a/app/cli/internal/trace/codex/hooks.go +++ b/app/cli/internal/trace/codex/hooks.go @@ -31,6 +31,8 @@ import ( const ( hookMarker = "chainloop trace hook codex" settingsFile = ".codex/hooks.json" + // Skip tracing when the CLI is missing, before any Go handler can run. + hookCommandPrefix = "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec " // Codex hook event names. eventSessionStart = "SessionStart" @@ -109,7 +111,7 @@ func (p *Provider) installHookEvents(repoRoot string, events []hookEvent) error changed = removeHookEntry(hooks, eventSessionEnd) } for _, h := range events { - if addHookEntry(hooks, h.event, h.command, h.matcher) { + if addHookEntry(hooks, h.event, hookCommandPrefix+h.command+"'", h.matcher) { changed = true } } @@ -319,6 +321,9 @@ func isChainloopHandler(handler any) bool { return false } command, _ := h["command"].(string) + if strings.HasPrefix(command, hookCommandPrefix) && strings.HasSuffix(command, "'") { + command = strings.TrimSuffix(strings.TrimPrefix(command, hookCommandPrefix), "'") + } return strings.HasPrefix(command, hookMarker+" ") } diff --git a/app/cli/internal/trace/codex/hooks_test.go b/app/cli/internal/trace/codex/hooks_test.go new file mode 100644 index 000000000..6a0cbac03 --- /dev/null +++ b/app/cli/internal/trace/codex/hooks_test.go @@ -0,0 +1,117 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInstalledHooksWithoutCLI(t *testing.T) { + shell, err := exec.LookPath("sh") + require.NoError(t, err) + bin := t.TempDir() + require.NoError(t, os.Symlink(shell, filepath.Join(bin, "sh"))) + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + settings, err := readJSONFile(p.SettingsFile(root)) + require.NoError(t, err) + + for _, event := range hookEvents { + t.Run(event.event, func(t *testing.T) { + command := installedHookCommand(t, settings, event.event) + cmd := exec.CommandContext(t.Context(), shell, "-c", command) //nolint:gosec // execute the installed hook to verify its shell behavior + cmd.Env = append(os.Environ(), "PATH="+bin) + cmd.Stdin = strings.NewReader(`{"session_id":"test"}`) + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + assert.Empty(t, out) + }) + } +} + +func TestInstalledHookPreservesIOAndExitStatus(t *testing.T) { + shell, err := exec.LookPath("sh") + require.NoError(t, err) + bin := t.TempDir() + require.NoError(t, os.Symlink(shell, filepath.Join(bin, "sh"))) + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + settings, err := readJSONFile(p.SettingsFile(root)) + require.NoError(t, err) + command := installedHookCommand(t, settings, eventPostToolUse) + stub := "#!/bin/sh\nIFS= read -r payload\nprintf '%s\\n%s\\n' \"$*\" \"$payload\"\nexit \"$HOOK_TEST_EXIT_CODE\"\n" + require.NoError(t, os.WriteFile(filepath.Join(bin, "chainloop"), []byte(stub), 0700)) //nolint:gosec // executable stub CLI in a temporary test directory + + for _, code := range []string{"0", "37"} { + t.Run("exit_"+code, func(t *testing.T) { + cmd := exec.CommandContext(t.Context(), shell, "-c", command) //nolint:gosec // execute the installed hook against a local stub CLI + cmd.Env = append(os.Environ(), "PATH="+bin, "HOOK_TEST_EXIT_CODE="+code) + cmd.Stdin = strings.NewReader("{\"session_id\":\"test\"}\n") + out, err := cmd.Output() + if code == "0" { + require.NoError(t, err) + } else { + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, 37, exitErr.ExitCode()) + } + assert.Equal(t, "trace hook codex post-tool-use\n{\"session_id\":\"test\"}\n", string(out)) + }) + } +} + +func TestInstallUpgradesUnguardedHooks(t *testing.T) { + root := t.TempDir() + p := New() + path := p.SettingsFile(root) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + legacy := `{"hooks":{"PostToolUse":[{"matcher":"^(apply_patch|Bash)$","hooks":[{"type":"command","command":"chainloop trace hook codex post-tool-use"},{"type":"command","command":"echo user-hook"}]}]}}` + require.NoError(t, os.WriteFile(path, []byte(legacy), 0600)) + require.NoError(t, p.InstallHooks(root)) + first, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(string(first), "chainloop trace hook codex post-tool-use")) + assert.Contains(t, string(first), "command -v chainloop") + assert.Contains(t, string(first), "echo user-hook") + require.NoError(t, p.InstallHooks(root)) + second, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.Equal(t, first, second) + require.NoError(t, p.UninstallHooks(root)) + remaining, err := os.ReadFile(p.SettingsFile(root)) + require.NoError(t, err) + assert.NotContains(t, string(remaining), hookMarker) + assert.Contains(t, string(remaining), "echo user-hook") +} + +func installedHookCommand(t *testing.T, settings map[string]any, event string) string { + t.Helper() + hooks := settings["hooks"].(map[string]any) + groups := hooks[event].([]any) + require.Len(t, groups, 1) + handlers := groups[0].(map[string]any)["hooks"].([]any) + require.Len(t, handlers, 1) + return handlers[0].(map[string]any)["command"].(string) +} From 499546244bea5ff6a2df0fe78005a8bb679d0597 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 14:14:54 +0200 Subject: [PATCH 3/8] fix(trace): address Codex provider review findings Chainloop-Trace-Sessions: 01a15e4e-4915-7882-8c86-3cc68f647126 Signed-off-by: Miguel Martinez Trivino Assisted-by: Codex --- README.md | 2 +- app/cli/internal/trace/README.md | 2 +- app/cli/internal/trace/codex/copy.go | 28 ++++ app/cli/internal/trace/codex/parse.go | 33 ++--- app/cli/internal/trace/codex/pricing.go | 5 +- app/cli/internal/trace/codex/provider_test.go | 120 ++++++++++++++++-- app/cli/internal/trace/state/session.go | 3 + 7 files changed, 164 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 69866e110..7cc44a352 100644 --- a/README.md +++ b/README.md @@ -354,7 +354,7 @@ You can also download the CLI from the [releases page](https://github.com/chainl ## For AI coding agents > [!NOTE] -> **If you are an AI coding agent setting up Chainloop in this repository:** Install the CLI with `curl -sfL https://dl.chainloop.dev/cli/install.sh | bash -s -- --oss`. Ask the user to run `chainloop auth login`, which opens a browser. Then run `chainloop trace init --project ` from the repository root. Add `--claude`, `--cursor`, `--opencode`, or `--codex` for the agents in use. The command writes `.chainloop.yml` and git hooks. Commit `.chainloop.yml`. The docs index for agents is https://docs.chainloop.dev/llms.txt. Any docs page is available as Markdown when you add `.md` to its URL. [Documentation](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=for-ai-coding-agents) +> **If you are an AI coding agent setting up Chainloop in this repository:** Install the CLI with `curl -sfL https://dl.chainloop.dev/cli/install.sh | bash -s -- --oss`. Ask the user to run `chainloop auth login`, which opens a browser. Then run `chainloop trace init --project ` from the repository root. Add `--claude`, `--cursor`, `--opencode`, or `--codex` for the agents in use. The command writes `.chainloop.yml` and Git hooks, plus `.codex/hooks.json` when using Codex. Commit `.chainloop.yml` and the generated `.codex/hooks.json`. Each clone must still run `chainloop trace init` to install its local Git hooks. The docs index for agents is https://docs.chainloop.dev/llms.txt. Any docs page is available as Markdown when you add `.md` to its URL. [Documentation](https://docs.chainloop.dev/ai-sessions?utm_source=github&utm_medium=readme&utm_campaign=chainloop-oss&utm_content=for-ai-coding-agents) Or paste this into Claude Code, Codex, or Cursor: diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 18222f050..038e42ddb 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -10,7 +10,7 @@ Claude Code has the most complete support. Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and `trace run --pi` do not exist, so no Pi session is recorded. Each Pi cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Pi support is added, set the Pi cells to **Unknown** and run the end-to-end suite to find the values. -Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Changing the hooks requires a new trust review. +Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --local --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Changing the hooks requires a new trust review. Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 19 feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts, skills tracking, spec source pointers, and pasted image pointers are not implemented. diff --git a/app/cli/internal/trace/codex/copy.go b/app/cli/internal/trace/codex/copy.go index 0aca8e56e..cc345d47d 100644 --- a/app/cli/internal/trace/codex/copy.go +++ b/app/cli/internal/trace/codex/copy.go @@ -47,7 +47,14 @@ func (p *Provider) CopySessionData(store *state.Store, loc trace.SessionLocation if store == nil || !state.ValidSessionID(loc.SessionID) { return errors.New("copy Codex session: invalid session") } + rec, err := store.LoadSessionRecord(loc.SessionID) + if err != nil { + return fmt.Errorf("copy Codex session: load session record: %w", err) + } home := codexHome() + if rec != nil && rec.CodexStorageRoot != "" { + home = rec.CodexStorageRoot + } if home == "" { return nil } @@ -108,6 +115,11 @@ func (p *Provider) CopySessionData(store *state.Store, loc trace.SessionLocation if json.Unmarshal(scanner.Bytes(), &header) != nil || header.Type != "session_meta" || header.Payload.ID != loc.SessionID { return errors.New("copy Codex session: rollout session ID mismatch") } + // Persist only after validating both storage containment and session identity. + // Do not infer a trusted root from the hook's transcript path. + if err := saveStorageRoot(store, rec, home); err != nil { + return fmt.Errorf("copy Codex session: save storage root: %w", err) + } if _, err := src.Seek(0, io.SeekStart); err != nil { return err } @@ -130,6 +142,22 @@ func (p *Provider) CopySessionData(store *state.Store, loc trace.SessionLocation return os.Rename(tmp.Name(), state.RawSessionPath(dir, loc.SessionID)) } +func saveStorageRoot(store *state.Store, rec *state.SessionRecord, home string) error { + if rec == nil || rec.CodexStorageRoot != "" { + return nil + } + root, err := filepath.EvalSymlinks(home) + if err != nil { + return err + } + root, err = filepath.Abs(root) + if err != nil { + return err + } + rec.CodexStorageRoot = root + return store.SaveSessionRecord(rec) +} + // Resolve symlinks as well as lexical traversal before opening a hook path. func insideStorage(path string, roots []string) bool { resolved, err := filepath.EvalSymlinks(path) diff --git a/app/cli/internal/trace/codex/parse.go b/app/cli/internal/trace/codex/parse.go index f7629fcce..1bfe968e0 100644 --- a/app/cli/internal/trace/codex/parse.go +++ b/app/cli/internal/trace/codex/parse.go @@ -45,14 +45,15 @@ type rolloutEntry struct { } type rolloutPayload struct { - ID string `json:"id"` - Type string `json:"type"` - Version string `json:"cli_version"` - ModelProvider string `json:"model_provider"` - Model string `json:"model"` - Role string `json:"role"` - Name string `json:"name"` - CallID string `json:"call_id"` + ID string `json:"id"` + Type string `json:"type"` + Version string `json:"cli_version"` + ModelProvider string `json:"model_provider"` + Model string `json:"model"` + Role string `json:"role"` + Name string `json:"name"` + CallID string `json:"call_id"` + ThreadUsage *tokenUsage `json:"thread_token_usage"` Info *struct { Total *tokenUsage `json:"total_token_usage"` } `json:"info"` @@ -74,7 +75,7 @@ func (p *Provider) ParseSession(ctx context.Context, opts *trace.ParseOpts) (*ai } defer f.Close() d := aicodingsession.Data{ - SchemaVersion: "0.1", + SchemaVersion: "v1", Agent: aicodingsession.Agent{Name: Name, Version: opts.AgentVersion}, Session: aicodingsession.Session{ID: opts.SessionID}, Model: &aicodingsession.Model{Primary: opts.Model}, @@ -180,17 +181,19 @@ func (p *rolloutParser) consume(e rolloutEntry, payload rolloutPayload, line int } } case "event_msg": - p.consumeUsage(payload, line) + if payload.Type == "token_count" && payload.Info != nil && payload.Info.Total != nil { + p.consumeUsage(*payload.Info.Total, line) + } + case "token_usage_record": + if payload.ThreadUsage != nil { + p.consumeUsage(*payload.ThreadUsage, line) + } } return nil } -func (p *rolloutParser) consumeUsage(payload rolloutPayload, line int) { - if payload.Type != "token_count" || payload.Info == nil || payload.Info.Total == nil { - return - } - total := *payload.Info.Total +func (p *rolloutParser) consumeUsage(total tokenUsage, line int) { if total.Input < 0 || total.Output < 0 || total.Cached < 0 || total.CacheWrite < 0 || total.Total < 0 { p.data.Warnings = append(p.data.Warnings, fmt.Sprintf("negative Codex token count on line %d", line)) return diff --git a/app/cli/internal/trace/codex/pricing.go b/app/cli/internal/trace/codex/pricing.go index 52e3d5eba..a52865df0 100644 --- a/app/cli/internal/trace/codex/pricing.go +++ b/app/cli/internal/trace/codex/pricing.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "strings" + "time" ) type modelPricing struct { @@ -52,7 +53,9 @@ func estimateCost(model string, u tokenUsage, warnings *[]string) float64 { if !ok { // Date-pinned model IDs use the same prices as their family alias. if len(name) > 11 && name[len(name)-11] == '-' { - price, ok = prices.Models[name[:len(name)-11]] + if _, err := time.Parse(time.DateOnly, name[len(name)-10:]); err == nil { + price, ok = prices.Models[name[:len(name)-11]] + } } } if !ok { diff --git a/app/cli/internal/trace/codex/provider_test.go b/app/cli/internal/trace/codex/provider_test.go index 60c8dfe7f..ebb858f2d 100644 --- a/app/cli/internal/trace/codex/provider_test.go +++ b/app/cli/internal/trace/codex/provider_test.go @@ -25,6 +25,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/internal/schemavalidators" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -128,6 +129,11 @@ func TestParseSession(t *testing.T) { assert.Equal(t, 1, d.Conversation.AssistantMessages) assert.Len(t, d.RawSession["main"], 12) assert.Empty(t, d.Warnings) + doc, err := json.Marshal(d) + require.NoError(t, err) + var data map[string]any + require.NoError(t, json.Unmarshal(doc, &data)) + require.NoError(t, schemavalidators.ValidateAICodingSession(data, schemavalidators.AICodingSessionVersion0_1)) } func TestCopySessionData(t *testing.T) { @@ -217,6 +223,7 @@ func TestUsageAcrossModels(t *testing.T) { extra := `{"timestamp":"2026-10-21T10:01:01Z","type":"turn_context","payload":{"model":"gpt-5"}} {"timestamp":"2026-10-21T10:01:02Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[]}} {"timestamp":"2026-10-21T10:01:03Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[]}} +{"timestamp":"2026-10-21T10:01:04Z","type":"token_usage_record","payload":{"usage":{"input_tokens":2000,"output_tokens":404,"total_tokens":2404},"thread_token_usage":{"input_tokens":4000,"cached_input_tokens":2000,"cache_write_input_tokens":200,"output_tokens":804,"total_tokens":4804}}} {"timestamp":"2026-10-21T10:01:04Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":4000,"cached_input_tokens":2000,"cache_write_input_tokens":200,"output_tokens":804,"total_tokens":4804}}}} ` raw = append(raw, extra...) @@ -231,17 +238,108 @@ func TestUsageAcrossModels(t *testing.T) { } func TestNativeRollout(t *testing.T) { - dir := t.TempDir() raw, err := os.ReadFile("testdata/native-session.jsonl") require.NoError(t, err) - require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // sanitized fixture and a fixed session ID - e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) - require.NoError(t, err) - assert.Equal(t, 19064, e.Data.Usage.InputTokens) - assert.Equal(t, 128, e.Data.Usage.OutputTokens) - assert.Equal(t, 19192, e.Data.Usage.TotalTokens) - assert.Equal(t, 19061, e.Data.Usage.CacheCreationInputTokens) - assert.NotEmpty(t, e.Data.Model.Primary) - assert.NotEmpty(t, e.Data.ToolsUsed.Summary) - assert.Empty(t, e.Data.Warnings) + lines := strings.SplitAfter(string(raw), "\n") + for _, tc := range []struct { + name, data string + }{ + {"before mirrored event", strings.Join(lines[:6], "")}, + {"with mirrored event", string(raw)}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), []byte(tc.data), 0600)) + e, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + require.NotNil(t, e.Data.Usage) + assert.Equal(t, 19064, e.Data.Usage.InputTokens) + assert.Equal(t, 128, e.Data.Usage.OutputTokens) + assert.Equal(t, 19192, e.Data.Usage.TotalTokens) + assert.Equal(t, 19061, e.Data.Usage.CacheCreationInputTokens) + assert.InDelta(t, 0.0489, e.Data.Usage.EstimatedCostUSD, 0.00001) + assert.NotEmpty(t, e.Data.Model.Primary) + assert.NotEmpty(t, e.Data.ToolsUsed.Summary) + assert.Empty(t, e.Data.Warnings) + }) + } +} + +func TestCopySessionDataPreservesStorageRoot(t *testing.T) { + for _, env := range []string{"unset", "different home"} { + t.Run(env, func(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + store := state.NewOutOfTreeStore(t.TempDir()) + require.NoError(t, store.SaveSessionRecord(&state.SessionRecord{SessionID: testSessionID, Provider: Name, Active: true})) + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + path := filepath.Join(home, "sessions", "rollout-time-"+testSessionID+".jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, raw, 0600)) //nolint:gosec // temporary Codex storage fixture and a fixed session ID + loc := trace.SessionLocation{SessionID: testSessionID, TranscriptPath: path} + require.NoError(t, New().CopySessionData(store, loc)) + rec, err := store.LoadSessionRecord(testSessionID) + require.NoError(t, err) + canonicalHome, err := filepath.EvalSymlinks(home) + require.NoError(t, err) + assert.Equal(t, canonicalHome, rec.CodexStorageRoot) + assert.True(t, rec.Active) + + updated := append(append([]byte(nil), raw...), []byte("\n{\"new_record\":true}\n")...) + require.NoError(t, os.WriteFile(path, updated, 0600)) //nolint:gosec // temporary Codex storage fixture and a fixed session ID + if env == "unset" { + require.NoError(t, os.Unsetenv("CODEX_HOME")) + } else { + t.Setenv("CODEX_HOME", t.TempDir()) + } + // A new provider instance represents a later Git hook process. + require.NoError(t, New().CopySessionData(store, loc)) + got, err := os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, updated, got) + + archived := filepath.Join(home, "archived_sessions", filepath.Base(path)) + require.NoError(t, os.MkdirAll(filepath.Dir(archived), 0700)) + require.NoError(t, os.Rename(path, archived)) + require.NoError(t, New().CopySessionData(store, loc)) + got, err = os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, updated, got) + + outside := filepath.Join(t.TempDir(), "outside.jsonl") + require.NoError(t, os.WriteFile(outside, raw, 0600)) //nolint:gosec // deliberately outside storage to verify rejection + assert.Error(t, New().CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: outside})) + link := filepath.Join(home, "sessions", "escaped.jsonl") + require.NoError(t, os.Symlink(outside, link)) + assert.Error(t, New().CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: link})) + }) + } +} + +func TestEstimateCostDateSuffix(t *testing.T) { + for _, tc := range []struct { + model string + cost float64 + warning bool + }{ + {"gpt-6-luna", 0.1, false}, + {"gpt-6-luna-2026-10-21", 0.1, false}, + {"openai/gpt-6-luna-2024-02-29", 0.1, false}, + {"gpt-6-luna-foo-123456", 0.25, true}, + {"gpt-6-luna-2026-02-29", 0.25, true}, + {"gpt-6-luna-2026-13-21", 0.25, true}, + {"gpt-6-luna-2026-1-21", 0.25, true}, + } { + t.Run(tc.model, func(t *testing.T) { + var warnings []string + assert.InDelta(t, tc.cost, estimateCost(tc.model, tokenUsage{Input: 1_000_000}, &warnings), 0.00001) + if tc.warning { + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0], "unknown Codex model") + } else { + assert.Empty(t, warnings) + } + }) + } } diff --git a/app/cli/internal/trace/state/session.go b/app/cli/internal/trace/state/session.go index 02bbd0dd3..11de110bf 100644 --- a/app/cli/internal/trace/state/session.go +++ b/app/cli/internal/trace/state/session.go @@ -44,6 +44,9 @@ type SessionRecord struct { // over Cwd to find the transcripts. Filled in by a later hook when the // first one did not carry it. Empty when the agent does not report it. TranscriptPath string `json:"transcript_path,omitempty"` + // CodexStorageRoot is the canonical Codex home validated while copying + // this session. Later pushes can refresh it without the original CODEX_HOME. + CodexStorageRoot string `json:"codex_storage_root,omitempty"` // Checkouts lists the roots of the other checkouts that this session // edited with a file tool, sorted. Shell hooks run in the session's own // checkout, and they also snapshot these checkouts, so that a command From 12cbb1c1b3e646c029434dfa3dd350c353d4f499 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 14:41:17 +0200 Subject: [PATCH 4/8] fix(trace): enforce the shared transcript contract Normalize Codex conversations and tool exchanges for the session viewer, including text blocks in tool results, and retain original source events. Validate transcript entries in the schema and check renderable messages and completed tool exchanges in the E2E suite. Align the Pi parser with the shared contract. Preserve newer session metadata when caching Codex storage, allow valid transcript copies with damaged optional metadata, and add a native Windows hook command. Promote the TOML dependency and identify the trusted E2E binary invocation for SAST. Refs #3657 Assisted-by: Codex Signed-off-by: Miguel Martinez Trivino Chainloop-Trace-Sessions: 01a15e4e-4915-7882-8c86-3cc68f647126 --- .codex/hooks.json | 5 + app/cli/internal/trace/PROVIDERS.md | 16 +- app/cli/internal/trace/README.md | 199 +++++++- app/cli/internal/trace/codex/copy.go | 18 +- .../trace/codex/copy_metadata_test.go | 68 +++ app/cli/internal/trace/codex/hooks.go | 10 +- app/cli/internal/trace/codex/hooks_test.go | 54 ++- app/cli/internal/trace/codex/parse.go | 6 +- app/cli/internal/trace/codex/transcript.go | 147 ++++++ .../internal/trace/codex/transcript_test.go | 131 ++++++ app/cli/internal/trace/e2e/checks.go | 82 +++- app/cli/internal/trace/e2e/codex.go | 2 +- app/cli/internal/trace/e2e/transcript_test.go | 67 +++ app/cli/internal/trace/pi/parse.go | 49 +- .../trace/pi/testdata/session-v3.golden.json | 304 ++++++++++--- app/cli/internal/trace/types.go | 34 +- docs/specs/issue-3657-codex-trace-provider.md | 4 +- go.mod | 2 +- .../ai-coding-session-0.1.schema.json | 428 +++++++++++++++++- internal/schemavalidators/transcript_test.go | 70 +++ .../materials/aicodingsession/redact_test.go | 2 +- 21 files changed, 1588 insertions(+), 110 deletions(-) create mode 100644 app/cli/internal/trace/codex/copy_metadata_test.go create mode 100644 app/cli/internal/trace/codex/transcript.go create mode 100644 app/cli/internal/trace/codex/transcript_test.go create mode 100644 app/cli/internal/trace/e2e/transcript_test.go create mode 100644 internal/schemavalidators/transcript_test.go diff --git a/.codex/hooks.json b/.codex/hooks.json index 198931de4..0ed3e3a71 100644 --- a/.codex/hooks.json +++ b/.codex/hooks.json @@ -5,6 +5,7 @@ "hooks": [ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex post-tool-use'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex post-tool-use)\"", "type": "command" } ], @@ -16,6 +17,7 @@ "hooks": [ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex pre-tool-use'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex pre-tool-use)\"", "type": "command" } ], @@ -27,6 +29,7 @@ "hooks": [ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-end'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex session-end)\"", "type": "command" } ] @@ -37,6 +40,7 @@ "hooks": [ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-start'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex session-start)\"", "type": "command" } ] @@ -47,6 +51,7 @@ "hooks": [ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex user-prompt-submit'", + "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex user-prompt-submit)\"", "type": "command" } ] diff --git a/app/cli/internal/trace/PROVIDERS.md b/app/cli/internal/trace/PROVIDERS.md index 34a5f4425..fe7e6a8d8 100644 --- a/app/cli/internal/trace/PROVIDERS.md +++ b/app/cli/internal/trace/PROVIDERS.md @@ -127,11 +127,25 @@ Fill the evidence (`pkg/attestation/crafter/materials/aicodingsession`) as follo | `ToolsUsed` | A count per tool, sorted by count, then by name. | | | | | `Conversation` | User and assistant message counts. | | | | | `Subagents` | One entry per subagent, with its tokens. | Yes. | No. | No. | -| `RawSession` | A map of streams. `main` is required: use an empty slice, not `nil`. Other keys are subagent streams. Each line must be one JSON object. If the agent has no line format, convert each message to the Claude line shape (`{type, uuid, timestamp, message: {role, content}}`) so the UI can show it. | Transcript lines. | Transcript lines. | Rebuilt by `buildRawSessionEntries`. | +| `RawSession` | A map of streams. `main` is required: use an empty slice, not `nil`. Other keys are subagent streams. Conversation and tool records MUST use the shared transcript format below, even when the agent stores native JSONL. | Transcript lines. | Transcript lines. | Rebuilt by `buildRawSessionEntries`. | | `Warnings` | Each data item that the parser could not read or that the agent does not give. | | Always "no token usage". | | The caller sets `Session.Mode`, `GitContext`, `CodeChanges`, `Spec`, and the skills. Do not set them in the provider. Claude sets `GitContext.Branch` and `WorkDir` from the transcript, but the push replaces all of `GitContext` when it can read the repository. +#### Transcript wire contract + +`trace.RawSessionEntry` defines the message envelope. The AI coding-session JSON schema validates it. Consumers MUST NOT need an agent-specific parser to read conversation text or tool exchanges. + +- A message MUST identify its role with `type: "user"` or `type: "assistant"`. Consumers also accept the Cursor `role` field when `type` is absent. +- `message.content` MUST be a string or an array of content blocks. A text block MUST have `type: "text"` and a string `text` field. +- A tool call MUST be an assistant content block with `type: "tool_use"`, a nonempty `id`, and a nonempty `name`. Its optional `input` MUST be an object. +- A tool result MUST be a user content block with `type: "tool_result"`, `tool_use_id`, and `content`. Its `tool_use_id` MUST match the call's `id`. This carrier is not a new user prompt. A call still running at capture time may have no result yet. +- Providers SHOULD include stable `uuid` values and RFC3339 `timestamp` values. They MUST preserve message order. +- A converted entry MAY retain its original native record in `source_event`. Unknown native metadata MAY remain as separate JSON objects. Neither can replace normalized conversation or tool records. +- A session with conversation messages and a stored transcript MUST contain a normalized message in `main`. Empty or metadata-only transcripts are valid only when no conversation was captured. + +The E2E transcript check MUST validate the schema and find renderable user and assistant content. It MUST find scenario work in normalized content, not just in native metadata. Codex also checks for a completed tool exchange with matching IDs. The final in-progress push call need not have a result yet. + Cost: - If the agent records the cost, use its value (OpenCode, Pi). diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 038e42ddb..b6e66b2cc 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -12,10 +12,14 @@ Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --local --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Changing the hooks requires a new trust review. -Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 19 feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts, skills tracking, spec source pointers, and pasted image pointers are not implemented. +Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 22 feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts, skills tracking, spec source pointers, and pasted image pointers are not implemented. Codex token usage comes from cumulative rollout counters. Cost is an estimate from the embedded standard API prices, with a warning for unknown models. Subscription billing and third-party provider prices can differ. Trace reads rollouts from `$CODEX_HOME/sessions` (default `~/.codex/sessions`) or `archived_sessions`. Their JSONL format is not a stable Codex interface. +In a linked Git worktree, Codex 0.163 reads project hooks from the primary checkout. Install the Codex hooks in that checkout too, then review and trust them in Codex. + +Codex evidence converts native messages, tool calls, and tool results to the [shared transcript wire contract](PROVIDERS.md#transcript-wire-contract). Each converted entry retains its original event in `source_event`. The transcript E2E check validates the schema, renderable messages, and a completed tool exchange. A nonempty native transcript alone does not pass. + ## Setup How Trace is added to a repository and to an agent. @@ -25,3 +29,196 @@ How Trace is added to a repository and to an agent. | Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Yes | Unknown | Yes | No | Yes | | Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Yes | Unknown | Yes | No | Yes | | More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | Unknown | + +Why not Yes: + +- **Hook installation and deletion, and Trace run wrapper, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **More than one agent in one repository, all agents:** not verified yet. The end-to-end suite needs a second agent to test it (spec issue-3635). + +## Session collection + +What the session evidence holds about the conversation. + +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | No | Yes | +| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | No | Yes | +| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | No | Yes | +| Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | No | No | +| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | No | Unknown | + +Why not Yes: + +- **Missed session start, Codex:** the scenario produces evidence, but its check also needs hook messages that Codex does not report in headless mode. +- **Transcript, OpenCode:** OpenCode has no transcript file. The provider runs `opencode export` (1.x) or `opencode session export` (2) and rebuilds the transcript from the export. It keeps the text and the inputs of the tool calls. It drops the tool outputs, the reasoning and the file parts. OpenCode 2 exports also have no agent version and no session slug. +- **Token usage and cost, Cursor:** the Cursor transcript does not hold token usage. Usage and cost are zero, and the evidence has a warning that tells so. (For OpenCode, the cost is the value that OpenCode reports. For Claude Code, Chainloop computes it from its price table.) +- **Model detection, Cursor:** the primary model comes from the session-start hook. The transcript does not tell the models, so the evidence does not list the other models. In auto mode, the hook names the model `default`, not the model that ran. +- **Subagents, Codex:** the initial provider records only the root session transcript. Child-session transcripts are not collected. +- **Subagents, Cursor:** the provider does not read subagent conversations. +- **Subagents, OpenCode:** a subagent runs in a child session with its own export. It is not part of the evidence of its parent session. +- **Missed session start, Cursor and OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). + +## Code attribution + +Which lines of a commit the AI wrote. + +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | Yes | +| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | Yes | +| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence reports the totals of the session in `ai_generated_lines_added` and `ai_generated_lines_removed`. A session with no commit sends no evidence, so it reports no totals (spec issue-3639). | Yes | Partial | Yes | Yes | No | Yes | +| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | Partial | +| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the removals of the session's commits, less the lines that a commit of the session wrote. The line counts of each file are the sum of its AI and human lines, and a file that the session created stays created when a later commit of the session changes it (spec issue-3639, issue 3674). | Partial | Partial | Partial | Partial | No | Partial | +| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | Partial | +| Line ranges | The line ranges of each file are positions in the file at the commit that the push sends. They cover the lines of the session that are still there, and each range says if its lines are AI or human. The AI ranges hold as many lines as the net AI lines added, and the human ranges as many lines as the net human lines added. A file that the pushed commit no longer holds has no ranges. A range with no attribution is an AI range at edit time. Older CLIs write only these ranges. This CLI writes them for a file that it cannot blame at the pushed commit, and for a file with edits from before line fingerprints (spec issue-3639, line ranges). | Partial | Partial | Partial | Partial | No | Partial | +| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | No | Partial | + +Why not Yes: + +- **Line attribution, partial commits and generated line counts, Cursor:** Cursor has only the `afterFileEdit` hook, which gives the edits but no snapshot before the edit. The provider rebuilds the content before the edit from the edits. +- **Net contribution and line ranges, all agents:** the push finds the last writer of each line with `git blame`. A later commit that only re-indents a line takes the line, because blame sees a change. For Cursor, the limits of line attribution apply too. +- **Session updates, all agents:** the update works only in the clone where the work happened, while the CLI keeps the trace state of the session (30 days), and only when the commits of the session are on the branch that the push sends. The push reads only the commits that are not on the default branch of the remote (`origin/HEAD`, or else `origin/main` or `origin/master`). After the commits of a session are pushed to that branch, a later commit does not update the session. +- **Renames and rewrites, all agents:** this works only in the clone where the work happened. Git finds a rename only when the file keeps at least half of its content. Two pushed commits can rename a file one after the other. A later change to the file under its middle name does not send the session again. A rename inside an amend or a squash of the session commit loses the AI lines: the new commit creates the file under its new name, so no line of it matches the trace state (issue 3686). The limits of session updates apply too. For Cursor, the limits of line attribution apply too. +- **Shell changes attribution, all agents:** the hook before a shell command records only a hash of each file, not its content. The file at the `HEAD` commit from before the command stands in for the content before the command. Uncommitted lines that an earlier agent edit recorded are not recorded again, but uncommitted human lines of a file that the command changes count as AI. + +## User experience + +What the user sees in the agent. + +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | Unknown | +| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | Unknown | +| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | Unknown | + +Why not Yes: + +- **Welcome message, Codex:** Codex does not report hook messages in its headless output, so the suite cannot verify the banner. +- **Session link after push and credentials warning, Codex:** these checks need a control plane. The suite uses local storage only. +- **Welcome message, Cursor and OpenCode:** the provider gives the session-start context to the model only. It has no channel that shows a message to the user at session start, so the welcome message is dropped. +- **Credentials warning, Cursor and OpenCode:** the provider has no channel to the user at session start. The model is told to tell the user, so the warning depends on the model. +- **Session link after push, Cursor:** the `postToolUse` hook runs after a shell command, but its response has no field that Cursor shows to the user. The link stays pending until it expires. +- **Session link after push, OpenCode:** the plugin hook fires after a shell command, but the response that shows a message to the user is not verified yet. The link stays pending until it expires. + +## Specs and skills + +What the session was asked to build, and which skills it used. + +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | No | Yes | +| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | Unknown | +| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | Yes | +| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Yes | Unknown | Yes | No | Yes | +| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | No | +| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | No | +| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | No | + +Why not Yes: + +- **Spec capture reminder, Codex:** Codex does not report hook messages in its headless output, so the suite cannot inspect the reminder. +- **Spec capture reminder, Cursor:** Cursor has no hook at each user prompt. The agent gets the session-start instruction only. +- **Spec source role, title, and description, OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Skills tracking, spec source pointers, and pasted image pointers, Codex:** these optional integrations are not implemented. +- **Skills tracking, Cursor:** the provider does not track skills. The evidence has no skill entries, which means "not recorded", not "no skill used". +- **Skills tracking, OpenCode:** only the skills that the model starts with the `skill` tool are counted. Skills that the user starts, and skills used in subagents, are not. A plugin installed by an earlier version does not send the skill folder for OpenCode 2: install the hooks again to update it. +- **Spec source pointers, Cursor and OpenCode:** the provider has no finder for the copies in its transcript yet. For OpenCode, the rebuilt transcript holds no tool outputs, so only the write copies would apply. +- **Pasted image pointers, Cursor and OpenCode:** the provider has no finder for pasted images yet. Each agent puts pasted images in a different block shape. The pasted images stay inline in the transcript. The agent is told not to capture them in the spec folder. + +## Security + +How the evidence is protected before it leaves the machine. + +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | No | Yes | +| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | No | Yes | +| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | No | Yes | + +Why not Yes: + +- **Media skipped by redaction, Cursor:** the Cursor transcript holds no tool outputs, so a file that the agent reads leaves no media in it. A pasted image stays inline in the transcript, but it is not verified that it is in the base64 media shape that the redaction skips. The headless Cursor CLI cannot attach an image, so the end-to-end suite cannot check it. +- **Media skipped by redaction, OpenCode:** the rebuilt transcript holds no media. + +## Git integration + +How the sessions are linked to the commits and attested. + +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | No | Yes | +| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | No | Unknown | +| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | No | Yes | +| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | No | Yes | + +### Switching storage modes + +`traceStorage` in `.chainloop.yml` is a committed repository policy. It is the same for every clone. When a repository uses `LOCAL`, Trace stores every session of every user on disk. To change the mode: + +- **To local:** run `chainloop trace init --local`. It writes `traceStorage: LOCAL`, installs the hooks, and does not connect to a control plane. Commit `.chainloop.yml` to share the change. +- **Back to the control plane:** run `chainloop trace init` without `--local`. It writes `traceStorage: CONTROL_PLANE` and sets up the workflow. The command tells you about the change. Commit `.chainloop.yml`. + +`tracePath` in `.chainloop.yml` sets the trace directory, where Chainloop Trace keeps its data in every mode. The default is `.git/chainloop-trace`. A relative path starts at the repository root. `chainloop trace init --trace-path ` writes it. A directory in the working tree must be in `.gitignore`. + +Run `chainloop trace init` one time for each repository. Git does not clone the git hooks in `.git`. In a clone, the first traced session installs them, so a teammate does not run init. + +Why not Yes: + +- **Attestation at push, Codex:** the suite uses local storage only and cannot verify control-plane uploads. +- **Shell commands in another checkout, Cursor:** a relative path in the command resolves against the first workspace root, not against the folder where the shell runs. + +## How each agent is hooked in + +| Agent | Hook installation | Hooks | +|-------|-------------------|-------| +| Claude Code | `.claude/settings.json` | Session start and end, user prompt, and before and after each file and shell tool and the Skill tool. | +| Cursor | `.cursor/hooks.json` | Session start and end, `afterFileEdit`, and `preToolUse` and `postToolUse` for the `Shell` tool. The `preToolUse` hook always allows the command. The provider installs no prompt hook. | +| Codex | `.codex/hooks.json` | `SessionStart`, `SessionEnd`, `UserPromptSubmit`, `PreToolUse` and `PostToolUse` for `Bash` and `apply_patch`. Patches use one snapshot per affected file, including both paths of a move. | +| OpenCode | `.opencode/plugins/chainloop-trace.ts` | Session start and end, user prompt, and before and after each tool. One plugin covers both versions: `server()` for 1.x (1.3.4 or later) and `setup()` for 2. The provider picks the export command from `opencode --version`. | + +All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. + +## End-to-end tests + +The `e2e/` package checks the tables against real agents and models (spec issue-3635). It creates throwaway repositories and uses Trace in local storage mode. It currently runs the Claude Code, Cursor, OpenCode 2, Pi, and Codex columns. Pi cells are **No**, so the suite skips them. + +The harnesses run as follows: + +- OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace. +- Cursor runs with `cursor-agent -p --output-format stream-json --force --trust` and an isolated `HOME`. Shell startup files restore the test CLI to the front of `PATH`, including after macOS `path_helper` runs. The isolated home has no stored login, so the run needs `CURSOR_API_KEY`. The headless CLI cannot attach a pasted image or report hook messages. +- Pi runs with `pi -p --mode json --approve`. +- Codex runs with `codex exec --json`, an isolated `CODEX_HOME`, no sandbox, and no hook trust prompt. The isolated configuration disables personal and bundled skills while keeping project skills available. For bundled OpenAI models on OpenRouter, the harness preserves native instructions and patch tools by aliasing the model metadata to OpenRouter's prefixed model ID. + +The configured OpenCode 2, Pi, and Codex models use OpenRouter and need `OPENROUTER_API_KEY`. A Codex model without the `openrouter/` prefix uses the default Codex provider and needs `CODEX_API_KEY`. The pinned Codex model is `openrouter/openai/gpt-6-luna`. GPT-5 Mini frequently ended the scripted task before committing and pushing, so it is not the default test model. + +Some scenarios add the git commands of a human after the session, for example a commit that replaces an AI line, an amend, or a rename, and then push again. They work on a feature branch, because the push reads only the commits that are not on the default branch of the remote. + +A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows: + +- A cell marked **No** or **Not applicable** is `skipped`. +- A feature that needs a control plane is `not covered`, because the suite uses local storage mode only. +- A row with no registered check is `no check`. + +`e2e/config.yaml` sets the version of each agent that the suite expects, and the models. The suite does not install the agent. It runs the installed version, and it logs a warning when that version is not the one in the configuration. The suite uses model tokens. To run it: + +```bash +make -C app/cli test-trace-e2e +``` + +To run Codex only, set `OPENROUTER_API_KEY` in your environment, then run: + +```bash +CHAINLOOP_E2E_HARNESS=Codex make -C app/cli test-trace-e2e +``` + +To retain the workspace and agent output for a failure: + +```bash +CHAINLOOP_E2E_HARNESS=Codex \ +CHAINLOOP_E2E_WORK_DIR="$(mktemp -d)" \ +make -C app/cli test-trace-e2e +``` + +Each scenario saves `harness-output.jsonl` next to its repository. The process fails when a **Yes** or **Partial** cell fails. Failures in **Unknown** cells appear in the result matrix without failing the process. + +`CHAINLOOP_E2E_HARNESS`, `CHAINLOOP_E2E_MODEL`, and `CHAINLOOP_E2E_FEATURE` limit a run to some columns, models, or features. Separate the values with `;`, because feature names can hold commas. Set `CHAINLOOP_E2E_WORK_DIR` to keep the workspaces and the agent output. The results go to `e2e/results/`, or to `CHAINLOOP_E2E_RESULTS_DIR`. Only a run with no filter becomes the previous result of the next comparison. diff --git a/app/cli/internal/trace/codex/copy.go b/app/cli/internal/trace/codex/copy.go index cc345d47d..9840fdcb2 100644 --- a/app/cli/internal/trace/codex/copy.go +++ b/app/cli/internal/trace/codex/copy.go @@ -49,7 +49,9 @@ func (p *Provider) CopySessionData(store *state.Store, loc trace.SessionLocation } rec, err := store.LoadSessionRecord(loc.SessionID) if err != nil { - return fmt.Errorf("copy Codex session: load session record: %w", err) + // The record is an optional storage hint. A damaged hint must not + // prevent copying a transcript validated against configured storage. + rec = nil } home := codexHome() if rec != nil && rec.CodexStorageRoot != "" { @@ -154,8 +156,18 @@ func saveStorageRoot(store *state.Store, rec *state.SessionRecord, home string) if err != nil { return err } - rec.CodexStorageRoot = root - return store.SaveSessionRecord(rec) + // Reload after discovery and validation so an overlapping hook's checkout + // registrations and active state are not replaced by the earlier snapshot. + current, err := store.LoadSessionRecord(rec.SessionID) + if err != nil || current == nil { + // This optional hint must not recreate a deleted or damaged record. + return nil + } + if current.CodexStorageRoot != "" { + return nil + } + current.CodexStorageRoot = root + return store.SaveSessionRecord(current) } // Resolve symlinks as well as lexical traversal before opening a hook path. diff --git a/app/cli/internal/trace/codex/copy_metadata_test.go b/app/cli/internal/trace/codex/copy_metadata_test.go new file mode 100644 index 000000000..48a5f092e --- /dev/null +++ b/app/cli/internal/trace/codex/copy_metadata_test.go @@ -0,0 +1,68 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "os" + "path/filepath" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStorageRootPreservesNewerMetadata(t *testing.T) { + store := state.NewOutOfTreeStore(t.TempDir()) + require.NoError(t, store.SaveSessionRecord(&state.SessionRecord{SessionID: testSessionID, Provider: Name, Active: true})) + stale, err := store.LoadSessionRecord(testSessionID) + require.NoError(t, err) + newer := *stale + newer.Active = false + newer.Checkouts = []string{t.TempDir()} + require.NoError(t, store.SaveSessionRecord(&newer)) + home := t.TempDir() + require.NoError(t, saveStorageRoot(store, stale, home)) + got, err := store.LoadSessionRecord(testSessionID) + require.NoError(t, err) + assert.False(t, got.Active) + assert.Equal(t, newer.Checkouts, got.Checkouts) + canonicalHome, err := filepath.EvalSymlinks(home) + require.NoError(t, err) + assert.Equal(t, canonicalHome, got.CodexStorageRoot) +} + +func TestCopyWithDamagedOptionalMetadata(t *testing.T) { + home := t.TempDir() + t.Setenv("CODEX_HOME", home) + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + source := filepath.Join(home, "sessions", "session.jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(source), 0700)) + require.NoError(t, os.WriteFile(source, raw, 0600)) //nolint:gosec // fixture copied into a private temporary Codex home + store := state.NewOutOfTreeStore(t.TempDir()) + require.NoError(t, store.SaveSessionRecord(&state.SessionRecord{SessionID: testSessionID, Provider: Name})) + record := filepath.Join(store.TraceDir(), "sessions", testSessionID+".json") + require.NoError(t, os.WriteFile(record, []byte("{"), 0600)) + require.NoError(t, New().CopySessionData(store, trace.SessionLocation{SessionID: testSessionID, TranscriptPath: source})) + got, err := os.ReadFile(state.RawSessionPath(store.RawSessionDir(), testSessionID)) + require.NoError(t, err) + assert.Equal(t, raw, got) + unchanged, err := os.ReadFile(record) + require.NoError(t, err) + assert.Equal(t, "{", string(unchanged)) +} diff --git a/app/cli/internal/trace/codex/hooks.go b/app/cli/internal/trace/codex/hooks.go index f2a10b216..ad6871db7 100644 --- a/app/cli/internal/trace/codex/hooks.go +++ b/app/cli/internal/trace/codex/hooks.go @@ -33,6 +33,7 @@ const ( settingsFile = ".codex/hooks.json" // Skip tracing when the CLI is missing, before any Go handler can run. hookCommandPrefix = "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec " + windowsHookPrefix = `cmd.exe /d /s /c "where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (` // Codex hook event names. eventSessionStart = "SessionStart" @@ -111,7 +112,7 @@ func (p *Provider) installHookEvents(repoRoot string, events []hookEvent) error changed = removeHookEntry(hooks, eventSessionEnd) } for _, h := range events { - if addHookEntry(hooks, h.event, hookCommandPrefix+h.command+"'", h.matcher) { + if addHookEntry(hooks, h.event, h.command, h.matcher) { changed = true } } @@ -252,7 +253,7 @@ func addHookEntry(hooks map[string]any, event, command, matcher string) bool { existingMatcher, _ := group["matcher"].(string) if len(handlers) == 1 && existingMatcher == matcher { handler, _ := handlers[0].(map[string]any) - if handler["command"] == command { + if handler["command"] == hookCommandPrefix+command+"'" && handler["commandWindows"] == windowsHookPrefix+command+`)"` { return false } } @@ -267,8 +268,9 @@ func buildHookEntry(command, matcher string) map[string]any { entry := map[string]any{ "hooks": []any{ map[string]any{ - "type": "command", - "command": command, + "type": "command", + "command": hookCommandPrefix + command + "'", + "commandWindows": windowsHookPrefix + command + `)"`, }, }, } diff --git a/app/cli/internal/trace/codex/hooks_test.go b/app/cli/internal/trace/codex/hooks_test.go index 6a0cbac03..4ccd16b23 100644 --- a/app/cli/internal/trace/codex/hooks_test.go +++ b/app/cli/internal/trace/codex/hooks_test.go @@ -19,6 +19,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" @@ -26,7 +27,12 @@ import ( "github.com/stretchr/testify/require" ) +const windowsOS = "windows" + func TestInstalledHooksWithoutCLI(t *testing.T) { + if runtime.GOOS == windowsOS { + t.Skip("covered by TestWindowsHookExecution") + } shell, err := exec.LookPath("sh") require.NoError(t, err) bin := t.TempDir() @@ -51,6 +57,9 @@ func TestInstalledHooksWithoutCLI(t *testing.T) { } func TestInstalledHookPreservesIOAndExitStatus(t *testing.T) { + if runtime.GOOS == windowsOS { + t.Skip("covered by TestWindowsHookExecution") + } shell, err := exec.LookPath("sh") require.NoError(t, err) bin := t.TempDir() @@ -92,7 +101,8 @@ func TestInstallUpgradesUnguardedHooks(t *testing.T) { require.NoError(t, p.InstallHooks(root)) first, err := os.ReadFile(p.SettingsFile(root)) require.NoError(t, err) - assert.Equal(t, 1, strings.Count(string(first), "chainloop trace hook codex post-tool-use")) + assert.Equal(t, 1, strings.Count(string(first), "exec chainloop trace hook codex post-tool-use")) + assert.NotContains(t, string(first), `"command": "chainloop trace hook codex post-tool-use"`) assert.Contains(t, string(first), "command -v chainloop") assert.Contains(t, string(first), "echo user-hook") require.NoError(t, p.InstallHooks(root)) @@ -115,3 +125,45 @@ func installedHookCommand(t *testing.T, settings map[string]any, event string) s require.Len(t, handlers, 1) return handlers[0].(map[string]any)["command"].(string) } + +func TestInstalledHooksWindowsOverride(t *testing.T) { + root := t.TempDir() + p := New() + require.NoError(t, p.InstallHooks(root)) + settings, err := readJSONFile(p.SettingsFile(root)) + require.NoError(t, err) + hooks := settings["hooks"].(map[string]any) + for _, event := range hookEvents { + groups := hooks[event.event].([]any) + handlers := groups[0].(map[string]any)["hooks"].([]any) + command := handlers[0].(map[string]any)["commandWindows"].(string) + assert.Contains(t, command, "where chainloop") + assert.Contains(t, command, event.command) + assert.NotContains(t, command, "sh -c") + } +} + +func TestWindowsHookExecution(t *testing.T) { + if runtime.GOOS != windowsOS { + t.Skip("requires the native Windows command interpreter") + } + shell, err := exec.LookPath("cmd.exe") + require.NoError(t, err) + bin := t.TempDir() + t.Setenv("PATH", bin+string(os.PathListSeparator)+filepath.Join(os.Getenv("SystemRoot"), "System32")) + command := windowsHookPrefix + "chainloop trace hook codex post-tool-use" + `)"` + missing := exec.CommandContext(t.Context(), shell, "/d", "/s", "/c", command) + out, err := missing.CombinedOutput() + require.NoError(t, err, string(out)) + assert.Empty(t, out) + stub := "@echo off\r\nset /p payload=\r\necho %*\r\necho %payload%\r\nexit /b 37\r\n" + require.NoError(t, os.WriteFile(filepath.Join(bin, "chainloop.cmd"), []byte(stub), 0600)) + available := exec.CommandContext(t.Context(), shell, "/d", "/s", "/c", command) + available.Stdin = strings.NewReader("{\"session_id\":\"test\"}\n") + out, err = available.Output() + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, 37, exitErr.ExitCode()) + assert.Contains(t, string(out), "trace hook codex post-tool-use") + assert.Contains(t, string(out), `{"session_id":"test"}`) +} diff --git a/app/cli/internal/trace/codex/parse.go b/app/cli/internal/trace/codex/parse.go index 1bfe968e0..a91e3750d 100644 --- a/app/cli/internal/trace/codex/parse.go +++ b/app/cli/internal/trace/codex/parse.go @@ -109,7 +109,11 @@ func (p *Provider) ParseSession(ctx context.Context, opts *trace.ParseOpts) (*ai d.Warnings = append(d.Warnings, fmt.Sprintf("invalid Codex rollout record on line %d", line)) continue } - d.RawSession["main"] = append(d.RawSession["main"], append(json.RawMessage(nil), raw...)) + entry, err := transcriptEntry(e, payload, append(json.RawMessage(nil), raw...), parser.model, line) + if err != nil { + return nil, err + } + d.RawSession["main"] = append(d.RawSession["main"], entry) if err := parser.consume(e, payload, line); err != nil { return nil, err } diff --git a/app/cli/internal/trace/codex/transcript.go b/app/cli/internal/trace/codex/transcript.go new file mode 100644 index 000000000..25ef182ac --- /dev/null +++ b/app/cli/internal/trace/codex/transcript.go @@ -0,0 +1,147 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" +) + +const ( + responseItem = "response_item" + customToolCall = "custom_tool_call" + functionCall = "function_call" + messageItem = "message" +) + +// transcriptEntry normalizes messages and tool exchanges for every consumer, +// while retaining their original event and all unknown metadata records. +func transcriptEntry(e rolloutEntry, payload rolloutPayload, raw json.RawMessage, model string, line int) (json.RawMessage, error) { + if e.Type != responseItem { + return raw, nil + } + switch payload.Type { + case messageItem, functionCall, customToolCall, "function_call_output", "custom_tool_call_output", "reasoning": + default: + return raw, nil + } + var item struct { + Content json.RawMessage `json:"content"` + Arguments string `json:"arguments"` + Input string `json:"input"` + Output json.RawMessage `json:"output"` + Summary []struct { + Text string `json:"text"` + } `json:"summary"` + } + if err := json.Unmarshal(e.Payload, &item); err != nil { + return nil, fmt.Errorf("read Codex transcript item on line %d: %w", line, err) + } + entry := trace.RawSessionEntry{ + UUID: payload.ID, + Timestamp: e.Timestamp, + SourceEvent: raw, + Message: trace.RawSessionMessage{Model: model}, + } + if entry.UUID == "" { + entry.UUID = fmt.Sprintf("codex-event-%d", line) + } + var blocks []any + switch payload.Type { + case messageItem: + if payload.Role != trace.RoleUser && payload.Role != trace.RoleAssistant { + return raw, nil + } + entry.Type = payload.Role + content, err := transcriptContent(item.Content) + if err != nil { + return nil, fmt.Errorf("read Codex message on line %d: %w", line, err) + } + entry.Message.Content = content + case functionCall, customToolCall: + entry.Type = trace.RoleAssistant + input := json.RawMessage(item.Arguments) + var object map[string]json.RawMessage + if payload.Type == customToolCall { + input, _ = json.Marshal(map[string]string{"input": item.Input}) + } else if json.Unmarshal(input, &object) != nil || object == nil { + input, _ = json.Marshal(map[string]string{"arguments": item.Arguments}) + } + blocks = append(blocks, trace.RawSessionToolUseBlock{ + Type: "tool_use", ID: payload.CallID, Name: payload.Name, Input: input, + }) + case "function_call_output", "custom_tool_call_output": + entry.Type = trace.RoleUser + output := item.Output + if len(output) == 0 { + output = json.RawMessage(`""`) + } + if bytes.HasPrefix(bytes.TrimSpace(output), []byte("[")) { + var err error + output, err = transcriptContent(output) + if err != nil { + return nil, fmt.Errorf("read Codex tool result on line %d: %w", line, err) + } + } + blocks = append(blocks, trace.RawSessionToolResultBlock{ + Type: "tool_result", ToolUseID: payload.CallID, Content: output, + }) + case "reasoning": + var summary []string + for _, part := range item.Summary { + if part.Text != "" { + summary = append(summary, part.Text) + } + } + if len(summary) == 0 { + return raw, nil + } + entry.Type = trace.RoleAssistant + blocks = append(blocks, map[string]string{"type": "thinking", "thinking": strings.Join(summary, "\n")}) + default: + return raw, nil + } + entry.Message.Role = entry.Type + if blocks != nil { + entry.Message.Content, _ = json.Marshal(blocks) + } + return json.Marshal(entry) +} + +func transcriptContent(content json.RawMessage) (json.RawMessage, error) { + if len(content) == 0 || bytes.Equal(bytes.TrimSpace(content), []byte("null")) { + return json.RawMessage(`[]`), nil + } + var text string + if json.Unmarshal(content, &text) == nil { + return content, nil + } + var blocks []map[string]any + if err := json.Unmarshal(content, &blocks); err != nil { + return nil, err + } + for _, block := range blocks { + switch block["type"] { + case "input_text", "output_text": + block["type"] = "text" + } + } + return json.Marshal(blocks) +} diff --git a/app/cli/internal/trace/codex/transcript_test.go b/app/cli/internal/trace/codex/transcript_test.go new file mode 100644 index 000000000..91a40a6d1 --- /dev/null +++ b/app/cli/internal/trace/codex/transcript_test.go @@ -0,0 +1,131 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codex + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/internal/schemavalidators" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranscriptContract(t *testing.T) { + raw, err := os.ReadFile("testdata/session.jsonl") + require.NoError(t, err) + dir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(dir, testSessionID), raw, 0600)) //nolint:gosec // temporary fixture path and a fixed session ID + evidence, err := New().ParseSession(t.Context(), &trace.ParseOpts{SessionDir: dir, SessionID: testSessionID}) + require.NoError(t, err) + lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + entries := evidence.Data.RawSession["main"] + require.Len(t, entries, len(lines)) + var texts []string + calls := map[string]trace.RawSessionToolUseBlock{} + results := map[string]string{} + for i, rawEntry := range entries { + var entry trace.RawSessionEntry + require.NoError(t, json.Unmarshal(rawEntry, &entry)) + if entry.Type != trace.RoleUser && entry.Type != trace.RoleAssistant { + assert.JSONEq(t, lines[i], string(rawEntry)) + continue + } + assert.NotEmpty(t, entry.UUID) + assert.Equal(t, entry.Type, entry.Message.Role) + assert.JSONEq(t, lines[i], string(entry.SourceEvent)) + var blocks []json.RawMessage + require.NoError(t, json.Unmarshal(entry.Message.Content, &blocks)) + for _, rawBlock := range blocks { + var block struct{ Type string } + require.NoError(t, json.Unmarshal(rawBlock, &block)) + switch block.Type { + case "text": + var text trace.RawSessionTextBlock + require.NoError(t, json.Unmarshal(rawBlock, &text)) + texts = append(texts, text.Text) + case "tool_use": + var call trace.RawSessionToolUseBlock + require.NoError(t, json.Unmarshal(rawBlock, &call)) + calls[call.ID] = call + case "tool_result": + var result trace.RawSessionToolResultBlock + require.NoError(t, json.Unmarshal(rawBlock, &result)) + var output string + require.NoError(t, json.Unmarshal(result.Content, &output)) + results[result.ToolUseID] = output + } + } + } + assert.Equal(t, []string{"Create hello.txt", "Created hello.txt"}, texts) + require.Len(t, calls, 2) + assert.Equal(t, "apply_patch", calls["call-1"].Name) + assert.Contains(t, string(calls["call-1"].Input), "hello.txt") + assert.Equal(t, "exec_command", calls["call-2"].Name) + assert.JSONEq(t, `{"cmd":"cat hello.txt"}`, string(calls["call-2"].Input)) + assert.Equal(t, map[string]string{"call-1": "Success", "call-2": "hello"}, results) + + encoded, err := json.Marshal(evidence.Data) + require.NoError(t, err) + var data map[string]any + require.NoError(t, json.Unmarshal(encoded, &data)) + require.NoError(t, schemavalidators.ValidateAICodingSession(data, "")) + // The original native-only evidence passed the old schema but is not renderable. + native := make([]any, 0, len(lines)) + for _, line := range lines { + var event any + require.NoError(t, json.Unmarshal([]byte(line), &event)) + native = append(native, event) + } + data["raw_session"] = map[string]any{"main": native} + require.Error(t, schemavalidators.ValidateAICodingSession(data, "")) +} + +func TestTranscriptReasoningAndUnknownEvents(t *testing.T) { + for _, tc := range []struct { + name, payload string + wantType string + }{ + {name: "reasoning summary", payload: `{"type":"reasoning","summary":[{"text":"Check the tests"}]}`, wantType: trace.RoleAssistant}, + {name: "encrypted reasoning", payload: `{"type":"reasoning","encrypted_content":"opaque"}`, wantType: responseItem}, + {name: "future event", payload: `{"type":"future_item","arguments":{"new":"shape"}}`, wantType: responseItem}, + {name: "non-JSON arguments", payload: `{"type":"function_call","name":"Bash","call_id":"c1","arguments":"not JSON"}`, wantType: trace.RoleAssistant}, + {name: "array tool output", payload: `{"type":"custom_tool_call_output","call_id":"c1","output":[{"type":"input_text","text":"Success"}]}`, wantType: trace.RoleUser}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := json.RawMessage(`{"type":"response_item","timestamp":"2026-10-21T10:00:00Z","payload":` + tc.payload + `}`) + var event rolloutEntry + var payload rolloutPayload + require.NoError(t, json.Unmarshal(raw, &event)) + require.NoError(t, json.Unmarshal(event.Payload, &payload)) + got, err := transcriptEntry(event, payload, raw, testModel, 1) + require.NoError(t, err) + var entry trace.RawSessionEntry + require.NoError(t, json.Unmarshal(got, &entry)) + assert.Equal(t, tc.wantType, entry.Type) + if tc.wantType == responseItem { + assert.JSONEq(t, string(raw), string(got)) + } else { + assert.JSONEq(t, string(raw), string(entry.SourceEvent)) + assert.Contains(t, string(entry.Message.Content), map[string]string{"reasoning summary": "Check the tests", "non-JSON arguments": "not JSON", "array tool output": `"type":"text"`}[tc.name]) + } + }) + } +} diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index d368e5a99..30cce9e31 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -17,6 +17,7 @@ package e2e import ( "bytes" + "encoding/json" "errors" "fmt" "io/fs" @@ -28,6 +29,7 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/internal/schemavalidators" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" ) @@ -146,16 +148,88 @@ func evidence(s *Session) *aicodingsession.Data { } func checkTranscript(s *Session) error { - main := evidence(s).RawSession["main"] + d := evidence(s) + main := d.RawSession["main"] if len(main) == 0 { return errors.New("the evidence has no main transcript") } + encoded, err := json.Marshal(d) + if err != nil { + return err + } + var document any + if err := json.Unmarshal(encoded, &document); err != nil { + return err + } + if err := schemavalidators.ValidateAICodingSession(document, ""); err != nil { + return fmt.Errorf("transcript does not satisfy the evidence contract: %w", err) + } + var user, assistant, showsWork bool + calls, results := map[string]bool{}, map[string]bool{} for _, msg := range main { - if bytes.Contains(msg, []byte("greeting.txt")) { - return nil + var entry trace.RawSessionEntry + if err := json.Unmarshal(msg, &entry); err != nil { + return err + } + role := entry.Type + if role == "" { + role = entry.Role + } + if role != trace.RoleUser && role != trace.RoleAssistant { + continue + } + readable := transcriptHasText(entry.Message.Content) + var blocks []struct { + Type, Text, ID, Name string + ToolUseID string `json:"tool_use_id"` + Content json.RawMessage `json:"content"` + } + if json.Unmarshal(entry.Message.Content, &blocks) == nil { + for _, block := range blocks { + switch block.Type { + case "tool_use": + calls[block.ID] = true + readable = true + case "tool_result": + results[block.ToolUseID] = transcriptHasText(block.Content) + } + } + } + user = user || role == trace.RoleUser && readable + assistant = assistant || role == trace.RoleAssistant && readable + showsWork = showsWork || bytes.Contains(entry.Message.Content, []byte("greeting.txt")) + } + if !user || !assistant { + return errors.New("the transcript has no renderable user and assistant messages") + } + if !showsWork { + return errors.New("the normalized transcript does not show the work of the session") + } + if d.Agent.Name == "codex" && d.ToolsUsed != nil && d.ToolsUsed.TotalInvocations > 0 { + for id := range calls { + if results[id] { + return nil + } + } + return errors.New("the Codex transcript has no matching tool call and result") + } + return nil +} + +func transcriptHasText(content json.RawMessage) bool { + var text string + if json.Unmarshal(content, &text) == nil { + return strings.TrimSpace(text) != "" + } + var blocks []trace.RawSessionTextBlock + if json.Unmarshal(content, &blocks) == nil { + for _, block := range blocks { + if block.Type == "text" && strings.TrimSpace(block.Text) != "" { + return true + } } } - return errors.New("the transcript does not show the work of the session") + return false } func checkTokenUsage(s *Session) error { diff --git a/app/cli/internal/trace/e2e/codex.go b/app/cli/internal/trace/e2e/codex.go index 98e5c74c5..565decb75 100644 --- a/app/cli/internal/trace/e2e/codex.go +++ b/app/cli/internal/trace/e2e/codex.go @@ -135,7 +135,7 @@ func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { } args = append(args, codexModelArgs(req.Model)...) if strings.HasPrefix(req.Model, codexOpenRouterPrefix+"openai/") { - export := exec.CommandContext(ctx, c.bin(), "debug", "models", "--bundled") //nolint:gosec // the harness binary under test + export := exec.CommandContext(ctx, c.bin(), "debug", "models", "--bundled") //nolint:gosec // the harness binary under test // nosemgrep export.Env = append(slices.Clone(req.Env), "CODEX_HOME="+codexHome) raw, err := export.Output() if err != nil { diff --git a/app/cli/internal/trace/e2e/transcript_test.go b/app/cli/internal/trace/e2e/transcript_test.go new file mode 100644 index 000000000..f554a7ac4 --- /dev/null +++ b/app/cli/internal/trace/e2e/transcript_test.go @@ -0,0 +1,67 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "testing" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/stretchr/testify/require" +) + +func TestTranscriptRequiresRenderableMessages(t *testing.T) { + const prompt = `{"type":"user","message":{"content":[{"type":"text","text":"Create greeting.txt"}]}}` + const answer = `{"type":"assistant","message":{"content":[{"type":"text","text":"Created greeting.txt"}]}}` + const call = `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"call-1","name":"apply_patch","input":{"input":"Create greeting.txt"}}]}}` + const result = `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","content":"Success"}]}}` + for _, tc := range []struct { + name string + entries []string + tools int + wantErr bool + }{ + {name: "messages", entries: []string{prompt, answer}}, + {name: "completed tool exchange", entries: []string{prompt, call, result}, tools: 1}, + {name: "native messages alone", entries: []string{`{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Create greeting.txt"}]}}`}, wantErr: true}, + {name: "filename only in metadata", entries: []string{`{"type":"user","message":{"content":"Hello"},"source_event":{"file":"greeting.txt"}}`, `{"type":"assistant","message":{"content":"Hello"}}`}, wantErr: true}, + {name: "no assistant", entries: []string{prompt}, wantErr: true}, + {name: "missing tool result", entries: []string{prompt, call}, tools: 1, wantErr: true}, + {name: "empty tool result", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","content":""}]}}`}, tools: 1, wantErr: true}, + {name: "unrelated tool result", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"other","content":"Success"}]}}`}, tools: 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var entries []json.RawMessage + for _, entry := range tc.entries { + entries = append(entries, json.RawMessage(entry)) + } + d := aicodingsession.Data{ + SchemaVersion: "v1", + Agent: aicodingsession.Agent{Name: "codex"}, + Session: aicodingsession.Session{ID: "test", StartedAt: "2026-10-21T10:00:00Z"}, + Conversation: &aicodingsession.Conversation{TotalMessages: 2}, + ToolsUsed: &aicodingsession.ToolsUsed{TotalInvocations: tc.tools}, + RawSession: map[string][]json.RawMessage{"main": entries}, + } + s := &Session{Stored: &StoredSession{Evidence: *aicodingsession.NewEvidence(d)}} + if tc.wantErr { + require.Error(t, checkTranscript(s)) + } else { + require.NoError(t, checkTranscript(s)) + } + }) + } +} diff --git a/app/cli/internal/trace/pi/parse.go b/app/cli/internal/trace/pi/parse.go index b188b0b56..a7d74e8b2 100644 --- a/app/cli/internal/trace/pi/parse.go +++ b/app/cli/internal/trace/pi/parse.go @@ -27,6 +27,7 @@ import ( "strconv" "time" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" ) @@ -279,7 +280,7 @@ func buildEvidence(parsed *parsedSession, agentVersion string) *aicodingsession. for _, entry := range parsed.Selected { if !isTreeMarker(entry) { - raw = append(raw, craftableRaw(entry.Raw)) + raw = append(raw, transcriptRaw(entry)) } if entry.Type != "message" { continue @@ -359,6 +360,52 @@ func isTreeMarker(entry *sessionEntry) bool { return entry.Type == "custom" && entry.CustomType == treeMarkerCustomType } +// transcriptRaw keeps native metadata and exposes messages in the shared format. +func transcriptRaw(entry *sessionEntry) json.RawMessage { + raw := craftableRaw(entry.Raw) + if entry.Type != "message" { + return raw + } + var fields map[string]json.RawMessage + var message map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil || json.Unmarshal(fields["message"], &message) != nil || message == nil { + return raw + } + role, _ := rawString(message["role"]) + if role != trace.RoleUser && role != trace.RoleAssistant && role != "toolResult" { + return raw + } + content := message["content"] + if len(content) == 0 || bytes.Equal(content, []byte("null")) { + content = json.RawMessage(`[]`) + } + if role == "toolResult" { + callID, _ := rawString(message["toolCallId"]) + content, _ = json.Marshal([]trace.RawSessionToolResultBlock{{Type: "tool_result", ToolUseID: callID, Content: content}}) + role = trace.RoleUser + } else { + var blocks []map[string]json.RawMessage + if json.Unmarshal(content, &blocks) == nil { + for _, block := range blocks { + if kind, _ := rawString(block["type"]); kind == "toolCall" { + block["type"] = json.RawMessage(`"tool_use"`) + block["input"] = block["arguments"] + delete(block, "arguments") + } + } + content, _ = json.Marshal(blocks) + } + } + message["content"] = content + message["role"], _ = json.Marshal(role) + fields["type"] = message["role"] + fields["uuid"] = fields["id"] + fields["source_event"] = raw + fields["message"], _ = json.Marshal(message) + normalized, _ := json.Marshal(fields) + return normalized +} + // craftableRaw preserves a native Pi entry unless encoding/json cannot decode // one of its numbers into the generic float64 representation used by the // evidence crafter. In that case only out-of-range numbers become null; the diff --git a/app/cli/internal/trace/pi/testdata/session-v3.golden.json b/app/cli/internal/trace/pi/testdata/session-v3.golden.json index eff98a6e2..75942dd8d 100644 --- a/app/cli/internal/trace/pi/testdata/session-v3.golden.json +++ b/app/cli/internal/trace/pi/testdata/session-v3.golden.json @@ -55,38 +55,47 @@ "raw_session": { "main": [ { - "type": "message", "id": "u0000001", + "message": { + "content": "start", + "role": "user" + }, "parentId": null, + "source_event": { + "type": "message", + "id": "u0000001", + "parentId": null, + "timestamp": "2026-01-02T03:04:01Z", + "message": { + "role": "user", + "content": "start" + } + }, "timestamp": "2026-01-02T03:04:01Z", - "message": { - "role": "user", - "content": "start" - } + "type": "user", + "uuid": "u0000001" }, { - "type": "message", "id": "a0000001", - "parentId": "u0000001", - "timestamp": "2026-01-02T03:04:02Z", "message": { - "role": "assistant", - "provider": "anthropic", - "model": "claude-sonnet-4", "content": [ { - "type": "text", - "text": "working" + "text": "working", + "type": "text" }, { - "type": "toolCall", "id": "call-1", - "name": "edit", - "arguments": { + "input": { "path": "a.go" - } + }, + "name": "edit", + "type": "tool_use" } ], + "model": "claude-sonnet-4", + "provider": "anthropic", + "role": "assistant", + "stopReason": "toolUse", "usage": { "input": 100, "output": 20, @@ -96,9 +105,48 @@ "cost": { "total": 0.1 } - }, - "stopReason": "toolUse" - } + } + }, + "parentId": "u0000001", + "source_event": { + "type": "message", + "id": "a0000001", + "parentId": "u0000001", + "timestamp": "2026-01-02T03:04:02Z", + "message": { + "role": "assistant", + "provider": "anthropic", + "model": "claude-sonnet-4", + "content": [ + { + "type": "text", + "text": "working" + }, + { + "type": "toolCall", + "id": "call-1", + "name": "edit", + "arguments": { + "path": "a.go" + } + } + ], + "usage": { + "input": 100, + "output": 20, + "cacheRead": 10, + "cacheWrite": 5, + "totalTokens": 135, + "cost": { + "total": 0.1 + } + }, + "stopReason": "toolUse" + } + }, + "timestamp": "2026-01-02T03:04:02Z", + "type": "assistant", + "uuid": "a0000001" }, { "type": "branch_summary", @@ -109,42 +157,51 @@ "summary": "left the abandoned branch" }, { - "type": "message", "id": "u0000003", + "message": { + "content": "continue", + "role": "user" + }, "parentId": "m0000001", + "source_event": { + "type": "message", + "id": "u0000003", + "parentId": "m0000001", + "timestamp": "2026-01-02T03:04:09Z", + "message": { + "role": "user", + "content": "continue" + } + }, "timestamp": "2026-01-02T03:04:09Z", - "message": { - "role": "user", - "content": "continue" - } + "type": "user", + "uuid": "u0000003" }, { - "type": "message", "id": "a0000003", - "parentId": "u0000003", - "timestamp": "2026-01-02T03:04:10Z", "message": { - "role": "assistant", - "provider": "openai", - "model": "gpt-5", "content": [ { - "type": "toolCall", "id": "call-2", - "name": "bash", - "arguments": { + "input": { "command": "go test ./..." - } + }, + "name": "bash", + "type": "tool_use" }, { - "type": "toolCall", "id": "call-3", - "name": "edit", - "arguments": { + "input": { "path": "b.go" - } + }, + "name": "edit", + "type": "tool_use" } ], + "model": "gpt-5", + "provider": "openai", + "role": "assistant", + "stopReason": "toolUse", "usage": { "input": 200, "output": 40, @@ -154,9 +211,52 @@ "cost": { "total": 0.2 } - }, - "stopReason": "toolUse" - } + } + }, + "parentId": "u0000003", + "source_event": { + "type": "message", + "id": "a0000003", + "parentId": "u0000003", + "timestamp": "2026-01-02T03:04:10Z", + "message": { + "role": "assistant", + "provider": "openai", + "model": "gpt-5", + "content": [ + { + "type": "toolCall", + "id": "call-2", + "name": "bash", + "arguments": { + "command": "go test ./..." + } + }, + { + "type": "toolCall", + "id": "call-3", + "name": "edit", + "arguments": { + "path": "b.go" + } + } + ], + "usage": { + "input": 200, + "output": 40, + "cacheRead": 20, + "cacheWrite": 0, + "totalTokens": 260, + "cost": { + "total": 0.2 + } + }, + "stopReason": "toolUse" + } + }, + "timestamp": "2026-01-02T03:04:10Z", + "type": "assistant", + "uuid": "a0000003" }, { "type": "compaction", @@ -177,34 +277,44 @@ "display": false }, { - "type": "message", "id": "u0000004", + "message": { + "content": "finish", + "role": "user" + }, "parentId": "cm000002", + "source_event": { + "type": "message", + "id": "u0000004", + "parentId": "cm000002", + "timestamp": "2026-01-02T03:04:15Z", + "message": { + "role": "user", + "content": "finish" + } + }, "timestamp": "2026-01-02T03:04:15Z", - "message": { - "role": "user", - "content": "finish" - } + "type": "user", + "uuid": "u0000004" }, { - "type": "message", "id": "a0000004", - "parentId": "u0000004", - "timestamp": "2026-01-02T03:04:16Z", "message": { - "role": "assistant", - "provider": "openai", - "model": "gpt-5", "content": [ { - "type": "toolCall", "id": "call-4", - "name": "write", - "arguments": { + "input": { "path": "c.go" - } + }, + "name": "write", + "type": "tool_use" } ], + "errorMessage": "failed", + "model": "gpt-5", + "provider": "openai", + "role": "assistant", + "stopReason": "error", "usage": { "input": -1, "output": 1.5, @@ -214,10 +324,45 @@ "cost": { "total": "bad" } - }, - "stopReason": "error", - "errorMessage": "failed" - } + } + }, + "parentId": "u0000004", + "source_event": { + "type": "message", + "id": "a0000004", + "parentId": "u0000004", + "timestamp": "2026-01-02T03:04:16Z", + "message": { + "role": "assistant", + "provider": "openai", + "model": "gpt-5", + "content": [ + { + "type": "toolCall", + "id": "call-4", + "name": "write", + "arguments": { + "path": "c.go" + } + } + ], + "usage": { + "input": -1, + "output": 1.5, + "cacheRead": "bad", + "cacheWrite": 2, + "totalTokens": 999999999999999999999999999, + "cost": { + "total": "bad" + } + }, + "stopReason": "error", + "errorMessage": "failed" + } + }, + "timestamp": "2026-01-02T03:04:16Z", + "type": "assistant", + "uuid": "a0000004" }, { "type": "future_entry", @@ -229,22 +374,41 @@ } }, { - "type": "message", "id": "a0000005", - "parentId": "q0000001", - "timestamp": "2026-01-02T03:04:18Z", "message": { - "role": "assistant", - "provider": "google", - "model": "gemini-2.5-pro", "content": [ { - "type": "text", - "text": "aborted response" + "text": "aborted response", + "type": "text" } ], + "model": "gemini-2.5-pro", + "provider": "google", + "role": "assistant", "stopReason": "aborted" - } + }, + "parentId": "q0000001", + "source_event": { + "type": "message", + "id": "a0000005", + "parentId": "q0000001", + "timestamp": "2026-01-02T03:04:18Z", + "message": { + "role": "assistant", + "provider": "google", + "model": "gemini-2.5-pro", + "content": [ + { + "type": "text", + "text": "aborted response" + } + ], + "stopReason": "aborted" + } + }, + "timestamp": "2026-01-02T03:04:18Z", + "type": "assistant", + "uuid": "a0000005" } ] }, diff --git a/app/cli/internal/trace/types.go b/app/cli/internal/trace/types.go index f776959ee..383acc41f 100644 --- a/app/cli/internal/trace/types.go +++ b/app/cli/internal/trace/types.go @@ -24,21 +24,23 @@ const ( AttributionAI = "ai" // AttributionHuman marks file changes produced by a human. AttributionHuman = "human" + + RoleUser = "user" + RoleAssistant = "assistant" ) -// RawSessionEntry is the normalized wire format most providers use for a -// single message in raw_session["main"]. A provider whose persisted format -// carries a native timeline may retain those objects instead, so valid unknown -// entries are not discarded. Consumers can select a renderer by Agent.Name. -// This shape mirrors Claude's JSONL transcript format, which the frontend was -// originally built against. +// RawSessionEntry is the shared wire format for messages in raw_session. +// Providers must normalize conversation and tool records into this shape. +// Native metadata events may remain alongside messages; they cannot replace +// normalized messages. SourceEvent preserves the original event when converting. type RawSessionEntry struct { - Type string `json:"type,omitempty"` - Role string `json:"role,omitempty"` - UUID string `json:"uuid,omitempty"` - Timestamp string `json:"timestamp,omitempty"` - IsMeta bool `json:"isMeta,omitempty"` - Message RawSessionMessage `json:"message"` + Type string `json:"type,omitempty"` + Role string `json:"role,omitempty"` + UUID string `json:"uuid,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + IsMeta bool `json:"isMeta,omitempty"` + Message RawSessionMessage `json:"message"` + SourceEvent json.RawMessage `json:"source_event,omitempty"` } // RawSessionMessage is the message body inside a RawSessionEntry. @@ -62,6 +64,14 @@ type RawSessionToolUseBlock struct { Input json.RawMessage `json:"input,omitempty"` } +// RawSessionToolResultBlock carries the result of a tool_use with the same ID. +// It belongs to a user-shaped entry, not to a new user prompt. +type RawSessionToolResultBlock struct { + Type string `json:"type"` + ToolUseID string `json:"tool_use_id"` + Content json.RawMessage `json:"content"` +} + // MustRawSessionEntry marshals a RawSessionEntry to json.RawMessage. // Panics only on impossible encoding failures (struct field bugs). func MustRawSessionEntry(e RawSessionEntry) json.RawMessage { diff --git a/docs/specs/issue-3657-codex-trace-provider.md b/docs/specs/issue-3657-codex-trace-provider.md index 75bb51022..a9e3fa944 100644 --- a/docs/specs/issue-3657-codex-trace-provider.md +++ b/docs/specs/issue-3657-codex-trace-provider.md @@ -50,7 +50,7 @@ Installation MUST preserve user hooks and produce identical content on repeated ### R-003: Session evidence -Trace MUST capture session identity, timing, agent version, conversations, tool calls, and model information from persisted Codex transcripts. Missing or unreadable data MUST produce a warning or parsing error. Repeated cumulative usage records MUST count tokens once. Cost MUST use embedded API prices and identify an unknown-model fallback. +Trace MUST capture session identity, timing, agent version, conversations, tool calls, and model information from persisted Codex transcripts. Conversation and tool records MUST use the shared transcript wire format in `app/cli/internal/trace/PROVIDERS.md`. The original converted event MUST remain in `source_event`. Missing or unreadable data MUST produce a warning or parsing error. Repeated cumulative usage records MUST count tokens once. Cost MUST use embedded API prices and identify an unknown-model fallback. ### R-004: Code attribution @@ -72,7 +72,7 @@ Native hook behavior follows Codex CLI 0.163.0. Codex requires users to trust pr The developer selects Codex during Trace initialization, reviews the installed hooks in Codex, and starts a session. Codex sends session and tool events to Trace. Trace records each patch path separately and uses working-tree snapshots for shell changes. -Trace copies the Codex rollout into private storage at session boundaries and before evidence assembly. The parser retains native JSONL records and derives summaries from metadata, response items, turn context, and cumulative token counters. Git hooks use the existing evidence pipeline. +Trace copies the Codex rollout into private storage at session boundaries and before evidence assembly. The parser normalizes conversation and tool records and preserves their original events in `source_event`. Native metadata records remain unchanged. It derives summaries from metadata, response items, turn context, and cumulative token counters. Git hooks use the existing evidence pipeline. ```mermaid flowchart LR diff --git a/go.mod b/go.mod index 2df6be5df..32aa47669 100644 --- a/go.mod +++ b/go.mod @@ -76,6 +76,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/openvex/go-vex v0.2.9 github.com/owenrumney/go-sarif/v3 v3.3.1 + github.com/pelletier/go-toml/v2 v2.4.3 github.com/posthog/posthog-go v1.34.0 github.com/prometheus/client_golang v1.25.0 github.com/prometheus/common v0.72.1 @@ -376,7 +377,6 @@ require ( github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pjbgf/sha1cd v0.7.0 // indirect diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index 02ad74465..96bfeb1c3 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -18,7 +18,9 @@ "agent": { "type": "object", "description": "AI agent provider information", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string", @@ -35,7 +37,11 @@ "session": { "type": "object", "description": "Session timing and identity", - "required": ["id", "started_at", "duration_seconds"], + "required": [ + "id", + "started_at", + "duration_seconds" + ], "properties": { "id": { "type": "string", @@ -205,7 +211,10 @@ "description": "Per-file change details", "items": { "type": "object", - "required": ["path", "status"], + "required": [ + "path", + "status" + ], "properties": { "path": { "type": "string", @@ -232,7 +241,10 @@ "description": "Contiguous line ranges", "items": { "type": "object", - "required": ["start", "end"], + "required": [ + "start", + "end" + ], "properties": { "start": { "type": "integer", @@ -331,9 +343,15 @@ }, "raw_session": { "type": "object", - "description": "Raw session conversation data keyed by thread name", + "description": "Transcript streams keyed by thread name. Conversation and tool events use the shared message format. Unknown native metadata may be retained alongside them; source_event preserves an original converted event.", + "required": [ + "main" + ], "additionalProperties": { - "type": "array" + "type": "array", + "items": { + "$ref": "#/definitions/transcriptEntry" + } } }, "warnings": { @@ -344,5 +362,401 @@ } } }, - "additionalProperties": false + "additionalProperties": false, + "definitions": { + "transcriptMessage": { + "type": "object", + "required": [ + "message" + ], + "anyOf": [ + { + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "user", + "assistant" + ] + } + } + }, + { + "required": [ + "role" + ], + "properties": { + "role": { + "enum": [ + "user", + "assistant" + ] + } + } + } + ], + "properties": { + "message": { + "type": "object", + "required": [ + "content" + ], + "properties": { + "role": { + "type": "string" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/transcriptBlock" + } + } + ] + } + } + } + } + }, + "transcriptEntry": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "role": { + "type": "string" + }, + "uuid": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "source_event": { + "type": "object" + } + }, + "allOf": [ + { + "if": { + "anyOf": [ + { + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "user", + "assistant" + ] + } + } + }, + { + "required": [ + "role" + ], + "properties": { + "role": { + "enum": [ + "user", + "assistant" + ] + } + } + } + ] + }, + "then": { + "anyOf": [ + { + "$ref": "#/definitions/transcriptMessage" + }, + { + "$ref": "#/definitions/transcriptError" + } + ] + } + }, + { + "if": { + "required": [ + "type", + "payload" + ], + "properties": { + "type": { + "const": "response_item" + }, + "payload": { + "anyOf": [ + { + "required": [ + "type", + "role" + ], + "properties": { + "type": { + "const": "message" + }, + "role": { + "enum": [ + "user", + "assistant" + ] + } + } + }, + { + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "function_call", + "custom_tool_call", + "function_call_output", + "custom_tool_call_output" + ] + } + } + } + ] + } + } + }, + "then": { + "$ref": "#/definitions/transcriptMessage" + } + }, + { + "if": { + "required": [ + "message" + ], + "properties": { + "message": { + "type": "object", + "required": [ + "role" + ], + "properties": { + "role": { + "enum": [ + "user", + "assistant" + ] + } + } + } + } + }, + "then": { + "anyOf": [ + { + "$ref": "#/definitions/transcriptMessage" + }, + { + "$ref": "#/definitions/transcriptError" + } + ] + } + } + ] + }, + "transcriptBlock": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "not": { + "enum": [ + "input_text", + "output_text" + ] + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "text" + } + } + }, + "then": { + "required": [ + "text" + ], + "properties": { + "text": { + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "tool_use" + } + } + }, + "then": { + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "input": { + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "tool_result" + } + } + }, + "then": { + "required": [ + "tool_use_id", + "content" + ], + "properties": { + "tool_use_id": { + "type": "string", + "minLength": 1 + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/transcriptBlock" + } + }, + { + "type": "object" + } + ] + } + } + } + } + ] + }, + "transcriptError": { + "type": "object", + "required": [ + "isApiErrorMessage", + "error" + ], + "properties": { + "isApiErrorMessage": { + "const": true + }, + "error": { + "type": "string", + "minLength": 1 + } + }, + "anyOf": [ + { + "required": [ + "type" + ], + "properties": { + "type": { + "const": "assistant" + } + } + }, + { + "required": [ + "role" + ], + "properties": { + "role": { + "const": "assistant" + } + } + } + ] + } + }, + "allOf": [ + { + "if": { + "required": [ + "raw_session", + "conversation" + ], + "properties": { + "conversation": { + "required": [ + "total_messages" + ], + "properties": { + "total_messages": { + "type": "integer", + "minimum": 1 + } + } + } + } + }, + "then": { + "properties": { + "raw_session": { + "properties": { + "main": { + "contains": { + "anyOf": [ + { + "$ref": "#/definitions/transcriptMessage" + }, + { + "$ref": "#/definitions/transcriptError" + } + ] + } + } + } + } + } + } + } + ] } diff --git a/internal/schemavalidators/transcript_test.go b/internal/schemavalidators/transcript_test.go new file mode 100644 index 000000000..78015b051 --- /dev/null +++ b/internal/schemavalidators/transcript_test.go @@ -0,0 +1,70 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemavalidators_test + +import ( + "encoding/json" + "os" + "testing" + + "github.com/chainloop-dev/chainloop/internal/schemavalidators" + "github.com/stretchr/testify/require" +) + +func TestTranscriptContract(t *testing.T) { + for _, tc := range []struct { + name, transcript string + messages int + wantErr bool + }{ + {name: "Claude text", transcript: `{"main":[{"type":"user","message":{"content":"Hello"}}]}`, messages: 1}, + {name: "Cursor role", transcript: `{"main":[{"role":"assistant","message":{"content":[{"type":"text","text":"Hello"}]}}]}`, messages: 1}, + {name: "assistant API error", transcript: `{"main":[{"type":"assistant","isApiErrorMessage":true,"error":"request failed"}]}`, messages: 1}, + {name: "normalized Codex with original event", transcript: `{"main":[{"type":"assistant","message":{"content":[{"type":"tool_use","id":"c1","name":"Bash","input":{"command":"pwd"}}]},"source_event":{"type":"response_item","payload":{"type":"function_call"}}},{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"c1","content":"/repo"}]}}]}`, messages: 1}, + {name: "unknown metadata alongside messages", transcript: `{"main":[{"type":"future_event","value":1},{"type":"user","message":{"content":"Hello"}}]}`, messages: 1}, + {name: "metadata without conversation", transcript: `{"main":[{"type":"session_meta","payload":{}}]}`}, + {name: "empty transcript without conversation", transcript: `{"main":[]}`}, + {name: "native conversation without normalized message", transcript: `{"main":[{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Hello"}]}}]}`, messages: 1, wantErr: true}, + {name: "native conversation without summary", transcript: `{"main":[{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Hello"}]}}]}`, wantErr: true}, + {name: "missing main", transcript: `{}`, wantErr: true}, + {name: "non-object entry", transcript: `{"main":["Hello"]}`, wantErr: true}, + {name: "message without content", transcript: `{"main":[{"type":"user","message":{}}]}`, messages: 1, wantErr: true}, + {name: "native role buried in message", transcript: `{"main":[{"type":"message","message":{"role":"user","content":"Hello"}}]}`, wantErr: true}, + {name: "content object", transcript: `{"main":[{"type":"assistant","message":{"content":{"text":"Hello"}}}]}`, messages: 1, wantErr: true}, + {name: "block without type", transcript: `{"main":[{"type":"user","message":{"content":[{"text":"Hello"}]}}]}`, messages: 1, wantErr: true}, + {name: "text block without text", transcript: `{"main":[{"type":"user","message":{"content":[{"type":"text"}]}}]}`, messages: 1, wantErr: true}, + {name: "call without id", transcript: `{"main":[{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash"}]}}]}`, messages: 1, wantErr: true}, + {name: "result without correlation id", transcript: `{"main":[{"type":"user","message":{"content":[{"type":"tool_result","content":"ok"}]}}]}`, messages: 1, wantErr: true}, + {name: "native text in tool output", transcript: `{"main":[{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"c1","content":[{"type":"input_text","text":"ok"}]}]}}]}`, messages: 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, err := os.ReadFile("testdata/ai_coding_session_valid.json") + require.NoError(t, err) + var data map[string]any + require.NoError(t, json.Unmarshal(fixture, &data)) + var transcript any + require.NoError(t, json.Unmarshal([]byte(tc.transcript), &transcript)) + data["raw_session"] = transcript + data["conversation"] = map[string]any{"total_messages": tc.messages} + err = schemavalidators.ValidateAICodingSession(data, "") + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/pkg/attestation/crafter/materials/aicodingsession/redact_test.go b/pkg/attestation/crafter/materials/aicodingsession/redact_test.go index cd72cd1e6..fbfc070a9 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/redact_test.go +++ b/pkg/attestation/crafter/materials/aicodingsession/redact_test.go @@ -278,7 +278,7 @@ func TestRedactSkipsMedia(t *testing.T) { }, { name: "text field named data is still scanned", - entry: `{"type":"user","message":{"role":"user","content":[{"type":"text","data":"` + media + `"}]}}`, + entry: `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"Text with extra metadata","data":"` + media + `"}]}}`, }, } From b9e3f8b15e3dcc9c9cc2a953e5613929e863a097 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 15:27:08 +0200 Subject: [PATCH 5/8] fix(trace): scope transcript validation to Codex Keep Codex normalization and viewer regressions while restoring the open raw_session schema and existing Pi parsing. Check renderable messages, valid tool calls, and matching readable results in the Codex E2E check without constraining other provider formats. Restore the original provider guidance and spec wording. Signed-off-by: Miguel Martinez Trivino Assisted-by: Codex --- app/cli/internal/trace/PROVIDERS.md | 16 +- app/cli/internal/trace/README.md | 2 +- .../internal/trace/codex/transcript_test.go | 16 - app/cli/internal/trace/e2e/checks.go | 29 +- app/cli/internal/trace/e2e/transcript_test.go | 16 + app/cli/internal/trace/pi/parse.go | 49 +- .../trace/pi/testdata/session-v3.golden.json | 304 +++---------- app/cli/internal/trace/types.go | 10 +- docs/specs/issue-3657-codex-trace-provider.md | 4 +- .../ai-coding-session-0.1.schema.json | 428 +----------------- internal/schemavalidators/transcript_test.go | 70 --- .../materials/aicodingsession/redact_test.go | 2 +- 12 files changed, 122 insertions(+), 824 deletions(-) delete mode 100644 internal/schemavalidators/transcript_test.go diff --git a/app/cli/internal/trace/PROVIDERS.md b/app/cli/internal/trace/PROVIDERS.md index fe7e6a8d8..34a5f4425 100644 --- a/app/cli/internal/trace/PROVIDERS.md +++ b/app/cli/internal/trace/PROVIDERS.md @@ -127,25 +127,11 @@ Fill the evidence (`pkg/attestation/crafter/materials/aicodingsession`) as follo | `ToolsUsed` | A count per tool, sorted by count, then by name. | | | | | `Conversation` | User and assistant message counts. | | | | | `Subagents` | One entry per subagent, with its tokens. | Yes. | No. | No. | -| `RawSession` | A map of streams. `main` is required: use an empty slice, not `nil`. Other keys are subagent streams. Conversation and tool records MUST use the shared transcript format below, even when the agent stores native JSONL. | Transcript lines. | Transcript lines. | Rebuilt by `buildRawSessionEntries`. | +| `RawSession` | A map of streams. `main` is required: use an empty slice, not `nil`. Other keys are subagent streams. Each line must be one JSON object. If the agent has no line format, convert each message to the Claude line shape (`{type, uuid, timestamp, message: {role, content}}`) so the UI can show it. | Transcript lines. | Transcript lines. | Rebuilt by `buildRawSessionEntries`. | | `Warnings` | Each data item that the parser could not read or that the agent does not give. | | Always "no token usage". | | The caller sets `Session.Mode`, `GitContext`, `CodeChanges`, `Spec`, and the skills. Do not set them in the provider. Claude sets `GitContext.Branch` and `WorkDir` from the transcript, but the push replaces all of `GitContext` when it can read the repository. -#### Transcript wire contract - -`trace.RawSessionEntry` defines the message envelope. The AI coding-session JSON schema validates it. Consumers MUST NOT need an agent-specific parser to read conversation text or tool exchanges. - -- A message MUST identify its role with `type: "user"` or `type: "assistant"`. Consumers also accept the Cursor `role` field when `type` is absent. -- `message.content` MUST be a string or an array of content blocks. A text block MUST have `type: "text"` and a string `text` field. -- A tool call MUST be an assistant content block with `type: "tool_use"`, a nonempty `id`, and a nonempty `name`. Its optional `input` MUST be an object. -- A tool result MUST be a user content block with `type: "tool_result"`, `tool_use_id`, and `content`. Its `tool_use_id` MUST match the call's `id`. This carrier is not a new user prompt. A call still running at capture time may have no result yet. -- Providers SHOULD include stable `uuid` values and RFC3339 `timestamp` values. They MUST preserve message order. -- A converted entry MAY retain its original native record in `source_event`. Unknown native metadata MAY remain as separate JSON objects. Neither can replace normalized conversation or tool records. -- A session with conversation messages and a stored transcript MUST contain a normalized message in `main`. Empty or metadata-only transcripts are valid only when no conversation was captured. - -The E2E transcript check MUST validate the schema and find renderable user and assistant content. It MUST find scenario work in normalized content, not just in native metadata. Codex also checks for a completed tool exchange with matching IDs. The final in-progress push call need not have a result yet. - Cost: - If the agent records the cost, use its value (OpenCode, Pi). diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index b6e66b2cc..8c39be810 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -18,7 +18,7 @@ Codex token usage comes from cumulative rollout counters. Cost is an estimate fr In a linked Git worktree, Codex 0.163 reads project hooks from the primary checkout. Install the Codex hooks in that checkout too, then review and trust them in Codex. -Codex evidence converts native messages, tool calls, and tool results to the [shared transcript wire contract](PROVIDERS.md#transcript-wire-contract). Each converted entry retains its original event in `source_event`. The transcript E2E check validates the schema, renderable messages, and a completed tool exchange. A nonempty native transcript alone does not pass. +Codex evidence converts native messages, tool calls, and tool results to the message shape used by the existing session viewer. Text uses `message.content`, calls use `tool_use` blocks, and results use `tool_result` blocks with matching IDs. Each converted entry retains its original event in `source_event`; native metadata stays unchanged. Codex regression and E2E checks require renderable messages and a completed tool exchange, including readable tool output. A nonempty native transcript alone does not pass. These checks apply to Codex; `raw_session` remains open in the shared evidence schema. ## Setup diff --git a/app/cli/internal/trace/codex/transcript_test.go b/app/cli/internal/trace/codex/transcript_test.go index 91a40a6d1..59838726e 100644 --- a/app/cli/internal/trace/codex/transcript_test.go +++ b/app/cli/internal/trace/codex/transcript_test.go @@ -23,7 +23,6 @@ import ( "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" - "github.com/chainloop-dev/chainloop/internal/schemavalidators" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -81,21 +80,6 @@ func TestTranscriptContract(t *testing.T) { assert.Equal(t, "exec_command", calls["call-2"].Name) assert.JSONEq(t, `{"cmd":"cat hello.txt"}`, string(calls["call-2"].Input)) assert.Equal(t, map[string]string{"call-1": "Success", "call-2": "hello"}, results) - - encoded, err := json.Marshal(evidence.Data) - require.NoError(t, err) - var data map[string]any - require.NoError(t, json.Unmarshal(encoded, &data)) - require.NoError(t, schemavalidators.ValidateAICodingSession(data, "")) - // The original native-only evidence passed the old schema but is not renderable. - native := make([]any, 0, len(lines)) - for _, line := range lines { - var event any - require.NoError(t, json.Unmarshal([]byte(line), &event)) - native = append(native, event) - } - data["raw_session"] = map[string]any{"main": native} - require.Error(t, schemavalidators.ValidateAICodingSession(data, "")) } func TestTranscriptReasoningAndUnknownEvents(t *testing.T) { diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index 30cce9e31..a8a5c75a2 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -27,9 +27,9 @@ import ( "strings" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/codex" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" - "github.com/chainloop-dev/chainloop/internal/schemavalidators" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" ) @@ -153,17 +153,17 @@ func checkTranscript(s *Session) error { if len(main) == 0 { return errors.New("the evidence has no main transcript") } - encoded, err := json.Marshal(d) - if err != nil { - return err - } - var document any - if err := json.Unmarshal(encoded, &document); err != nil { - return err - } - if err := schemavalidators.ValidateAICodingSession(document, ""); err != nil { - return fmt.Errorf("transcript does not satisfy the evidence contract: %w", err) + // Other providers retain their existing native transcript checks. Codex + // converts its rollout to the message shape used by the session viewer. + if d.Agent.Name != codex.Name { + for _, msg := range main { + if bytes.Contains(msg, []byte("greeting.txt")) { + return nil + } + } + return errors.New("the transcript does not show the work of the session") } + var user, assistant, showsWork bool calls, results := map[string]bool{}, map[string]bool{} for _, msg := range main { @@ -183,11 +183,16 @@ func checkTranscript(s *Session) error { Type, Text, ID, Name string ToolUseID string `json:"tool_use_id"` Content json.RawMessage `json:"content"` + Input json.RawMessage `json:"input"` } if json.Unmarshal(entry.Message.Content, &blocks) == nil { for _, block := range blocks { switch block.Type { case "tool_use": + var input map[string]json.RawMessage + if block.ID == "" || block.Name == "" || json.Unmarshal(block.Input, &input) != nil || input == nil { + return errors.New("the Codex transcript has a malformed tool call") + } calls[block.ID] = true readable = true case "tool_result": @@ -205,7 +210,7 @@ func checkTranscript(s *Session) error { if !showsWork { return errors.New("the normalized transcript does not show the work of the session") } - if d.Agent.Name == "codex" && d.ToolsUsed != nil && d.ToolsUsed.TotalInvocations > 0 { + if d.ToolsUsed != nil && d.ToolsUsed.TotalInvocations > 0 { for id := range calls { if results[id] { return nil diff --git a/app/cli/internal/trace/e2e/transcript_test.go b/app/cli/internal/trace/e2e/transcript_test.go index f554a7ac4..f7c498ffc 100644 --- a/app/cli/internal/trace/e2e/transcript_test.go +++ b/app/cli/internal/trace/e2e/transcript_test.go @@ -41,6 +41,10 @@ func TestTranscriptRequiresRenderableMessages(t *testing.T) { {name: "no assistant", entries: []string{prompt}, wantErr: true}, {name: "missing tool result", entries: []string{prompt, call}, tools: 1, wantErr: true}, {name: "empty tool result", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","content":""}]}}`}, tools: 1, wantErr: true}, + {name: "missing tool name", entries: []string{prompt, `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"call-1","input":{}}]}}`, result}, tools: 1, wantErr: true}, + {name: "missing tool ID", entries: []string{prompt, `{"type":"assistant","message":{"content":[{"type":"tool_use","name":"apply_patch","input":{}}]}}`, result}, tools: 1, wantErr: true}, + {name: "invalid tool input", entries: []string{prompt, `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"call-1","name":"apply_patch","input":"native arguments"}]}}`, result}, tools: 1, wantErr: true}, + {name: "native tool output", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"call-1","content":[{"type":"output_text","text":"Success"}]}]}}`}, tools: 1, wantErr: true}, {name: "unrelated tool result", entries: []string{prompt, call, `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"other","content":"Success"}]}}`}, tools: 1, wantErr: true}, } { t.Run(tc.name, func(t *testing.T) { @@ -65,3 +69,15 @@ func TestTranscriptRequiresRenderableMessages(t *testing.T) { }) } } + +// The Codex renderer checks must not impose a format on other providers. +func TestTranscriptPreservesOtherProviderFormats(t *testing.T) { + d := aicodingsession.Data{ + Agent: aicodingsession.Agent{Name: "pi"}, + RawSession: map[string][]json.RawMessage{"main": { + json.RawMessage(`{"type":"message","message":{"role":"user","content":"Create greeting.txt"}}`), + }}, + } + s := &Session{Stored: &StoredSession{Evidence: *aicodingsession.NewEvidence(d)}} + require.NoError(t, checkTranscript(s)) +} diff --git a/app/cli/internal/trace/pi/parse.go b/app/cli/internal/trace/pi/parse.go index a7d74e8b2..b188b0b56 100644 --- a/app/cli/internal/trace/pi/parse.go +++ b/app/cli/internal/trace/pi/parse.go @@ -27,7 +27,6 @@ import ( "strconv" "time" - "github.com/chainloop-dev/chainloop/app/cli/internal/trace" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" ) @@ -280,7 +279,7 @@ func buildEvidence(parsed *parsedSession, agentVersion string) *aicodingsession. for _, entry := range parsed.Selected { if !isTreeMarker(entry) { - raw = append(raw, transcriptRaw(entry)) + raw = append(raw, craftableRaw(entry.Raw)) } if entry.Type != "message" { continue @@ -360,52 +359,6 @@ func isTreeMarker(entry *sessionEntry) bool { return entry.Type == "custom" && entry.CustomType == treeMarkerCustomType } -// transcriptRaw keeps native metadata and exposes messages in the shared format. -func transcriptRaw(entry *sessionEntry) json.RawMessage { - raw := craftableRaw(entry.Raw) - if entry.Type != "message" { - return raw - } - var fields map[string]json.RawMessage - var message map[string]json.RawMessage - if json.Unmarshal(raw, &fields) != nil || json.Unmarshal(fields["message"], &message) != nil || message == nil { - return raw - } - role, _ := rawString(message["role"]) - if role != trace.RoleUser && role != trace.RoleAssistant && role != "toolResult" { - return raw - } - content := message["content"] - if len(content) == 0 || bytes.Equal(content, []byte("null")) { - content = json.RawMessage(`[]`) - } - if role == "toolResult" { - callID, _ := rawString(message["toolCallId"]) - content, _ = json.Marshal([]trace.RawSessionToolResultBlock{{Type: "tool_result", ToolUseID: callID, Content: content}}) - role = trace.RoleUser - } else { - var blocks []map[string]json.RawMessage - if json.Unmarshal(content, &blocks) == nil { - for _, block := range blocks { - if kind, _ := rawString(block["type"]); kind == "toolCall" { - block["type"] = json.RawMessage(`"tool_use"`) - block["input"] = block["arguments"] - delete(block, "arguments") - } - } - content, _ = json.Marshal(blocks) - } - } - message["content"] = content - message["role"], _ = json.Marshal(role) - fields["type"] = message["role"] - fields["uuid"] = fields["id"] - fields["source_event"] = raw - fields["message"], _ = json.Marshal(message) - normalized, _ := json.Marshal(fields) - return normalized -} - // craftableRaw preserves a native Pi entry unless encoding/json cannot decode // one of its numbers into the generic float64 representation used by the // evidence crafter. In that case only out-of-range numbers become null; the diff --git a/app/cli/internal/trace/pi/testdata/session-v3.golden.json b/app/cli/internal/trace/pi/testdata/session-v3.golden.json index 75942dd8d..eff98a6e2 100644 --- a/app/cli/internal/trace/pi/testdata/session-v3.golden.json +++ b/app/cli/internal/trace/pi/testdata/session-v3.golden.json @@ -55,47 +55,38 @@ "raw_session": { "main": [ { + "type": "message", "id": "u0000001", - "message": { - "content": "start", - "role": "user" - }, "parentId": null, - "source_event": { - "type": "message", - "id": "u0000001", - "parentId": null, - "timestamp": "2026-01-02T03:04:01Z", - "message": { - "role": "user", - "content": "start" - } - }, "timestamp": "2026-01-02T03:04:01Z", - "type": "user", - "uuid": "u0000001" + "message": { + "role": "user", + "content": "start" + } }, { + "type": "message", "id": "a0000001", + "parentId": "u0000001", + "timestamp": "2026-01-02T03:04:02Z", "message": { + "role": "assistant", + "provider": "anthropic", + "model": "claude-sonnet-4", "content": [ { - "text": "working", - "type": "text" + "type": "text", + "text": "working" }, { + "type": "toolCall", "id": "call-1", - "input": { - "path": "a.go" - }, "name": "edit", - "type": "tool_use" + "arguments": { + "path": "a.go" + } } ], - "model": "claude-sonnet-4", - "provider": "anthropic", - "role": "assistant", - "stopReason": "toolUse", "usage": { "input": 100, "output": 20, @@ -105,48 +96,9 @@ "cost": { "total": 0.1 } - } - }, - "parentId": "u0000001", - "source_event": { - "type": "message", - "id": "a0000001", - "parentId": "u0000001", - "timestamp": "2026-01-02T03:04:02Z", - "message": { - "role": "assistant", - "provider": "anthropic", - "model": "claude-sonnet-4", - "content": [ - { - "type": "text", - "text": "working" - }, - { - "type": "toolCall", - "id": "call-1", - "name": "edit", - "arguments": { - "path": "a.go" - } - } - ], - "usage": { - "input": 100, - "output": 20, - "cacheRead": 10, - "cacheWrite": 5, - "totalTokens": 135, - "cost": { - "total": 0.1 - } - }, - "stopReason": "toolUse" - } - }, - "timestamp": "2026-01-02T03:04:02Z", - "type": "assistant", - "uuid": "a0000001" + }, + "stopReason": "toolUse" + } }, { "type": "branch_summary", @@ -157,51 +109,42 @@ "summary": "left the abandoned branch" }, { + "type": "message", "id": "u0000003", - "message": { - "content": "continue", - "role": "user" - }, "parentId": "m0000001", - "source_event": { - "type": "message", - "id": "u0000003", - "parentId": "m0000001", - "timestamp": "2026-01-02T03:04:09Z", - "message": { - "role": "user", - "content": "continue" - } - }, "timestamp": "2026-01-02T03:04:09Z", - "type": "user", - "uuid": "u0000003" + "message": { + "role": "user", + "content": "continue" + } }, { + "type": "message", "id": "a0000003", + "parentId": "u0000003", + "timestamp": "2026-01-02T03:04:10Z", "message": { + "role": "assistant", + "provider": "openai", + "model": "gpt-5", "content": [ { + "type": "toolCall", "id": "call-2", - "input": { - "command": "go test ./..." - }, "name": "bash", - "type": "tool_use" + "arguments": { + "command": "go test ./..." + } }, { + "type": "toolCall", "id": "call-3", - "input": { - "path": "b.go" - }, "name": "edit", - "type": "tool_use" + "arguments": { + "path": "b.go" + } } ], - "model": "gpt-5", - "provider": "openai", - "role": "assistant", - "stopReason": "toolUse", "usage": { "input": 200, "output": 40, @@ -211,52 +154,9 @@ "cost": { "total": 0.2 } - } - }, - "parentId": "u0000003", - "source_event": { - "type": "message", - "id": "a0000003", - "parentId": "u0000003", - "timestamp": "2026-01-02T03:04:10Z", - "message": { - "role": "assistant", - "provider": "openai", - "model": "gpt-5", - "content": [ - { - "type": "toolCall", - "id": "call-2", - "name": "bash", - "arguments": { - "command": "go test ./..." - } - }, - { - "type": "toolCall", - "id": "call-3", - "name": "edit", - "arguments": { - "path": "b.go" - } - } - ], - "usage": { - "input": 200, - "output": 40, - "cacheRead": 20, - "cacheWrite": 0, - "totalTokens": 260, - "cost": { - "total": 0.2 - } - }, - "stopReason": "toolUse" - } - }, - "timestamp": "2026-01-02T03:04:10Z", - "type": "assistant", - "uuid": "a0000003" + }, + "stopReason": "toolUse" + } }, { "type": "compaction", @@ -277,44 +177,34 @@ "display": false }, { + "type": "message", "id": "u0000004", - "message": { - "content": "finish", - "role": "user" - }, "parentId": "cm000002", - "source_event": { - "type": "message", - "id": "u0000004", - "parentId": "cm000002", - "timestamp": "2026-01-02T03:04:15Z", - "message": { - "role": "user", - "content": "finish" - } - }, "timestamp": "2026-01-02T03:04:15Z", - "type": "user", - "uuid": "u0000004" + "message": { + "role": "user", + "content": "finish" + } }, { + "type": "message", "id": "a0000004", + "parentId": "u0000004", + "timestamp": "2026-01-02T03:04:16Z", "message": { + "role": "assistant", + "provider": "openai", + "model": "gpt-5", "content": [ { + "type": "toolCall", "id": "call-4", - "input": { - "path": "c.go" - }, "name": "write", - "type": "tool_use" + "arguments": { + "path": "c.go" + } } ], - "errorMessage": "failed", - "model": "gpt-5", - "provider": "openai", - "role": "assistant", - "stopReason": "error", "usage": { "input": -1, "output": 1.5, @@ -324,45 +214,10 @@ "cost": { "total": "bad" } - } - }, - "parentId": "u0000004", - "source_event": { - "type": "message", - "id": "a0000004", - "parentId": "u0000004", - "timestamp": "2026-01-02T03:04:16Z", - "message": { - "role": "assistant", - "provider": "openai", - "model": "gpt-5", - "content": [ - { - "type": "toolCall", - "id": "call-4", - "name": "write", - "arguments": { - "path": "c.go" - } - } - ], - "usage": { - "input": -1, - "output": 1.5, - "cacheRead": "bad", - "cacheWrite": 2, - "totalTokens": 999999999999999999999999999, - "cost": { - "total": "bad" - } - }, - "stopReason": "error", - "errorMessage": "failed" - } - }, - "timestamp": "2026-01-02T03:04:16Z", - "type": "assistant", - "uuid": "a0000004" + }, + "stopReason": "error", + "errorMessage": "failed" + } }, { "type": "future_entry", @@ -374,41 +229,22 @@ } }, { + "type": "message", "id": "a0000005", + "parentId": "q0000001", + "timestamp": "2026-01-02T03:04:18Z", "message": { + "role": "assistant", + "provider": "google", + "model": "gemini-2.5-pro", "content": [ { - "text": "aborted response", - "type": "text" + "type": "text", + "text": "aborted response" } ], - "model": "gemini-2.5-pro", - "provider": "google", - "role": "assistant", "stopReason": "aborted" - }, - "parentId": "q0000001", - "source_event": { - "type": "message", - "id": "a0000005", - "parentId": "q0000001", - "timestamp": "2026-01-02T03:04:18Z", - "message": { - "role": "assistant", - "provider": "google", - "model": "gemini-2.5-pro", - "content": [ - { - "type": "text", - "text": "aborted response" - } - ], - "stopReason": "aborted" - } - }, - "timestamp": "2026-01-02T03:04:18Z", - "type": "assistant", - "uuid": "a0000005" + } } ] }, diff --git a/app/cli/internal/trace/types.go b/app/cli/internal/trace/types.go index 383acc41f..c74595334 100644 --- a/app/cli/internal/trace/types.go +++ b/app/cli/internal/trace/types.go @@ -29,10 +29,12 @@ const ( RoleAssistant = "assistant" ) -// RawSessionEntry is the shared wire format for messages in raw_session. -// Providers must normalize conversation and tool records into this shape. -// Native metadata events may remain alongside messages; they cannot replace -// normalized messages. SourceEvent preserves the original event when converting. +// RawSessionEntry is the normalized wire format most providers use for a +// single message in raw_session["main"]. A provider whose persisted format +// carries a native timeline may retain those objects instead, so valid unknown +// entries are not discarded. Consumers can select a renderer by Agent.Name. +// This shape mirrors Claude's JSONL transcript format, which the frontend was +// originally built against. type RawSessionEntry struct { Type string `json:"type,omitempty"` Role string `json:"role,omitempty"` diff --git a/docs/specs/issue-3657-codex-trace-provider.md b/docs/specs/issue-3657-codex-trace-provider.md index a9e3fa944..75bb51022 100644 --- a/docs/specs/issue-3657-codex-trace-provider.md +++ b/docs/specs/issue-3657-codex-trace-provider.md @@ -50,7 +50,7 @@ Installation MUST preserve user hooks and produce identical content on repeated ### R-003: Session evidence -Trace MUST capture session identity, timing, agent version, conversations, tool calls, and model information from persisted Codex transcripts. Conversation and tool records MUST use the shared transcript wire format in `app/cli/internal/trace/PROVIDERS.md`. The original converted event MUST remain in `source_event`. Missing or unreadable data MUST produce a warning or parsing error. Repeated cumulative usage records MUST count tokens once. Cost MUST use embedded API prices and identify an unknown-model fallback. +Trace MUST capture session identity, timing, agent version, conversations, tool calls, and model information from persisted Codex transcripts. Missing or unreadable data MUST produce a warning or parsing error. Repeated cumulative usage records MUST count tokens once. Cost MUST use embedded API prices and identify an unknown-model fallback. ### R-004: Code attribution @@ -72,7 +72,7 @@ Native hook behavior follows Codex CLI 0.163.0. Codex requires users to trust pr The developer selects Codex during Trace initialization, reviews the installed hooks in Codex, and starts a session. Codex sends session and tool events to Trace. Trace records each patch path separately and uses working-tree snapshots for shell changes. -Trace copies the Codex rollout into private storage at session boundaries and before evidence assembly. The parser normalizes conversation and tool records and preserves their original events in `source_event`. Native metadata records remain unchanged. It derives summaries from metadata, response items, turn context, and cumulative token counters. Git hooks use the existing evidence pipeline. +Trace copies the Codex rollout into private storage at session boundaries and before evidence assembly. The parser retains native JSONL records and derives summaries from metadata, response items, turn context, and cumulative token counters. Git hooks use the existing evidence pipeline. ```mermaid flowchart LR diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index 96bfeb1c3..02ad74465 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -18,9 +18,7 @@ "agent": { "type": "object", "description": "AI agent provider information", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string", @@ -37,11 +35,7 @@ "session": { "type": "object", "description": "Session timing and identity", - "required": [ - "id", - "started_at", - "duration_seconds" - ], + "required": ["id", "started_at", "duration_seconds"], "properties": { "id": { "type": "string", @@ -211,10 +205,7 @@ "description": "Per-file change details", "items": { "type": "object", - "required": [ - "path", - "status" - ], + "required": ["path", "status"], "properties": { "path": { "type": "string", @@ -241,10 +232,7 @@ "description": "Contiguous line ranges", "items": { "type": "object", - "required": [ - "start", - "end" - ], + "required": ["start", "end"], "properties": { "start": { "type": "integer", @@ -343,15 +331,9 @@ }, "raw_session": { "type": "object", - "description": "Transcript streams keyed by thread name. Conversation and tool events use the shared message format. Unknown native metadata may be retained alongside them; source_event preserves an original converted event.", - "required": [ - "main" - ], + "description": "Raw session conversation data keyed by thread name", "additionalProperties": { - "type": "array", - "items": { - "$ref": "#/definitions/transcriptEntry" - } + "type": "array" } }, "warnings": { @@ -362,401 +344,5 @@ } } }, - "additionalProperties": false, - "definitions": { - "transcriptMessage": { - "type": "object", - "required": [ - "message" - ], - "anyOf": [ - { - "required": [ - "type" - ], - "properties": { - "type": { - "enum": [ - "user", - "assistant" - ] - } - } - }, - { - "required": [ - "role" - ], - "properties": { - "role": { - "enum": [ - "user", - "assistant" - ] - } - } - } - ], - "properties": { - "message": { - "type": "object", - "required": [ - "content" - ], - "properties": { - "role": { - "type": "string" - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "$ref": "#/definitions/transcriptBlock" - } - } - ] - } - } - } - } - }, - "transcriptEntry": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "role": { - "type": "string" - }, - "uuid": { - "type": "string" - }, - "timestamp": { - "type": "string" - }, - "source_event": { - "type": "object" - } - }, - "allOf": [ - { - "if": { - "anyOf": [ - { - "required": [ - "type" - ], - "properties": { - "type": { - "enum": [ - "user", - "assistant" - ] - } - } - }, - { - "required": [ - "role" - ], - "properties": { - "role": { - "enum": [ - "user", - "assistant" - ] - } - } - } - ] - }, - "then": { - "anyOf": [ - { - "$ref": "#/definitions/transcriptMessage" - }, - { - "$ref": "#/definitions/transcriptError" - } - ] - } - }, - { - "if": { - "required": [ - "type", - "payload" - ], - "properties": { - "type": { - "const": "response_item" - }, - "payload": { - "anyOf": [ - { - "required": [ - "type", - "role" - ], - "properties": { - "type": { - "const": "message" - }, - "role": { - "enum": [ - "user", - "assistant" - ] - } - } - }, - { - "required": [ - "type" - ], - "properties": { - "type": { - "enum": [ - "function_call", - "custom_tool_call", - "function_call_output", - "custom_tool_call_output" - ] - } - } - } - ] - } - } - }, - "then": { - "$ref": "#/definitions/transcriptMessage" - } - }, - { - "if": { - "required": [ - "message" - ], - "properties": { - "message": { - "type": "object", - "required": [ - "role" - ], - "properties": { - "role": { - "enum": [ - "user", - "assistant" - ] - } - } - } - } - }, - "then": { - "anyOf": [ - { - "$ref": "#/definitions/transcriptMessage" - }, - { - "$ref": "#/definitions/transcriptError" - } - ] - } - } - ] - }, - "transcriptBlock": { - "type": "object", - "required": [ - "type" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "not": { - "enum": [ - "input_text", - "output_text" - ] - } - } - }, - "allOf": [ - { - "if": { - "properties": { - "type": { - "const": "text" - } - } - }, - "then": { - "required": [ - "text" - ], - "properties": { - "text": { - "type": "string" - } - } - } - }, - { - "if": { - "properties": { - "type": { - "const": "tool_use" - } - } - }, - "then": { - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "input": { - "type": "object" - } - } - } - }, - { - "if": { - "properties": { - "type": { - "const": "tool_result" - } - } - }, - "then": { - "required": [ - "tool_use_id", - "content" - ], - "properties": { - "tool_use_id": { - "type": "string", - "minLength": 1 - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "$ref": "#/definitions/transcriptBlock" - } - }, - { - "type": "object" - } - ] - } - } - } - } - ] - }, - "transcriptError": { - "type": "object", - "required": [ - "isApiErrorMessage", - "error" - ], - "properties": { - "isApiErrorMessage": { - "const": true - }, - "error": { - "type": "string", - "minLength": 1 - } - }, - "anyOf": [ - { - "required": [ - "type" - ], - "properties": { - "type": { - "const": "assistant" - } - } - }, - { - "required": [ - "role" - ], - "properties": { - "role": { - "const": "assistant" - } - } - } - ] - } - }, - "allOf": [ - { - "if": { - "required": [ - "raw_session", - "conversation" - ], - "properties": { - "conversation": { - "required": [ - "total_messages" - ], - "properties": { - "total_messages": { - "type": "integer", - "minimum": 1 - } - } - } - } - }, - "then": { - "properties": { - "raw_session": { - "properties": { - "main": { - "contains": { - "anyOf": [ - { - "$ref": "#/definitions/transcriptMessage" - }, - { - "$ref": "#/definitions/transcriptError" - } - ] - } - } - } - } - } - } - } - ] + "additionalProperties": false } diff --git a/internal/schemavalidators/transcript_test.go b/internal/schemavalidators/transcript_test.go deleted file mode 100644 index 78015b051..000000000 --- a/internal/schemavalidators/transcript_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// -// Copyright 2026 The Chainloop Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package schemavalidators_test - -import ( - "encoding/json" - "os" - "testing" - - "github.com/chainloop-dev/chainloop/internal/schemavalidators" - "github.com/stretchr/testify/require" -) - -func TestTranscriptContract(t *testing.T) { - for _, tc := range []struct { - name, transcript string - messages int - wantErr bool - }{ - {name: "Claude text", transcript: `{"main":[{"type":"user","message":{"content":"Hello"}}]}`, messages: 1}, - {name: "Cursor role", transcript: `{"main":[{"role":"assistant","message":{"content":[{"type":"text","text":"Hello"}]}}]}`, messages: 1}, - {name: "assistant API error", transcript: `{"main":[{"type":"assistant","isApiErrorMessage":true,"error":"request failed"}]}`, messages: 1}, - {name: "normalized Codex with original event", transcript: `{"main":[{"type":"assistant","message":{"content":[{"type":"tool_use","id":"c1","name":"Bash","input":{"command":"pwd"}}]},"source_event":{"type":"response_item","payload":{"type":"function_call"}}},{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"c1","content":"/repo"}]}}]}`, messages: 1}, - {name: "unknown metadata alongside messages", transcript: `{"main":[{"type":"future_event","value":1},{"type":"user","message":{"content":"Hello"}}]}`, messages: 1}, - {name: "metadata without conversation", transcript: `{"main":[{"type":"session_meta","payload":{}}]}`}, - {name: "empty transcript without conversation", transcript: `{"main":[]}`}, - {name: "native conversation without normalized message", transcript: `{"main":[{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Hello"}]}}]}`, messages: 1, wantErr: true}, - {name: "native conversation without summary", transcript: `{"main":[{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Hello"}]}}]}`, wantErr: true}, - {name: "missing main", transcript: `{}`, wantErr: true}, - {name: "non-object entry", transcript: `{"main":["Hello"]}`, wantErr: true}, - {name: "message without content", transcript: `{"main":[{"type":"user","message":{}}]}`, messages: 1, wantErr: true}, - {name: "native role buried in message", transcript: `{"main":[{"type":"message","message":{"role":"user","content":"Hello"}}]}`, wantErr: true}, - {name: "content object", transcript: `{"main":[{"type":"assistant","message":{"content":{"text":"Hello"}}}]}`, messages: 1, wantErr: true}, - {name: "block without type", transcript: `{"main":[{"type":"user","message":{"content":[{"text":"Hello"}]}}]}`, messages: 1, wantErr: true}, - {name: "text block without text", transcript: `{"main":[{"type":"user","message":{"content":[{"type":"text"}]}}]}`, messages: 1, wantErr: true}, - {name: "call without id", transcript: `{"main":[{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash"}]}}]}`, messages: 1, wantErr: true}, - {name: "result without correlation id", transcript: `{"main":[{"type":"user","message":{"content":[{"type":"tool_result","content":"ok"}]}}]}`, messages: 1, wantErr: true}, - {name: "native text in tool output", transcript: `{"main":[{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"c1","content":[{"type":"input_text","text":"ok"}]}]}}]}`, messages: 1, wantErr: true}, - } { - t.Run(tc.name, func(t *testing.T) { - fixture, err := os.ReadFile("testdata/ai_coding_session_valid.json") - require.NoError(t, err) - var data map[string]any - require.NoError(t, json.Unmarshal(fixture, &data)) - var transcript any - require.NoError(t, json.Unmarshal([]byte(tc.transcript), &transcript)) - data["raw_session"] = transcript - data["conversation"] = map[string]any{"total_messages": tc.messages} - err = schemavalidators.ValidateAICodingSession(data, "") - if tc.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} diff --git a/pkg/attestation/crafter/materials/aicodingsession/redact_test.go b/pkg/attestation/crafter/materials/aicodingsession/redact_test.go index fbfc070a9..cd72cd1e6 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/redact_test.go +++ b/pkg/attestation/crafter/materials/aicodingsession/redact_test.go @@ -278,7 +278,7 @@ func TestRedactSkipsMedia(t *testing.T) { }, { name: "text field named data is still scanned", - entry: `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"Text with extra metadata","data":"` + media + `"}]}}`, + entry: `{"type":"user","message":{"role":"user","content":[{"type":"text","data":"` + media + `"}]}}`, }, } From fb6000177af43af7775806e1d68796b1490af9db Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 15:30:47 +0200 Subject: [PATCH 6/8] fix(trace): label Codex hooks in the hooks menu Set a descriptive Chainloop Trace status message for each Codex lifecycle hook. Upgrade existing unlabeled hooks on init while preserving user hooks and idempotence. Signed-off-by: Miguel Martinez Trivino Assisted-by: Codex --- .codex/hooks.json | 15 ++++++++---- app/cli/internal/trace/README.md | 2 +- app/cli/internal/trace/codex/hooks.go | 28 ++++++++++++---------- app/cli/internal/trace/codex/hooks_test.go | 19 +++++++++++++++ 4 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.codex/hooks.json b/.codex/hooks.json index 0ed3e3a71..ce7406e54 100644 --- a/.codex/hooks.json +++ b/.codex/hooks.json @@ -6,7 +6,8 @@ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex post-tool-use'", "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex post-tool-use)\"", - "type": "command" + "type": "command", + "statusMessage": "Chainloop Trace: record tool changes" } ], "matcher": "^(apply_patch|Bash)$" @@ -18,7 +19,8 @@ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex pre-tool-use'", "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex pre-tool-use)\"", - "type": "command" + "type": "command", + "statusMessage": "Chainloop Trace: snapshot files before changes" } ], "matcher": "^(apply_patch|Bash)$" @@ -30,7 +32,8 @@ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-end'", "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex session-end)\"", - "type": "command" + "type": "command", + "statusMessage": "Chainloop Trace: finish session" } ] } @@ -41,7 +44,8 @@ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex session-start'", "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex session-start)\"", - "type": "command" + "type": "command", + "statusMessage": "Chainloop Trace: start session" } ] } @@ -52,7 +56,8 @@ { "command": "sh -c 'if ! command -v chainloop >/dev/null 2>&1; then exit 0; fi; exec chainloop trace hook codex user-prompt-submit'", "commandWindows": "cmd.exe /d /s /c \"where chainloop >nul 2>nul & if errorlevel 1 (exit /b 0) else (chainloop trace hook codex user-prompt-submit)\"", - "type": "command" + "type": "command", + "statusMessage": "Chainloop Trace: capture prompt context" } ] } diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 8c39be810..f37a606fb 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -10,7 +10,7 @@ Claude Code has the most complete support. Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and `trace run --pi` do not exist, so no Pi session is recorded. Each Pi cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Pi support is added, set the Pi cells to **Unknown** and run the end-to-end suite to find the values. -Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --local --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Changing the hooks requires a new trust review. +Codex uses native hooks in `.codex/hooks.json` (Codex CLI 0.163.0). Run `chainloop trace init --codex` or `chainloop trace run --local --codex -- codex exec …`. Review and trust the installed hooks in Codex before starting a session. See the [official hooks documentation](https://learn.chatgpt.com/docs/hooks). Installed hooks have a `Chainloop Trace` status label in Codex. Re-running init adds labels to existing Chainloop hooks. Changing the hooks requires a new trust review. Codex was tested with CLI 0.163.0 and `openrouter/openai/gpt-6-luna`: 22 feature checks passed with no failures. The tables keep unverified features at **Unknown**. Subagent transcripts, skills tracking, spec source pointers, and pasted image pointers are not implemented. diff --git a/app/cli/internal/trace/codex/hooks.go b/app/cli/internal/trace/codex/hooks.go index ad6871db7..40b2a0099 100644 --- a/app/cli/internal/trace/codex/hooks.go +++ b/app/cli/internal/trace/codex/hooks.go @@ -57,17 +57,18 @@ var commandTools = []string{"Bash"} var hookToolMatcher = "^(" + strings.Join(append(append([]string{}, fileWritingTools...), commandTools...), "|") + ")$" type hookEvent struct { - event string - command string - matcher string // optional: tool name matcher + event string + command string + matcher string // optional: tool name matcher + statusMessage string } var hookEvents = []hookEvent{ - {eventSessionStart, "chainloop trace hook codex session-start", ""}, - {eventUserPromptSubmit, "chainloop trace hook codex user-prompt-submit", ""}, - {eventPreToolUse, "chainloop trace hook codex pre-tool-use", hookToolMatcher}, - {eventPostToolUse, "chainloop trace hook codex post-tool-use", hookToolMatcher}, - {eventSessionEnd, "chainloop trace hook codex session-end", ""}, + {eventSessionStart, "chainloop trace hook codex session-start", "", "Chainloop Trace: start session"}, + {eventUserPromptSubmit, "chainloop trace hook codex user-prompt-submit", "", "Chainloop Trace: capture prompt context"}, + {eventPreToolUse, "chainloop trace hook codex pre-tool-use", hookToolMatcher, "Chainloop Trace: snapshot files before changes"}, + {eventPostToolUse, "chainloop trace hook codex post-tool-use", hookToolMatcher, "Chainloop Trace: record tool changes"}, + {eventSessionEnd, "chainloop trace hook codex session-end", "", "Chainloop Trace: finish session"}, } // SettingsFile returns the absolute path to .codex/hooks.json for the given repo. @@ -112,7 +113,7 @@ func (p *Provider) installHookEvents(repoRoot string, events []hookEvent) error changed = removeHookEntry(hooks, eventSessionEnd) } for _, h := range events { - if addHookEntry(hooks, h.event, h.command, h.matcher) { + if addHookEntry(hooks, h.event, h.command, h.matcher, h.statusMessage) { changed = true } } @@ -242,7 +243,7 @@ func patchPaths(patch, cwd string) []string { return paths } -func addHookEntry(hooks map[string]any, event, command, matcher string) bool { +func addHookEntry(hooks map[string]any, event, command, matcher, statusMessage string) bool { entries, _ := hooks[event].([]any) for _, entry := range entries { group, ok := entry.(map[string]any) @@ -253,22 +254,23 @@ func addHookEntry(hooks map[string]any, event, command, matcher string) bool { existingMatcher, _ := group["matcher"].(string) if len(handlers) == 1 && existingMatcher == matcher { handler, _ := handlers[0].(map[string]any) - if handler["command"] == hookCommandPrefix+command+"'" && handler["commandWindows"] == windowsHookPrefix+command+`)"` { + if handler["command"] == hookCommandPrefix+command+"'" && handler["commandWindows"] == windowsHookPrefix+command+`)"` && handler["statusMessage"] == statusMessage { return false } } } removeHookEntry(hooks, event) entries, _ = hooks[event].([]any) - hooks[event] = append(entries, buildHookEntry(command, matcher)) + hooks[event] = append(entries, buildHookEntry(command, matcher, statusMessage)) return true } -func buildHookEntry(command, matcher string) map[string]any { +func buildHookEntry(command, matcher, statusMessage string) map[string]any { entry := map[string]any{ "hooks": []any{ map[string]any{ "type": "command", + "statusMessage": statusMessage, "command": hookCommandPrefix + command + "'", "commandWindows": windowsHookPrefix + command + `)"`, }, diff --git a/app/cli/internal/trace/codex/hooks_test.go b/app/cli/internal/trace/codex/hooks_test.go index 4ccd16b23..3b8243287 100644 --- a/app/cli/internal/trace/codex/hooks_test.go +++ b/app/cli/internal/trace/codex/hooks_test.go @@ -105,6 +105,25 @@ func TestInstallUpgradesUnguardedHooks(t *testing.T) { assert.NotContains(t, string(first), `"command": "chainloop trace hook codex post-tool-use"`) assert.Contains(t, string(first), "command -v chainloop") assert.Contains(t, string(first), "echo user-hook") + // Existing guarded hooks also gain labels on init, without duplicating them. + settings, err := readJSONFile(path) + require.NoError(t, err) + for _, groups := range settings["hooks"].(map[string]any) { + for _, group := range groups.([]any) { + for _, handler := range group.(map[string]any)["hooks"].([]any) { + if isChainloopHandler(handler) { + delete(handler.(map[string]any), "statusMessage") + } + } + } + } + require.NoError(t, writeJSONFile(path, settings)) + require.NoError(t, p.InstallHooks(root)) + upgraded, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, first, upgraded) + assert.Contains(t, string(upgraded), "Chainloop Trace: snapshot files before changes") + require.NoError(t, p.InstallHooks(root)) second, err := os.ReadFile(p.SettingsFile(root)) require.NoError(t, err) From 1b98faa91d58b6d0e47dc1be72013fbe0636ced8 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 15:34:25 +0200 Subject: [PATCH 7/8] docs(trace): qualify Codex tool exchange checks Signed-off-by: Miguel Martinez Trivino Assisted-by: Codex --- app/cli/internal/trace/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index f37a606fb..c253b456e 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -18,7 +18,7 @@ Codex token usage comes from cumulative rollout counters. Cost is an estimate fr In a linked Git worktree, Codex 0.163 reads project hooks from the primary checkout. Install the Codex hooks in that checkout too, then review and trust them in Codex. -Codex evidence converts native messages, tool calls, and tool results to the message shape used by the existing session viewer. Text uses `message.content`, calls use `tool_use` blocks, and results use `tool_result` blocks with matching IDs. Each converted entry retains its original event in `source_event`; native metadata stays unchanged. Codex regression and E2E checks require renderable messages and a completed tool exchange, including readable tool output. A nonempty native transcript alone does not pass. These checks apply to Codex; `raw_session` remains open in the shared evidence schema. +Codex evidence converts native messages, tool calls, and tool results to the message shape used by the existing session viewer. Text uses `message.content`, calls use `tool_use` blocks, and results use `tool_result` blocks with matching IDs. Each converted entry retains its original event in `source_event`; native metadata stays unchanged. Codex regression and E2E checks require renderable messages. When tools are used, they also require a completed exchange with readable tool output. A nonempty native transcript alone does not pass. These checks apply to Codex; `raw_session` remains open in the shared evidence schema. ## Setup From 67fa3b3bdd4832f913f8a0768df227c39a912e10 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Oct 2026 16:04:05 +0200 Subject: [PATCH 8/8] docs(trace): record verified Codex user messages Mark the welcome banner and session link after push as supported after manual verification in Codex CLI 0.163.0. Document that session start runs after the first prompt, and distinguish interactive verification from local E2E coverage. Signed-off-by: Miguel Martinez Trivino Assisted-by: Codex --- app/cli/internal/trace/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index c253b456e..a9fee17de 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -85,16 +85,17 @@ Why not Yes: What the user sees in the agent. +The Codex welcome message and session link after push were verified manually in the interactive CLI 0.163.0. Codex runs its session-start hook after the first user message, so the welcome message appears then. The local-mode E2E suite does not verify these interactive messages. + | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | |---------|---------------|-------------|--------|--------------|------------|---|---| -| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | Unknown | -| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | Unknown | +| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | Yes | +| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | Yes | | Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | Unknown | Why not Yes: -- **Welcome message, Codex:** Codex does not report hook messages in its headless output, so the suite cannot verify the banner. -- **Session link after push and credentials warning, Codex:** these checks need a control plane. The suite uses local storage only. +- **Credentials warning, Codex:** this check needs a control plane. The suite uses local storage only, and the warning has not been verified manually. - **Welcome message, Cursor and OpenCode:** the provider gives the session-start context to the model only. It has no channel that shows a message to the user at session start, so the welcome message is dropped. - **Credentials warning, Cursor and OpenCode:** the provider has no channel to the user at session start. The model is told to tell the user, so the warning depends on the model. - **Session link after push, Cursor:** the `postToolUse` hook runs after a shell command, but its response has no field that Cursor shows to the user. The link stays pending until it expires.