feat(trace): check net contribution, session updates and rewrites end to end

app/cli/internal/trace/README.md

@@ -67,8 +67,8 @@ Why not Yes:
 
 - **Line attribution, partial commits and generated line counts, Cursor:** Cursor has only the `afterFileEdit` hook, which gives the edits but no snapshot before the edit. The provider rebuilds the content before the edit from the edits.
 - **Net contribution and line ranges, all agents:** the push finds the last writer of each line with `git blame`. A later commit that only re-indents a line takes the line, because blame sees a change. For Cursor, the limits of line attribution apply too.
-- **Session updates, all agents:** the update works only in the clone where the work happened, while the CLI keeps the trace state of the session (30 days), and only when the commits of the session are on the branch that the push sends.
-- **Renames and rewrites, all agents:** this works only in the clone where the work happened. Git finds a rename only when the file keeps at least half of its content. Two pushed commits can rename a file one after the other. A later change to the file under its middle name does not send the session again. For Cursor, the limits of line attribution apply too.
+- **Session updates, all agents:** the update works only in the clone where the work happened, while the CLI keeps the trace state of the session (30 days), and only when the commits of the session are on the branch that the push sends. The push reads only the commits that are not on the default branch of the remote (`origin/HEAD`, or else `origin/main` or `origin/master`). After the commits of a session are pushed to that branch, a later commit does not update the session.
+- **Renames and rewrites, all agents:** this works only in the clone where the work happened. Git finds a rename only when the file keeps at least half of its content. Two pushed commits can rename a file one after the other. A later change to the file under its middle name does not send the session again. A rename inside an amend or a squash of the session commit loses the AI lines: the new commit creates the file under its new name, so no line of it matches the trace state (issue 3686). The limits of session updates apply too. For Cursor, the limits of line attribution apply too.
 - **Shell changes attribution, all agents:** the hook before a shell command records only a hash of each file, not its content. The file at the `HEAD` commit from before the command stands in for the content before the command. Uncommitted lines that an earlier agent edit recorded are not recorded again, but uncommitted human lines of a file that the command changes count as AI.
 
 ## User experience
@@ -165,6 +165,8 @@ All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`.
 
 The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code, Cursor, OpenCode 2, Pi, and Codex columns. The Pi and Codex cells are **No**, so the suite skips them until Pi and Codex support is added. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. Pi runs with `pi -p --mode json --approve`, with its configuration and session folders next to the workspace. Codex runs with `codex exec --json`, with its `CODEX_HOME` folder next to the workspace, no sandbox, and no hook trust prompt. Cursor runs with `cursor-agent -p --output-format stream-json --force --trust`, with its `HOME` folder next to the workspace, so the Cursor configuration and hooks of the user do not load. Cursor runs its shell tool in a login shell, and on macOS `path_helper` puts the system folders first in `PATH`. So the harness writes shell startup files in that `HOME` that set `PATH` back. Otherwise a `chainloop` in `/usr/local/bin` runs the git hooks instead of the CLI under test. The Cursor CLI then has no stored login, so the run needs `CURSOR_API_KEY`. The headless Cursor CLI cannot attach a pasted image, and its output does not show the hook messages. The OpenCode 2, Pi, and Codex models in `e2e/config.yaml` are OpenRouter models, so the run needs `OPENROUTER_API_KEY`. A Codex model with no `openrouter/` prefix runs on the default Codex provider, which needs `CODEX_API_KEY`.
 
+Some scenarios add the git commands of a human after the session, for example a commit that replaces an AI line, an amend, or a rename, and then push again. They work on a feature branch, because the push reads only the commits that are not on the default branch of the remote.
+
 A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows:
 
 - A cell marked **No** or **Not applicable** is `skipped`.

app/cli/internal/trace/e2e/checks.go

@@ -47,8 +47,11 @@ type Session struct {
 	Output   *RunOutput
 	// Stored is the session that the push stored in local storage mode.
 	Stored *StoredSession
-	// HeadMessage is the message of the commit that the agent made.
+	// HeadMessage is the message of the last commit of the checkout: the
+	// commit that the agent made, or the commit of the follow-up.
 	HeadMessage string
+	// HeadSHA is the last commit of the checkout.
+	HeadSHA string
 }
 
 // Check verifies one feature on the result of one scenario session.
@@ -95,11 +98,17 @@ var features = map[string]Feature{
 	"Subagents":            {Checks: []Check{{scenarioSubagent, checkSubagents}}},
 
 	// Code attribution
-	"Line attribution":          {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("greeting.txt")}}},
+	"Line attribution":          {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI(greetingFile)}}},
 	"Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}},
 	"Partial commits":           {Checks: []Check{{scenarioPartial, checkBothCommitsCredited}}},
-	"Line ranges":               {Checks: []Check{{scenarioEditAndCommit, checkLineRanges("greeting.txt", 3)}}},
+	"Line ranges":               {Checks: []Check{{scenarioEditAndCommit, checkLineRanges(greetingFile, 3)}}},
 	"Generated line counts":     {Checks: []Check{{scenarioEditAndCommit, checkGeneratedLineCounts}}},
+	"Net contribution":          {Checks: []Check{{scenarioHumanUpdate, checkNetContribution}}},
+	"Session updates":           {Checks: []Check{{scenarioHumanUpdate, checkSessionUpdated}}},
+	"Renames and rewrites": {Checks: []Check{
+		{scenarioAmend, checkAmended},
+		{scenarioRename, checkRenamed},
+	}},
 
 	// User experience
 	"Welcome message":         {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}},

app/cli/internal/trace/e2e/checks_more.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/chainloop-dev/chainloop/app/cli/internal/trace"
+	tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git"
 	"github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer"
 	"github.com/chainloop-dev/chainloop/app/cli/internal/trace/state"
 	"github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession"
@@ -243,3 +244,101 @@ func checkLineRanges(path string, lines int) func(s *Session) error {
 		return fmt.Errorf("the code changes do not list %s", path)
 	}
 }
+
+// fileChange returns the change of a file in the evidence.
+func fileChange(s *Session, path string) (*aicodingsession.FileChange, error) {
+	cc := evidence(s).CodeChanges
+	if cc == nil {
+		return nil, errors.New("the evidence has no code changes")
+	}
+	for i := range cc.Files {
+		if cc.Files[i].Path == path {
+			return &cc.Files[i], nil
+		}
+	}
+	return nil, fmt.Errorf("the code changes do not list %s", path)
+}
+
+// headInSession checks that the last commit of the checkout is in the commit
+// list of the session. Each entry of the list is the SHA and the subject of a
+// commit.
+func headInSession(s *Session) error {
+	gc := evidence(s).GitContext
+	if gc != nil && slices.ContainsFunc(gc.Commits, func(c string) bool {
+		sha, _, _ := strings.Cut(c, " ")
+		return sha == s.HeadSHA
+	}) {
+		return nil
+	}
+	return fmt.Errorf("the commit list of the session does not hold the last commit %s", s.HeadSHA)
+}
+
+// checkNetContribution checks that the line that the human commit replaced
+// no longer counts as AI, that each line counts once, and that the file that
+// the session created stays created.
+func checkNetContribution(s *Session) error {
+	f, err := fileChange(s, greetingFile)
+	if err != nil {
+		return err
+	}
+	if f.AILinesAdded != 2 || f.HumanLinesAdded != 1 {
+		return fmt.Errorf("%s: %d AI lines and %d human lines added, want 2 and 1", greetingFile, f.AILinesAdded, f.HumanLinesAdded)
+	}
+	if f.LinesAdded != f.AILinesAdded+f.HumanLinesAdded {
+		return fmt.Errorf("%s: %d lines added, want the sum of the AI and human lines", greetingFile, f.LinesAdded)
+	}
+	if f.Status != tracegit.StatusCreated {
+		return fmt.Errorf("%s: status %q, want created", greetingFile, f.Status)
+	}
+	return nil
+}
+
+// checkSessionUpdated checks that the push after the human commit stored the
+// session again, with the human commit in its commit list and the replacing
+// line as a human line.
+func checkSessionUpdated(s *Session) error {
+	if err := headInSession(s); err != nil {
+		return err
+	}
+	f, err := fileChange(s, greetingFile)
+	if err != nil {
+		return err
+	}
+	if f.HumanLinesAdded != 1 {
+		return fmt.Errorf("%s: %d human lines added, want the 1 replacing line", greetingFile, f.HumanLinesAdded)
+	}
+	return nil
+}
+
+// checkAmended checks that the session follows its amended commit, with the
+// lines of the agent as AI lines and the line that the amend added as a human
+// line.
+func checkAmended(s *Session) error {
+	if err := headInSession(s); err != nil {
+		return err
+	}
+	f, err := fileChange(s, greetingFile)
+	if err != nil {
+		return err
+	}
+	if f.AILinesAdded != 3 || f.HumanLinesAdded != 1 {
+		return fmt.Errorf("%s: %d AI lines and %d human lines added, want 3 and 1", greetingFile, f.AILinesAdded, f.HumanLinesAdded)
+	}
+	return nil
+}
+
+// checkRenamed checks that the lines of the agent stay AI lines after a later
+// commit renamed their file. The evidence lists the file under its path in
+// the commit of the session or under its new path.
+func checkRenamed(s *Session) error {
+	f, err := fileChange(s, greetingFile)
+	if err != nil {
+		if f, err = fileChange(s, renamedFile); err != nil {
+			return err
+		}
+	}
+	if f.AILinesAdded != 3 || f.HumanLinesAdded != 0 {
+		return fmt.Errorf("%s: %d AI lines and %d human lines added, want 3 and 0", f.Path, f.AILinesAdded, f.HumanLinesAdded)
+	}
+	return nil
+}

app/cli/internal/trace/e2e/checks_test.go

@@ -19,6 +19,8 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/chainloop-dev/chainloop/app/cli/internal/trace"
+	tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git"
 	"github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,3 +57,183 @@ func TestCheckModel(t *testing.T) {
 		})
 	}
 }
+
+const (
+	testAgentCommit = "aaaa"
+	testHumanCommit = "bbbb"
+)
+
+// sessionWith returns a session whose stored evidence has the given commits
+// and file changes. Its last commit is testHumanCommit. The evidence lists each
+// commit with its subject, as the push writes it.
+func sessionWith(commits []string, files ...aicodingsession.FileChange) *Session {
+	s := &Session{Stored: &StoredSession{}, HeadSHA: testHumanCommit}
+	d := &s.Stored.Evidence.Data
+	d.GitContext = &aicodingsession.GitContext{}
+	for _, c := range commits {
+		d.GitContext.Commits = append(d.GitContext.Commits, c+" a subject")
+	}
+	d.CodeChanges = &aicodingsession.CodeChanges{Files: files}
+	return s
+}
+
+// updatedGreeting is the file of the update scenario after the human commit
+// replaced one of its three AI lines.
+func updatedGreeting() aicodingsession.FileChange {
+	return aicodingsession.FileChange{
+		Path: greetingFile, Status: tracegit.StatusCreated, Attribution: trace.AttributionAI,
+		LinesAdded: 3, AILinesAdded: 2, HumanLinesAdded: 1,
+	}
+}
+
+func TestCheckNetContribution(t *testing.T) {
+	cases := []struct {
+		name    string
+		file    aicodingsession.FileChange
+		wantErr string
+	}{
+		{name: "replaced line no longer counts", file: updatedGreeting()},
+		{
+			name: "replaced line still counts as AI",
+			file: func() aicodingsession.FileChange {
+				f := updatedGreeting()
+				f.AILinesAdded, f.HumanLinesAdded = 3, 0
+				return f
+			}(),
+			wantErr: "3 AI lines",
+		},
+		{
+			name: "line counted twice",
+			file: func() aicodingsession.FileChange {
+				f := updatedGreeting()
+				f.LinesAdded = 4
+				return f
+			}(),
+			wantErr: "4 lines added",
+		},
+		{
+			name: "file no longer created",
+			file: func() aicodingsession.FileChange {
+				f := updatedGreeting()
+				f.Status = "modified"
+				return f
+			}(),
+			wantErr: "status",
+		},
+		{name: "file not listed", file: aicodingsession.FileChange{Path: "other.txt"}, wantErr: "do not list"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := checkNetContribution(sessionWith([]string{testAgentCommit, testHumanCommit}, tc.file))
+			if tc.wantErr == "" {
+				assert.NoError(t, err)
+				return
+			}
+			assert.ErrorContains(t, err, tc.wantErr)
+		})
+	}
+}
+
+func TestCheckSessionUpdated(t *testing.T) {
+	cases := []struct {
+		name    string
+		commits []string
+		file    aicodingsession.FileChange
+		wantErr string
+	}{
+		{name: "human commit joins the session", commits: []string{testAgentCommit, testHumanCommit}, file: updatedGreeting()},
+		{name: "human commit not in the session", commits: []string{testAgentCommit}, file: updatedGreeting(), wantErr: "commit list"},
+		{
+			name:    "replacing line not human",
+			commits: []string{testAgentCommit, testHumanCommit},
+			file: func() aicodingsession.FileChange {
+				f := updatedGreeting()
+				f.HumanLinesAdded = 0
+				return f
+			}(),
+			wantErr: "human lines",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := checkSessionUpdated(sessionWith(tc.commits, tc.file))
+			if tc.wantErr == "" {
+				assert.NoError(t, err)
+				return
+			}
+			assert.ErrorContains(t, err, tc.wantErr)
+		})
+	}
+}
+
+func TestCheckAmended(t *testing.T) {
+	amended := aicodingsession.FileChange{
+		Path: greetingFile, Status: tracegit.StatusCreated, Attribution: trace.AttributionAI,
+		LinesAdded: 4, AILinesAdded: 3, HumanLinesAdded: 1,
+	}
+	cases := []struct {
+		name    string
+		commits []string
+		file    aicodingsession.FileChange
+		wantErr string
+	}{
+		{name: "split follows the amend", commits: []string{testHumanCommit}, file: amended},
+		{name: "amended commit not in the session", commits: []string{testAgentCommit}, file: amended, wantErr: "commit list"},
+		{
+			name:    "agent lines lost in the amend",
+			commits: []string{testHumanCommit},
+			file: func() aicodingsession.FileChange {
+				f := amended
+				f.AILinesAdded, f.HumanLinesAdded = 0, 4
+				return f
+			}(),
+			wantErr: "0 AI lines",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := checkAmended(sessionWith(tc.commits, tc.file))
+			if tc.wantErr == "" {
+				assert.NoError(t, err)
+				return
+			}
+			assert.ErrorContains(t, err, tc.wantErr)
+		})
+	}
+}
+
+func TestCheckRenamed(t *testing.T) {
+	aiFile := func(path string) aicodingsession.FileChange {
+		return aicodingsession.FileChange{
+			Path: path, Status: tracegit.StatusCreated, Attribution: trace.AttributionAI, LinesAdded: 3, AILinesAdded: 3,
+		}
+	}
+	cases := []struct {
+		name    string
+		file    aicodingsession.FileChange
+		wantErr string
+	}{
+		{name: "listed under the path of the session commit", file: aiFile(greetingFile)},
+		{name: "listed under the new path", file: aiFile(renamedFile)},
+		{
+			name: "lines lost in the rename",
+			file: func() aicodingsession.FileChange {
+				f := aiFile(renamedFile)
+				f.AILinesAdded, f.HumanLinesAdded = 0, 3
+				return f
+			}(),
+			wantErr: "0 AI lines",
+		},
+		{name: "file not listed", file: aiFile("other.txt"), wantErr: "do not list"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := checkRenamed(sessionWith(nil, tc.file))
+			if tc.wantErr == "" {
+				assert.NoError(t, err)
+				return
+			}
+			assert.ErrorContains(t, err, tc.wantErr)
+		})
+	}
+}

app/cli/internal/trace/e2e/runner.go

@@ -255,9 +255,19 @@ func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Sc
 	}
 
 	s.Checkout = checkout
+	if sc.FollowUp != nil {
+		if err := sc.FollowUp(ctx, ws, checkout); err != nil {
+			return nil, fmt.Errorf("follow-up of the session: %w", err)
+		}
+	}
 	if s.HeadMessage, err = ws.run(ctx, checkout, "git", "log", "-1", "--format=%B"); err != nil {
 		return nil, err
 	}
+	head, err := ws.run(ctx, checkout, "git", "rev-parse", "HEAD")
+	if err != nil {
+		return nil, err
+	}
+	s.HeadSHA = strings.TrimSpace(head)
 
 	stored, err := StoredSessions(checkout)
 	if err != nil {