From 2a6ce7b3e96d23c849dba36a69a1b9afa89be331 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 9 Aug 2026 18:14:56 -0600 Subject: [PATCH] feat(bundle): give the bundle an evidence-only view for downstream writers A bundle has two readers with opposite needs. An auditor cites the session and needs every source byte. A writer must reach its own conclusion and therefore must not be able to read the conclusion the session already reached, which the transcript holds verbatim: on one measured report, 42 of 80 substantive lines were recoverable from the transcript of the session that wrote it, including the shell call that authored it. A check for the earlier report FILE passes while its CONTENT is right there. `traces bundle-view --view evidence-only --out ` projects an assembled bundle instead of policing it. The projection is an artifact-to-artifact command, like `inspect` and `export`, so the writer's copy is derivable from the auditor's copy and carries its manifest hash; `--view` on `bundle` is refused so nobody can ask for a projection and receive the full record. The view carries an explicit allow-list: the session index, the policy evidence, and the structured ledger records. Every other path is excluded by name in `manifest.excluded` with its SHA-256 and one of four rules, and an unrecognised path is denied by default, so the view can narrow by accident but never widen. Before anything is written, each candidate is compared against each excluded file for shared eight-word runs of prose, with identifiers and this package's own constants scrubbed from both sides and JSON compared by value rather than by serialized form. A repo ledger file that repeats one is dropped and recorded; a traces-derived artifact that repeats one is a defect in the derivation and no view is written. `manifest.schemaVersion` moves to 2 and every manifest now names its `view`, so a reader can never mistake one copy for the other. --- README.md | 32 +- skills/inspect-agent-traces/SKILL.md | 27 +- src/bundle-view.ts | 526 +++++++++++++++++++++++++++ src/bundle.ts | 170 ++++++++- src/cli.ts | 66 +++- src/evidence.ts | 11 +- src/index.ts | 1 + tests/bundle-view.test.ts | 435 ++++++++++++++++++++++ tests/bundle.test.ts | 6 + tests/cli.test.ts | 65 ++++ 10 files changed, 1309 insertions(+), 30 deletions(-) create mode 100644 src/bundle-view.ts create mode 100644 tests/bundle-view.test.ts diff --git a/README.md b/README.md index 52e5f81..2ae894e 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Emitting the contract is the supported way to integrate a new system. The adapte - [Watch a run tree](#watch-a-run-tree) - [Improvement engine](#improvement-engine) - [Session index](#session-index) -- [Session bundle](#session-bundle) +- [Session bundle](#session-bundle) · [Two views](#two-views-two-consumers) - [Policy-mining evidence](#policy-mining-evidence) - [Upload to the Intelligence Platform](#upload-to-the-intelligence-platform) - [Trace analysts](#trace-analysts) @@ -245,6 +245,7 @@ traces analyze --otlp results/sessions --out report.md traces convert --harness claude-code --last 1 --otlp-out spans.jsonl # OTLP only traces index --all --since 24h --out session-index.json traces bundle --harness claude-code --session --out bundle-dir # one session's durable evidence dir +traces bundle-view bundle-dir --view evidence-only --out writer-dir # same session, without its own words traces inspect session-index.json --out inspection-report.md traces evidence --harness codex --last 20 --out policy-evidence.jsonl traces evidence --harness codex-exec --session /tmp/codex.jsonl --cwd "$PWD" --out policy-evidence.jsonl @@ -472,11 +473,38 @@ The bundle directory: - `derived/` — `session-index.json`, the deterministic `report.md`, `evidence.jsonl`, and the OTLP span artifact - `ledger/` — the repo's `.evolve` slices for the session window: `experiments.jsonl` and `skill-runs.jsonl` rows inside the padded span window, `current.json` / `scorecard.json` / `progress.md` copied whole, the latest flat `handoff-*.md`, and reflections dated inside the window - `repo/git-log.txt` — commits in the session window from the session's cwd -- `manifest.json` — SHA-256 + byte count per file, provenance (session id, harness, cwd, transcript hash, session window), every absent artifact with its reason, per-slice row counts, and the bundle's known limits +- `manifest.json` — `view`, SHA-256 + byte count per file, provenance (session id, harness, cwd, transcript hash, session window), every absent artifact with its reason, per-slice row counts, and the bundle's known limits A missing transcript fails the assembly loudly. Every optional input that is absent (no subagents, no `.evolve`, no git repo) is recorded in `manifest.absent` with the probed path — a fact, not an error. +### Two views, two consumers + +A bundle has two kinds of reader, and they need opposite things. + +- An **auditor** cites the session and needs every source byte. That is the FULL view — what `traces bundle` writes, `manifest.view: "full"`. +- A **writer** must reach its own conclusion about the session, so it must not be able to read the conclusion that session already reached. That is the EVIDENCE-ONLY view, `manifest.view: "evidence-only"`. + +The full bundle cannot serve the writer. Its transcript holds every word the session wrote, including the shell call that authored an earlier report, so a check that looks for the report FILE passes while the report CONTENT is right there. Projecting is the fix, not grepping: + +```bash +traces bundle --harness claude-code --session --out bundle-dir +traces bundle-view bundle-dir --view evidence-only --out writer-dir +``` + +The evidence-only view carries an explicit allow-list — `derived/session-index.json`, `derived/evidence.jsonl`, and the structured `ledger/` records (`current.json`, `scorecard.json`, `skill-runs.jsonl`). Everything else is excluded **by name** in `manifest.excluded`, with its SHA-256 and one of four rules: + +| rule | what it covers | +|---|---| +| `session-source` | `session/` — the transcript and every subagent transcript | +| `session-text-derived` | `derived/report.md` (quotes the first prompt line and evidence excerpts) and `derived/trace.otlp.jsonl` (spans carry the full prompt and response text) | +| `authored-prose` | `ledger/progress.md`, `ledger/handoff-*.md`, `ledger/reflections/`, `ledger/experiments.jsonl` free-text, `repo/git-log.txt` | +| `not-allow-listed` | any other path, including one a later version of `traces` adds — the view can only narrow by accident, never widen | + +The allow-list is the structure. A content check is the proof it is right: before anything is written, every candidate is compared against every excluded file for shared 8-word runs of prose, with identifiers (paths, URLs, hashes) and this package's own constant strings scrubbed from both sides first. A shared run in a repo ledger file drops that file with `rule: "content-signature"` and the match count; a shared run in an artifact `traces` derives is a defect in the derivation and no view is written at all. `manifest.projection.leakCheck` records the width, the sources compared, and the result — always `matches: 0`, because a match means no view exists. + +A projected view also carries the full bundle's `manifest.json` hash, so an auditor can prove which record it came from. Projecting a view of a view is refused. + ## Policy-mining evidence `traces` does **not** emit benchmark campaign cells. It emits normalized coding-agent session evidence that another system can mine. diff --git a/skills/inspect-agent-traces/SKILL.md b/skills/inspect-agent-traces/SKILL.md index cb08f6e..a6cbbbe 100644 --- a/skills/inspect-agent-traces/SKILL.md +++ b/skills/inspect-agent-traces/SKILL.md @@ -22,10 +22,9 @@ traces analyze --otlp results/sessions --out .traces/all.md # a directory of e Use `--harness` only for coding agents whose on-disk format we do not control. Those adapters are the legacy edge, not the way to integrate a system you own. -A directory is a run directory: it holds the span export beside raw event, stream and -SDK logs that are also `*.jsonl`. Only the OTLP files are read, and the rest are listed -with what they actually hold. An `otlp/` subdirectory, when the producer made one, is -read on its own. +A run directory holds the span export beside raw event, stream and SDK logs that are +also `*.jsonl`. Only the OTLP files are read; the rest are listed with what they hold. +An `otlp/` subdirectory, when present, is read on its own. Any section headed `inputs incomplete`, or carrying an `Inputs incomplete` line above its table, is computed from a field the trace does not carry everywhere. Report those @@ -73,7 +72,7 @@ traces improve --harness codex --current --latest-turn --workflow \ `improve` writes findings, evidence, a report, and spans. It does not edit an agent, repository, memory store, or knowledge base. -Write one session's durable evidence directory when a later reader must cite the session: +Write one session's durable evidence directory for a later reader: ```bash traces bundle --harness claude-code --session --out .traces/bundle @@ -81,12 +80,24 @@ traces bundle --harness claude-code --session --out .traces/bundle `bundle` copies the transcript, the derived report and spans, the `.evolve` ledger rows inside the session window, and a `manifest.json` with a SHA-256 per file. It spends no model call. -Use it when the live stores can rotate before the reader arrives. A missing transcript stops the assembly. An absent optional input is recorded in `manifest.absent` with the probed path. -The bundle holds the whole session transcript. -Do not give a bundle to a writer that must not see an earlier conclusion, because the transcript holds the text of every file the session wrote. +## Pick the view for the reader + +That bundle is the FULL view (`manifest.view: "full"`), and it holds the whole session transcript. +Never give it to a writer that must not see an earlier conclusion: the transcript holds the text of every file the session wrote, so a check for the earlier report FILE passes while its CONTENT is still readable. + +Project the writer's copy instead: + +```bash +traces bundle-view .traces/bundle --view evidence-only --out .traces/writer +``` + +That view carries `derived/session-index.json`, `derived/evidence.jsonl`, and the structured `ledger/` records. +It excludes the transcripts, the report, the spans, and every prose ledger file by name, with rules and hashes in `manifest.excluded`. +Before writing, it compares each carried file against each excluded file for shared 8-word runs of prose, and drops any file that repeats one. +`manifest.view` names which copy you hold. ## Report diff --git a/src/bundle-view.ts b/src/bundle-view.ts new file mode 100644 index 0000000..97e50fe --- /dev/null +++ b/src/bundle-view.ts @@ -0,0 +1,526 @@ +/** + * `traces bundle-view` — project a full session bundle down to one view. + * + * A bundle has two consumers with opposite needs. An AUDITOR cites the + * session and needs every source byte. A WRITER must reach its own conclusion + * about the session and therefore must not be able to read the conclusion the + * session already reached — which the transcript holds verbatim, including the + * shell call that authored an earlier report. + * + * Serving both from one directory cannot work, and policing it by grep cannot + * work either: a check that looks for the earlier report FILE passes while its + * CONTENT sits in the transcript. So the two consumers get two directories. + * The evidence-only view is a projection of the full bundle: an explicit + * allow-list of counted artifacts, every other path excluded by name with its + * reason and its hash, and a content check that proves no excluded file's text + * survived into the view. + * + * The allow-list is the structure. The content check is the proof that the + * structure is right: before anything is written, every candidate is compared + * against every excluded file, and a candidate that repeats one is dropped + * from the view and named in the manifest. So the written view never carries + * text from a file the view says it removed. + */ + +import { createHash } from 'node:crypto' +import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { + hashSessionBundleFiles, + listSessionBundleFiles, + readSessionBundleManifest, + writeSessionBundleManifest, + type SessionBundleExclusion, + type SessionBundleExclusionRule, + type SessionBundleLeakCheck, + type SessionBundleManifest, + type SessionBundleView, +} from './bundle.js' +import { POLICY_EVIDENCE_NOTE } from './evidence.js' + +/** + * Bundle-relative paths the evidence-only view carries. Everything else is + * excluded, so a path added to the bundle later is dropped until someone puts + * it here on purpose: the view can only get narrower by accident, never wider. + * + * Each entry states what the artifact is MADE of, because that is the reason + * it is safe: counts, names, identifiers, timestamps and hashes, derived by + * code from the spans — never a run of words the session wrote. + * + * `owned` marks the artifacts whose schema THIS package defines and whose every + * field it derives. Those two classes fail differently under the content check: + * an owned artifact that repeats session text is a defect in traces and stops + * the projection, while a repo ledger file is free-form by convention and its + * text is a fact about that repo, so it is dropped from the view and named in + * the manifest instead. + */ +interface AllowListEntry { + readonly path: string + readonly owned: boolean + readonly carries: string +} + +const EVIDENCE_ONLY_ALLOW_LIST: readonly AllowListEntry[] = [ + { + path: 'derived/session-index.json', + owned: true, + carries: 'per-session counts, tool names, models, and nearby-context file metadata', + }, + { + path: 'derived/evidence.jsonl', + owned: true, + carries: 'execution statistics, token and tool totals, loop and error counts', + }, + { + path: 'ledger/current.json', + owned: false, + carries: 'the repo ledger current-state record', + }, + { + path: 'ledger/scorecard.json', + owned: false, + carries: 'the repo ledger scorecard rows', + }, + { + path: 'ledger/skill-runs.jsonl', + owned: false, + carries: 'skill-run rows: skill name, timestamp, and outcome fields', + }, +] + +interface ExclusionRuleEntry { + readonly matches: (path: string) => boolean + readonly rule: SessionBundleExclusionRule + readonly reason: string +} + +/** + * Why each excluded path is excluded, in the words a reader of the manifest + * needs. The last entry is the default deny, and it must stay last. + */ +const EXCLUSION_RULES: readonly ExclusionRuleEntry[] = [ + { + matches: (path) => path === 'session/transcript.jsonl' || path.startsWith('session/'), + rule: 'session-source', + reason: + 'raw session text: the transcript and any subagent transcript hold every word the session wrote, ' + + 'including the shell call that authored an earlier report', + }, + { + matches: (path) => path === 'derived/report.md', + rule: 'session-text-derived', + reason: 'the deterministic report quotes the session verbatim: the first prompt line and evidence excerpts per finding', + }, + { + matches: (path) => path === 'derived/trace.otlp.jsonl', + rule: 'session-text-derived', + reason: 'spans carry input.value, output.value and content: the full prompt and response text of every turn', + }, + { + matches: (path) => path === 'ledger/progress.md', + rule: 'authored-prose', + reason: 'progress notes are written at session close and restate what the session concluded', + }, + { + matches: (path) => /^ledger\/handoff-.*\.md$/.test(path), + rule: 'authored-prose', + reason: 'a handoff is the session conclusion in prose', + }, + { + matches: (path) => path.startsWith('ledger/reflections/'), + rule: 'authored-prose', + reason: 'a reflection is the session conclusion in prose', + }, + { + matches: (path) => path === 'ledger/experiments.jsonl', + rule: 'authored-prose', + reason: 'experiment rows carry free-text verdict and note fields that restate what the session concluded', + }, + { + matches: (path) => path === 'repo/git-log.txt', + rule: 'authored-prose', + reason: 'commit subjects are prose the session authored', + }, + { + matches: () => true, + rule: 'not-allow-listed', + reason: 'the evidence-only view carries an explicit allow-list of counted artifacts, and this path is not on it', + }, +] + +/** + * Words per content signature: the width at which a shared run reads as a + * quotation rather than a coincidence. + * + * Measured over one real 170-file bundle: the deterministic report shares 509 + * of its 747 runs with the transcript, while the two counted artifacts the + * view carries hold no eight-word run of prose at all. + */ +const SIGNATURE_WORDS = 8 + +/** + * How many of those words must be letters for the run to count as prose. + * + * `input 0 output 0 reasoning 0 cached 0` is a token-usage block, not a + * sentence, so any two files that ever printed one would share it. Requiring 7 + * letters of 8 keeps the quotations and drops the structural runs: on that same + * bundle it cut one ledger file's recorded overlap from 47 runs to the 22 that + * are sentences. + */ +const MIN_WORD_TOKENS = 7 + +/** + * Identifiers are not prose. A session file path, a URL, a UUID and a SHA + * appear on both sides of every comparison because both sides describe the + * same session, and counting those as leaked text would exclude the counted + * artifacts and leave the view empty. Scrubbing them from BOTH sides before + * signing leaves only words someone chose. + */ +function scrubIdentifiers(line: string): string { + return line + .replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, ' ') + .replace(/\/?(?:[\w.@+~-]+\/)+[\w.@+~-]*/g, ' ') + .replace(/\b[0-9a-f-]{8,}\b/gi, ' ') +} + +/** + * Prose this package writes into every derived artifact, whatever the session. + * + * A constant is not session text, but it reads as one to a content check when + * the session happened to work on this package's own source. Subtracting these + * from the comparison keeps the check about words the SESSION produced. Each + * entry is imported from the code that emits it, so the two cannot drift. + */ +const PACKAGE_CONSTANT_PROSE: readonly string[] = [POLICY_EVIDENCE_NOTE] + +function stripPackageConstants(text: string): string { + let out = text + for (const constant of PACKAGE_CONSTANT_PROSE) out = out.split(constant).join(' ') + return out +} + +function collectJsonStrings(value: unknown, into: string[]): void { + if (typeof value === 'string') into.push(value) + else if (Array.isArray(value)) for (const item of value) collectJsonStrings(item, into) + else if (value !== null && typeof value === 'object') { + // Values only. A field NAME is schema this package or a ledger convention + // chose, so `stddev null min null max` would otherwise read as a shared + // sentence between a counted artifact and any transcript that printed one. + for (const item of Object.values(value)) collectJsonStrings(item, into) + } +} + +/** + * The text of one line that a person or a model could have written. + * + * A JSON line yields its string VALUES; anything else yields the line itself. + * Probing with a parse is the point: "this line is not JSON" is an answer, not + * a failure, and the line is then compared as plain text. + */ +function proseFragments(line: string): string[] { + const trimmed = line.trim() + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + const parsed: unknown = JSON.parse(trimmed) + const strings: string[] = [] + collectJsonStrings(parsed, strings) + return strings + } catch { + // not JSON after all → compared as plain text below + } + } + return [line] +} + +/** + * Distinct signatures in one text: every run of `SIGNATURE_WORDS` consecutive + * words, lowercased, taken WITHIN one line's prose. Signatures never span a + * newline, so memory stays bounded by the longest line rather than the file, + * and a structure whose every line is a key and a number produces no + * signatures at all — which is the true statement about it: it holds no prose. + */ +export function sessionTextSignatures(text: string): Set { + const out = new Set() + for (const line of stripPackageConstants(text).split('\n')) { + for (const fragment of proseFragments(line)) { + const words = scrubIdentifiers(fragment) + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() + .split(' ') + .filter(Boolean) + for (let i = 0; i + SIGNATURE_WORDS <= words.length; i++) { + const window = words.slice(i, i + SIGNATURE_WORDS) + if (window.filter((word) => /^[a-z]+$/.test(word)).length < MIN_WORD_TOKENS) continue + out.add(window.join(' ')) + } + } + } + return out +} + +export interface SessionTextLeak { + /** Bundle-relative path of the carried file that repeats excluded text. */ + readonly viewPath: string + /** Bundle-relative paths of the excluded files the repeated text came from. */ + readonly sources: readonly string[] + /** Distinct signatures this file shares with those sources. */ + readonly signatures: number + /** + * One matched signature, for a diagnostic message only. It is a fragment of + * the excluded text, so it must never be written into a view artifact. + */ + readonly example: string +} + +export interface FindSessionTextLeaksOptions { + /** Directory holding the carried files. */ + readonly viewDir: string + /** Directory holding the excluded files. */ + readonly sourceDir: string + /** Bundle-relative paths under `viewDir` to check. */ + readonly carriedPaths: readonly string[] + /** Bundle-relative paths under `sourceDir` whose text must not appear in the view. */ + readonly excludedPaths: readonly string[] +} + +export interface SessionTextLeakReport { + /** One entry per carried file that repeats excluded text. Empty is the pass. */ + readonly leaks: readonly SessionTextLeak[] + readonly carried: readonly { readonly path: string; readonly signatures: number }[] + readonly comparedSources: readonly string[] + readonly sourceSignatures: number +} + +/** + * Compare a set of carried files against the files a view excluded. + * + * The signature index is built from the CARRIED files, which are small, and + * the excluded files are then read one at a time and tested against it — the + * cheap direction, since a transcript can be orders of magnitude larger than + * everything a view keeps. Every excluded file is scanned to the end, so the + * per-file counts are totals rather than "however many we saw before stopping". + */ +export async function findSessionTextLeaks( + opts: FindSessionTextLeaksOptions, +): Promise { + const index = new Map() + const carried: { path: string; signatures: number }[] = [] + for (const path of opts.carriedPaths) { + const signatures = sessionTextSignatures(await readFile(join(opts.viewDir, path), 'utf8')) + carried.push({ path, signatures: signatures.size }) + for (const signature of signatures) if (!index.has(signature)) index.set(signature, path) + } + + const hits = new Map; sources: Set; example: string }>() + const comparedSources: string[] = [] + let sourceSignatures = 0 + for (const sourcePath of opts.excludedPaths) { + const signatures = sessionTextSignatures(await readFile(join(opts.sourceDir, sourcePath), 'utf8')) + comparedSources.push(sourcePath) + sourceSignatures += signatures.size + for (const signature of signatures) { + const viewPath = index.get(signature) + if (viewPath === undefined) continue + const hit = hits.get(viewPath) ?? { matched: new Set(), sources: new Set(), example: signature } + hit.matched.add(signature) + hit.sources.add(sourcePath) + hits.set(viewPath, hit) + } + } + const leaks: SessionTextLeak[] = [...hits].map(([viewPath, hit]) => ({ + viewPath, + sources: [...hit.sources].sort(), + signatures: hit.matched.size, + example: hit.example, + })) + return { leaks, carried, comparedSources, sourceSignatures } +} + +export type EvidenceOnlyVerdict = + | { readonly carried: true; readonly owned: boolean; readonly carries: string } + | { readonly carried: false; readonly rule: SessionBundleExclusionRule; readonly reason: string } + +/** Classify one bundle-relative path against the evidence-only view. */ +export function classifyEvidenceOnlyPath(path: string): EvidenceOnlyVerdict { + const allowed = EVIDENCE_ONLY_ALLOW_LIST.find((entry) => entry.path === path) + if (allowed) return { carried: true, owned: allowed.owned, carries: allowed.carries } + const rule = EXCLUSION_RULES.find((entry) => entry.matches(path))! + return { carried: false, rule: rule.rule, reason: rule.reason } +} + +/** + * Limits of the evidence-only view, carried in its manifest so a reader meets + * the boundary in the artifact instead of discovering it. + */ +const EVIDENCE_ONLY_LIMITS: readonly string[] = [ + 'the exclusion is verbatim-tight, not paraphrase-tight: a conclusion restated in different words inside a carried artifact is not detectable by content signatures', + 'ledger prose (progress, handoffs, reflections) and experiment free-text fields are excluded whole; a measured experiment result is readable only from the full bundle, under audit rules', + 'the view carries counts, names, identifiers, timestamps and hashes; a question that needs the words of a turn cannot be answered from it', +] + +export interface ProjectSessionBundleOptions { + /** Directory holding the FULL bundle to project. */ + readonly bundleDir: string + /** View directory to create. Must be new or empty: one directory, one view. */ + readonly outDir: string + readonly view: SessionBundleView + readonly generatedAt?: string +} + +export interface ProjectSessionBundleResult { + readonly directory: string + readonly manifestPath: string + readonly manifest: SessionBundleManifest + readonly leakCheck: SessionBundleLeakCheck +} + +/** + * Project a full bundle into a narrower view. + * + * Every file is verified against the source manifest's hash first: a bundle + * whose bytes have drifted from its own manifest cannot be projected, because + * the classification is only meaningful over the files this package wrote. + * The content check then runs before anything is written, so a candidate that + * repeats an excluded source is dropped instead of copied. + */ +export async function projectSessionBundle( + opts: ProjectSessionBundleOptions, +): Promise { + if (opts.view !== 'evidence-only') { + throw new Error( + `unsupported bundle view ${JSON.stringify(opts.view)}. ` + + 'The full view IS the bundle `traces bundle` assembles, so there is nothing to project into it; ' + + 'the projectable view is "evidence-only".', + ) + } + const generatedAt = opts.generatedAt ?? new Date().toISOString() + const sourceDir = resolve(opts.bundleDir) + const source = await readSessionBundleManifest(sourceDir) + if (source.view !== 'full') { + throw new Error( + `${sourceDir} is already the ${source.view} view — project from the full bundle, not from a view of it`, + ) + } + + const outDir = resolve(opts.outDir) + await mkdir(outDir, { recursive: true }) + if ((await readdir(outDir)).length > 0) { + throw new Error( + `view output directory ${outDir} is not empty — pass a new directory so one directory holds exactly one view`, + ) + } + + const sourceHashes = new Map(source.files.map((file) => [file.path, file] as const)) + const carriedPaths: string[] = [] + const carries = new Map() + const excluded: SessionBundleExclusion[] = [] + const candidates: { + path: string + bytes: Buffer + sha256: string + carries: string + owned: boolean + }[] = [] + for (const path of await listSessionBundleFiles(sourceDir)) { + // manifest.json is rewritten for the view, never copied: it names the view + // it describes, and a copied one would name the wrong view. + if (path === 'manifest.json') continue + const bytes = await readFile(join(sourceDir, path)) + const sha256 = createHash('sha256').update(bytes).digest('hex') + const recorded = sourceHashes.get(path) + if (!recorded) { + throw new Error( + `${join(sourceDir, path)} is not listed in the bundle manifest — the bundle changed after assembly ` + + 'and cannot be projected; re-run `traces bundle`', + ) + } + if (recorded.sha256 !== sha256) { + throw new Error( + `${join(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha256} on disk) — ` + + 'the bundle changed after assembly and cannot be projected; re-run `traces bundle`', + ) + } + const verdict = classifyEvidenceOnlyPath(path) + if (!verdict.carried) { + excluded.push({ path, bytes: bytes.length, sha256, rule: verdict.rule, reason: verdict.reason }) + continue + } + candidates.push({ path, bytes, sha256, carries: verdict.carries, owned: verdict.owned }) + } + + // The content check runs BEFORE anything is written, so a file that repeats + // an excluded source never reaches the view directory at all. + const report = await findSessionTextLeaks({ + viewDir: sourceDir, + sourceDir, + carriedPaths: candidates.map((candidate) => candidate.path), + excludedPaths: excluded.map((entry) => entry.path), + }) + const leakByPath = new Map(report.leaks.map((leak) => [leak.viewPath, leak] as const)) + const ownedLeak = report.leaks.find((leak) => candidates.some((c) => c.path === leak.viewPath && c.owned)) + if (ownedLeak) { + throw new Error( + `${ownedLeak.viewPath} repeats ${ownedLeak.signatures} ${SIGNATURE_WORDS}-word run(s) found in ` + + `${ownedLeak.sources.join(', ')}, for example "${ownedLeak.example}". That artifact is derived by ` + + 'traces and must carry counts only, so this is a defect in the derivation, not a property of the ' + + 'session. No view was written.', + ) + } + + for (const candidate of candidates) { + const leak = leakByPath.get(candidate.path) + if (leak) { + excluded.push({ + path: candidate.path, + bytes: candidate.bytes.length, + sha256: candidate.sha256, + rule: 'content-signature', + reason: + `the content check found ${leak.signatures} ${SIGNATURE_WORDS}-word run(s) this file shares with ` + + `${leak.sources.join(', ')}; carrying it would put back text the view removes`, + contentMatch: { signatures: leak.signatures, sources: leak.sources }, + }) + continue + } + const target = join(outDir, candidate.path) + await mkdir(dirname(target), { recursive: true }) + await writeFile(target, candidate.bytes) + carriedPaths.push(candidate.path) + carries.set(candidate.path, candidate.carries) + } + + const leakCheck: SessionBundleLeakCheck = { + signatureWords: SIGNATURE_WORDS, + comparedSources: report.comparedSources, + sourceSignatures: report.sourceSignatures, + carried: report.carried + .filter((entry) => carries.has(entry.path)) + .map((entry) => ({ + path: entry.path, + carries: carries.get(entry.path)!, + signatures: entry.signatures, + })), + matches: 0, + } + const sourceManifestBytes = await readFile(join(sourceDir, 'manifest.json')) + const manifest: SessionBundleManifest = { + schemaVersion: 2, + kind: 'traces.session_bundle', + view: 'evidence-only', + createdAt: generatedAt, + provenance: source.provenance, + files: await hashSessionBundleFiles(outDir), + absent: source.absent, + excluded, + ledgerSlices: source.ledgerSlices.filter((slice) => carriedPaths.includes(slice.path)), + knownLimits: [...source.knownLimits, ...EVIDENCE_ONLY_LIMITS], + projection: { + sourceDirectory: sourceDir, + sourceManifestSha256: createHash('sha256').update(sourceManifestBytes).digest('hex'), + leakCheck, + }, + } + const manifestPath = await writeSessionBundleManifest(outDir, manifest) + return { directory: outDir, manifestPath, manifest, leakCheck } +} diff --git a/src/bundle.ts b/src/bundle.ts index 3a16748..76d4dc0 100644 --- a/src/bundle.ts +++ b/src/bundle.ts @@ -15,6 +15,10 @@ * manifest (`absent`, with the reason), never a silent gap and never an * error: a session without subagents or without an `.evolve` ledger is a * complete bundle of a smaller session. + * + * This module assembles the FULL view. `bundle-view.ts` projects that full + * bundle down to the evidence-only view for a consumer that must not read the + * session's own words; `manifest.view` names which of the two a directory is. */ import { createHash } from 'node:crypto' @@ -66,9 +70,83 @@ export interface SessionBundleLedgerSlice { readonly unparseableTsRows?: number } +/** + * Which projection of a session a bundle directory holds. + * + * `full` is the auditor's copy: every source byte, so a claim can be checked + * against the exact text it cites. `evidence-only` is the writer's copy: the + * counted artifacts, with the session's own words removed, so a writer that + * must reach its own conclusion cannot read an earlier one instead. + */ +export type SessionBundleView = 'full' | 'evidence-only' + +export type SessionBundleExclusionRule = + /** The session's raw text: transcript and subagent transcripts. */ + | 'session-source' + /** A derived artifact that replays session text (prompt lines, excerpts, span content). */ + | 'session-text-derived' + /** A file whose body is prose someone authored: ledger notes, handoffs, commit subjects. */ + | 'authored-prose' + /** An allow-listed file dropped because its own text repeats an excluded source. */ + | 'content-signature' + /** Default deny: the evidence-only view carries an allow-list, and this path is not on it. */ + | 'not-allow-listed' + +export interface SessionBundleExclusion { + /** Bundle-relative path in the FULL bundle this view was projected from. */ + readonly path: string + readonly bytes: number + /** SHA-256 of the excluded bytes, so an auditor can prove which file was dropped. */ + readonly sha256: string + readonly rule: SessionBundleExclusionRule + readonly reason: string + /** + * Present when `rule` is `content-signature`: how much text the file shared + * with which excluded sources. Counts and paths only — quoting the matched + * text here would carry into the view the very text the exclusion removes. + */ + readonly contentMatch?: { + readonly signatures: number + readonly sources: readonly string[] + } +} + +export interface SessionBundleCarriedSignatures { + readonly path: string + /** What the artifact is made of, which is the reason it is safe to carry. */ + readonly carries: string + /** Distinct content signatures the carried file holds at all. 0 = no prose run of that length. */ + readonly signatures: number +} + +/** + * The projection's own proof that no excluded file's text survived into the + * view. `matches` is typed `0` because a match aborts the projection: a view + * directory carrying a leak is never written, so the artifact cannot record one. + */ +export interface SessionBundleLeakCheck { + /** Consecutive words per content signature. */ + readonly signatureWords: number + /** Excluded files whose text supplied the signatures compared against. */ + readonly comparedSources: readonly string[] + readonly sourceSignatures: number + readonly carried: readonly SessionBundleCarriedSignatures[] + readonly matches: 0 +} + +export interface SessionBundleProjection { + /** Absolute directory of the full bundle this view was projected from. */ + readonly sourceDirectory: string + /** SHA-256 of that bundle's `manifest.json` bytes. */ + readonly sourceManifestSha256: string + readonly leakCheck: SessionBundleLeakCheck +} + export interface SessionBundleManifest { - readonly schemaVersion: 1 + readonly schemaVersion: 2 readonly kind: 'traces.session_bundle' + /** Which projection this directory is. A reader must branch on this, never on the file list. */ + readonly view: SessionBundleView readonly createdAt: string readonly provenance: { readonly sessionId: string @@ -89,8 +167,12 @@ export interface SessionBundleManifest { } readonly files: readonly SessionBundleFile[] readonly absent: readonly SessionBundleAbsence[] + /** Files the view deliberately dropped. Empty in the `full` view: it drops nothing. */ + readonly excluded: readonly SessionBundleExclusion[] readonly ledgerSlices: readonly SessionBundleLedgerSlice[] readonly knownLimits: readonly string[] + /** Present only in a projected view: where it came from, and the leak check that passed. */ + readonly projection?: SessionBundleProjection } export interface SessionBundleResult { @@ -211,15 +293,20 @@ async function runGitLog(cwd: string, sinceIso: string, untilIso: string): Promi } } -async function walkBundleFiles(root: string): Promise { +/** + * Every file under a bundle directory, as sorted bundle-relative + * `/`-separated paths. Both the manifest writer and the view projector walk a + * bundle through here, so neither can see a file the other does not. + */ +export async function listSessionBundleFiles(root: string): Promise { const out: string[] = [] - const pending = [root] + const pending = [resolve(root)] while (pending.length > 0) { const dir = pending.pop()! for (const entry of await readdir(dir, { withFileTypes: true })) { const path = join(dir, entry.name) if (entry.isDirectory()) pending.push(path) - else if (entry.isFile()) out.push(path) + else if (entry.isFile()) out.push(relative(resolve(root), path).split(sep).join('/')) } } return out.sort() @@ -461,18 +548,11 @@ export async function assembleSessionBundle(opts: AssembleSessionBundleOptions): } // manifest.json — sha256 per file, written LAST so it covers every byte. - const files: SessionBundleFile[] = [] - for (const path of await walkBundleFiles(outDir)) { - const bytes = await readFile(path) - files.push({ - path: relative(outDir, path).split(sep).join('/'), - bytes: bytes.length, - sha256: sha256Hex(bytes), - }) - } + const files = await hashSessionBundleFiles(outDir) const manifest: SessionBundleManifest = { - schemaVersion: 1, + schemaVersion: 2, kind: 'traces.session_bundle', + view: 'full', createdAt: generatedAt, provenance: { sessionId: ref.sessionId, @@ -486,12 +566,72 @@ export async function assembleSessionBundle(opts: AssembleSessionBundleOptions): }, files, absent, + // The full view is the whole record: it drops nothing, and says so rather + // than leaving a reader to infer completeness from a missing field. + excluded: [], ledgerSlices: slices, knownLimits: KNOWN_LIMITS, } + const manifestPath = await writeSessionBundleManifest(outDir, manifest) + return { directory: outDir, manifestPath, manifest } +} + +/** SHA-256 + byte count for every file under a bundle directory, manifest included when present. */ +export async function hashSessionBundleFiles(root: string): Promise { + const files: SessionBundleFile[] = [] + for (const path of await listSessionBundleFiles(root)) { + const bytes = await readFile(join(root, path)) + files.push({ path, bytes: bytes.length, sha256: sha256Hex(bytes) }) + } + return files +} + +export async function writeSessionBundleManifest( + outDir: string, + manifest: SessionBundleManifest, +): Promise { const manifestPath = join(outDir, 'manifest.json') await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8') - return { directory: outDir, manifestPath, manifest } + return manifestPath +} + +/** + * Read a bundle's manifest and prove it is one. A directory that merely looks + * like a bundle must not be projected: the view guarantee is only as good as + * the classification, and the classification only holds over paths this + * package wrote. + */ +export async function readSessionBundleManifest(bundleDir: string): Promise { + const path = join(resolve(bundleDir), 'manifest.json') + let text: string + try { + text = await readFile(path, 'utf8') + } catch (error) { + if (isMissingPathError(error)) { + throw new Error(`no manifest.json at ${path} — ${bundleDir} is not a traces session bundle`) + } + throw error + } + let parsed: unknown + try { + parsed = JSON.parse(text) + } catch (error) { + throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`) + } + const manifest = parsed as Partial + if (manifest?.kind !== 'traces.session_bundle') { + throw new Error(`${path} has kind ${JSON.stringify(manifest?.kind)}, expected "traces.session_bundle"`) + } + if (manifest.schemaVersion !== 2) { + throw new Error( + `${path} is bundle schemaVersion ${JSON.stringify(manifest.schemaVersion)}; this traces reads 2. ` + + 'Re-run `traces bundle` to assemble it with the view-aware manifest.', + ) + } + if (manifest.view !== 'full' && manifest.view !== 'evidence-only') { + throw new Error(`${path} has view ${JSON.stringify(manifest.view)}, expected "full" or "evidence-only"`) + } + return manifest as SessionBundleManifest } let cachedVersion: string | undefined diff --git a/src/cli.ts b/src/cli.ts index 108b57f..ae819b4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,6 +14,7 @@ * traces convert [--harness claude-code] [--last 1] --otlp-out spans.jsonl * traces index [--harness claude-code] [--last 20] --out session-index.json * traces bundle --harness claude-code --session --out + * traces bundle-view --view evidence-only --out * traces inspect session-index.json [--out inspection-report.md] * traces export --out spans.openinference.jsonl * traces import-codetracebench --trajectory-dir --out --revision <40-or-64-character-hex> @@ -108,7 +109,8 @@ import { type SessionWorkflowIssue, type SessionWorkflowSummary, } from './session-workflow.js' -import { assembleSessionBundle } from './bundle.js' +import { assembleSessionBundle, type SessionBundleView } from './bundle.js' +import { projectSessionBundle } from './bundle-view.js' import { buildSessionIndexFromRows, serializeSessionIndex, writeSessionIndexFile } from './session-index.js' import { sessionReportSource } from './report.js' import type { ReportSource } from './report.js' @@ -154,6 +156,8 @@ interface Args { config?: string format?: string mode?: string + /** `bundle-view`: which projection of a bundle to write. */ + view?: string replay: boolean /** `--once`: a single pass instead of a live tail. */ once: boolean @@ -241,6 +245,7 @@ function parseArgs(argv: string[]): Args { case '--model': a.model = next(); break case '--config': a.config = next(); break case '--mode': a.mode = next(); break + case '--view': a.view = next(); break case '--metadata': a.metadata = next(); break case '--attr': { const v = next(); if (v) a.attrs.push(v); break } case '--interval': a.interval = Number(next()); a.intervalExplicit = true; break @@ -354,6 +359,20 @@ function applyCurrentSessionSelection(args: Args): Args { return { ...args, session: sessionId } } +/** + * `--view` names a projection of an assembled bundle, so it belongs to + * `bundle-view` alone. Accepting it silently on `bundle` would let someone ask + * for the evidence-only view and receive the full one, which is the exact + * mistake the two views exist to make impossible. + */ +function validateViewSelection(args: Args): Args { + if (args.view === undefined || args.command === 'bundle-view') return args + throw new Error( + `--view is not supported by ${args.command}. A view is a projection of an assembled bundle: ` + + 'run `traces bundle ... --out ` first, then `traces bundle-view --view evidence-only --out `.', + ) +} + function validateWorkflowSelection(args: Args): Args { if (!args.workflow && !args.latestTurn) return args if (!WORKFLOW_COMMANDS.has(args.command)) { @@ -785,11 +804,37 @@ async function cmdBundle(args: Args): Promise { }) const { manifest } = result console.log( - `session bundle → ${result.directory} (${manifest.files.length} file(s), ` + + `session bundle (${manifest.view} view) → ${result.directory} (${manifest.files.length} file(s), ` + `${manifest.ledgerSlices.length} ledger slice(s), ${manifest.absent.length} recorded absent)`, ) } +const PROJECTABLE_VIEWS: readonly SessionBundleView[] = ['evidence-only'] + +async function cmdBundleView(args: Args): Promise { + if (!args.input) { + throw new Error('bundle-view needs the bundle directory to project; run `traces bundle --out ` first') + } + if (!args.view) { + throw new Error(`bundle-view needs --view <${PROJECTABLE_VIEWS.join('|')}>`) + } + if (!PROJECTABLE_VIEWS.includes(args.view as SessionBundleView)) { + throw new Error(`unknown view "${args.view}"; projectable views: ${PROJECTABLE_VIEWS.join(', ')}`) + } + if (!args.out) throw new Error('bundle-view needs --out — a new or empty directory for the view') + const result = await projectSessionBundle({ + bundleDir: args.input, + outDir: args.out, + view: args.view as SessionBundleView, + }) + const { manifest, leakCheck } = result + console.log( + `${manifest.view} view → ${result.directory} (${manifest.files.length} file(s) carried, ` + + `${manifest.excluded.length} excluded, ${leakCheck.matches} session-text match(es) against ` + + `${leakCheck.sourceSignatures} signature(s) from ${leakCheck.comparedSources.length} excluded source(s))`, + ) +} + async function cmdInspect(args: Args): Promise { if (!args.input) throw new Error('inspect needs an index file; run `traces index --out session-index.json` first') const index = await readSessionIndexFile(args.input) @@ -1538,7 +1583,14 @@ Commands: bundle Assemble one session's durable evidence directory: transcript + subagents, derived index/report/evidence/OTLP, the repo's .evolve ledger sliced to the session window, git log, and a sha256 manifest - (needs --session and --out ) + (needs --session and --out ). This is the FULL + view: it holds the session's own words, for a reader who cites them + bundle-view + Project an assembled bundle into a narrower view for a consumer that + must NOT read the session's own words. --view evidence-only carries + the counted artifacts and structured ledger rows, excludes the + transcript, the report, the spans and every prose file by name, and + refuses to write a view whose text repeats an excluded source inspect Read a session index and print ranked improvement findings export Convert evidence/events files to OpenInference JSONL for HALO import-codetracebench @@ -1592,6 +1644,9 @@ Options: --metadata analyze/export file: attach JSON object fields as span attributes --attr analyze/export file: attach one span attribute (repeatable) --mode stream: visualizer | findings | agent (default visualizer) + --view bundle-view: which projection to write. evidence-only is the + writer's view; the full view is the bundle itself, so there is + nothing to project into it --replay, --once stream: scan once and exit (default for positional input / --session) --once watch : print ONE snapshot and exit, for scripts and agents --no-spans stream: omit per-span pulse events @@ -1667,7 +1722,9 @@ async function main(): Promise { else usage() return } - const args = validateOtlpSelection(validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs))) + const args = validateOtlpSelection( + validateViewSelection(validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs))), + ) switch (args.command) { case 'help': if (args.input === 'import-codetracebench') usageImportCodeTraceBench() @@ -1683,6 +1740,7 @@ async function main(): Promise { case 'convert': await cmdConvert(args); break case 'index': await cmdIndex(args); break case 'bundle': await cmdBundle(args); break + case 'bundle-view': await cmdBundleView(args); break case 'inspect': await cmdInspect(args); break case 'export': await cmdExport(args); break case 'import-codetracebench': await cmdImportCodeTraceBench(args); break diff --git a/src/evidence.ts b/src/evidence.ts index aadca16..9871226 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -14,6 +14,15 @@ import { type ScanOptions, scanSessions } from './session-source.js' import { describeSessionRelationship, type SessionRole } from './session-relationship.js' import type { SessionRef } from './types.js' +/** + * The one sentence of prose every policy-evidence record carries. It is the + * same for every session, so a content check that compares an evidence file + * against a transcript must subtract it: a session that reads or writes this + * package's source would otherwise look like it leaked its own text. + */ +export const POLICY_EVIDENCE_NOTE = + 'This is normalized coding-agent session evidence for downstream policy mining; it is not an eval campaign cell.' + export interface PolicyEvidenceToolSummary { readonly name: string readonly calls: number @@ -220,7 +229,7 @@ export async function buildPolicyEvidenceRecord( ...(opts.otlpPath ? { otlpPath: opts.otlpPath } : {}), ...(opts.sourceSha256 ? { sourceSha256: opts.sourceSha256 } : {}), notCampaignCell: true, - note: 'This is normalized coding-agent session evidence for downstream policy mining; it is not an eval campaign cell.', + note: POLICY_EVIDENCE_NOTE, }, } } diff --git a/src/index.ts b/src/index.ts index 3f9e1aa..2e06599 100644 --- a/src/index.ts +++ b/src/index.ts @@ -89,6 +89,7 @@ export * from './execution.js' // shared execution accounting over normalized sp export * from './evidence.js' // policy-evidence JSONL for downstream miners export * from './session-index.js' // collectSessionIndex() — reusable session catalog export * from './bundle.js' // assembleSessionBundle() — one session's durable evidence dir +export * from './bundle-view.js' // projectSessionBundle() — the evidence-only view for a writer export * from './inspect.js' // inspectSessionIndex() — ranked findings from a session catalog export * from './file-export.js' // convert evidence/events files to OpenInference JSONL export * from './chat-trajectory.js' // generic chat trajectory to stable step spans diff --git a/tests/bundle-view.test.ts b/tests/bundle-view.test.ts new file mode 100644 index 0000000..aae4a63 --- /dev/null +++ b/tests/bundle-view.test.ts @@ -0,0 +1,435 @@ +import { createHash } from 'node:crypto' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { ClaudeAdapter } from '../src/adapters/claude.js' +import { + assembleSessionBundle, + listSessionBundleFiles, + writeSessionBundleManifest, + type SessionBundleManifest, +} from '../src/bundle.js' +import { + classifyEvidenceOnlyPath, + findSessionTextLeaks, + projectSessionBundle, + sessionTextSignatures, +} from '../src/bundle-view.js' +import type { SessionRef } from '../src/types.js' + +const dir = mkdtempSync(join(tmpdir(), 'traces-bundle-view-')) +afterAll(() => rmSync(dir, { recursive: true, force: true })) + +let outSeq = 0 +function newDir(prefix: string): string { + outSeq += 1 + return join(dir, `${prefix}-${outSeq}`) +} + +/** + * One sentence the session wrote and then copied into a ledger file, which is + * the leak this view exists to stop: the file is not the transcript, but the + * words are. + */ +const LEAKED_SENTENCE = + 'The retry budget was the wrong lever and the queue depth is what actually decides tail latency here.' + +/** A sentence only the transcript holds, used to prove the view carries none of it. */ +const TRANSCRIPT_ONLY_SENTENCE = + 'I rewrote the scheduler so that every queued item carries the deadline it was admitted under.' + +function refFor(path: string, cwd: string | null = null): SessionRef { + return { + harness: 'claude-code', + sessionId: 'view-fixture', + path, + cwd, + mtimeMs: Date.parse('2026-01-01T00:00:05Z'), + } +} + +function writeTranscript(path: string): void { + writeFileSync( + path, + [ + { + type: 'user', + uuid: 'root-user', + sessionId: 'view-fixture', + timestamp: '2026-01-01T00:00:00Z', + message: { role: 'user', content: `${LEAKED_SENTENCE} ${TRANSCRIPT_ONLY_SENTENCE}` }, + }, + { + type: 'assistant', + uuid: 'root-assistant', + sessionId: 'view-fixture', + timestamp: '2026-01-01T00:00:01Z', + message: { + id: 'root-message', + role: 'assistant', + content: [{ type: 'tool_use', id: 'call-one', name: 'Bash', input: { command: "cat > REPORT.md <<'EOF'" } }], + }, + }, + ].map((event) => JSON.stringify(event)).join('\n'), + ) +} + +function writeChild(subDir: string): void { + mkdirSync(subDir, { recursive: true }) + writeFileSync( + join(subDir, 'agent-worker.jsonl'), + [ + { + type: 'user', + uuid: 'child-user', + timestamp: '2026-01-01T00:00:02Z', + isSidechain: true, + message: { role: 'user', content: 'worker TASK' }, + }, + { + type: 'assistant', + uuid: 'child-assistant', + timestamp: '2026-01-01T00:00:03Z', + message: { id: 'child-message', role: 'assistant', content: 'worker ANSWER' }, + }, + ].map((event) => JSON.stringify(event)).join('\n'), + ) + writeFileSync(join(subDir, 'agent-worker.meta.json'), JSON.stringify({ agentType: 'worker', toolUseId: 'call-one' })) +} + +/** + * A `.evolve` ledger with both kinds of allow-listed file: `current.json` + * repeats a sentence from the transcript, `scorecard.json` and + * `skill-runs.jsonl` hold structured rows only. + */ +async function writeEvolveFixture(root: string): Promise { + const evolve = join(root, '.evolve') + await mkdir(join(evolve, 'reflections'), { recursive: true }) + await writeFile( + join(evolve, 'experiments.jsonl'), + `${JSON.stringify({ ts: '2026-01-01T00:00:03Z', round: 1, verdict: 'KEEP', note: LEAKED_SENTENCE })}\n`, + 'utf8', + ) + await writeFile( + join(evolve, 'skill-runs.jsonl'), + `${JSON.stringify({ skill: '/verify', ts: '2026-01-01T00:00:04Z', exitCode: 0 })}\n`, + 'utf8', + ) + await writeFile(join(evolve, 'current.json'), `${JSON.stringify({ mode: 'research', decision: LEAKED_SENTENCE })}\n`, 'utf8') + await writeFile(join(evolve, 'scorecard.json'), `${JSON.stringify({ flows: [{ flow: 'latency', score: 0.9 }] })}\n`, 'utf8') + await writeFile(join(evolve, 'progress.md'), `# Progress\n\n${LEAKED_SENTENCE}\n`, 'utf8') + await writeFile(join(evolve, 'handoff-2026-01-02-latest.md'), `# Handoff\n\n${LEAKED_SENTENCE}\n`, 'utf8') + await writeFile(join(evolve, 'reflections', '2026-01-01.md'), `# Reflection\n\n${LEAKED_SENTENCE}\n`, 'utf8') +} + +async function assembleFullBundle(): Promise<{ bundleDir: string; manifest: SessionBundleManifest }> { + const ctxRoot = newDir('ctx') + await mkdir(ctxRoot, { recursive: true }) + await writeEvolveFixture(ctxRoot) + const transcript = join(dir, `session-${outSeq}.jsonl`) + writeTranscript(transcript) + writeChild(join(dir, `session-${outSeq}`, 'subagents')) + + const bundleDir = newDir('bundle') + const { manifest } = await assembleSessionBundle({ + adapter: new ClaudeAdapter(), + ref: refFor(transcript, ctxRoot), + outDir: bundleDir, + generatedAt: '2026-01-01T01:00:00.000Z', + }) + return { bundleDir, manifest } +} + +/** A bundle written by hand, so a test can control the bytes of a derived artifact. */ +async function writeSyntheticBundle(files: Record): Promise { + const bundleDir = newDir('synthetic') + for (const [path, content] of Object.entries(files)) { + const target = join(bundleDir, path) + await mkdir(join(target, '..'), { recursive: true }) + await writeFile(target, content, 'utf8') + } + const manifest: SessionBundleManifest = { + schemaVersion: 2, + kind: 'traces.session_bundle', + view: 'full', + createdAt: '2026-01-01T01:00:00.000Z', + provenance: { + sessionId: 'synthetic', + harness: 'claude-code', + cwd: null, + transcriptPath: join(bundleDir, 'session', 'transcript.jsonl'), + transcriptSha256: createHash('sha256').update(files['session/transcript.jsonl'] ?? '').digest('hex'), + contextRoot: null, + tracesVersion: '0.0.0-test', + sessionWindow: { firstSpanAt: null, lastSpanAt: null, padMs: 0 }, + }, + files: await Promise.all( + (await listSessionBundleFiles(bundleDir)).map(async (path) => { + const bytes = await readFile(join(bundleDir, path)) + return { path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') } + }), + ), + absent: [], + excluded: [], + ledgerSlices: [], + knownLimits: [], + } + await writeSessionBundleManifest(bundleDir, manifest) + return bundleDir +} + +describe('evidence-only bundle view', () => { + it('carries the counted artifacts, excludes every prose source by name, and names its own view', async () => { + const { bundleDir, manifest: full } = await assembleFullBundle() + + // The bundle `traces bundle` writes says which view it is, and that it drops nothing. + expect(full.schemaVersion).toBe(2) + expect(full.view).toBe('full') + expect(full.excluded).toEqual([]) + expect(full.projection).toBeUndefined() + + const outDir = newDir('view') + const { manifest, leakCheck } = await projectSessionBundle({ + bundleDir, + outDir, + view: 'evidence-only', + generatedAt: '2026-01-01T02:00:00.000Z', + }) + + expect(manifest.schemaVersion).toBe(2) + expect(manifest.view).toBe('evidence-only') + expect(manifest.createdAt).toBe('2026-01-01T02:00:00.000Z') + + // What is on disk, and nothing else. + const written = await listSessionBundleFiles(outDir) + expect(written).toEqual([ + 'derived/evidence.jsonl', + 'derived/session-index.json', + 'ledger/scorecard.json', + 'ledger/skill-runs.jsonl', + 'manifest.json', + ]) + expect(manifest.files.map((file) => file.path)).toEqual(written.filter((path) => path !== 'manifest.json')) + + // The session's own words are gone, by name and by rule. + const excludedByPath = new Map(manifest.excluded.map((entry) => [entry.path, entry] as const)) + expect(excludedByPath.get('session/transcript.jsonl')?.rule).toBe('session-source') + expect(excludedByPath.get('session/subagents/agent-worker.jsonl')?.rule).toBe('session-source') + expect(excludedByPath.get('derived/report.md')?.rule).toBe('session-text-derived') + expect(excludedByPath.get('derived/trace.otlp.jsonl')?.rule).toBe('session-text-derived') + expect(excludedByPath.get('ledger/progress.md')?.rule).toBe('authored-prose') + expect(excludedByPath.get('ledger/handoff-2026-01-02-latest.md')?.rule).toBe('authored-prose') + expect(excludedByPath.get('ledger/reflections/2026-01-01.md')?.rule).toBe('authored-prose') + expect(excludedByPath.get('ledger/experiments.jsonl')?.rule).toBe('authored-prose') + for (const entry of manifest.excluded) { + expect(entry.reason.length).toBeGreaterThan(0) + expect(entry.sha256).toMatch(/^[0-9a-f]{64}$/) + } + + // Every excluded hash is the full bundle's own hash for that file, so an + // auditor can prove which bytes were dropped without holding them. + const fullByPath = new Map(full.files.map((file) => [file.path, file] as const)) + for (const entry of manifest.excluded) { + expect(entry.sha256).toBe(fullByPath.get(entry.path)!.sha256) + expect(entry.bytes).toBe(fullByPath.get(entry.path)!.bytes) + } + // Carried bytes are the full bundle's bytes, unchanged. + for (const file of manifest.files) expect(file.sha256).toBe(fullByPath.get(file.path)!.sha256) + + // The projection points back at exactly the bundle it came from. + const sourceManifestBytes = await readFile(join(bundleDir, 'manifest.json')) + expect(manifest.projection!.sourceDirectory).toBe(bundleDir) + expect(manifest.projection!.sourceManifestSha256).toBe( + createHash('sha256').update(sourceManifestBytes).digest('hex'), + ) + expect(manifest.projection!.leakCheck).toEqual(leakCheck) + expect(leakCheck.matches).toBe(0) + expect(leakCheck.signatureWords).toBe(8) + // Every path excluded by classification supplied signatures. A path + // excluded BY the check cannot also be one of its sources. + expect(leakCheck.comparedSources).toEqual( + manifest.excluded.filter((entry) => entry.rule !== 'content-signature').map((entry) => entry.path), + ) + expect(leakCheck.carried.map((entry) => entry.path)).toEqual([ + 'derived/evidence.jsonl', + 'derived/session-index.json', + 'ledger/scorecard.json', + 'ledger/skill-runs.jsonl', + ]) + for (const entry of leakCheck.carried) expect(entry.carries.length).toBeGreaterThan(0) + + // Ledger slices are narrowed to what survived, and the view states its own limits. + expect(manifest.ledgerSlices.map((slice) => slice.path)).toEqual(['ledger/skill-runs.jsonl', 'ledger/scorecard.json']) + expect(manifest.knownLimits).toEqual(expect.arrayContaining([...full.knownLimits])) + expect(manifest.knownLimits.some((limit) => limit.includes('paraphrase-tight'))).toBe(true) + }) + + it('carries no content signature from any excluded source', async () => { + const { bundleDir } = await assembleFullBundle() + const outDir = newDir('view') + const { manifest } = await projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' }) + + // The check the guarantee rests on, run over the view that was actually + // written, against every file the view says it dropped. + const report = await findSessionTextLeaks({ + viewDir: outDir, + sourceDir: bundleDir, + carriedPaths: (await listSessionBundleFiles(outDir)).filter((path) => path !== 'manifest.json'), + excludedPaths: manifest.excluded.map((entry) => entry.path), + }) + expect(report.leaks).toEqual([]) + expect(report.comparedSources.length).toBe(manifest.excluded.length) + expect(report.sourceSignatures).toBeGreaterThan(0) + + // The same read the naive check makes, and the reason it is not enough on + // its own: neither the sentence nor the authoring call survives anywhere. + const viewText = ( + await Promise.all( + (await listSessionBundleFiles(outDir)).map((path) => readFile(join(outDir, path), 'utf8')), + ) + ).join('\n') + expect(viewText).not.toContain(LEAKED_SENTENCE) + expect(viewText).not.toContain(TRANSCRIPT_ONLY_SENTENCE) + expect(viewText).not.toContain("cat > REPORT.md <<'EOF'") + }) + + it('drops an allow-listed ledger file whose own text repeats an excluded source', async () => { + const { bundleDir } = await assembleFullBundle() + const outDir = newDir('view') + const { manifest } = await projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' }) + + const dropped = manifest.excluded.find((entry) => entry.path === 'ledger/current.json')! + expect(dropped.rule).toBe('content-signature') + expect(dropped.contentMatch!.signatures).toBeGreaterThan(0) + expect(dropped.contentMatch!.sources).toContain('session/transcript.jsonl') + // The record counts the overlap; it never quotes it, or the manifest would + // carry the text the view removed. + expect(JSON.stringify(dropped)).not.toContain(LEAKED_SENTENCE) + expect(dropped.reason).toContain('content check') + }) + + it('refuses to write a view when a traces-derived artifact repeats session text', async () => { + const bundleDir = await writeSyntheticBundle({ + 'session/transcript.jsonl': `${JSON.stringify({ message: { content: LEAKED_SENTENCE } })}\n`, + 'derived/evidence.jsonl': `${JSON.stringify({ kind: 'traces.policy_evidence.session', note: LEAKED_SENTENCE })}\n`, + 'derived/report.md': '# report\n', + }) + const outDir = newDir('view') + + await expect(projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' })).rejects.toThrow( + /derived\/evidence\.jsonl repeats \d+ 8-word run\(s\).*defect in the derivation/s, + ) + // Nothing was written, so a caller cannot mistake a partial directory for a view. + await expect(listSessionBundleFiles(outDir)).resolves.toEqual([]) + }) + + it('drops a path nobody put on the allow-list', async () => { + const bundleDir = await writeSyntheticBundle({ + 'session/transcript.jsonl': `${JSON.stringify({ message: { content: 'hello' } })}\n`, + 'derived/session-index.json': `${JSON.stringify({ kind: 'traces.session_index' })}\n`, + 'derived/future-artifact.md': 'a section a later version of traces added\n', + }) + const outDir = newDir('view') + const { manifest } = await projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' }) + + const unknown = manifest.excluded.find((entry) => entry.path === 'derived/future-artifact.md')! + expect(unknown.rule).toBe('not-allow-listed') + expect(await listSessionBundleFiles(outDir)).toEqual(['derived/session-index.json', 'manifest.json']) + }) + + it('refuses a bundle whose bytes drifted from its manifest', async () => { + const bundleDir = await writeSyntheticBundle({ + 'session/transcript.jsonl': `${JSON.stringify({ message: { content: 'hello' } })}\n`, + 'derived/session-index.json': `${JSON.stringify({ kind: 'traces.session_index' })}\n`, + }) + await writeFile(join(bundleDir, 'derived', 'session-index.json'), '{"kind":"tampered"}\n', 'utf8') + + await expect( + projectSessionBundle({ bundleDir, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/does not match its manifest hash/) + }) + + it('refuses a non-bundle directory, an older manifest, a view of a view, and an unknown view', async () => { + const empty = newDir('empty') + await mkdir(empty, { recursive: true }) + await expect( + projectSessionBundle({ bundleDir: empty, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/is not a traces session bundle/) + + const v1 = newDir('v1') + await mkdir(v1, { recursive: true }) + await writeFile( + join(v1, 'manifest.json'), + JSON.stringify({ schemaVersion: 1, kind: 'traces.session_bundle' }), + 'utf8', + ) + await expect( + projectSessionBundle({ bundleDir: v1, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/schemaVersion 1; this traces reads 2/) + + const { bundleDir } = await assembleFullBundle() + const viewDir = newDir('view') + await projectSessionBundle({ bundleDir, outDir: viewDir, view: 'evidence-only' }) + await expect( + projectSessionBundle({ bundleDir: viewDir, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/already the evidence-only view/) + await expect( + projectSessionBundle({ bundleDir, outDir: newDir('view'), view: 'full' }), + ).rejects.toThrow(/unsupported bundle view "full"/) + await expect( + projectSessionBundle({ bundleDir, outDir: viewDir, view: 'evidence-only' }), + ).rejects.toThrow(/is not empty/) + }) +}) + +describe('classifyEvidenceOnlyPath', () => { + it('carries only the allow-list and denies everything else', () => { + for (const path of [ + 'derived/session-index.json', + 'derived/evidence.jsonl', + 'ledger/current.json', + 'ledger/scorecard.json', + 'ledger/skill-runs.jsonl', + ]) { + expect(classifyEvidenceOnlyPath(path)).toMatchObject({ carried: true }) + } + expect(classifyEvidenceOnlyPath('session/transcript.jsonl')).toMatchObject({ carried: false, rule: 'session-source' }) + expect(classifyEvidenceOnlyPath('session/subagents/a.jsonl')).toMatchObject({ carried: false, rule: 'session-source' }) + expect(classifyEvidenceOnlyPath('derived/report.md')).toMatchObject({ carried: false, rule: 'session-text-derived' }) + expect(classifyEvidenceOnlyPath('derived/trace.otlp.jsonl')).toMatchObject({ carried: false, rule: 'session-text-derived' }) + expect(classifyEvidenceOnlyPath('ledger/progress.md')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('ledger/handoff-2026-01-01-x.md')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('ledger/reflections/2026-01-01.md')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('ledger/experiments.jsonl')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('repo/git-log.txt')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('derived/anything-new.json')).toMatchObject({ carried: false, rule: 'not-allow-listed' }) + }) +}) + +describe('sessionTextSignatures', () => { + it('signs prose and ignores identifiers, structure, and package constants', () => { + expect([...sessionTextSignatures(LEAKED_SENTENCE)].length).toBeGreaterThan(0) + + // A shared session path is the same identifier on both sides, not shared prose. + const path = '/home/u/.claude/projects/-home-u-code-app/2f1c9d40-1111-2222-3333-444455556666.jsonl' + expect(sessionTextSignatures(path).size).toBe(0) + + // A counted artifact's key/number structure is not a sentence. + const counted = JSON.stringify({ + tokenUsage: { input: { n: 0, mean: null, p50: null, p95: null, stddev: null, min: null, max: null } }, + }) + expect(sessionTextSignatures(counted).size).toBe(0) + + // A JSON line signs its VALUES, so a field name can never collide with prose. + const row = JSON.stringify({ decision: LEAKED_SENTENCE }) + expect([...sessionTextSignatures(row)]).toEqual([...sessionTextSignatures(LEAKED_SENTENCE)]) + + // Signatures never span a line boundary: ten words on one line sign, the + // same ten words split across two lines of five do not. + const ten = 'the queue depth decides tail latency far more than the' + expect(sessionTextSignatures(ten).size).toBeGreaterThan(0) + expect(sessionTextSignatures(ten.split(' ').slice(0, 5).join(' ') + '\n' + ten.split(' ').slice(5).join(' ')).size).toBe(0) + }) +}) diff --git a/tests/bundle.test.ts b/tests/bundle.test.ts index ab59f60..54378aa 100644 --- a/tests/bundle.test.ts +++ b/tests/bundle.test.ts @@ -136,6 +136,12 @@ describe('assembleSessionBundle', () => { expect(directory).toBe(outDir) expect(manifest.kind).toBe('traces.session_bundle') + expect(manifest.schemaVersion).toBe(2) + // `traces bundle` assembles the auditor's copy, and says so: a reader + // never has to infer which view a directory is from its file list. + expect(manifest.view).toBe('full') + expect(manifest.excluded).toEqual([]) + expect(manifest.projection).toBeUndefined() expect(manifest.createdAt).toBe('2026-01-01T01:00:00.000Z') expect(manifest.provenance.sessionId).toBe('bundle-fixture') expect(manifest.provenance.harness).toBe('claude-code') diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 078fd4d..08c12a4 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -939,3 +939,68 @@ describe('traces analyze --llm failure surfacing', () => { expect(reportText).toContain('DSPY-BRIDGE-FAILURE: ValueError: analyze input must contain exactly') }, 60_000) }) + +describe('traces bundle + bundle-view', () => { + it('assembles the full view, then projects the writer view over the same session', async () => { + const root = await mkdtemp(join(tmpdir(), 'traces-cli-bundle-')) + const leaked = 'The retry budget was the wrong lever and the queue depth decides tail latency here.' + const transcript = join(root, 'session.jsonl') + await writeFile( + transcript, + [ + { + type: 'user', + uuid: 'u1', + sessionId: 'cli-bundle', + timestamp: '2026-01-01T00:00:00Z', + message: { role: 'user', content: leaked }, + }, + { + type: 'assistant', + uuid: 'a1', + sessionId: 'cli-bundle', + timestamp: '2026-01-01T00:00:01Z', + message: { id: 'm1', role: 'assistant', content: 'understood' }, + }, + ].map((row) => JSON.stringify(row)).join('\n'), + 'utf8', + ) + + const run = (args: string[]) => + execFileAsync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], { + cwd: process.cwd(), + env: { ...process.env, NO_COLOR: '1', FORCE_COLOR: '' }, + maxBuffer: 10 * 1024 * 1024, + timeout: 120_000, + }) + + const bundleDir = join(root, 'bundle') + const bundled = await run([ + 'bundle', '--harness', 'claude-code', '--session', transcript, '--out', bundleDir, + ]) + expect(bundled.stdout).toContain('session bundle (full view) →') + + const viewDir = join(root, 'view') + const projected = await run([ + 'bundle-view', bundleDir, '--view', 'evidence-only', '--out', viewDir, + ]) + expect(projected.stdout).toContain('evidence-only view →') + expect(projected.stdout).toContain('0 session-text match(es)') + + const manifest = JSON.parse(await readFile(join(viewDir, 'manifest.json'), 'utf8')) as { + view: string + excluded: { path: string; rule: string }[] + } + expect(manifest.view).toBe('evidence-only') + expect(manifest.excluded.map((entry) => entry.path)).toContain('session/transcript.jsonl') + await expect(readFile(join(viewDir, 'session', 'transcript.jsonl'), 'utf8')).rejects.toThrow() + await expect(readFile(join(viewDir, 'derived', 'report.md'), 'utf8')).rejects.toThrow() + expect(await readFile(join(bundleDir, 'derived', 'report.md'), 'utf8')).toContain(leaked) + expect(await readFile(join(viewDir, 'derived', 'evidence.jsonl'), 'utf8')).not.toContain(leaked) + + // --view belongs to bundle-view alone, so nobody can ask `bundle` for a + // projection and silently receive the full record. + await expect(run(['bundle', '--session', transcript, '--view', 'evidence-only', '--out', join(root, 'x')])) + .rejects.toThrow(/--view is not supported by bundle/) + }, 180_000) +})