diff --git a/README.md b/README.md index 52e5f81..2ae894e 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Emitting the contract is the supported way to integrate a new system. The adapte - [Watch a run tree](#watch-a-run-tree) - [Improvement engine](#improvement-engine) - [Session index](#session-index) -- [Session bundle](#session-bundle) +- [Session bundle](#session-bundle) · [Two views](#two-views-two-consumers) - [Policy-mining evidence](#policy-mining-evidence) - [Upload to the Intelligence Platform](#upload-to-the-intelligence-platform) - [Trace analysts](#trace-analysts) @@ -245,6 +245,7 @@ traces analyze --otlp results/sessions --out report.md traces convert --harness claude-code --last 1 --otlp-out spans.jsonl # OTLP only traces index --all --since 24h --out session-index.json traces bundle --harness claude-code --session --out bundle-dir # one session's durable evidence dir +traces bundle-view bundle-dir --view evidence-only --out writer-dir # same session, without its own words traces inspect session-index.json --out inspection-report.md traces evidence --harness codex --last 20 --out policy-evidence.jsonl traces evidence --harness codex-exec --session /tmp/codex.jsonl --cwd "$PWD" --out policy-evidence.jsonl @@ -472,11 +473,38 @@ The bundle directory: - `derived/` — `session-index.json`, the deterministic `report.md`, `evidence.jsonl`, and the OTLP span artifact - `ledger/` — the repo's `.evolve` slices for the session window: `experiments.jsonl` and `skill-runs.jsonl` rows inside the padded span window, `current.json` / `scorecard.json` / `progress.md` copied whole, the latest flat `handoff-*.md`, and reflections dated inside the window - `repo/git-log.txt` — commits in the session window from the session's cwd -- `manifest.json` — SHA-256 + byte count per file, provenance (session id, harness, cwd, transcript hash, session window), every absent artifact with its reason, per-slice row counts, and the bundle's known limits +- `manifest.json` — `view`, SHA-256 + byte count per file, provenance (session id, harness, cwd, transcript hash, session window), every absent artifact with its reason, per-slice row counts, and the bundle's known limits A missing transcript fails the assembly loudly. Every optional input that is absent (no subagents, no `.evolve`, no git repo) is recorded in `manifest.absent` with the probed path — a fact, not an error. +### Two views, two consumers + +A bundle has two kinds of reader, and they need opposite things. + +- An **auditor** cites the session and needs every source byte. That is the FULL view — what `traces bundle` writes, `manifest.view: "full"`. +- A **writer** must reach its own conclusion about the session, so it must not be able to read the conclusion that session already reached. That is the EVIDENCE-ONLY view, `manifest.view: "evidence-only"`. + +The full bundle cannot serve the writer. Its transcript holds every word the session wrote, including the shell call that authored an earlier report, so a check that looks for the report FILE passes while the report CONTENT is right there. Projecting is the fix, not grepping: + +```bash +traces bundle --harness claude-code --session --out bundle-dir +traces bundle-view bundle-dir --view evidence-only --out writer-dir +``` + +The evidence-only view carries an explicit allow-list — `derived/session-index.json`, `derived/evidence.jsonl`, and the structured `ledger/` records (`current.json`, `scorecard.json`, `skill-runs.jsonl`). Everything else is excluded **by name** in `manifest.excluded`, with its SHA-256 and one of four rules: + +| rule | what it covers | +|---|---| +| `session-source` | `session/` — the transcript and every subagent transcript | +| `session-text-derived` | `derived/report.md` (quotes the first prompt line and evidence excerpts) and `derived/trace.otlp.jsonl` (spans carry the full prompt and response text) | +| `authored-prose` | `ledger/progress.md`, `ledger/handoff-*.md`, `ledger/reflections/`, `ledger/experiments.jsonl` free-text, `repo/git-log.txt` | +| `not-allow-listed` | any other path, including one a later version of `traces` adds — the view can only narrow by accident, never widen | + +The allow-list is the structure. A content check is the proof it is right: before anything is written, every candidate is compared against every excluded file for shared 8-word runs of prose, with identifiers (paths, URLs, hashes) and this package's own constant strings scrubbed from both sides first. A shared run in a repo ledger file drops that file with `rule: "content-signature"` and the match count; a shared run in an artifact `traces` derives is a defect in the derivation and no view is written at all. `manifest.projection.leakCheck` records the width, the sources compared, and the result — always `matches: 0`, because a match means no view exists. + +A projected view also carries the full bundle's `manifest.json` hash, so an auditor can prove which record it came from. Projecting a view of a view is refused. + ## Policy-mining evidence `traces` does **not** emit benchmark campaign cells. It emits normalized coding-agent session evidence that another system can mine. diff --git a/skills/inspect-agent-traces/SKILL.md b/skills/inspect-agent-traces/SKILL.md index cb08f6e..a6cbbbe 100644 --- a/skills/inspect-agent-traces/SKILL.md +++ b/skills/inspect-agent-traces/SKILL.md @@ -22,10 +22,9 @@ traces analyze --otlp results/sessions --out .traces/all.md # a directory of e Use `--harness` only for coding agents whose on-disk format we do not control. Those adapters are the legacy edge, not the way to integrate a system you own. -A directory is a run directory: it holds the span export beside raw event, stream and -SDK logs that are also `*.jsonl`. Only the OTLP files are read, and the rest are listed -with what they actually hold. An `otlp/` subdirectory, when the producer made one, is -read on its own. +A run directory holds the span export beside raw event, stream and SDK logs that are +also `*.jsonl`. Only the OTLP files are read; the rest are listed with what they hold. +An `otlp/` subdirectory, when present, is read on its own. Any section headed `inputs incomplete`, or carrying an `Inputs incomplete` line above its table, is computed from a field the trace does not carry everywhere. Report those @@ -73,7 +72,7 @@ traces improve --harness codex --current --latest-turn --workflow \ `improve` writes findings, evidence, a report, and spans. It does not edit an agent, repository, memory store, or knowledge base. -Write one session's durable evidence directory when a later reader must cite the session: +Write one session's durable evidence directory for a later reader: ```bash traces bundle --harness claude-code --session --out .traces/bundle @@ -81,12 +80,24 @@ traces bundle --harness claude-code --session --out .traces/bundle `bundle` copies the transcript, the derived report and spans, the `.evolve` ledger rows inside the session window, and a `manifest.json` with a SHA-256 per file. It spends no model call. -Use it when the live stores can rotate before the reader arrives. A missing transcript stops the assembly. An absent optional input is recorded in `manifest.absent` with the probed path. -The bundle holds the whole session transcript. -Do not give a bundle to a writer that must not see an earlier conclusion, because the transcript holds the text of every file the session wrote. +## Pick the view for the reader + +That bundle is the FULL view (`manifest.view: "full"`), and it holds the whole session transcript. +Never give it to a writer that must not see an earlier conclusion: the transcript holds the text of every file the session wrote, so a check for the earlier report FILE passes while its CONTENT is still readable. + +Project the writer's copy instead: + +```bash +traces bundle-view .traces/bundle --view evidence-only --out .traces/writer +``` + +That view carries `derived/session-index.json`, `derived/evidence.jsonl`, and the structured `ledger/` records. +It excludes the transcripts, the report, the spans, and every prose ledger file by name, with rules and hashes in `manifest.excluded`. +Before writing, it compares each carried file against each excluded file for shared 8-word runs of prose, and drops any file that repeats one. +`manifest.view` names which copy you hold. ## Report diff --git a/src/bundle-view.ts b/src/bundle-view.ts new file mode 100644 index 0000000..97e50fe --- /dev/null +++ b/src/bundle-view.ts @@ -0,0 +1,526 @@ +/** + * `traces bundle-view` — project a full session bundle down to one view. + * + * A bundle has two consumers with opposite needs. An AUDITOR cites the + * session and needs every source byte. A WRITER must reach its own conclusion + * about the session and therefore must not be able to read the conclusion the + * session already reached — which the transcript holds verbatim, including the + * shell call that authored an earlier report. + * + * Serving both from one directory cannot work, and policing it by grep cannot + * work either: a check that looks for the earlier report FILE passes while its + * CONTENT sits in the transcript. So the two consumers get two directories. + * The evidence-only view is a projection of the full bundle: an explicit + * allow-list of counted artifacts, every other path excluded by name with its + * reason and its hash, and a content check that proves no excluded file's text + * survived into the view. + * + * The allow-list is the structure. The content check is the proof that the + * structure is right: before anything is written, every candidate is compared + * against every excluded file, and a candidate that repeats one is dropped + * from the view and named in the manifest. So the written view never carries + * text from a file the view says it removed. + */ + +import { createHash } from 'node:crypto' +import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { + hashSessionBundleFiles, + listSessionBundleFiles, + readSessionBundleManifest, + writeSessionBundleManifest, + type SessionBundleExclusion, + type SessionBundleExclusionRule, + type SessionBundleLeakCheck, + type SessionBundleManifest, + type SessionBundleView, +} from './bundle.js' +import { POLICY_EVIDENCE_NOTE } from './evidence.js' + +/** + * Bundle-relative paths the evidence-only view carries. Everything else is + * excluded, so a path added to the bundle later is dropped until someone puts + * it here on purpose: the view can only get narrower by accident, never wider. + * + * Each entry states what the artifact is MADE of, because that is the reason + * it is safe: counts, names, identifiers, timestamps and hashes, derived by + * code from the spans — never a run of words the session wrote. + * + * `owned` marks the artifacts whose schema THIS package defines and whose every + * field it derives. Those two classes fail differently under the content check: + * an owned artifact that repeats session text is a defect in traces and stops + * the projection, while a repo ledger file is free-form by convention and its + * text is a fact about that repo, so it is dropped from the view and named in + * the manifest instead. + */ +interface AllowListEntry { + readonly path: string + readonly owned: boolean + readonly carries: string +} + +const EVIDENCE_ONLY_ALLOW_LIST: readonly AllowListEntry[] = [ + { + path: 'derived/session-index.json', + owned: true, + carries: 'per-session counts, tool names, models, and nearby-context file metadata', + }, + { + path: 'derived/evidence.jsonl', + owned: true, + carries: 'execution statistics, token and tool totals, loop and error counts', + }, + { + path: 'ledger/current.json', + owned: false, + carries: 'the repo ledger current-state record', + }, + { + path: 'ledger/scorecard.json', + owned: false, + carries: 'the repo ledger scorecard rows', + }, + { + path: 'ledger/skill-runs.jsonl', + owned: false, + carries: 'skill-run rows: skill name, timestamp, and outcome fields', + }, +] + +interface ExclusionRuleEntry { + readonly matches: (path: string) => boolean + readonly rule: SessionBundleExclusionRule + readonly reason: string +} + +/** + * Why each excluded path is excluded, in the words a reader of the manifest + * needs. The last entry is the default deny, and it must stay last. + */ +const EXCLUSION_RULES: readonly ExclusionRuleEntry[] = [ + { + matches: (path) => path === 'session/transcript.jsonl' || path.startsWith('session/'), + rule: 'session-source', + reason: + 'raw session text: the transcript and any subagent transcript hold every word the session wrote, ' + + 'including the shell call that authored an earlier report', + }, + { + matches: (path) => path === 'derived/report.md', + rule: 'session-text-derived', + reason: 'the deterministic report quotes the session verbatim: the first prompt line and evidence excerpts per finding', + }, + { + matches: (path) => path === 'derived/trace.otlp.jsonl', + rule: 'session-text-derived', + reason: 'spans carry input.value, output.value and content: the full prompt and response text of every turn', + }, + { + matches: (path) => path === 'ledger/progress.md', + rule: 'authored-prose', + reason: 'progress notes are written at session close and restate what the session concluded', + }, + { + matches: (path) => /^ledger\/handoff-.*\.md$/.test(path), + rule: 'authored-prose', + reason: 'a handoff is the session conclusion in prose', + }, + { + matches: (path) => path.startsWith('ledger/reflections/'), + rule: 'authored-prose', + reason: 'a reflection is the session conclusion in prose', + }, + { + matches: (path) => path === 'ledger/experiments.jsonl', + rule: 'authored-prose', + reason: 'experiment rows carry free-text verdict and note fields that restate what the session concluded', + }, + { + matches: (path) => path === 'repo/git-log.txt', + rule: 'authored-prose', + reason: 'commit subjects are prose the session authored', + }, + { + matches: () => true, + rule: 'not-allow-listed', + reason: 'the evidence-only view carries an explicit allow-list of counted artifacts, and this path is not on it', + }, +] + +/** + * Words per content signature: the width at which a shared run reads as a + * quotation rather than a coincidence. + * + * Measured over one real 170-file bundle: the deterministic report shares 509 + * of its 747 runs with the transcript, while the two counted artifacts the + * view carries hold no eight-word run of prose at all. + */ +const SIGNATURE_WORDS = 8 + +/** + * How many of those words must be letters for the run to count as prose. + * + * `input 0 output 0 reasoning 0 cached 0` is a token-usage block, not a + * sentence, so any two files that ever printed one would share it. Requiring 7 + * letters of 8 keeps the quotations and drops the structural runs: on that same + * bundle it cut one ledger file's recorded overlap from 47 runs to the 22 that + * are sentences. + */ +const MIN_WORD_TOKENS = 7 + +/** + * Identifiers are not prose. A session file path, a URL, a UUID and a SHA + * appear on both sides of every comparison because both sides describe the + * same session, and counting those as leaked text would exclude the counted + * artifacts and leave the view empty. Scrubbing them from BOTH sides before + * signing leaves only words someone chose. + */ +function scrubIdentifiers(line: string): string { + return line + .replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, ' ') + .replace(/\/?(?:[\w.@+~-]+\/)+[\w.@+~-]*/g, ' ') + .replace(/\b[0-9a-f-]{8,}\b/gi, ' ') +} + +/** + * Prose this package writes into every derived artifact, whatever the session. + * + * A constant is not session text, but it reads as one to a content check when + * the session happened to work on this package's own source. Subtracting these + * from the comparison keeps the check about words the SESSION produced. Each + * entry is imported from the code that emits it, so the two cannot drift. + */ +const PACKAGE_CONSTANT_PROSE: readonly string[] = [POLICY_EVIDENCE_NOTE] + +function stripPackageConstants(text: string): string { + let out = text + for (const constant of PACKAGE_CONSTANT_PROSE) out = out.split(constant).join(' ') + return out +} + +function collectJsonStrings(value: unknown, into: string[]): void { + if (typeof value === 'string') into.push(value) + else if (Array.isArray(value)) for (const item of value) collectJsonStrings(item, into) + else if (value !== null && typeof value === 'object') { + // Values only. A field NAME is schema this package or a ledger convention + // chose, so `stddev null min null max` would otherwise read as a shared + // sentence between a counted artifact and any transcript that printed one. + for (const item of Object.values(value)) collectJsonStrings(item, into) + } +} + +/** + * The text of one line that a person or a model could have written. + * + * A JSON line yields its string VALUES; anything else yields the line itself. + * Probing with a parse is the point: "this line is not JSON" is an answer, not + * a failure, and the line is then compared as plain text. + */ +function proseFragments(line: string): string[] { + const trimmed = line.trim() + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + const parsed: unknown = JSON.parse(trimmed) + const strings: string[] = [] + collectJsonStrings(parsed, strings) + return strings + } catch { + // not JSON after all → compared as plain text below + } + } + return [line] +} + +/** + * Distinct signatures in one text: every run of `SIGNATURE_WORDS` consecutive + * words, lowercased, taken WITHIN one line's prose. Signatures never span a + * newline, so memory stays bounded by the longest line rather than the file, + * and a structure whose every line is a key and a number produces no + * signatures at all — which is the true statement about it: it holds no prose. + */ +export function sessionTextSignatures(text: string): Set { + const out = new Set() + for (const line of stripPackageConstants(text).split('\n')) { + for (const fragment of proseFragments(line)) { + const words = scrubIdentifiers(fragment) + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() + .split(' ') + .filter(Boolean) + for (let i = 0; i + SIGNATURE_WORDS <= words.length; i++) { + const window = words.slice(i, i + SIGNATURE_WORDS) + if (window.filter((word) => /^[a-z]+$/.test(word)).length < MIN_WORD_TOKENS) continue + out.add(window.join(' ')) + } + } + } + return out +} + +export interface SessionTextLeak { + /** Bundle-relative path of the carried file that repeats excluded text. */ + readonly viewPath: string + /** Bundle-relative paths of the excluded files the repeated text came from. */ + readonly sources: readonly string[] + /** Distinct signatures this file shares with those sources. */ + readonly signatures: number + /** + * One matched signature, for a diagnostic message only. It is a fragment of + * the excluded text, so it must never be written into a view artifact. + */ + readonly example: string +} + +export interface FindSessionTextLeaksOptions { + /** Directory holding the carried files. */ + readonly viewDir: string + /** Directory holding the excluded files. */ + readonly sourceDir: string + /** Bundle-relative paths under `viewDir` to check. */ + readonly carriedPaths: readonly string[] + /** Bundle-relative paths under `sourceDir` whose text must not appear in the view. */ + readonly excludedPaths: readonly string[] +} + +export interface SessionTextLeakReport { + /** One entry per carried file that repeats excluded text. Empty is the pass. */ + readonly leaks: readonly SessionTextLeak[] + readonly carried: readonly { readonly path: string; readonly signatures: number }[] + readonly comparedSources: readonly string[] + readonly sourceSignatures: number +} + +/** + * Compare a set of carried files against the files a view excluded. + * + * The signature index is built from the CARRIED files, which are small, and + * the excluded files are then read one at a time and tested against it — the + * cheap direction, since a transcript can be orders of magnitude larger than + * everything a view keeps. Every excluded file is scanned to the end, so the + * per-file counts are totals rather than "however many we saw before stopping". + */ +export async function findSessionTextLeaks( + opts: FindSessionTextLeaksOptions, +): Promise { + const index = new Map() + const carried: { path: string; signatures: number }[] = [] + for (const path of opts.carriedPaths) { + const signatures = sessionTextSignatures(await readFile(join(opts.viewDir, path), 'utf8')) + carried.push({ path, signatures: signatures.size }) + for (const signature of signatures) if (!index.has(signature)) index.set(signature, path) + } + + const hits = new Map; sources: Set; example: string }>() + const comparedSources: string[] = [] + let sourceSignatures = 0 + for (const sourcePath of opts.excludedPaths) { + const signatures = sessionTextSignatures(await readFile(join(opts.sourceDir, sourcePath), 'utf8')) + comparedSources.push(sourcePath) + sourceSignatures += signatures.size + for (const signature of signatures) { + const viewPath = index.get(signature) + if (viewPath === undefined) continue + const hit = hits.get(viewPath) ?? { matched: new Set(), sources: new Set(), example: signature } + hit.matched.add(signature) + hit.sources.add(sourcePath) + hits.set(viewPath, hit) + } + } + const leaks: SessionTextLeak[] = [...hits].map(([viewPath, hit]) => ({ + viewPath, + sources: [...hit.sources].sort(), + signatures: hit.matched.size, + example: hit.example, + })) + return { leaks, carried, comparedSources, sourceSignatures } +} + +export type EvidenceOnlyVerdict = + | { readonly carried: true; readonly owned: boolean; readonly carries: string } + | { readonly carried: false; readonly rule: SessionBundleExclusionRule; readonly reason: string } + +/** Classify one bundle-relative path against the evidence-only view. */ +export function classifyEvidenceOnlyPath(path: string): EvidenceOnlyVerdict { + const allowed = EVIDENCE_ONLY_ALLOW_LIST.find((entry) => entry.path === path) + if (allowed) return { carried: true, owned: allowed.owned, carries: allowed.carries } + const rule = EXCLUSION_RULES.find((entry) => entry.matches(path))! + return { carried: false, rule: rule.rule, reason: rule.reason } +} + +/** + * Limits of the evidence-only view, carried in its manifest so a reader meets + * the boundary in the artifact instead of discovering it. + */ +const EVIDENCE_ONLY_LIMITS: readonly string[] = [ + 'the exclusion is verbatim-tight, not paraphrase-tight: a conclusion restated in different words inside a carried artifact is not detectable by content signatures', + 'ledger prose (progress, handoffs, reflections) and experiment free-text fields are excluded whole; a measured experiment result is readable only from the full bundle, under audit rules', + 'the view carries counts, names, identifiers, timestamps and hashes; a question that needs the words of a turn cannot be answered from it', +] + +export interface ProjectSessionBundleOptions { + /** Directory holding the FULL bundle to project. */ + readonly bundleDir: string + /** View directory to create. Must be new or empty: one directory, one view. */ + readonly outDir: string + readonly view: SessionBundleView + readonly generatedAt?: string +} + +export interface ProjectSessionBundleResult { + readonly directory: string + readonly manifestPath: string + readonly manifest: SessionBundleManifest + readonly leakCheck: SessionBundleLeakCheck +} + +/** + * Project a full bundle into a narrower view. + * + * Every file is verified against the source manifest's hash first: a bundle + * whose bytes have drifted from its own manifest cannot be projected, because + * the classification is only meaningful over the files this package wrote. + * The content check then runs before anything is written, so a candidate that + * repeats an excluded source is dropped instead of copied. + */ +export async function projectSessionBundle( + opts: ProjectSessionBundleOptions, +): Promise { + if (opts.view !== 'evidence-only') { + throw new Error( + `unsupported bundle view ${JSON.stringify(opts.view)}. ` + + 'The full view IS the bundle `traces bundle` assembles, so there is nothing to project into it; ' + + 'the projectable view is "evidence-only".', + ) + } + const generatedAt = opts.generatedAt ?? new Date().toISOString() + const sourceDir = resolve(opts.bundleDir) + const source = await readSessionBundleManifest(sourceDir) + if (source.view !== 'full') { + throw new Error( + `${sourceDir} is already the ${source.view} view — project from the full bundle, not from a view of it`, + ) + } + + const outDir = resolve(opts.outDir) + await mkdir(outDir, { recursive: true }) + if ((await readdir(outDir)).length > 0) { + throw new Error( + `view output directory ${outDir} is not empty — pass a new directory so one directory holds exactly one view`, + ) + } + + const sourceHashes = new Map(source.files.map((file) => [file.path, file] as const)) + const carriedPaths: string[] = [] + const carries = new Map() + const excluded: SessionBundleExclusion[] = [] + const candidates: { + path: string + bytes: Buffer + sha256: string + carries: string + owned: boolean + }[] = [] + for (const path of await listSessionBundleFiles(sourceDir)) { + // manifest.json is rewritten for the view, never copied: it names the view + // it describes, and a copied one would name the wrong view. + if (path === 'manifest.json') continue + const bytes = await readFile(join(sourceDir, path)) + const sha256 = createHash('sha256').update(bytes).digest('hex') + const recorded = sourceHashes.get(path) + if (!recorded) { + throw new Error( + `${join(sourceDir, path)} is not listed in the bundle manifest — the bundle changed after assembly ` + + 'and cannot be projected; re-run `traces bundle`', + ) + } + if (recorded.sha256 !== sha256) { + throw new Error( + `${join(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha256} on disk) — ` + + 'the bundle changed after assembly and cannot be projected; re-run `traces bundle`', + ) + } + const verdict = classifyEvidenceOnlyPath(path) + if (!verdict.carried) { + excluded.push({ path, bytes: bytes.length, sha256, rule: verdict.rule, reason: verdict.reason }) + continue + } + candidates.push({ path, bytes, sha256, carries: verdict.carries, owned: verdict.owned }) + } + + // The content check runs BEFORE anything is written, so a file that repeats + // an excluded source never reaches the view directory at all. + const report = await findSessionTextLeaks({ + viewDir: sourceDir, + sourceDir, + carriedPaths: candidates.map((candidate) => candidate.path), + excludedPaths: excluded.map((entry) => entry.path), + }) + const leakByPath = new Map(report.leaks.map((leak) => [leak.viewPath, leak] as const)) + const ownedLeak = report.leaks.find((leak) => candidates.some((c) => c.path === leak.viewPath && c.owned)) + if (ownedLeak) { + throw new Error( + `${ownedLeak.viewPath} repeats ${ownedLeak.signatures} ${SIGNATURE_WORDS}-word run(s) found in ` + + `${ownedLeak.sources.join(', ')}, for example "${ownedLeak.example}". That artifact is derived by ` + + 'traces and must carry counts only, so this is a defect in the derivation, not a property of the ' + + 'session. No view was written.', + ) + } + + for (const candidate of candidates) { + const leak = leakByPath.get(candidate.path) + if (leak) { + excluded.push({ + path: candidate.path, + bytes: candidate.bytes.length, + sha256: candidate.sha256, + rule: 'content-signature', + reason: + `the content check found ${leak.signatures} ${SIGNATURE_WORDS}-word run(s) this file shares with ` + + `${leak.sources.join(', ')}; carrying it would put back text the view removes`, + contentMatch: { signatures: leak.signatures, sources: leak.sources }, + }) + continue + } + const target = join(outDir, candidate.path) + await mkdir(dirname(target), { recursive: true }) + await writeFile(target, candidate.bytes) + carriedPaths.push(candidate.path) + carries.set(candidate.path, candidate.carries) + } + + const leakCheck: SessionBundleLeakCheck = { + signatureWords: SIGNATURE_WORDS, + comparedSources: report.comparedSources, + sourceSignatures: report.sourceSignatures, + carried: report.carried + .filter((entry) => carries.has(entry.path)) + .map((entry) => ({ + path: entry.path, + carries: carries.get(entry.path)!, + signatures: entry.signatures, + })), + matches: 0, + } + const sourceManifestBytes = await readFile(join(sourceDir, 'manifest.json')) + const manifest: SessionBundleManifest = { + schemaVersion: 2, + kind: 'traces.session_bundle', + view: 'evidence-only', + createdAt: generatedAt, + provenance: source.provenance, + files: await hashSessionBundleFiles(outDir), + absent: source.absent, + excluded, + ledgerSlices: source.ledgerSlices.filter((slice) => carriedPaths.includes(slice.path)), + knownLimits: [...source.knownLimits, ...EVIDENCE_ONLY_LIMITS], + projection: { + sourceDirectory: sourceDir, + sourceManifestSha256: createHash('sha256').update(sourceManifestBytes).digest('hex'), + leakCheck, + }, + } + const manifestPath = await writeSessionBundleManifest(outDir, manifest) + return { directory: outDir, manifestPath, manifest, leakCheck } +} diff --git a/src/bundle.ts b/src/bundle.ts index 3a16748..76d4dc0 100644 --- a/src/bundle.ts +++ b/src/bundle.ts @@ -15,6 +15,10 @@ * manifest (`absent`, with the reason), never a silent gap and never an * error: a session without subagents or without an `.evolve` ledger is a * complete bundle of a smaller session. + * + * This module assembles the FULL view. `bundle-view.ts` projects that full + * bundle down to the evidence-only view for a consumer that must not read the + * session's own words; `manifest.view` names which of the two a directory is. */ import { createHash } from 'node:crypto' @@ -66,9 +70,83 @@ export interface SessionBundleLedgerSlice { readonly unparseableTsRows?: number } +/** + * Which projection of a session a bundle directory holds. + * + * `full` is the auditor's copy: every source byte, so a claim can be checked + * against the exact text it cites. `evidence-only` is the writer's copy: the + * counted artifacts, with the session's own words removed, so a writer that + * must reach its own conclusion cannot read an earlier one instead. + */ +export type SessionBundleView = 'full' | 'evidence-only' + +export type SessionBundleExclusionRule = + /** The session's raw text: transcript and subagent transcripts. */ + | 'session-source' + /** A derived artifact that replays session text (prompt lines, excerpts, span content). */ + | 'session-text-derived' + /** A file whose body is prose someone authored: ledger notes, handoffs, commit subjects. */ + | 'authored-prose' + /** An allow-listed file dropped because its own text repeats an excluded source. */ + | 'content-signature' + /** Default deny: the evidence-only view carries an allow-list, and this path is not on it. */ + | 'not-allow-listed' + +export interface SessionBundleExclusion { + /** Bundle-relative path in the FULL bundle this view was projected from. */ + readonly path: string + readonly bytes: number + /** SHA-256 of the excluded bytes, so an auditor can prove which file was dropped. */ + readonly sha256: string + readonly rule: SessionBundleExclusionRule + readonly reason: string + /** + * Present when `rule` is `content-signature`: how much text the file shared + * with which excluded sources. Counts and paths only — quoting the matched + * text here would carry into the view the very text the exclusion removes. + */ + readonly contentMatch?: { + readonly signatures: number + readonly sources: readonly string[] + } +} + +export interface SessionBundleCarriedSignatures { + readonly path: string + /** What the artifact is made of, which is the reason it is safe to carry. */ + readonly carries: string + /** Distinct content signatures the carried file holds at all. 0 = no prose run of that length. */ + readonly signatures: number +} + +/** + * The projection's own proof that no excluded file's text survived into the + * view. `matches` is typed `0` because a match aborts the projection: a view + * directory carrying a leak is never written, so the artifact cannot record one. + */ +export interface SessionBundleLeakCheck { + /** Consecutive words per content signature. */ + readonly signatureWords: number + /** Excluded files whose text supplied the signatures compared against. */ + readonly comparedSources: readonly string[] + readonly sourceSignatures: number + readonly carried: readonly SessionBundleCarriedSignatures[] + readonly matches: 0 +} + +export interface SessionBundleProjection { + /** Absolute directory of the full bundle this view was projected from. */ + readonly sourceDirectory: string + /** SHA-256 of that bundle's `manifest.json` bytes. */ + readonly sourceManifestSha256: string + readonly leakCheck: SessionBundleLeakCheck +} + export interface SessionBundleManifest { - readonly schemaVersion: 1 + readonly schemaVersion: 2 readonly kind: 'traces.session_bundle' + /** Which projection this directory is. A reader must branch on this, never on the file list. */ + readonly view: SessionBundleView readonly createdAt: string readonly provenance: { readonly sessionId: string @@ -89,8 +167,12 @@ export interface SessionBundleManifest { } readonly files: readonly SessionBundleFile[] readonly absent: readonly SessionBundleAbsence[] + /** Files the view deliberately dropped. Empty in the `full` view: it drops nothing. */ + readonly excluded: readonly SessionBundleExclusion[] readonly ledgerSlices: readonly SessionBundleLedgerSlice[] readonly knownLimits: readonly string[] + /** Present only in a projected view: where it came from, and the leak check that passed. */ + readonly projection?: SessionBundleProjection } export interface SessionBundleResult { @@ -211,15 +293,20 @@ async function runGitLog(cwd: string, sinceIso: string, untilIso: string): Promi } } -async function walkBundleFiles(root: string): Promise { +/** + * Every file under a bundle directory, as sorted bundle-relative + * `/`-separated paths. Both the manifest writer and the view projector walk a + * bundle through here, so neither can see a file the other does not. + */ +export async function listSessionBundleFiles(root: string): Promise { const out: string[] = [] - const pending = [root] + const pending = [resolve(root)] while (pending.length > 0) { const dir = pending.pop()! for (const entry of await readdir(dir, { withFileTypes: true })) { const path = join(dir, entry.name) if (entry.isDirectory()) pending.push(path) - else if (entry.isFile()) out.push(path) + else if (entry.isFile()) out.push(relative(resolve(root), path).split(sep).join('/')) } } return out.sort() @@ -461,18 +548,11 @@ export async function assembleSessionBundle(opts: AssembleSessionBundleOptions): } // manifest.json — sha256 per file, written LAST so it covers every byte. - const files: SessionBundleFile[] = [] - for (const path of await walkBundleFiles(outDir)) { - const bytes = await readFile(path) - files.push({ - path: relative(outDir, path).split(sep).join('/'), - bytes: bytes.length, - sha256: sha256Hex(bytes), - }) - } + const files = await hashSessionBundleFiles(outDir) const manifest: SessionBundleManifest = { - schemaVersion: 1, + schemaVersion: 2, kind: 'traces.session_bundle', + view: 'full', createdAt: generatedAt, provenance: { sessionId: ref.sessionId, @@ -486,12 +566,72 @@ export async function assembleSessionBundle(opts: AssembleSessionBundleOptions): }, files, absent, + // The full view is the whole record: it drops nothing, and says so rather + // than leaving a reader to infer completeness from a missing field. + excluded: [], ledgerSlices: slices, knownLimits: KNOWN_LIMITS, } + const manifestPath = await writeSessionBundleManifest(outDir, manifest) + return { directory: outDir, manifestPath, manifest } +} + +/** SHA-256 + byte count for every file under a bundle directory, manifest included when present. */ +export async function hashSessionBundleFiles(root: string): Promise { + const files: SessionBundleFile[] = [] + for (const path of await listSessionBundleFiles(root)) { + const bytes = await readFile(join(root, path)) + files.push({ path, bytes: bytes.length, sha256: sha256Hex(bytes) }) + } + return files +} + +export async function writeSessionBundleManifest( + outDir: string, + manifest: SessionBundleManifest, +): Promise { const manifestPath = join(outDir, 'manifest.json') await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8') - return { directory: outDir, manifestPath, manifest } + return manifestPath +} + +/** + * Read a bundle's manifest and prove it is one. A directory that merely looks + * like a bundle must not be projected: the view guarantee is only as good as + * the classification, and the classification only holds over paths this + * package wrote. + */ +export async function readSessionBundleManifest(bundleDir: string): Promise { + const path = join(resolve(bundleDir), 'manifest.json') + let text: string + try { + text = await readFile(path, 'utf8') + } catch (error) { + if (isMissingPathError(error)) { + throw new Error(`no manifest.json at ${path} — ${bundleDir} is not a traces session bundle`) + } + throw error + } + let parsed: unknown + try { + parsed = JSON.parse(text) + } catch (error) { + throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`) + } + const manifest = parsed as Partial + if (manifest?.kind !== 'traces.session_bundle') { + throw new Error(`${path} has kind ${JSON.stringify(manifest?.kind)}, expected "traces.session_bundle"`) + } + if (manifest.schemaVersion !== 2) { + throw new Error( + `${path} is bundle schemaVersion ${JSON.stringify(manifest.schemaVersion)}; this traces reads 2. ` + + 'Re-run `traces bundle` to assemble it with the view-aware manifest.', + ) + } + if (manifest.view !== 'full' && manifest.view !== 'evidence-only') { + throw new Error(`${path} has view ${JSON.stringify(manifest.view)}, expected "full" or "evidence-only"`) + } + return manifest as SessionBundleManifest } let cachedVersion: string | undefined diff --git a/src/cli.ts b/src/cli.ts index 108b57f..ae819b4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,6 +14,7 @@ * traces convert [--harness claude-code] [--last 1] --otlp-out spans.jsonl * traces index [--harness claude-code] [--last 20] --out session-index.json * traces bundle --harness claude-code --session --out + * traces bundle-view --view evidence-only --out * traces inspect session-index.json [--out inspection-report.md] * traces export --out spans.openinference.jsonl * traces import-codetracebench --trajectory-dir --out --revision <40-or-64-character-hex> @@ -108,7 +109,8 @@ import { type SessionWorkflowIssue, type SessionWorkflowSummary, } from './session-workflow.js' -import { assembleSessionBundle } from './bundle.js' +import { assembleSessionBundle, type SessionBundleView } from './bundle.js' +import { projectSessionBundle } from './bundle-view.js' import { buildSessionIndexFromRows, serializeSessionIndex, writeSessionIndexFile } from './session-index.js' import { sessionReportSource } from './report.js' import type { ReportSource } from './report.js' @@ -154,6 +156,8 @@ interface Args { config?: string format?: string mode?: string + /** `bundle-view`: which projection of a bundle to write. */ + view?: string replay: boolean /** `--once`: a single pass instead of a live tail. */ once: boolean @@ -241,6 +245,7 @@ function parseArgs(argv: string[]): Args { case '--model': a.model = next(); break case '--config': a.config = next(); break case '--mode': a.mode = next(); break + case '--view': a.view = next(); break case '--metadata': a.metadata = next(); break case '--attr': { const v = next(); if (v) a.attrs.push(v); break } case '--interval': a.interval = Number(next()); a.intervalExplicit = true; break @@ -354,6 +359,20 @@ function applyCurrentSessionSelection(args: Args): Args { return { ...args, session: sessionId } } +/** + * `--view` names a projection of an assembled bundle, so it belongs to + * `bundle-view` alone. Accepting it silently on `bundle` would let someone ask + * for the evidence-only view and receive the full one, which is the exact + * mistake the two views exist to make impossible. + */ +function validateViewSelection(args: Args): Args { + if (args.view === undefined || args.command === 'bundle-view') return args + throw new Error( + `--view is not supported by ${args.command}. A view is a projection of an assembled bundle: ` + + 'run `traces bundle ... --out ` first, then `traces bundle-view --view evidence-only --out `.', + ) +} + function validateWorkflowSelection(args: Args): Args { if (!args.workflow && !args.latestTurn) return args if (!WORKFLOW_COMMANDS.has(args.command)) { @@ -785,11 +804,37 @@ async function cmdBundle(args: Args): Promise { }) const { manifest } = result console.log( - `session bundle → ${result.directory} (${manifest.files.length} file(s), ` + + `session bundle (${manifest.view} view) → ${result.directory} (${manifest.files.length} file(s), ` + `${manifest.ledgerSlices.length} ledger slice(s), ${manifest.absent.length} recorded absent)`, ) } +const PROJECTABLE_VIEWS: readonly SessionBundleView[] = ['evidence-only'] + +async function cmdBundleView(args: Args): Promise { + if (!args.input) { + throw new Error('bundle-view needs the bundle directory to project; run `traces bundle --out ` first') + } + if (!args.view) { + throw new Error(`bundle-view needs --view <${PROJECTABLE_VIEWS.join('|')}>`) + } + if (!PROJECTABLE_VIEWS.includes(args.view as SessionBundleView)) { + throw new Error(`unknown view "${args.view}"; projectable views: ${PROJECTABLE_VIEWS.join(', ')}`) + } + if (!args.out) throw new Error('bundle-view needs --out — a new or empty directory for the view') + const result = await projectSessionBundle({ + bundleDir: args.input, + outDir: args.out, + view: args.view as SessionBundleView, + }) + const { manifest, leakCheck } = result + console.log( + `${manifest.view} view → ${result.directory} (${manifest.files.length} file(s) carried, ` + + `${manifest.excluded.length} excluded, ${leakCheck.matches} session-text match(es) against ` + + `${leakCheck.sourceSignatures} signature(s) from ${leakCheck.comparedSources.length} excluded source(s))`, + ) +} + async function cmdInspect(args: Args): Promise { if (!args.input) throw new Error('inspect needs an index file; run `traces index --out session-index.json` first') const index = await readSessionIndexFile(args.input) @@ -1538,7 +1583,14 @@ Commands: bundle Assemble one session's durable evidence directory: transcript + subagents, derived index/report/evidence/OTLP, the repo's .evolve ledger sliced to the session window, git log, and a sha256 manifest - (needs --session and --out ) + (needs --session and --out ). This is the FULL + view: it holds the session's own words, for a reader who cites them + bundle-view + Project an assembled bundle into a narrower view for a consumer that + must NOT read the session's own words. --view evidence-only carries + the counted artifacts and structured ledger rows, excludes the + transcript, the report, the spans and every prose file by name, and + refuses to write a view whose text repeats an excluded source inspect Read a session index and print ranked improvement findings export Convert evidence/events files to OpenInference JSONL for HALO import-codetracebench @@ -1592,6 +1644,9 @@ Options: --metadata analyze/export file: attach JSON object fields as span attributes --attr analyze/export file: attach one span attribute (repeatable) --mode stream: visualizer | findings | agent (default visualizer) + --view bundle-view: which projection to write. evidence-only is the + writer's view; the full view is the bundle itself, so there is + nothing to project into it --replay, --once stream: scan once and exit (default for positional input / --session) --once watch : print ONE snapshot and exit, for scripts and agents --no-spans stream: omit per-span pulse events @@ -1667,7 +1722,9 @@ async function main(): Promise { else usage() return } - const args = validateOtlpSelection(validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs))) + const args = validateOtlpSelection( + validateViewSelection(validateWorkflowSelection(applyCurrentSessionSelection(parsedArgs))), + ) switch (args.command) { case 'help': if (args.input === 'import-codetracebench') usageImportCodeTraceBench() @@ -1683,6 +1740,7 @@ async function main(): Promise { case 'convert': await cmdConvert(args); break case 'index': await cmdIndex(args); break case 'bundle': await cmdBundle(args); break + case 'bundle-view': await cmdBundleView(args); break case 'inspect': await cmdInspect(args); break case 'export': await cmdExport(args); break case 'import-codetracebench': await cmdImportCodeTraceBench(args); break diff --git a/src/evidence.ts b/src/evidence.ts index aadca16..9871226 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -14,6 +14,15 @@ import { type ScanOptions, scanSessions } from './session-source.js' import { describeSessionRelationship, type SessionRole } from './session-relationship.js' import type { SessionRef } from './types.js' +/** + * The one sentence of prose every policy-evidence record carries. It is the + * same for every session, so a content check that compares an evidence file + * against a transcript must subtract it: a session that reads or writes this + * package's source would otherwise look like it leaked its own text. + */ +export const POLICY_EVIDENCE_NOTE = + 'This is normalized coding-agent session evidence for downstream policy mining; it is not an eval campaign cell.' + export interface PolicyEvidenceToolSummary { readonly name: string readonly calls: number @@ -220,7 +229,7 @@ export async function buildPolicyEvidenceRecord( ...(opts.otlpPath ? { otlpPath: opts.otlpPath } : {}), ...(opts.sourceSha256 ? { sourceSha256: opts.sourceSha256 } : {}), notCampaignCell: true, - note: 'This is normalized coding-agent session evidence for downstream policy mining; it is not an eval campaign cell.', + note: POLICY_EVIDENCE_NOTE, }, } } diff --git a/src/index.ts b/src/index.ts index 3f9e1aa..2e06599 100644 --- a/src/index.ts +++ b/src/index.ts @@ -89,6 +89,7 @@ export * from './execution.js' // shared execution accounting over normalized sp export * from './evidence.js' // policy-evidence JSONL for downstream miners export * from './session-index.js' // collectSessionIndex() — reusable session catalog export * from './bundle.js' // assembleSessionBundle() — one session's durable evidence dir +export * from './bundle-view.js' // projectSessionBundle() — the evidence-only view for a writer export * from './inspect.js' // inspectSessionIndex() — ranked findings from a session catalog export * from './file-export.js' // convert evidence/events files to OpenInference JSONL export * from './chat-trajectory.js' // generic chat trajectory to stable step spans diff --git a/tests/bundle-view.test.ts b/tests/bundle-view.test.ts new file mode 100644 index 0000000..aae4a63 --- /dev/null +++ b/tests/bundle-view.test.ts @@ -0,0 +1,435 @@ +import { createHash } from 'node:crypto' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { ClaudeAdapter } from '../src/adapters/claude.js' +import { + assembleSessionBundle, + listSessionBundleFiles, + writeSessionBundleManifest, + type SessionBundleManifest, +} from '../src/bundle.js' +import { + classifyEvidenceOnlyPath, + findSessionTextLeaks, + projectSessionBundle, + sessionTextSignatures, +} from '../src/bundle-view.js' +import type { SessionRef } from '../src/types.js' + +const dir = mkdtempSync(join(tmpdir(), 'traces-bundle-view-')) +afterAll(() => rmSync(dir, { recursive: true, force: true })) + +let outSeq = 0 +function newDir(prefix: string): string { + outSeq += 1 + return join(dir, `${prefix}-${outSeq}`) +} + +/** + * One sentence the session wrote and then copied into a ledger file, which is + * the leak this view exists to stop: the file is not the transcript, but the + * words are. + */ +const LEAKED_SENTENCE = + 'The retry budget was the wrong lever and the queue depth is what actually decides tail latency here.' + +/** A sentence only the transcript holds, used to prove the view carries none of it. */ +const TRANSCRIPT_ONLY_SENTENCE = + 'I rewrote the scheduler so that every queued item carries the deadline it was admitted under.' + +function refFor(path: string, cwd: string | null = null): SessionRef { + return { + harness: 'claude-code', + sessionId: 'view-fixture', + path, + cwd, + mtimeMs: Date.parse('2026-01-01T00:00:05Z'), + } +} + +function writeTranscript(path: string): void { + writeFileSync( + path, + [ + { + type: 'user', + uuid: 'root-user', + sessionId: 'view-fixture', + timestamp: '2026-01-01T00:00:00Z', + message: { role: 'user', content: `${LEAKED_SENTENCE} ${TRANSCRIPT_ONLY_SENTENCE}` }, + }, + { + type: 'assistant', + uuid: 'root-assistant', + sessionId: 'view-fixture', + timestamp: '2026-01-01T00:00:01Z', + message: { + id: 'root-message', + role: 'assistant', + content: [{ type: 'tool_use', id: 'call-one', name: 'Bash', input: { command: "cat > REPORT.md <<'EOF'" } }], + }, + }, + ].map((event) => JSON.stringify(event)).join('\n'), + ) +} + +function writeChild(subDir: string): void { + mkdirSync(subDir, { recursive: true }) + writeFileSync( + join(subDir, 'agent-worker.jsonl'), + [ + { + type: 'user', + uuid: 'child-user', + timestamp: '2026-01-01T00:00:02Z', + isSidechain: true, + message: { role: 'user', content: 'worker TASK' }, + }, + { + type: 'assistant', + uuid: 'child-assistant', + timestamp: '2026-01-01T00:00:03Z', + message: { id: 'child-message', role: 'assistant', content: 'worker ANSWER' }, + }, + ].map((event) => JSON.stringify(event)).join('\n'), + ) + writeFileSync(join(subDir, 'agent-worker.meta.json'), JSON.stringify({ agentType: 'worker', toolUseId: 'call-one' })) +} + +/** + * A `.evolve` ledger with both kinds of allow-listed file: `current.json` + * repeats a sentence from the transcript, `scorecard.json` and + * `skill-runs.jsonl` hold structured rows only. + */ +async function writeEvolveFixture(root: string): Promise { + const evolve = join(root, '.evolve') + await mkdir(join(evolve, 'reflections'), { recursive: true }) + await writeFile( + join(evolve, 'experiments.jsonl'), + `${JSON.stringify({ ts: '2026-01-01T00:00:03Z', round: 1, verdict: 'KEEP', note: LEAKED_SENTENCE })}\n`, + 'utf8', + ) + await writeFile( + join(evolve, 'skill-runs.jsonl'), + `${JSON.stringify({ skill: '/verify', ts: '2026-01-01T00:00:04Z', exitCode: 0 })}\n`, + 'utf8', + ) + await writeFile(join(evolve, 'current.json'), `${JSON.stringify({ mode: 'research', decision: LEAKED_SENTENCE })}\n`, 'utf8') + await writeFile(join(evolve, 'scorecard.json'), `${JSON.stringify({ flows: [{ flow: 'latency', score: 0.9 }] })}\n`, 'utf8') + await writeFile(join(evolve, 'progress.md'), `# Progress\n\n${LEAKED_SENTENCE}\n`, 'utf8') + await writeFile(join(evolve, 'handoff-2026-01-02-latest.md'), `# Handoff\n\n${LEAKED_SENTENCE}\n`, 'utf8') + await writeFile(join(evolve, 'reflections', '2026-01-01.md'), `# Reflection\n\n${LEAKED_SENTENCE}\n`, 'utf8') +} + +async function assembleFullBundle(): Promise<{ bundleDir: string; manifest: SessionBundleManifest }> { + const ctxRoot = newDir('ctx') + await mkdir(ctxRoot, { recursive: true }) + await writeEvolveFixture(ctxRoot) + const transcript = join(dir, `session-${outSeq}.jsonl`) + writeTranscript(transcript) + writeChild(join(dir, `session-${outSeq}`, 'subagents')) + + const bundleDir = newDir('bundle') + const { manifest } = await assembleSessionBundle({ + adapter: new ClaudeAdapter(), + ref: refFor(transcript, ctxRoot), + outDir: bundleDir, + generatedAt: '2026-01-01T01:00:00.000Z', + }) + return { bundleDir, manifest } +} + +/** A bundle written by hand, so a test can control the bytes of a derived artifact. */ +async function writeSyntheticBundle(files: Record): Promise { + const bundleDir = newDir('synthetic') + for (const [path, content] of Object.entries(files)) { + const target = join(bundleDir, path) + await mkdir(join(target, '..'), { recursive: true }) + await writeFile(target, content, 'utf8') + } + const manifest: SessionBundleManifest = { + schemaVersion: 2, + kind: 'traces.session_bundle', + view: 'full', + createdAt: '2026-01-01T01:00:00.000Z', + provenance: { + sessionId: 'synthetic', + harness: 'claude-code', + cwd: null, + transcriptPath: join(bundleDir, 'session', 'transcript.jsonl'), + transcriptSha256: createHash('sha256').update(files['session/transcript.jsonl'] ?? '').digest('hex'), + contextRoot: null, + tracesVersion: '0.0.0-test', + sessionWindow: { firstSpanAt: null, lastSpanAt: null, padMs: 0 }, + }, + files: await Promise.all( + (await listSessionBundleFiles(bundleDir)).map(async (path) => { + const bytes = await readFile(join(bundleDir, path)) + return { path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') } + }), + ), + absent: [], + excluded: [], + ledgerSlices: [], + knownLimits: [], + } + await writeSessionBundleManifest(bundleDir, manifest) + return bundleDir +} + +describe('evidence-only bundle view', () => { + it('carries the counted artifacts, excludes every prose source by name, and names its own view', async () => { + const { bundleDir, manifest: full } = await assembleFullBundle() + + // The bundle `traces bundle` writes says which view it is, and that it drops nothing. + expect(full.schemaVersion).toBe(2) + expect(full.view).toBe('full') + expect(full.excluded).toEqual([]) + expect(full.projection).toBeUndefined() + + const outDir = newDir('view') + const { manifest, leakCheck } = await projectSessionBundle({ + bundleDir, + outDir, + view: 'evidence-only', + generatedAt: '2026-01-01T02:00:00.000Z', + }) + + expect(manifest.schemaVersion).toBe(2) + expect(manifest.view).toBe('evidence-only') + expect(manifest.createdAt).toBe('2026-01-01T02:00:00.000Z') + + // What is on disk, and nothing else. + const written = await listSessionBundleFiles(outDir) + expect(written).toEqual([ + 'derived/evidence.jsonl', + 'derived/session-index.json', + 'ledger/scorecard.json', + 'ledger/skill-runs.jsonl', + 'manifest.json', + ]) + expect(manifest.files.map((file) => file.path)).toEqual(written.filter((path) => path !== 'manifest.json')) + + // The session's own words are gone, by name and by rule. + const excludedByPath = new Map(manifest.excluded.map((entry) => [entry.path, entry] as const)) + expect(excludedByPath.get('session/transcript.jsonl')?.rule).toBe('session-source') + expect(excludedByPath.get('session/subagents/agent-worker.jsonl')?.rule).toBe('session-source') + expect(excludedByPath.get('derived/report.md')?.rule).toBe('session-text-derived') + expect(excludedByPath.get('derived/trace.otlp.jsonl')?.rule).toBe('session-text-derived') + expect(excludedByPath.get('ledger/progress.md')?.rule).toBe('authored-prose') + expect(excludedByPath.get('ledger/handoff-2026-01-02-latest.md')?.rule).toBe('authored-prose') + expect(excludedByPath.get('ledger/reflections/2026-01-01.md')?.rule).toBe('authored-prose') + expect(excludedByPath.get('ledger/experiments.jsonl')?.rule).toBe('authored-prose') + for (const entry of manifest.excluded) { + expect(entry.reason.length).toBeGreaterThan(0) + expect(entry.sha256).toMatch(/^[0-9a-f]{64}$/) + } + + // Every excluded hash is the full bundle's own hash for that file, so an + // auditor can prove which bytes were dropped without holding them. + const fullByPath = new Map(full.files.map((file) => [file.path, file] as const)) + for (const entry of manifest.excluded) { + expect(entry.sha256).toBe(fullByPath.get(entry.path)!.sha256) + expect(entry.bytes).toBe(fullByPath.get(entry.path)!.bytes) + } + // Carried bytes are the full bundle's bytes, unchanged. + for (const file of manifest.files) expect(file.sha256).toBe(fullByPath.get(file.path)!.sha256) + + // The projection points back at exactly the bundle it came from. + const sourceManifestBytes = await readFile(join(bundleDir, 'manifest.json')) + expect(manifest.projection!.sourceDirectory).toBe(bundleDir) + expect(manifest.projection!.sourceManifestSha256).toBe( + createHash('sha256').update(sourceManifestBytes).digest('hex'), + ) + expect(manifest.projection!.leakCheck).toEqual(leakCheck) + expect(leakCheck.matches).toBe(0) + expect(leakCheck.signatureWords).toBe(8) + // Every path excluded by classification supplied signatures. A path + // excluded BY the check cannot also be one of its sources. + expect(leakCheck.comparedSources).toEqual( + manifest.excluded.filter((entry) => entry.rule !== 'content-signature').map((entry) => entry.path), + ) + expect(leakCheck.carried.map((entry) => entry.path)).toEqual([ + 'derived/evidence.jsonl', + 'derived/session-index.json', + 'ledger/scorecard.json', + 'ledger/skill-runs.jsonl', + ]) + for (const entry of leakCheck.carried) expect(entry.carries.length).toBeGreaterThan(0) + + // Ledger slices are narrowed to what survived, and the view states its own limits. + expect(manifest.ledgerSlices.map((slice) => slice.path)).toEqual(['ledger/skill-runs.jsonl', 'ledger/scorecard.json']) + expect(manifest.knownLimits).toEqual(expect.arrayContaining([...full.knownLimits])) + expect(manifest.knownLimits.some((limit) => limit.includes('paraphrase-tight'))).toBe(true) + }) + + it('carries no content signature from any excluded source', async () => { + const { bundleDir } = await assembleFullBundle() + const outDir = newDir('view') + const { manifest } = await projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' }) + + // The check the guarantee rests on, run over the view that was actually + // written, against every file the view says it dropped. + const report = await findSessionTextLeaks({ + viewDir: outDir, + sourceDir: bundleDir, + carriedPaths: (await listSessionBundleFiles(outDir)).filter((path) => path !== 'manifest.json'), + excludedPaths: manifest.excluded.map((entry) => entry.path), + }) + expect(report.leaks).toEqual([]) + expect(report.comparedSources.length).toBe(manifest.excluded.length) + expect(report.sourceSignatures).toBeGreaterThan(0) + + // The same read the naive check makes, and the reason it is not enough on + // its own: neither the sentence nor the authoring call survives anywhere. + const viewText = ( + await Promise.all( + (await listSessionBundleFiles(outDir)).map((path) => readFile(join(outDir, path), 'utf8')), + ) + ).join('\n') + expect(viewText).not.toContain(LEAKED_SENTENCE) + expect(viewText).not.toContain(TRANSCRIPT_ONLY_SENTENCE) + expect(viewText).not.toContain("cat > REPORT.md <<'EOF'") + }) + + it('drops an allow-listed ledger file whose own text repeats an excluded source', async () => { + const { bundleDir } = await assembleFullBundle() + const outDir = newDir('view') + const { manifest } = await projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' }) + + const dropped = manifest.excluded.find((entry) => entry.path === 'ledger/current.json')! + expect(dropped.rule).toBe('content-signature') + expect(dropped.contentMatch!.signatures).toBeGreaterThan(0) + expect(dropped.contentMatch!.sources).toContain('session/transcript.jsonl') + // The record counts the overlap; it never quotes it, or the manifest would + // carry the text the view removed. + expect(JSON.stringify(dropped)).not.toContain(LEAKED_SENTENCE) + expect(dropped.reason).toContain('content check') + }) + + it('refuses to write a view when a traces-derived artifact repeats session text', async () => { + const bundleDir = await writeSyntheticBundle({ + 'session/transcript.jsonl': `${JSON.stringify({ message: { content: LEAKED_SENTENCE } })}\n`, + 'derived/evidence.jsonl': `${JSON.stringify({ kind: 'traces.policy_evidence.session', note: LEAKED_SENTENCE })}\n`, + 'derived/report.md': '# report\n', + }) + const outDir = newDir('view') + + await expect(projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' })).rejects.toThrow( + /derived\/evidence\.jsonl repeats \d+ 8-word run\(s\).*defect in the derivation/s, + ) + // Nothing was written, so a caller cannot mistake a partial directory for a view. + await expect(listSessionBundleFiles(outDir)).resolves.toEqual([]) + }) + + it('drops a path nobody put on the allow-list', async () => { + const bundleDir = await writeSyntheticBundle({ + 'session/transcript.jsonl': `${JSON.stringify({ message: { content: 'hello' } })}\n`, + 'derived/session-index.json': `${JSON.stringify({ kind: 'traces.session_index' })}\n`, + 'derived/future-artifact.md': 'a section a later version of traces added\n', + }) + const outDir = newDir('view') + const { manifest } = await projectSessionBundle({ bundleDir, outDir, view: 'evidence-only' }) + + const unknown = manifest.excluded.find((entry) => entry.path === 'derived/future-artifact.md')! + expect(unknown.rule).toBe('not-allow-listed') + expect(await listSessionBundleFiles(outDir)).toEqual(['derived/session-index.json', 'manifest.json']) + }) + + it('refuses a bundle whose bytes drifted from its manifest', async () => { + const bundleDir = await writeSyntheticBundle({ + 'session/transcript.jsonl': `${JSON.stringify({ message: { content: 'hello' } })}\n`, + 'derived/session-index.json': `${JSON.stringify({ kind: 'traces.session_index' })}\n`, + }) + await writeFile(join(bundleDir, 'derived', 'session-index.json'), '{"kind":"tampered"}\n', 'utf8') + + await expect( + projectSessionBundle({ bundleDir, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/does not match its manifest hash/) + }) + + it('refuses a non-bundle directory, an older manifest, a view of a view, and an unknown view', async () => { + const empty = newDir('empty') + await mkdir(empty, { recursive: true }) + await expect( + projectSessionBundle({ bundleDir: empty, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/is not a traces session bundle/) + + const v1 = newDir('v1') + await mkdir(v1, { recursive: true }) + await writeFile( + join(v1, 'manifest.json'), + JSON.stringify({ schemaVersion: 1, kind: 'traces.session_bundle' }), + 'utf8', + ) + await expect( + projectSessionBundle({ bundleDir: v1, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/schemaVersion 1; this traces reads 2/) + + const { bundleDir } = await assembleFullBundle() + const viewDir = newDir('view') + await projectSessionBundle({ bundleDir, outDir: viewDir, view: 'evidence-only' }) + await expect( + projectSessionBundle({ bundleDir: viewDir, outDir: newDir('view'), view: 'evidence-only' }), + ).rejects.toThrow(/already the evidence-only view/) + await expect( + projectSessionBundle({ bundleDir, outDir: newDir('view'), view: 'full' }), + ).rejects.toThrow(/unsupported bundle view "full"/) + await expect( + projectSessionBundle({ bundleDir, outDir: viewDir, view: 'evidence-only' }), + ).rejects.toThrow(/is not empty/) + }) +}) + +describe('classifyEvidenceOnlyPath', () => { + it('carries only the allow-list and denies everything else', () => { + for (const path of [ + 'derived/session-index.json', + 'derived/evidence.jsonl', + 'ledger/current.json', + 'ledger/scorecard.json', + 'ledger/skill-runs.jsonl', + ]) { + expect(classifyEvidenceOnlyPath(path)).toMatchObject({ carried: true }) + } + expect(classifyEvidenceOnlyPath('session/transcript.jsonl')).toMatchObject({ carried: false, rule: 'session-source' }) + expect(classifyEvidenceOnlyPath('session/subagents/a.jsonl')).toMatchObject({ carried: false, rule: 'session-source' }) + expect(classifyEvidenceOnlyPath('derived/report.md')).toMatchObject({ carried: false, rule: 'session-text-derived' }) + expect(classifyEvidenceOnlyPath('derived/trace.otlp.jsonl')).toMatchObject({ carried: false, rule: 'session-text-derived' }) + expect(classifyEvidenceOnlyPath('ledger/progress.md')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('ledger/handoff-2026-01-01-x.md')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('ledger/reflections/2026-01-01.md')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('ledger/experiments.jsonl')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('repo/git-log.txt')).toMatchObject({ carried: false, rule: 'authored-prose' }) + expect(classifyEvidenceOnlyPath('derived/anything-new.json')).toMatchObject({ carried: false, rule: 'not-allow-listed' }) + }) +}) + +describe('sessionTextSignatures', () => { + it('signs prose and ignores identifiers, structure, and package constants', () => { + expect([...sessionTextSignatures(LEAKED_SENTENCE)].length).toBeGreaterThan(0) + + // A shared session path is the same identifier on both sides, not shared prose. + const path = '/home/u/.claude/projects/-home-u-code-app/2f1c9d40-1111-2222-3333-444455556666.jsonl' + expect(sessionTextSignatures(path).size).toBe(0) + + // A counted artifact's key/number structure is not a sentence. + const counted = JSON.stringify({ + tokenUsage: { input: { n: 0, mean: null, p50: null, p95: null, stddev: null, min: null, max: null } }, + }) + expect(sessionTextSignatures(counted).size).toBe(0) + + // A JSON line signs its VALUES, so a field name can never collide with prose. + const row = JSON.stringify({ decision: LEAKED_SENTENCE }) + expect([...sessionTextSignatures(row)]).toEqual([...sessionTextSignatures(LEAKED_SENTENCE)]) + + // Signatures never span a line boundary: ten words on one line sign, the + // same ten words split across two lines of five do not. + const ten = 'the queue depth decides tail latency far more than the' + expect(sessionTextSignatures(ten).size).toBeGreaterThan(0) + expect(sessionTextSignatures(ten.split(' ').slice(0, 5).join(' ') + '\n' + ten.split(' ').slice(5).join(' ')).size).toBe(0) + }) +}) diff --git a/tests/bundle.test.ts b/tests/bundle.test.ts index ab59f60..54378aa 100644 --- a/tests/bundle.test.ts +++ b/tests/bundle.test.ts @@ -136,6 +136,12 @@ describe('assembleSessionBundle', () => { expect(directory).toBe(outDir) expect(manifest.kind).toBe('traces.session_bundle') + expect(manifest.schemaVersion).toBe(2) + // `traces bundle` assembles the auditor's copy, and says so: a reader + // never has to infer which view a directory is from its file list. + expect(manifest.view).toBe('full') + expect(manifest.excluded).toEqual([]) + expect(manifest.projection).toBeUndefined() expect(manifest.createdAt).toBe('2026-01-01T01:00:00.000Z') expect(manifest.provenance.sessionId).toBe('bundle-fixture') expect(manifest.provenance.harness).toBe('claude-code') diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 078fd4d..08c12a4 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -939,3 +939,68 @@ describe('traces analyze --llm failure surfacing', () => { expect(reportText).toContain('DSPY-BRIDGE-FAILURE: ValueError: analyze input must contain exactly') }, 60_000) }) + +describe('traces bundle + bundle-view', () => { + it('assembles the full view, then projects the writer view over the same session', async () => { + const root = await mkdtemp(join(tmpdir(), 'traces-cli-bundle-')) + const leaked = 'The retry budget was the wrong lever and the queue depth decides tail latency here.' + const transcript = join(root, 'session.jsonl') + await writeFile( + transcript, + [ + { + type: 'user', + uuid: 'u1', + sessionId: 'cli-bundle', + timestamp: '2026-01-01T00:00:00Z', + message: { role: 'user', content: leaked }, + }, + { + type: 'assistant', + uuid: 'a1', + sessionId: 'cli-bundle', + timestamp: '2026-01-01T00:00:01Z', + message: { id: 'm1', role: 'assistant', content: 'understood' }, + }, + ].map((row) => JSON.stringify(row)).join('\n'), + 'utf8', + ) + + const run = (args: string[]) => + execFileAsync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], { + cwd: process.cwd(), + env: { ...process.env, NO_COLOR: '1', FORCE_COLOR: '' }, + maxBuffer: 10 * 1024 * 1024, + timeout: 120_000, + }) + + const bundleDir = join(root, 'bundle') + const bundled = await run([ + 'bundle', '--harness', 'claude-code', '--session', transcript, '--out', bundleDir, + ]) + expect(bundled.stdout).toContain('session bundle (full view) →') + + const viewDir = join(root, 'view') + const projected = await run([ + 'bundle-view', bundleDir, '--view', 'evidence-only', '--out', viewDir, + ]) + expect(projected.stdout).toContain('evidence-only view →') + expect(projected.stdout).toContain('0 session-text match(es)') + + const manifest = JSON.parse(await readFile(join(viewDir, 'manifest.json'), 'utf8')) as { + view: string + excluded: { path: string; rule: string }[] + } + expect(manifest.view).toBe('evidence-only') + expect(manifest.excluded.map((entry) => entry.path)).toContain('session/transcript.jsonl') + await expect(readFile(join(viewDir, 'session', 'transcript.jsonl'), 'utf8')).rejects.toThrow() + await expect(readFile(join(viewDir, 'derived', 'report.md'), 'utf8')).rejects.toThrow() + expect(await readFile(join(bundleDir, 'derived', 'report.md'), 'utf8')).toContain(leaked) + expect(await readFile(join(viewDir, 'derived', 'evidence.jsonl'), 'utf8')).not.toContain(leaked) + + // --view belongs to bundle-view alone, so nobody can ask `bundle` for a + // projection and silently receive the full record. + await expect(run(['bundle', '--session', transcript, '--view', 'evidence-only', '--out', join(root, 'x')])) + .rejects.toThrow(/--view is not supported by bundle/) + }, 180_000) +})