From a552d6a6dcc72e223bf445c96f7c711231023496 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Mon, 26 Oct 2026 15:08:38 +0800 Subject: [PATCH] feat(dev): split one turn's prompt into fixed block, history, tool results and task (#796) Rebased onto 2d3eb88c (main) with no conflict: main's nine merges since d04712bb do not touch any of this PR's six files. One commit, the same content as 9b673294. - core/dev/turn-split.mjs and its mirror fold each call's surface and split the provider-reported prompt into the fixed block, history, tool results and task. - core/test/turn-split.test.js and its mirror: ten tests, including the --days row. - docs/reports/2026-10-26-turn-input-split.md: the measurement and its provenance. - docs/subsystems/ds-bot-core.md: names the script, core/dev/ is owned by that page. --- core/dev/turn-split.mjs | 483 +++++++++++++++++++ core/test/turn-split.test.js | 214 ++++++++ docs/reports/2026-10-26-turn-input-split.md | 199 ++++++++ docs/subsystems/ds-bot-core.md | 11 + patches/ds-bot/files/dev/turn-split.mjs | 483 +++++++++++++++++++ patches/ds-bot/files/test/turn-split.test.js | 214 ++++++++ 6 files changed, 1604 insertions(+) create mode 100644 core/dev/turn-split.mjs create mode 100644 core/test/turn-split.test.js create mode 100644 docs/reports/2026-10-26-turn-input-split.md create mode 100644 patches/ds-bot/files/dev/turn-split.mjs create mode 100644 patches/ds-bot/files/test/turn-split.test.js diff --git a/core/dev/turn-split.mjs b/core/dev/turn-split.mjs new file mode 100644 index 00000000..6f3c6e45 --- /dev/null +++ b/core/dev/turn-split.mjs @@ -0,0 +1,483 @@ +// Splits one real turn's prompt into the four parts #796 asks about: the fixed block +// (system prompt plus tool schemas, re-sent on every call), the history, the tool results, +// and the task (the message that opened the turn). +// +// node dev/turn-split.mjs [--sessions ] [--session ] [--turn ] +// [--tz ] [--top ] [--json] +// +// `--sessions` is the DSH session root (default `$DSH_HOME/sessions`); the script walks it +// for `session.v4.jsonl.zstd` files. Every Session log carries, per call, the provider's own +// usage (`inputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `outputTokens`) and the exact +// surface sent: the surviving `system/message`, the `request/header` tool list, and every +// message and `tool/result` appended so far. The prompt total is therefore measured, not +// estimated; the split across the four parts is derived from the characters each part holds +// and the fixed block is measured separately, by fitting one Session's prompts against its +// message characters (the fixed block is the intercept, because it is constant per call). +// +// Fresh input, cache read and cache write are never added into one another: `prompt` is +// their sum and is its own column, as on the usage page (`docs/reports/2026-10-25-token-spend.md`). +// +// A character count is printed as a character count. A token figure for one part is +// `derived` and carries its formula, because a Chinese system prompt and an ASCII tool +// schema do not share one characters-per-token ratio (see the report this script backs, +// `docs/reports/2026-10-26-turn-input-split.md`). +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { homedir } from 'node:os' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { zstdDecompressSync } from 'node:zlib' + +const USAGE = 'usage: node dev/turn-split.mjs [--sessions ] [--session ] [--turn ] [--tz ] [--days] [--top ] [--json]' +const ZSTD_MAGIC = 4247762216 + +function die(message) { + console.error(`turn-split: ${message}`) + process.exit(1) +} + +function parseArgs(argv) { + const flags = { sessions: null, session: null, turn: null, tz: 'UTC', days: false, top: 5, json: false } + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + const value = () => { + const next = argv[index + 1] + if (next === undefined || next.startsWith('--')) die(`${arg} needs a value`) + index += 1 + return next + } + switch (arg) { + case '--sessions': flags.sessions = value(); break + case '--session': flags.session = value(); break + case '--turn': flags.turn = Number(value()); break + case '--tz': flags.tz = value(); break + case '--top': flags.top = Number(value()); break + case '--days': flags.days = true; break + case '--json': flags.json = true; break + case '--help': case '-h': console.log(USAGE); process.exit(0); break + default: die(`unknown argument ${arg}\n${USAGE}`) + } + } + if (!Number.isSafeInteger(flags.top) || flags.top < 1) die('--top must be a whole number') + if (flags.turn !== null && !Number.isSafeInteger(flags.turn)) die('--turn must be a whole number') + return flags +} + +// A Session artifact is a concatenation of independently decodable Zstandard frames, one +// per appended batch. Node's one-shot decoder stops after the first frame, so the frame +// ranges are found by walking the container format (no decompression) and each frame is +// decoded on its own. This mirrors the reader in the Session backend. +export function scanZstdFrames(buffer) { + const frames = [] + let offset = 0 + while (offset < buffer.length) { + const start = offset + if (buffer.length - offset < 4) return { frames, tornStart: start } + if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) throw new Error(`corrupt Zstandard session log: invalid frame magic at byte ${offset}`) + offset += 4 + if (offset === buffer.length) return { frames, tornStart: start } + const descriptor = buffer.readUInt8(offset) + offset += 1 + if ((descriptor & 24) !== 0) throw new Error(`corrupt Zstandard session log: reserved frame-header bit at byte ${offset - 1}`) + const contentSizeFlag = descriptor >>> 6 + const singleSegment = (descriptor & 32) !== 0 + const checksum = (descriptor & 4) !== 0 + const dictionaryFlag = descriptor & 3 + const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag + const contentSizeBytes = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag + offset += (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes + for (;;) { + if (buffer.length - offset < 3) return { frames, tornStart: start } + const blockHeader = buffer.readUIntLE(offset, 3) + offset += 3 + const lastBlock = (blockHeader & 1) !== 0 + const blockType = (blockHeader >>> 1) & 3 + const blockSize = blockHeader >>> 3 + if (blockType === 3) throw new Error(`corrupt Zstandard session log: reserved block type at byte ${offset - 3}`) + offset += blockType === 1 ? 1 : blockSize + if (offset > buffer.length) return { frames, tornStart: start } + if (lastBlock) break + } + if (checksum) offset += 4 + if (offset > buffer.length) return { frames, tornStart: start } + frames.push({ start, end: offset }) + } + return { frames } +} + +// The provider's reported usage is the last `usage` chunk of a settled attempt. +export function usageOf(stream) { + if (!Array.isArray(stream)) return undefined + for (let index = stream.length - 1; index >= 0; index -= 1) { + const record = stream[index] + if (record?.type === 'chunk' && record.chunk?.type === 'usage') return record.chunk.usage + } + return undefined +} + +function textOf(blocks) { + if (typeof blocks === 'string') return blocks + if (!Array.isArray(blocks)) return '' + let text = '' + for (const block of blocks) { + if (!block || typeof block !== 'object') continue + if (typeof block.text === 'string') text += block.text + else if (block.type === 'tool-call') text += JSON.stringify(block) + } + return text +} + +const count = (value) => (Number.isSafeInteger(value) && value > 0 ? value : 0) + +// One Session's events, folded into the surface each call sent and the usage it reported. +// A message belongs to the turn that was open when it arrived; a message admitted just +// before `turn/start` (the runtime context, the time context, the user's own words) is the +// turn's task, so it is attributed to that turn when the turn opens. +export function foldSessionEvents(events) { + const steps = [] + const turns = new Map() + let system = '' + let tools = '' + let turn = null + let open = false + let pending = 0 + const messages = [] + let toolCallNames = new Map() + const largestTool = new Map() + const toolTotal = new Map() + for (const event of events) { + const data = event?.data ?? {} + switch (event?.type) { + case 'system/message': + system = textOf(data.message?.content) + break + case 'request/header': + tools = JSON.stringify(data.header?.tools ?? []) + break + case 'turn/start': + turn = data.turn ?? null + open = true + for (let index = messages.length - pending; index < messages.length; index += 1) messages[index].turn = turn + pending = 0 + break + case 'turn/end': + open = false + break + case 'compaction/end': + messages.length = 0 + pending = 0 + break + case 'tool/call': + toolCallNames.set(data.callId, data.name ?? '') + break + case 'user/message': { + const entry = { role: 'user', turn: open ? turn : null, chars: textOf(data.content).length } + messages.push(entry) + if (!open) pending += 1 + break + } + case 'tool/result': { + const callId = data.message?.toolCallId ?? data.message?.source?.callId + const resultTurn = open ? turn : (data.turn ?? null) + const resultChars = textOf(data.message?.content).length + largestTool.set(resultTurn, Math.max(largestTool.get(resultTurn) ?? 0, resultChars)) + toolTotal.set(resultTurn, (toolTotal.get(resultTurn) ?? 0) + resultChars) + messages.push({ + role: 'tool', + turn: open ? turn : (data.turn ?? null), + chars: textOf(data.message?.content).length, + name: toolCallNames.get(callId) ?? data.message?.toolName ?? '', + }) + break + } + case 'assistant/message': { + const usage = usageOf(data.stream) + const current = data.turn ?? null + if (usage) { + const chars = { tool: 0, assistant: 0, task: 0, history: 0 } + const toolCharsByName = {} + for (const message of messages) { + if (message.role === 'tool') { + chars.tool += message.chars + toolCharsByName[message.name] = (toolCharsByName[message.name] ?? 0) + message.chars + } + else if (message.role === 'assistant') chars.assistant += message.chars + else if (message.role === 'user') { + if (message.turn === current) chars.task += message.chars + else chars.history += message.chars + } + } + const step = { + turn: current, + step: data.step ?? null, + time: event.time, + input: count(usage.inputTokens), + cacheRead: count(usage.cacheReadTokens), + cacheWrite: count(usage.cacheWriteTokens), + output: count(usage.outputTokens), + systemChars: system.length, + toolsChars: tools.length, + messageChars: chars.task + chars.history + chars.assistant + chars.tool, + chars: { task: chars.task, history: chars.history + chars.assistant, tool: chars.tool }, + toolCharsByName, + } + step.prompt = step.input + step.cacheRead + step.cacheWrite + steps.push(step) + const aggregate = turns.get(current) ?? { turn: current, calls: 0, prompt: 0, output: 0, first: step, last: step } + aggregate.calls += 1 + aggregate.prompt += step.prompt + aggregate.output += step.output + aggregate.last = step + turns.set(current, aggregate) + } + let assistantChars = 0 + for (const block of (data.message?.content ?? [])) { + if (block.type === 'tool-call') assistantChars += JSON.stringify(block).length + else if (typeof block.text === 'string') assistantChars += block.text.length + } + messages.push({ role: 'assistant', turn: current, chars: assistantChars }) + break + } + default: + break + } + } + return { steps, turns: [...turns.values()], largestTool, toolTotal, systemChars: system.length, toolsChars: tools.length } +} + +// The fixed block is constant for every call of one Session, so it is that Session's +// intercept when its provider-reported prompts are fitted against its message characters: +// `prompt = fixed + k * messageChars`. `k` is the message cost per character and +// `charsPerToken` its inverse. Fewer than eight calls, or no variation in the message +// characters, leaves the Session without a fit rather than inventing one. +export function fitFixed(steps) { + const usable = steps.filter((step) => step.prompt > 0 && step.messageChars > 0) + if (usable.length < 8) return null + const n = usable.length + const meanX = usable.reduce((sum, step) => sum + step.messageChars, 0) / n + const meanY = usable.reduce((sum, step) => sum + step.prompt, 0) / n + let sxx = 0 + let sxy = 0 + for (const step of usable) { + sxx += (step.messageChars - meanX) ** 2 + sxy += (step.messageChars - meanX) * (step.prompt - meanY) + } + if (sxx === 0) return null + const slope = sxy / sxx + const intercept = meanY - slope * meanX + let residual = 0 + let total = 0 + for (const step of usable) { + residual += (step.prompt - intercept - slope * step.messageChars) ** 2 + total += (step.prompt - meanY) ** 2 + } + return { + n, + fixedTokens: intercept, + charsPerToken: slope > 0 ? 1 / slope : null, + r2: total > 0 ? 1 - residual / total : 1, + fixedChars: usable[0].systemChars + usable[0].toolsChars, + } +} + +// One call's prompt, split into the four parts. The fixed block is the measured intercept; +// what is left after it is distributed over the parts by their character share, which is +// the only per-part evidence the log holds. +export function splitStep(step, fit) { + const fixed = Math.max(0, Math.min(fit.fixedTokens, step.prompt)) + const rest = Math.max(0, step.prompt - fixed) + const total = step.messageChars + const share = (chars) => (total > 0 ? rest * chars / total : 0) + return { fixed, task: share(step.chars.task), history: share(step.chars.history), tool: share(step.chars.tool) } +} + +export function readSessionFile(file) { + const buffer = readFileSync(file) + const { frames } = scanZstdFrames(buffer) + const events = [] + for (const { start, end } of frames) { + let text + try { + text = zstdDecompressSync(buffer.subarray(start, end)).toString('utf8') + } catch { + continue + } + for (const line of text.split('\n')) { + if (!line.trim()) continue + try { + events.push(JSON.parse(line)) + } catch { + // A torn or foreign line is skipped: one bad line must not hide the Session. + } + } + } + return events +} + +export function findSessionFiles(root) { + const files = [] + const walk = (dir, depth) => { + let entries + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + const path = join(dir, entry.name) + if (entry.isDirectory()) { + if (depth > 0) walk(path, depth - 1) + } else if (entry.name === 'session.v4.jsonl.zstd') files.push(path) + } + } + walk(root, 2) + return files.sort() +} + +const localDay = (time, tz) => { + if (!Number.isFinite(time)) return 'unknown' + const parts = new Intl.DateTimeFormat('en-CA', { timeZone: tz, year: 'numeric', month: '2-digit', day: '2-digit' }).formatToParts(new Date(time)) + const part = Object.fromEntries(parts.map(({ type, value }) => [type, value])) + return `${part.year}-${part.month}-${part.day}` +} +const integer = (value) => Math.round(value).toLocaleString('en-US') + +function main(argv) { + const flags = parseArgs(argv) + const root = flags.sessions === null ? join(process.env.DSH_HOME ?? join(homedir(), '.dsh'), 'sessions') : flags.sessions + let files + try { + statSync(root) + files = findSessionFiles(resolve(root)) + } catch { + die(`no session root at ${resolve(root)}`) + } + if (files.length === 0) die(`no session.v4.jsonl.zstd under ${resolve(root)}`) + const sessions = [] + for (const file of files) { + const id = file.split('/').at(-2) + if (flags.session !== null && !id.startsWith(flags.session)) continue + const folded = foldSessionEvents(readSessionFile(file)) + if (folded.steps.length === 0) continue + sessions.push({ id, file, ...folded, fixed: fitFixed(folded.steps) }) + } + if (sessions.length === 0) die('no Session in the sample carried a provider usage report') + const rows = [] + for (const session of sessions) { + if (!session.fixed) continue + for (const turn of session.turns) { + const steps = session.steps.filter((step) => step.turn === turn.turn) + const total = { prompt: 0, fixed: 0, history: 0, tool: 0, task: 0 } + for (const step of steps) total.prompt += step.prompt + for (const step of steps) { + const split = splitStep(step, session.fixed) + total.fixed += split.fixed + total.task += split.task + total.history += split.history + total.tool += split.tool + } + const calls = steps.length + if (calls === 0) continue + rows.push({ + session: session.id, + turn: turn.turn, + day: localDay(turn.first.time, flags.tz), + calls, + prompt: total.prompt, + output: turn.output, + toolAdded: steps.at(-1).chars.tool, + largestTool: session.largestTool?.get(turn.turn) ?? 0, + toolTotal: session.toolTotal?.get(turn.turn) ?? 0, + fixed: total.fixed, + history: total.history, + tool: total.tool, + task: total.task, + first: steps[0].prompt, + last: steps.at(-1).prompt, + max: Math.max(...steps.map((step) => step.prompt)), + fit: session.fixed, + systemChars: session.systemChars, + toolsChars: session.toolsChars, + steps, + toolNames: (() => { + const names = new Map() + for (const step of steps) for (const [name, chars] of Object.entries(step.toolCharsByName ?? {})) names.set(name, (names.get(name) ?? 0) + chars) + return names + })(), + }) + } + } + const fits = sessions.map((session) => session.fixed).filter(Boolean) + const median = (values) => (values.length === 0 ? 0 : values.slice().sort((a, b) => a - b)[Math.floor((values.length - 1) / 2)]) + if (flags.days) { + const days = new Map() + for (const session of sessions) { + for (const step of session.steps) { + const day = localDay(step.time, flags.tz) + const row = days.get(day) ?? { day, calls: 0, prompt: 0, input: 0, cacheRead: 0, cacheWrite: 0, output: 0, fixed: 0 } + row.calls += 1 + row.prompt += step.prompt + row.input += step.input + row.cacheRead += step.cacheRead + row.cacheWrite += step.cacheWrite + row.output += step.output + row.fixed += session.fixed ? splitStep(step, session.fixed).fixed : 0 + days.set(day, row) + } + } + const rowsOut = [...days.values()].sort((a, b) => a.day.localeCompare(b.day)) + if (flags.json) { + console.log(JSON.stringify({ sample: { root: resolve(root), sessions: sessions.length, tz: flags.tz }, days: rowsOut }, null, 2)) + return + } + console.log(`sample: ${sessions.length} Sessions under ${resolve(root)}, zone ${flags.tz}`) + console.log(`Sessions with a fixed-block fit: ${fits.length}; fixed tokens p50 ${integer(median(fits.map((fit) => fit.fixedTokens)))}`) + console.log('') + console.log('day calls prompt fresh input cache read cache write output fixed share') + for (const row of rowsOut) { + console.log(`${row.day} ${String(row.calls).padStart(6)} ${integer(row.prompt).padStart(13)} ${integer(row.input).padStart(13)} ${integer(row.cacheRead).padStart(13)} ${integer(row.cacheWrite).padStart(13)} ${integer(row.output).padStart(12)} ${(row.fixed / row.prompt * 100).toFixed(1).padStart(11)}%`) + } + return + } + if (rows.length === 0) die('no turn in the sample could be split (every Session needs 8 calls)') + if (flags.turn !== null) { + const only = rows.filter((row) => row.turn === flags.turn) + if (only.length === 0) die(`no turn ${flags.turn} in the sample`) + rows.length = 0 + rows.push(...only) + } + rows.sort((a, b) => b.prompt - a.prompt) + const top = rows.slice(0, flags.top) + if (flags.json) { + console.log(JSON.stringify({ sample: { root: resolve(root), sessions: sessions.length, tz: flags.tz }, turns: top.map((row) => ({ ...row, steps: undefined, toolNames: undefined, fit: row.fit })) }, null, 2)) + return + } + console.log(`sample: ${sessions.length} Sessions under ${resolve(root)}, zone ${flags.tz}`) + console.log('') + console.log('## Fixed block per call (measured: the intercept of each Session\'s prompt fit)') + console.log(`Sessions with a fit: ${fits.length}; fixed tokens p50 ${integer(median(fits.map((fit) => fit.fixedTokens)))}; ` + + `fixed chars p50 ${integer(median(fits.map((fit) => fit.fixedChars)))}; message chars/token p50 ${median(fits.map((fit) => fit.charsPerToken)).toFixed(2)}; ` + + `R2 p50 ${median(fits.map((fit) => fit.r2)).toFixed(3)}`) + console.log('') + console.log('## Top turns by prompt') + console.log('session turn calls prompt fixed history tool task first last max') + for (const row of top) { + const share = (value) => `${(value / row.prompt * 100).toFixed(1)}%` + console.log(`${row.session.padEnd(32)} ${String(row.turn).padStart(4)} ${String(row.calls).padStart(6)} ${integer(row.prompt).padStart(8)} ` + + `${share(row.fixed).padStart(7)} ${share(row.history).padStart(7)} ${share(row.tool).padStart(7)} ${share(row.task).padStart(7)} ` + + `${integer(row.first).padStart(8)} ${integer(row.last).padStart(8)} ${integer(row.max).padStart(8)}`) + } + const first = top[0] + if (first) { + console.log('') + console.log(`## Turn ${first.turn} of ${first.session}: one call's surface`) + console.log(`system prompt ${integer(first.systemChars)} chars; tool schemas ${integer(first.toolsChars)} chars; ` + + `fixed block ${integer(first.fit.fixedTokens)} tokens (${(first.fit.fixedChars / first.fit.fixedTokens).toFixed(2)} chars/token)`) + console.log(`calls ${first.calls}; provider prompt ${integer(first.prompt)} tokens (${integer(first.prompt / first.calls)} per call); output ${integer(first.output)} tokens`) + console.log(`tool text added by the turn ${integer(first.toolTotal)} chars; still on the last call's surface ${integer(first.toolAdded)} chars; largest single result ${integer(first.largestTool)} chars`) + if (first.toolNames.size > 0) { + console.log(`tool text read again on every call, by tool: ${[...first.toolNames].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([name, chars]) => `${name} ${integer(chars)} chars`).join(', ')}`) + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main(process.argv.slice(2)) diff --git a/core/test/turn-split.test.js b/core/test/turn-split.test.js new file mode 100644 index 00000000..9e49dab7 --- /dev/null +++ b/core/test/turn-split.test.js @@ -0,0 +1,214 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { zstdCompressSync } from 'node:zlib' + +import { fitFixed, foldSessionEvents, scanZstdFrames, splitStep } from '../dev/turn-split.mjs' + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..') +const SCRIPT = join(ROOT, 'core', 'dev', 'turn-split.mjs') + +const usage = (input, cacheRead, output) => ({ + type: 'chunk', + chunk: { type: 'usage', usage: { inputTokens: input, cacheReadTokens: cacheRead, outputTokens: output } }, +}) +const system = (text) => ({ type: 'system/message', data: { message: { role: 'system', content: [{ type: 'text', text }] } } }) +const header = (tools) => ({ type: 'request/header', data: { header: { tools } } }) +const user = (text) => ({ type: 'user/message', data: { content: [{ type: 'text', text }] } }) +const toolResult = (callId, name, text) => ({ + type: 'tool/result', + data: { message: { role: 'tool', toolCallId: callId, content: [{ type: 'text', text }] } }, +}) +const toolCall = (callId, name) => ({ type: 'tool/call', data: { callId, name } }) +const assistant = (turn, step, input, cacheRead, output, text = 'ok') => ({ + type: 'assistant/message', + time: 1_792_900_000_000 + turn * 100_000 + step * 1000, + data: { turn, step, message: { role: 'assistant', content: [{ type: 'text', text }] }, stream: [{ type: 'chunk', chunk: { type: 'finish' } }, usage(input, cacheRead, output)] }, +}) + +// One turn that reads a 1000-character tool result, then a second turn whose task is short. +function fixtureEvents() { + return [ + system('S'.repeat(100)), + header([{ name: 'bash', description: 'd'.repeat(900) }]), + user('context before the turn'), + { type: 'turn/start', data: { turn: 1 } }, + user('do the thing'), + assistant(1, 1, 100, 900, 10), + toolCall('call-1', 'bash'), + toolResult('call-1', 'bash', 'T'.repeat(1000)), + assistant(1, 2, 50, 1950, 20), + { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } }, + { type: 'turn/start', data: { turn: 2 } }, + user('next'), + assistant(2, 1, 40, 2960, 5), + ] +} + +test('a turn separates its task from the history that came before it', () => { + const folded = foldSessionEvents(fixtureEvents()) + assert.equal(folded.steps.length, 3) + const [first, , third] = folded.steps + // Everything admitted before the turn opens — the injected context and the user's words — + // is that turn's task; nothing before it counts as history yet. + assert.equal(first.chars.task, 'context before the turn'.length + 'do the thing'.length) + assert.equal(first.chars.history, 0) + assert.equal(first.chars.tool, 0) + // Turn 2 opens with a short task; turn 1's two user messages and its two "ok" replies are + // history by then. + assert.equal(third.chars.task, 'next'.length) + assert.equal(third.chars.history, 'context before the turn'.length + 'do the thing'.length + 2 * 'ok'.length) +}) + +test('a tool result is its own part, never folded into the history', () => { + const folded = foldSessionEvents(fixtureEvents()) + const second = folded.steps[1] + assert.equal(second.chars.tool, 1000) + // The kill: counting tool text as history would move all 1000 characters there. + assert.ok(second.chars.tool >= 1000) + // What the assistant itself wrote is history; the turn's task stays its own part. + assert.equal(second.chars.history, 'ok'.length) +}) + +test('the fixed block is the intercept of the Session\'s prompt fit', () => { + const steps = Array.from({ length: 10 }, (_, index) => ({ + prompt: 48_000 + Math.round((index + 1) * 3_320), + messageChars: (index + 1) * 10_000, + systemChars: 20_000, + toolsChars: 110_000, + })) + const fit = fitFixed(steps) + assert.ok(fit) + assert.ok(Math.abs(fit.fixedTokens - 48_000) <= 1) + assert.ok(Math.abs(fit.charsPerToken - 10000 / 3320) < 1e-9) + assert.ok(fit.r2 > 0.999) + assert.equal(fit.n, 10) + // The kill: a fit that reported no fixed block would put every prompt token in the parts. + assert.ok(fit.fixedTokens > 0) +}) + +test('a Session with too few calls gets no fit instead of a guess', () => { + const steps = Array.from({ length: 7 }, (_, index) => ({ prompt: 1000 + index, messageChars: index * 10, systemChars: 1, toolsChars: 1 })) + assert.equal(fitFixed(steps), null) + const flat = Array.from({ length: 12 }, () => ({ prompt: 1000, messageChars: 10, systemChars: 1, toolsChars: 1 })) + assert.equal(fitFixed(flat), null) +}) + +test('a split accounts for the whole prompt and never claims the fixed block twice', () => { + const fit = { fixedTokens: 48_000, charsPerToken: 3.32, r2: 1, n: 100, fixedChars: 130_000 } + const step = { prompt: 148_000, messageChars: 100_000, chars: { task: 10_000, history: 30_000, tool: 60_000 } } + const split = splitStep(step, fit) + const sum = split.fixed + split.task + split.history + split.tool + assert.ok(Math.abs(sum - step.prompt) <= 1) + assert.equal(split.fixed, 48_000) + // A short prompt is all fixed block, not a negative remainder. + const tiny = splitStep({ prompt: 30_000, messageChars: 1000, chars: { task: 1000, history: 0, tool: 0 } }, fit) + assert.equal(tiny.fixed, 30_000) + assert.equal(tiny.task + tiny.history + tiny.tool, 0) +}) + +test('the frame walker finds every frame of a concatenated log', (t) => { + if (typeof zstdCompressSync !== 'function') { + t.skip('this Node has no zstd in node:zlib (needs 22.15+)') + return + } + const first = zstdCompressSync(Buffer.from('{"type":"a"}\n')) + const second = zstdCompressSync(Buffer.from('{"type":"b"}\n')) + const buffer = Buffer.concat([first, second]) + const { frames } = scanZstdFrames(buffer) + assert.equal(frames.length, 2) + assert.deepEqual(frames[0], { start: 0, end: first.length }) + assert.deepEqual(frames[1], { start: first.length, end: buffer.length }) +}) + +function sessionFile(events) { + const bytes = [events.map((event) => JSON.stringify(event)).join('\n') + '\n', JSON.stringify({ type: 'session/title', data: { title: 'x' } })] + return Buffer.concat(bytes.map((text) => zstdCompressSync(Buffer.from(text)))) +} + +function sample(t) { + const dir = mkdtempSync(join(tmpdir(), 'turn-split-')) + const workspace = join(dir, 'sessions', '--workspace--') + mkdirSync(workspace, { recursive: true }) + // Twelve calls with a growing message surface: the fit needs variation and eight calls. + const events = [system('S'.repeat(100)), header([{ name: 'bash', description: 'd'.repeat(900) }]), { type: 'turn/start', data: { turn: 1 } }] + for (let step = 1; step <= 12; step += 1) { + events.push(user(`task ${step} `.repeat(step * 10))) + events.push(assistant(1, step, 100, 48_000 + step * 500, 10)) + events.push(toolCall(`call-${step}`, 'bash')) + events.push(toolResult(`call-${step}`, 'bash', 'T'.repeat(step * 400))) + } + const folder = join(workspace, 'session-fixture') + mkdirSync(folder) + writeFileSync(join(folder, 'session.v4.jsonl.zstd'), sessionFile(events)) + t.after(() => rmSync(dir, { recursive: true, force: true })) + return join(dir, 'sessions') +} + +test('the CLI prints a split for the sampled turn', (t) => { + if (typeof zstdCompressSync !== 'function') { + t.skip('this Node has no zstd in node:zlib (needs 22.15+)') + return + } + const root = sample(t) + const result = spawnSync(process.execPath, [SCRIPT, '--sessions', root], { encoding: 'utf8' }) + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /Sessions with a fit: 1/) + assert.match(result.stdout, /fixed block \d[\d,]* tokens/) +}) + +test('the --days table divides each day by the same measured fixed block', (t) => { + if (typeof zstdCompressSync !== 'function') { + t.skip('this Node has no zstd in node:zlib (needs 22.15+)') + return + } + const root = sample(t) + const result = spawnSync(process.execPath, [SCRIPT, '--sessions', root, '--days'], { encoding: 'utf8' }) + assert.equal(result.status, 0, result.stderr) + // The fixture is twelve calls of 48,100 + 500 × step: 616,200 prompt over one local day. + const row = result.stdout.split('\n').find((line) => /^\d{4}-\d{2}-\d{2}\s/.test(line)) + assert.ok(row, `the --days table printed no day row:\n${result.stdout}`) + const cells = row.trim().split(/\s+/) + assert.equal(cells[1], '12') + assert.equal(cells[2], '616,200') + // Every call's fixed block comes from the fixture's own fit (49,459 tokens), so 96.1% of + // the day's prompt is the same block re-sent. The kill: `row.fixed += 0` at `turn-split.mjs` + // prints 0.0% here and leaves every other test green, which is how this path went unpinned. + assert.equal(cells.at(-1), '96.1%') +}) + +test('a missing session root is an error that names the path', () => { + const missing = join(tmpdir(), 'turn-split-no-such-root') + const result = spawnSync(process.execPath, [SCRIPT, '--sessions', missing], { encoding: 'utf8' }) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /no session root at/) + assert.match(result.stderr, new RegExp(missing.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))) +}) + +// The report this script backs ships a number only in a table row whose last cell names +// where it came from: the provider's usage, a direct count in the log, or a formula. +const REPORT = join(ROOT, 'docs', 'reports', '2026-10-26-turn-input-split.md') +const PROVENANCE = /^(provider usage|measured: .+|derived: .+)$/ + +test('every row of the committed split report carries its provenance', () => { + const lines = readFileSync(REPORT, 'utf8').split(/\r?\n/) + const rows = [] + let header = null + for (const line of lines) { + const cells = line.startsWith('|') ? line.split('|').slice(1, -1).map((cell) => cell.trim()) : null + if (cells === null) { header = null; continue } + if (header === null) { header = cells; continue } + if (cells.every((cell) => /^:?-{3,}:?$/.test(cell))) continue + assert.equal(header.at(-1), 'Source', `a table starting with ${header[0]} has no Source column`) + rows.push({ header, cells }) + } + assert.ok(rows.length > 0, 'the report has no tables') + for (const row of rows) { + assert.match(row.cells.at(-1), PROVENANCE, `no provenance: | ${row.cells.join(' | ')} |`) + assert.equal(row.cells.length, row.header.length, `a row has ${row.cells.length} cells, its table ${row.header.length}: | ${row.cells.join(' | ')} |`) + } +}) diff --git a/docs/reports/2026-10-26-turn-input-split.md b/docs/reports/2026-10-26-turn-input-split.md new file mode 100644 index 00000000..834ba6dd --- /dev/null +++ b/docs/reports/2026-10-26-turn-input-split.md @@ -0,0 +1,199 @@ +# One turn's input, split by part + +**This host, 2026-10-26.** [#796](https://github.com/ThinkFlowLab/FlyBot/issues/796) asked where a turn's input tokens go: the fixed prompt, the history, the tool results, the task. The [2026-10-25 page](2026-10-25-token-spend.md) answered the ledger questions and said the composition was **not measured**, because the ledger's own JSON holds the prompt total and no request. This page measures it: the DSH Session logs carry the exact surface each call sent (the surviving system prompt, the `request/header` tool list, every message and every `tool/result`) together with the provider's own `usage` for that call, so the split can be read per call without a replay. + +Nothing here changes what is sent to a model. No prompt, notice, roster, memory or tool output was edited to produce these numbers. + +## Run + +- Label: one real turn, this host's own Session logs +- Host: `DSH_HOME=/data/dsh`; logs under `/data/dsh/sessions/--data-dsh-bot-workspace--/` +- FlyBot: this PR's head on `dev/796-turn-input-split`, on base `d04712bbb3cc48b09cf1645341adcb687631d6d0` — the tree the reader ran from is **this PR's head**, not the base: `core/dev/turn-split.mjs` does not exist on `d04712bb`. +- DSH: 0.2.0-rc.2 harness (`@deepseek-ai/dsh` 0.2.1-rc.2) +- Route: `opencsitool` / `DeepSeek-V4.1-Flash-1017` +- Sample: 165 Session artifacts, `2026-10-20T07:34:24Z .. 2026-10-26T05:05Z`; 137 of them carry enough calls for a fixed-block fit + +## Method + +Every Session log records, per settled call, the provider's `usage` chunk (`inputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `outputTokens`) and the surface that call sent. Fresh input, cache read and cache write are never added into one another: `prompt` is their sum and stays its own column. + +The four parts: + +| Part | What it is | Source | +| --- | --- | --- | +| Fixed block | the surviving `system/message` plus the `request/header` tool list | measured: the intercept of one Session's fit, `prompt = fixed + k × messageChars`, over its own provider-reported prompts | +| History | earlier assistant text and every user message of earlier turns | derived: its character share of what is left after the fixed block | +| Tool results | every `tool/result` content on the surface | derived: its character share of what is left after the fixed block | +| Task | the user/runtime/time messages admitted when the turn opened | derived: its character share of what is left after the fixed block | + +The intercept is the fixed block because it is constant for every call of one Session while the message characters vary. Across the 137 fitted Sessions the fit explains the data well (R² p50 **0.986**, p10 0.921), so the split is anchored on measured prompts, not on a characters-per-token guess. A character count is printed as a character count; a per-part token figure is `derived` and says so. + +```sh +# the day table and the fixed block, one line per local day +node core/dev/turn-split.mjs --sessions /data/dsh/sessions --tz Asia/Shanghai --days + +# one real turn, and the surface of its last call +node core/dev/turn-split.mjs --sessions /data/dsh/sessions \ + --session session-2413164c --turn 62 --top 1 +``` + +The same fold is what `core/dev/turn-split.mjs` implements; `core/test/turn-split.test.js` pins the four rules that matter (a tool result is never history, a turn's own messages are its task, the fit recovers a known intercept, a split never claims the fixed block twice). + +On this host the usage ledger (`/data/dsh/bot/usage.sqlite`) holds only the calls made outside a Session — 944 rows, all `memory`, `compaction` or `image` — so the per-turn numbers come from the logs' provider usage, which is the same source the ledger folds Session calls from. + +## 1. The days + +`--days`, zone `Asia/Shanghai`, every Session artifact on this host. Fresh input, cache read and output are the provider's own columns; the fixed share is `Σ fixed / Σ prompt` with each call's fixed block from its Session's fit. + +That share is a **lower bound**: a Session with no usable fit (fewer than eight calls, or no variation in its message characters — 137 of the 165 artifacts here have one) contributes every call to `prompt` and **0** to `fixed`, so the share can only be understated by the Sessions the fit does not reach. + +| Day | Calls | Prompt | Fresh input | Cache read | Cache write | Output | Fixed share | Source | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 2026-10-20 | 453 | 41,165,813 | 2,292,597 | 38,873,216 | 0 | 287,983 | 34.4% | provider usage | +| 2026-10-21 | 4,993 | 640,046,030 | 33,714,382 | 606,331,648 | 0 | 3,111,678 | 36.0% | provider usage | +| 2026-10-22 | 10,031 | 1,411,115,770 | 40,504,954 | 1,370,610,816 | 0 | 5,649,482 | 35.5% | provider usage | +| 2026-10-23 | 10,747 | 1,591,205,919 | 28,640,799 | 1,562,565,120 | 0 | 6,233,105 | 34.6% | provider usage | +| 2026-10-24 | 8,946 | 1,304,251,151 | 22,564,495 | 1,281,686,656 | 0 | 5,086,165 | 35.9% | provider usage | +| 2026-10-25 | 11,131 | 1,645,130,705 | 26,546,513 | 1,618,584,192 | 0 | 5,869,633 | 36.5% | provider usage | +| 2026-10-26 (partial) | 5,881 | 877,351,671 | 18,430,071 | 858,921,600 | 0 | 3,447,510 | 35.3% | provider usage | + +A working day is **~1.6 billion prompt tokens**, of which **~36%** is the same fixed block re-sent on every call. Cache reads are 98% of it: the block is cheap per token, but it is paid on every call of every turn. + +## 2. The fixed block, per call + +137 Sessions fit; the block is the intercept of each one. + +| Statistic | p10 | p50 | p90 | Source | +| --- | --- | --- | --- | --- | +| Fixed tokens per call | 34,210 | 48,583 | 56,239 | derived: intercept of the Session's prompt fit | +| Fixed characters per call | 125,377 | 132,270 | 142,655 | measured: system prompt + tool schemas on the surface | +| Message characters per token | 2.85 | 3.32 | 3.62 | derived: 1 / slope of the same fit | +| Fit R² | 0.921 | 0.986 | 0.995 | derived: least squares over the Session's calls | + +The fixed block is dense — 2.7 characters per token against 3.32 for the messages — because it is mostly a JSON tool schema plus a Chinese system prompt: no prose padding to average out. + +## 3. One real turn + +Turn 62 of `session-2413164c` (Dev2, group **FlyBot 交付组**), `2026-10-25T10:01:33.511Z .. 10:14:17.073Z` — 12.7 minutes, 125 tool calls (117 `bash`), opened by a 225-character `[Continue task]` card. + +| Measure | Value | Source | +| --- | --- | --- | +| Calls | 124 | provider usage | +| Prompt, whole turn | 17,538,593 | provider usage | +| Prompt per call | 141,440 | derived: prompt / calls | +| Output, whole turn | 49,804 | provider usage | +| First call prompt | 215,263 | provider usage | +| Largest call prompt | 248,706 (call 30) | provider usage | +| Last call prompt | 156,754 | provider usage | + +The turn's own split: + +| Part | Tokens | Share | Source | +| --- | --- | --- | --- | +| Fixed block | 7,184,620 | 41.0% | derived: 57,940 × 124 calls (fit intercept) | +| History | 5,087,027 | 29.0% | derived: character share of prompt − fixed | +| Tool results | 4,222,042 | 24.1% | derived: character share of prompt − fixed | +| Task | 1,044,904 | 6.0% | derived: character share of prompt − fixed | +| Sum | 17,538,593 | 100.0% | provider usage | + +On the largest call (248,706 tokens) the split is fixed **57,940** (23.3%), history **121,005** (48.6%), tool results **69,694** (28.0%), task **66** (0.03%). The bigger the call, the smaller the fixed block's share — and the larger the price of everything the turn has already read. + +## 4. What the fixed block is made of + +The same turn's surface, last call. Characters are measured; the token column splits the measured fixed block (57,940 tokens) by character share. + +| Piece | Characters | Tokens (derived) | Source | +| --- | --- | --- | --- | +| Tool schemas (113 tools) | 115,020 | 45,596 | measured: characters on the surface | +| System prompt, total | 31,138 | 12,344 | measured: characters on the surface | +| — How your Bot team works | 8,980 | 3,560 | measured: characters on the surface | +| — Team memory (`team/MEMORY.md`) | 6,891 | 2,732 | measured: characters on the surface | +| — MCP resource servers (tool docs) | 4,636 | 1,838 | measured: characters on the surface | +| — `MEMORY.md` | 3,992 | 1,583 | measured: characters on the surface | +| — This group chat (notice, roster, order) | 2,588 | 1,026 | measured: characters on the surface | +| — Your memory (header and rules) | 2,168 | 859 | measured: characters on the surface | +| — Your identity | 1,763 | 699 | measured: characters on the surface | +| — preamble | 121 | 48 | measured: characters on the surface | + +The tool schemas are **79%** of the fixed block's characters. The group notice plus roster plus speaking order is 2,588 characters — 8% of the system prompt and 1.8% of the whole fixed block. The part people assume is expensive is the small part. Memory (own + team) is 13,051 characters, 42% of the system prompt. + +## 5. Tool results + +| Measure | Value | Source | +| --- | --- | --- | +| Tool results in the turn | 125 | measured: `tool/result` count | +| Characters added across the turn | 228,571 | measured: sum of `tool/result` content | +| Still on the last call's surface | 160,307 | measured: the checkpoint dropped the rest | +| Largest single result | 25,541 | measured: one `bash` output | +| Share of `bash` | 98.5% | derived: 225,095 / 228,571 | +| Carried in the turn's prompts | 4,222,042 tokens | derived: character share of prompt − fixed | + +One 25,541-character `bash` result is re-read on every later call of the turn. The turn ran 117 shell commands; that is where the 24% of prompt tokens under "tool results" comes from, not from the GitHub connector (2,278 characters in this turn). + +## 6. Growth and the checkpoint + +Same turn, prompt per call (every 10th call, plus the checkpoint): + +| Call | Prompt | Source | +| --- | --- | --- | +| 1 | 215,263 | provider usage | +| 10 | 225,840 | provider usage | +| 20 | 239,531 | provider usage | +| 30 | 248,706 | provider usage | +| 31 (checkpoint) | 58,844 | provider usage | +| 61 | 103,339 | provider usage | +| 91 | 119,876 | provider usage | +| 124 | 156,754 | provider usage | + +Growth is +33,443 tokens over the first 29 calls (+1,153 per call); the checkpoint at call 31 cuts it by 76.3%, and the surface then grows back at +1,053 per call. Host-wide, the 165 Sessions hold 233 checkpoint boundaries: + +| Measure | Value | Source | +| --- | --- | --- | +| Checkpoint boundaries with a call on each side | 233 | measured: `compaction/end` between two calls | +| Prompt before, median | 235,992 | provider usage | +| Prompt after, median | 55,208 | provider usage | +| Drop, median | 76.4% | derived: 1 − after / before | + +A checkpoint buys back roughly twenty minutes of growth at this rate; the fixed block is not affected by it at all. + +## 7. Fan-out in the group + +FlyBot 交付组, 2026-10-25, 511 user messages in the room (one per `turn/start` of the room's own relay Session): + +| Measure | p50 | p90 | max | Source | +| --- | --- | --- | --- | --- | +| Member Bots that started a turn within 10 minutes | 6 | 9 | 13 | measured: member Sessions whose log names the room | +| Member turns started within 10 minutes | 13 | 27 | 92 | measured: `turn/start` in those Sessions | +| Calls in the room's member Sessions, whole day | 11,082 | — | — | provider usage | +| Prompt in those calls, whole day | 1,638,385,969 | — | — | provider usage | +| Fixed block in those calls, whole day | 600,005,904 (36.6%) | — | — | derived: Σ fit intercept | +| Fixed block per user message | 1,174,180 | — | — | derived: 600,005,904 / 511 | + +One message in the delivery group costs its members **21.7 calls** and **1.17 million fixed-block tokens** on average, before any of them reads the message. The window is 10 minutes and a round can run longer, so the p90 rows over-count overlapping rounds; the day totals do not. + +## 8. Retries and repeated work + +| Measure | 2026-10-25 | 2026-10-26 (partial) | Source | +| --- | --- | --- | --- | +| `llm/retry` events | 290 | 553 | measured: log events | +| Attempts carrying a non-zero usage chunk | 0 | 0 | measured: `assistant/attempt` usage chunks | +| Attempt records, all days | 2,932 | — | measured: `assistant/attempt` events | + +The logs keep one `usage` row per settled call, and an `assistant/attempt` record does carry a usage chunk — with every field zero. Zero usage is what an attempt that never reached the provider reports, so the tokens a retried attempt spent are not reconstructable from this sample; the retry counts are, and they are not zero. The row counts the non-zero chunks, which is why it reads 0. + +## What this report does not say + +- It proposes no saving. Which of #796's candidates to take is the owner's decision, and every one of them changes what every Bot is sent. +- It is not a price. The logs have no cost column, and a cache read is counted at face value. +- It does not compare with the 2026-10-23 or 2026-10-25 pages: those read `usage.sqlite` over other windows, this reads Session logs; no delta is printed. +- The per-part token split inside a call is `derived` from character share, not an ablated replay. An ablation through `core/dev/llm-tap.mjs` per category remains the only way to price a part without the character model, and it costs one real provider call per variant. +- It does not measure the memory-review or checkpoint model calls (`memory` / `compaction` rows in the ledger); they are outside a Session and outside this sample. + +## Provenance + +- `provider usage` — a `usage` chunk the provider reported for that call, summed or listed here. +- `measured: ` — counted directly in the log (events, characters, tool results); no model involved. +- `derived: ` — computed from the rows by the formula in the cell. Every per-part token figure is derived this way. +- `not measured` — no number exists in this sample for it. + +Every row of every table above carries its provenance in the last cell. diff --git a/docs/subsystems/ds-bot-core.md b/docs/subsystems/ds-bot-core.md index f41b2e73..f29f2732 100644 --- a/docs/subsystems/ds-bot-core.md +++ b/docs/subsystems/ds-bot-core.md @@ -42,6 +42,7 @@ holds. Task bookkeeping does not imply runtime execution or tool permission. | `scripts/ds-bot-patches.mjs` | generate / check patches. | | `core/dev/eval/` | User-scenario runner (#358): pure rubrics (`rubric.mjs`), canonical journey specs (`specs.mjs`), optional model judge (`judge.mjs`), grader hashing (#398, `guard.mjs`) and a CLI (`run.mjs`). Dev only, not part of `npm test`. | | `core/dev/token-report.mjs` | Reads the usage ledger into per-Bot, per-Session and per-day token numbers and each Session's prompt growth, keeping fresh input, cache read and cache write apart (#799). Read-only; `--db`, `--from`, `--to`, `--tz`, `--bot`, `--top`, `--growth`, `--json`. | +| `core/dev/turn-split.mjs` | Reads Session logs into the split of one turn's prompt: the fixed block (system prompt plus tool schemas) measured as each Session's fit intercept, then history, tool results and task by character share (#796). Read-only; `--sessions`, `--session`, `--turn`, `--tz`, `--days`, `--top`, `--json`. | | `core/dev/experience/` | Offline experience library (#874 Step 2): JSON lessons under `repos///` and `stacks//`. The read API is `core/src/host/experience.js` (see [core-host](core-host.md)). `check.mjs` validates the tree and can add one entry file. Not part of the npm `files` list, and not injected into a harness tier. | | `core/src/host/hooks.js` (approvals) | Held sends bound to their owner — provenance, owner-bound decisions, stale revalidation. Chief replacement invalidates sender-side holds eagerly on `setMain` (not via `staleReason`). The authority contract lives in [team](team.md) (#52). | @@ -97,6 +98,16 @@ path when it is missing. The report it produced and its sample are in `core/test/token-report.test.js` asserts hand-computed sums on a fixture ledger, the missing-`--db` exit, and that every row of that report still carries its provenance. +The composition the ledger cannot show is read from the Session logs instead. +`core/dev/turn-split.mjs` (#796) folds one Session's events into the surface each call +sent — the surviving system prompt, the `request/header` tool list, every message and every +`tool/result` — and splits that call's provider-reported prompt into the fixed block, +history, tool results and task. The fixed block is measured, not assumed: it is the +intercept of `prompt = fixed + k × messageChars` fitted over the same Session's own calls, +so a per-part token figure stays a share of a measured total. The report it backs is +[docs/reports/2026-10-26-turn-input-split.md](../../docs/reports/2026-10-26-turn-input-split.md) +and `core/test/turn-split.test.js` pins the four rules behind the split. + ## Common mistakes - Hand-editing `ds-bot/` or `client.js`. diff --git a/patches/ds-bot/files/dev/turn-split.mjs b/patches/ds-bot/files/dev/turn-split.mjs new file mode 100644 index 00000000..6f3c6e45 --- /dev/null +++ b/patches/ds-bot/files/dev/turn-split.mjs @@ -0,0 +1,483 @@ +// Splits one real turn's prompt into the four parts #796 asks about: the fixed block +// (system prompt plus tool schemas, re-sent on every call), the history, the tool results, +// and the task (the message that opened the turn). +// +// node dev/turn-split.mjs [--sessions ] [--session ] [--turn ] +// [--tz ] [--top ] [--json] +// +// `--sessions` is the DSH session root (default `$DSH_HOME/sessions`); the script walks it +// for `session.v4.jsonl.zstd` files. Every Session log carries, per call, the provider's own +// usage (`inputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `outputTokens`) and the exact +// surface sent: the surviving `system/message`, the `request/header` tool list, and every +// message and `tool/result` appended so far. The prompt total is therefore measured, not +// estimated; the split across the four parts is derived from the characters each part holds +// and the fixed block is measured separately, by fitting one Session's prompts against its +// message characters (the fixed block is the intercept, because it is constant per call). +// +// Fresh input, cache read and cache write are never added into one another: `prompt` is +// their sum and is its own column, as on the usage page (`docs/reports/2026-10-25-token-spend.md`). +// +// A character count is printed as a character count. A token figure for one part is +// `derived` and carries its formula, because a Chinese system prompt and an ASCII tool +// schema do not share one characters-per-token ratio (see the report this script backs, +// `docs/reports/2026-10-26-turn-input-split.md`). +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { homedir } from 'node:os' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { zstdDecompressSync } from 'node:zlib' + +const USAGE = 'usage: node dev/turn-split.mjs [--sessions ] [--session ] [--turn ] [--tz ] [--days] [--top ] [--json]' +const ZSTD_MAGIC = 4247762216 + +function die(message) { + console.error(`turn-split: ${message}`) + process.exit(1) +} + +function parseArgs(argv) { + const flags = { sessions: null, session: null, turn: null, tz: 'UTC', days: false, top: 5, json: false } + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + const value = () => { + const next = argv[index + 1] + if (next === undefined || next.startsWith('--')) die(`${arg} needs a value`) + index += 1 + return next + } + switch (arg) { + case '--sessions': flags.sessions = value(); break + case '--session': flags.session = value(); break + case '--turn': flags.turn = Number(value()); break + case '--tz': flags.tz = value(); break + case '--top': flags.top = Number(value()); break + case '--days': flags.days = true; break + case '--json': flags.json = true; break + case '--help': case '-h': console.log(USAGE); process.exit(0); break + default: die(`unknown argument ${arg}\n${USAGE}`) + } + } + if (!Number.isSafeInteger(flags.top) || flags.top < 1) die('--top must be a whole number') + if (flags.turn !== null && !Number.isSafeInteger(flags.turn)) die('--turn must be a whole number') + return flags +} + +// A Session artifact is a concatenation of independently decodable Zstandard frames, one +// per appended batch. Node's one-shot decoder stops after the first frame, so the frame +// ranges are found by walking the container format (no decompression) and each frame is +// decoded on its own. This mirrors the reader in the Session backend. +export function scanZstdFrames(buffer) { + const frames = [] + let offset = 0 + while (offset < buffer.length) { + const start = offset + if (buffer.length - offset < 4) return { frames, tornStart: start } + if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) throw new Error(`corrupt Zstandard session log: invalid frame magic at byte ${offset}`) + offset += 4 + if (offset === buffer.length) return { frames, tornStart: start } + const descriptor = buffer.readUInt8(offset) + offset += 1 + if ((descriptor & 24) !== 0) throw new Error(`corrupt Zstandard session log: reserved frame-header bit at byte ${offset - 1}`) + const contentSizeFlag = descriptor >>> 6 + const singleSegment = (descriptor & 32) !== 0 + const checksum = (descriptor & 4) !== 0 + const dictionaryFlag = descriptor & 3 + const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag + const contentSizeBytes = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag + offset += (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes + for (;;) { + if (buffer.length - offset < 3) return { frames, tornStart: start } + const blockHeader = buffer.readUIntLE(offset, 3) + offset += 3 + const lastBlock = (blockHeader & 1) !== 0 + const blockType = (blockHeader >>> 1) & 3 + const blockSize = blockHeader >>> 3 + if (blockType === 3) throw new Error(`corrupt Zstandard session log: reserved block type at byte ${offset - 3}`) + offset += blockType === 1 ? 1 : blockSize + if (offset > buffer.length) return { frames, tornStart: start } + if (lastBlock) break + } + if (checksum) offset += 4 + if (offset > buffer.length) return { frames, tornStart: start } + frames.push({ start, end: offset }) + } + return { frames } +} + +// The provider's reported usage is the last `usage` chunk of a settled attempt. +export function usageOf(stream) { + if (!Array.isArray(stream)) return undefined + for (let index = stream.length - 1; index >= 0; index -= 1) { + const record = stream[index] + if (record?.type === 'chunk' && record.chunk?.type === 'usage') return record.chunk.usage + } + return undefined +} + +function textOf(blocks) { + if (typeof blocks === 'string') return blocks + if (!Array.isArray(blocks)) return '' + let text = '' + for (const block of blocks) { + if (!block || typeof block !== 'object') continue + if (typeof block.text === 'string') text += block.text + else if (block.type === 'tool-call') text += JSON.stringify(block) + } + return text +} + +const count = (value) => (Number.isSafeInteger(value) && value > 0 ? value : 0) + +// One Session's events, folded into the surface each call sent and the usage it reported. +// A message belongs to the turn that was open when it arrived; a message admitted just +// before `turn/start` (the runtime context, the time context, the user's own words) is the +// turn's task, so it is attributed to that turn when the turn opens. +export function foldSessionEvents(events) { + const steps = [] + const turns = new Map() + let system = '' + let tools = '' + let turn = null + let open = false + let pending = 0 + const messages = [] + let toolCallNames = new Map() + const largestTool = new Map() + const toolTotal = new Map() + for (const event of events) { + const data = event?.data ?? {} + switch (event?.type) { + case 'system/message': + system = textOf(data.message?.content) + break + case 'request/header': + tools = JSON.stringify(data.header?.tools ?? []) + break + case 'turn/start': + turn = data.turn ?? null + open = true + for (let index = messages.length - pending; index < messages.length; index += 1) messages[index].turn = turn + pending = 0 + break + case 'turn/end': + open = false + break + case 'compaction/end': + messages.length = 0 + pending = 0 + break + case 'tool/call': + toolCallNames.set(data.callId, data.name ?? '') + break + case 'user/message': { + const entry = { role: 'user', turn: open ? turn : null, chars: textOf(data.content).length } + messages.push(entry) + if (!open) pending += 1 + break + } + case 'tool/result': { + const callId = data.message?.toolCallId ?? data.message?.source?.callId + const resultTurn = open ? turn : (data.turn ?? null) + const resultChars = textOf(data.message?.content).length + largestTool.set(resultTurn, Math.max(largestTool.get(resultTurn) ?? 0, resultChars)) + toolTotal.set(resultTurn, (toolTotal.get(resultTurn) ?? 0) + resultChars) + messages.push({ + role: 'tool', + turn: open ? turn : (data.turn ?? null), + chars: textOf(data.message?.content).length, + name: toolCallNames.get(callId) ?? data.message?.toolName ?? '', + }) + break + } + case 'assistant/message': { + const usage = usageOf(data.stream) + const current = data.turn ?? null + if (usage) { + const chars = { tool: 0, assistant: 0, task: 0, history: 0 } + const toolCharsByName = {} + for (const message of messages) { + if (message.role === 'tool') { + chars.tool += message.chars + toolCharsByName[message.name] = (toolCharsByName[message.name] ?? 0) + message.chars + } + else if (message.role === 'assistant') chars.assistant += message.chars + else if (message.role === 'user') { + if (message.turn === current) chars.task += message.chars + else chars.history += message.chars + } + } + const step = { + turn: current, + step: data.step ?? null, + time: event.time, + input: count(usage.inputTokens), + cacheRead: count(usage.cacheReadTokens), + cacheWrite: count(usage.cacheWriteTokens), + output: count(usage.outputTokens), + systemChars: system.length, + toolsChars: tools.length, + messageChars: chars.task + chars.history + chars.assistant + chars.tool, + chars: { task: chars.task, history: chars.history + chars.assistant, tool: chars.tool }, + toolCharsByName, + } + step.prompt = step.input + step.cacheRead + step.cacheWrite + steps.push(step) + const aggregate = turns.get(current) ?? { turn: current, calls: 0, prompt: 0, output: 0, first: step, last: step } + aggregate.calls += 1 + aggregate.prompt += step.prompt + aggregate.output += step.output + aggregate.last = step + turns.set(current, aggregate) + } + let assistantChars = 0 + for (const block of (data.message?.content ?? [])) { + if (block.type === 'tool-call') assistantChars += JSON.stringify(block).length + else if (typeof block.text === 'string') assistantChars += block.text.length + } + messages.push({ role: 'assistant', turn: current, chars: assistantChars }) + break + } + default: + break + } + } + return { steps, turns: [...turns.values()], largestTool, toolTotal, systemChars: system.length, toolsChars: tools.length } +} + +// The fixed block is constant for every call of one Session, so it is that Session's +// intercept when its provider-reported prompts are fitted against its message characters: +// `prompt = fixed + k * messageChars`. `k` is the message cost per character and +// `charsPerToken` its inverse. Fewer than eight calls, or no variation in the message +// characters, leaves the Session without a fit rather than inventing one. +export function fitFixed(steps) { + const usable = steps.filter((step) => step.prompt > 0 && step.messageChars > 0) + if (usable.length < 8) return null + const n = usable.length + const meanX = usable.reduce((sum, step) => sum + step.messageChars, 0) / n + const meanY = usable.reduce((sum, step) => sum + step.prompt, 0) / n + let sxx = 0 + let sxy = 0 + for (const step of usable) { + sxx += (step.messageChars - meanX) ** 2 + sxy += (step.messageChars - meanX) * (step.prompt - meanY) + } + if (sxx === 0) return null + const slope = sxy / sxx + const intercept = meanY - slope * meanX + let residual = 0 + let total = 0 + for (const step of usable) { + residual += (step.prompt - intercept - slope * step.messageChars) ** 2 + total += (step.prompt - meanY) ** 2 + } + return { + n, + fixedTokens: intercept, + charsPerToken: slope > 0 ? 1 / slope : null, + r2: total > 0 ? 1 - residual / total : 1, + fixedChars: usable[0].systemChars + usable[0].toolsChars, + } +} + +// One call's prompt, split into the four parts. The fixed block is the measured intercept; +// what is left after it is distributed over the parts by their character share, which is +// the only per-part evidence the log holds. +export function splitStep(step, fit) { + const fixed = Math.max(0, Math.min(fit.fixedTokens, step.prompt)) + const rest = Math.max(0, step.prompt - fixed) + const total = step.messageChars + const share = (chars) => (total > 0 ? rest * chars / total : 0) + return { fixed, task: share(step.chars.task), history: share(step.chars.history), tool: share(step.chars.tool) } +} + +export function readSessionFile(file) { + const buffer = readFileSync(file) + const { frames } = scanZstdFrames(buffer) + const events = [] + for (const { start, end } of frames) { + let text + try { + text = zstdDecompressSync(buffer.subarray(start, end)).toString('utf8') + } catch { + continue + } + for (const line of text.split('\n')) { + if (!line.trim()) continue + try { + events.push(JSON.parse(line)) + } catch { + // A torn or foreign line is skipped: one bad line must not hide the Session. + } + } + } + return events +} + +export function findSessionFiles(root) { + const files = [] + const walk = (dir, depth) => { + let entries + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + const path = join(dir, entry.name) + if (entry.isDirectory()) { + if (depth > 0) walk(path, depth - 1) + } else if (entry.name === 'session.v4.jsonl.zstd') files.push(path) + } + } + walk(root, 2) + return files.sort() +} + +const localDay = (time, tz) => { + if (!Number.isFinite(time)) return 'unknown' + const parts = new Intl.DateTimeFormat('en-CA', { timeZone: tz, year: 'numeric', month: '2-digit', day: '2-digit' }).formatToParts(new Date(time)) + const part = Object.fromEntries(parts.map(({ type, value }) => [type, value])) + return `${part.year}-${part.month}-${part.day}` +} +const integer = (value) => Math.round(value).toLocaleString('en-US') + +function main(argv) { + const flags = parseArgs(argv) + const root = flags.sessions === null ? join(process.env.DSH_HOME ?? join(homedir(), '.dsh'), 'sessions') : flags.sessions + let files + try { + statSync(root) + files = findSessionFiles(resolve(root)) + } catch { + die(`no session root at ${resolve(root)}`) + } + if (files.length === 0) die(`no session.v4.jsonl.zstd under ${resolve(root)}`) + const sessions = [] + for (const file of files) { + const id = file.split('/').at(-2) + if (flags.session !== null && !id.startsWith(flags.session)) continue + const folded = foldSessionEvents(readSessionFile(file)) + if (folded.steps.length === 0) continue + sessions.push({ id, file, ...folded, fixed: fitFixed(folded.steps) }) + } + if (sessions.length === 0) die('no Session in the sample carried a provider usage report') + const rows = [] + for (const session of sessions) { + if (!session.fixed) continue + for (const turn of session.turns) { + const steps = session.steps.filter((step) => step.turn === turn.turn) + const total = { prompt: 0, fixed: 0, history: 0, tool: 0, task: 0 } + for (const step of steps) total.prompt += step.prompt + for (const step of steps) { + const split = splitStep(step, session.fixed) + total.fixed += split.fixed + total.task += split.task + total.history += split.history + total.tool += split.tool + } + const calls = steps.length + if (calls === 0) continue + rows.push({ + session: session.id, + turn: turn.turn, + day: localDay(turn.first.time, flags.tz), + calls, + prompt: total.prompt, + output: turn.output, + toolAdded: steps.at(-1).chars.tool, + largestTool: session.largestTool?.get(turn.turn) ?? 0, + toolTotal: session.toolTotal?.get(turn.turn) ?? 0, + fixed: total.fixed, + history: total.history, + tool: total.tool, + task: total.task, + first: steps[0].prompt, + last: steps.at(-1).prompt, + max: Math.max(...steps.map((step) => step.prompt)), + fit: session.fixed, + systemChars: session.systemChars, + toolsChars: session.toolsChars, + steps, + toolNames: (() => { + const names = new Map() + for (const step of steps) for (const [name, chars] of Object.entries(step.toolCharsByName ?? {})) names.set(name, (names.get(name) ?? 0) + chars) + return names + })(), + }) + } + } + const fits = sessions.map((session) => session.fixed).filter(Boolean) + const median = (values) => (values.length === 0 ? 0 : values.slice().sort((a, b) => a - b)[Math.floor((values.length - 1) / 2)]) + if (flags.days) { + const days = new Map() + for (const session of sessions) { + for (const step of session.steps) { + const day = localDay(step.time, flags.tz) + const row = days.get(day) ?? { day, calls: 0, prompt: 0, input: 0, cacheRead: 0, cacheWrite: 0, output: 0, fixed: 0 } + row.calls += 1 + row.prompt += step.prompt + row.input += step.input + row.cacheRead += step.cacheRead + row.cacheWrite += step.cacheWrite + row.output += step.output + row.fixed += session.fixed ? splitStep(step, session.fixed).fixed : 0 + days.set(day, row) + } + } + const rowsOut = [...days.values()].sort((a, b) => a.day.localeCompare(b.day)) + if (flags.json) { + console.log(JSON.stringify({ sample: { root: resolve(root), sessions: sessions.length, tz: flags.tz }, days: rowsOut }, null, 2)) + return + } + console.log(`sample: ${sessions.length} Sessions under ${resolve(root)}, zone ${flags.tz}`) + console.log(`Sessions with a fixed-block fit: ${fits.length}; fixed tokens p50 ${integer(median(fits.map((fit) => fit.fixedTokens)))}`) + console.log('') + console.log('day calls prompt fresh input cache read cache write output fixed share') + for (const row of rowsOut) { + console.log(`${row.day} ${String(row.calls).padStart(6)} ${integer(row.prompt).padStart(13)} ${integer(row.input).padStart(13)} ${integer(row.cacheRead).padStart(13)} ${integer(row.cacheWrite).padStart(13)} ${integer(row.output).padStart(12)} ${(row.fixed / row.prompt * 100).toFixed(1).padStart(11)}%`) + } + return + } + if (rows.length === 0) die('no turn in the sample could be split (every Session needs 8 calls)') + if (flags.turn !== null) { + const only = rows.filter((row) => row.turn === flags.turn) + if (only.length === 0) die(`no turn ${flags.turn} in the sample`) + rows.length = 0 + rows.push(...only) + } + rows.sort((a, b) => b.prompt - a.prompt) + const top = rows.slice(0, flags.top) + if (flags.json) { + console.log(JSON.stringify({ sample: { root: resolve(root), sessions: sessions.length, tz: flags.tz }, turns: top.map((row) => ({ ...row, steps: undefined, toolNames: undefined, fit: row.fit })) }, null, 2)) + return + } + console.log(`sample: ${sessions.length} Sessions under ${resolve(root)}, zone ${flags.tz}`) + console.log('') + console.log('## Fixed block per call (measured: the intercept of each Session\'s prompt fit)') + console.log(`Sessions with a fit: ${fits.length}; fixed tokens p50 ${integer(median(fits.map((fit) => fit.fixedTokens)))}; ` + + `fixed chars p50 ${integer(median(fits.map((fit) => fit.fixedChars)))}; message chars/token p50 ${median(fits.map((fit) => fit.charsPerToken)).toFixed(2)}; ` + + `R2 p50 ${median(fits.map((fit) => fit.r2)).toFixed(3)}`) + console.log('') + console.log('## Top turns by prompt') + console.log('session turn calls prompt fixed history tool task first last max') + for (const row of top) { + const share = (value) => `${(value / row.prompt * 100).toFixed(1)}%` + console.log(`${row.session.padEnd(32)} ${String(row.turn).padStart(4)} ${String(row.calls).padStart(6)} ${integer(row.prompt).padStart(8)} ` + + `${share(row.fixed).padStart(7)} ${share(row.history).padStart(7)} ${share(row.tool).padStart(7)} ${share(row.task).padStart(7)} ` + + `${integer(row.first).padStart(8)} ${integer(row.last).padStart(8)} ${integer(row.max).padStart(8)}`) + } + const first = top[0] + if (first) { + console.log('') + console.log(`## Turn ${first.turn} of ${first.session}: one call's surface`) + console.log(`system prompt ${integer(first.systemChars)} chars; tool schemas ${integer(first.toolsChars)} chars; ` + + `fixed block ${integer(first.fit.fixedTokens)} tokens (${(first.fit.fixedChars / first.fit.fixedTokens).toFixed(2)} chars/token)`) + console.log(`calls ${first.calls}; provider prompt ${integer(first.prompt)} tokens (${integer(first.prompt / first.calls)} per call); output ${integer(first.output)} tokens`) + console.log(`tool text added by the turn ${integer(first.toolTotal)} chars; still on the last call's surface ${integer(first.toolAdded)} chars; largest single result ${integer(first.largestTool)} chars`) + if (first.toolNames.size > 0) { + console.log(`tool text read again on every call, by tool: ${[...first.toolNames].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([name, chars]) => `${name} ${integer(chars)} chars`).join(', ')}`) + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main(process.argv.slice(2)) diff --git a/patches/ds-bot/files/test/turn-split.test.js b/patches/ds-bot/files/test/turn-split.test.js new file mode 100644 index 00000000..9e49dab7 --- /dev/null +++ b/patches/ds-bot/files/test/turn-split.test.js @@ -0,0 +1,214 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { zstdCompressSync } from 'node:zlib' + +import { fitFixed, foldSessionEvents, scanZstdFrames, splitStep } from '../dev/turn-split.mjs' + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..') +const SCRIPT = join(ROOT, 'core', 'dev', 'turn-split.mjs') + +const usage = (input, cacheRead, output) => ({ + type: 'chunk', + chunk: { type: 'usage', usage: { inputTokens: input, cacheReadTokens: cacheRead, outputTokens: output } }, +}) +const system = (text) => ({ type: 'system/message', data: { message: { role: 'system', content: [{ type: 'text', text }] } } }) +const header = (tools) => ({ type: 'request/header', data: { header: { tools } } }) +const user = (text) => ({ type: 'user/message', data: { content: [{ type: 'text', text }] } }) +const toolResult = (callId, name, text) => ({ + type: 'tool/result', + data: { message: { role: 'tool', toolCallId: callId, content: [{ type: 'text', text }] } }, +}) +const toolCall = (callId, name) => ({ type: 'tool/call', data: { callId, name } }) +const assistant = (turn, step, input, cacheRead, output, text = 'ok') => ({ + type: 'assistant/message', + time: 1_792_900_000_000 + turn * 100_000 + step * 1000, + data: { turn, step, message: { role: 'assistant', content: [{ type: 'text', text }] }, stream: [{ type: 'chunk', chunk: { type: 'finish' } }, usage(input, cacheRead, output)] }, +}) + +// One turn that reads a 1000-character tool result, then a second turn whose task is short. +function fixtureEvents() { + return [ + system('S'.repeat(100)), + header([{ name: 'bash', description: 'd'.repeat(900) }]), + user('context before the turn'), + { type: 'turn/start', data: { turn: 1 } }, + user('do the thing'), + assistant(1, 1, 100, 900, 10), + toolCall('call-1', 'bash'), + toolResult('call-1', 'bash', 'T'.repeat(1000)), + assistant(1, 2, 50, 1950, 20), + { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } }, + { type: 'turn/start', data: { turn: 2 } }, + user('next'), + assistant(2, 1, 40, 2960, 5), + ] +} + +test('a turn separates its task from the history that came before it', () => { + const folded = foldSessionEvents(fixtureEvents()) + assert.equal(folded.steps.length, 3) + const [first, , third] = folded.steps + // Everything admitted before the turn opens — the injected context and the user's words — + // is that turn's task; nothing before it counts as history yet. + assert.equal(first.chars.task, 'context before the turn'.length + 'do the thing'.length) + assert.equal(first.chars.history, 0) + assert.equal(first.chars.tool, 0) + // Turn 2 opens with a short task; turn 1's two user messages and its two "ok" replies are + // history by then. + assert.equal(third.chars.task, 'next'.length) + assert.equal(third.chars.history, 'context before the turn'.length + 'do the thing'.length + 2 * 'ok'.length) +}) + +test('a tool result is its own part, never folded into the history', () => { + const folded = foldSessionEvents(fixtureEvents()) + const second = folded.steps[1] + assert.equal(second.chars.tool, 1000) + // The kill: counting tool text as history would move all 1000 characters there. + assert.ok(second.chars.tool >= 1000) + // What the assistant itself wrote is history; the turn's task stays its own part. + assert.equal(second.chars.history, 'ok'.length) +}) + +test('the fixed block is the intercept of the Session\'s prompt fit', () => { + const steps = Array.from({ length: 10 }, (_, index) => ({ + prompt: 48_000 + Math.round((index + 1) * 3_320), + messageChars: (index + 1) * 10_000, + systemChars: 20_000, + toolsChars: 110_000, + })) + const fit = fitFixed(steps) + assert.ok(fit) + assert.ok(Math.abs(fit.fixedTokens - 48_000) <= 1) + assert.ok(Math.abs(fit.charsPerToken - 10000 / 3320) < 1e-9) + assert.ok(fit.r2 > 0.999) + assert.equal(fit.n, 10) + // The kill: a fit that reported no fixed block would put every prompt token in the parts. + assert.ok(fit.fixedTokens > 0) +}) + +test('a Session with too few calls gets no fit instead of a guess', () => { + const steps = Array.from({ length: 7 }, (_, index) => ({ prompt: 1000 + index, messageChars: index * 10, systemChars: 1, toolsChars: 1 })) + assert.equal(fitFixed(steps), null) + const flat = Array.from({ length: 12 }, () => ({ prompt: 1000, messageChars: 10, systemChars: 1, toolsChars: 1 })) + assert.equal(fitFixed(flat), null) +}) + +test('a split accounts for the whole prompt and never claims the fixed block twice', () => { + const fit = { fixedTokens: 48_000, charsPerToken: 3.32, r2: 1, n: 100, fixedChars: 130_000 } + const step = { prompt: 148_000, messageChars: 100_000, chars: { task: 10_000, history: 30_000, tool: 60_000 } } + const split = splitStep(step, fit) + const sum = split.fixed + split.task + split.history + split.tool + assert.ok(Math.abs(sum - step.prompt) <= 1) + assert.equal(split.fixed, 48_000) + // A short prompt is all fixed block, not a negative remainder. + const tiny = splitStep({ prompt: 30_000, messageChars: 1000, chars: { task: 1000, history: 0, tool: 0 } }, fit) + assert.equal(tiny.fixed, 30_000) + assert.equal(tiny.task + tiny.history + tiny.tool, 0) +}) + +test('the frame walker finds every frame of a concatenated log', (t) => { + if (typeof zstdCompressSync !== 'function') { + t.skip('this Node has no zstd in node:zlib (needs 22.15+)') + return + } + const first = zstdCompressSync(Buffer.from('{"type":"a"}\n')) + const second = zstdCompressSync(Buffer.from('{"type":"b"}\n')) + const buffer = Buffer.concat([first, second]) + const { frames } = scanZstdFrames(buffer) + assert.equal(frames.length, 2) + assert.deepEqual(frames[0], { start: 0, end: first.length }) + assert.deepEqual(frames[1], { start: first.length, end: buffer.length }) +}) + +function sessionFile(events) { + const bytes = [events.map((event) => JSON.stringify(event)).join('\n') + '\n', JSON.stringify({ type: 'session/title', data: { title: 'x' } })] + return Buffer.concat(bytes.map((text) => zstdCompressSync(Buffer.from(text)))) +} + +function sample(t) { + const dir = mkdtempSync(join(tmpdir(), 'turn-split-')) + const workspace = join(dir, 'sessions', '--workspace--') + mkdirSync(workspace, { recursive: true }) + // Twelve calls with a growing message surface: the fit needs variation and eight calls. + const events = [system('S'.repeat(100)), header([{ name: 'bash', description: 'd'.repeat(900) }]), { type: 'turn/start', data: { turn: 1 } }] + for (let step = 1; step <= 12; step += 1) { + events.push(user(`task ${step} `.repeat(step * 10))) + events.push(assistant(1, step, 100, 48_000 + step * 500, 10)) + events.push(toolCall(`call-${step}`, 'bash')) + events.push(toolResult(`call-${step}`, 'bash', 'T'.repeat(step * 400))) + } + const folder = join(workspace, 'session-fixture') + mkdirSync(folder) + writeFileSync(join(folder, 'session.v4.jsonl.zstd'), sessionFile(events)) + t.after(() => rmSync(dir, { recursive: true, force: true })) + return join(dir, 'sessions') +} + +test('the CLI prints a split for the sampled turn', (t) => { + if (typeof zstdCompressSync !== 'function') { + t.skip('this Node has no zstd in node:zlib (needs 22.15+)') + return + } + const root = sample(t) + const result = spawnSync(process.execPath, [SCRIPT, '--sessions', root], { encoding: 'utf8' }) + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /Sessions with a fit: 1/) + assert.match(result.stdout, /fixed block \d[\d,]* tokens/) +}) + +test('the --days table divides each day by the same measured fixed block', (t) => { + if (typeof zstdCompressSync !== 'function') { + t.skip('this Node has no zstd in node:zlib (needs 22.15+)') + return + } + const root = sample(t) + const result = spawnSync(process.execPath, [SCRIPT, '--sessions', root, '--days'], { encoding: 'utf8' }) + assert.equal(result.status, 0, result.stderr) + // The fixture is twelve calls of 48,100 + 500 × step: 616,200 prompt over one local day. + const row = result.stdout.split('\n').find((line) => /^\d{4}-\d{2}-\d{2}\s/.test(line)) + assert.ok(row, `the --days table printed no day row:\n${result.stdout}`) + const cells = row.trim().split(/\s+/) + assert.equal(cells[1], '12') + assert.equal(cells[2], '616,200') + // Every call's fixed block comes from the fixture's own fit (49,459 tokens), so 96.1% of + // the day's prompt is the same block re-sent. The kill: `row.fixed += 0` at `turn-split.mjs` + // prints 0.0% here and leaves every other test green, which is how this path went unpinned. + assert.equal(cells.at(-1), '96.1%') +}) + +test('a missing session root is an error that names the path', () => { + const missing = join(tmpdir(), 'turn-split-no-such-root') + const result = spawnSync(process.execPath, [SCRIPT, '--sessions', missing], { encoding: 'utf8' }) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /no session root at/) + assert.match(result.stderr, new RegExp(missing.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))) +}) + +// The report this script backs ships a number only in a table row whose last cell names +// where it came from: the provider's usage, a direct count in the log, or a formula. +const REPORT = join(ROOT, 'docs', 'reports', '2026-10-26-turn-input-split.md') +const PROVENANCE = /^(provider usage|measured: .+|derived: .+)$/ + +test('every row of the committed split report carries its provenance', () => { + const lines = readFileSync(REPORT, 'utf8').split(/\r?\n/) + const rows = [] + let header = null + for (const line of lines) { + const cells = line.startsWith('|') ? line.split('|').slice(1, -1).map((cell) => cell.trim()) : null + if (cells === null) { header = null; continue } + if (header === null) { header = cells; continue } + if (cells.every((cell) => /^:?-{3,}:?$/.test(cell))) continue + assert.equal(header.at(-1), 'Source', `a table starting with ${header[0]} has no Source column`) + rows.push({ header, cells }) + } + assert.ok(rows.length > 0, 'the report has no tables') + for (const row of rows) { + assert.match(row.cells.at(-1), PROVENANCE, `no provenance: | ${row.cells.join(' | ')} |`) + assert.equal(row.cells.length, row.header.length, `a row has ${row.cells.length} cells, its table ${row.header.length}: | ${row.cells.join(' | ')} |`) + } +})