diff --git a/docs/docs/configure/learn.md b/docs/docs/configure/learn.md index 31ef056111..67a80b0f0c 100644 --- a/docs/docs/configure/learn.md +++ b/docs/docs/configure/learn.md @@ -222,7 +222,21 @@ altimate-code learn disable ### status -Shows whether learning is on, whether automatic promotion is on with its last promotion and last held-back reason, lesson counts (approved, candidate, retired), open signals, pending recoveries, the last reflection and its summary, and the resolved limits. `--json` also includes the last reflection's token use and estimated cost. +Shows whether learning is on, whether automatic promotion is on with its last promotion and last held-back reason, lesson counts (approved, candidate, retired), open signals, pending recoveries, the last reflection and its summary, the resolved limits, and lesson recurrence. `--json` also includes the last reflection's token use and estimated cost. + +**Recurrence** answers "was this lesson shown, and did the user still correct the same thing?". It reads the newest 200 session snapshots and the recorded corrections. It makes no model calls and writes nothing. For each shown lesson it counts sessions where a matching correction came: + +| Count | Meaning | +|---|---| +| `recurred` | after the lesson was shown, before the agent started answering the corrected request | +| `during` | while the agent was answering, for example a file-hook lesson added mid-turn; the lesson may have arrived too late | +| `unknown` | in an order that cannot be told, for example older corrections or a request replayed after compaction | +| `ambiguous` | when the correction matched more than one lesson the session was shown in its current text | +| `earlier_version` | when the session was shown only an earlier version of the lesson text; it never counts for the current text | + +A correction matches a lesson when it names one of the lesson's full backtick identifiers plus another word from the lesson, or contains at least 3 of the lesson's words and 60% of them. A prefix or suffix identifier such as `_at` or `stg_` never matches on its own. Lessons first shown after the correction are ignored for it. The text output lists only lessons with a matching correction and summarizes the rest as "no observed correction". That is not proof the agent followed them. Corrections are recorded only while capture is on. Status also lists up to 5 identifiers that were corrected in 2 or more sessions with no matching current lesson. They are taken from your local corrections and never leave the project. If the analysis fails, status shows `Recurrence unavailable` (`recurrence_error` in JSON), and the rest of the status still prints. + +When the current text of a lesson has `recurred` in 2 or more sessions, reflection marks it `r:N` in the playbook it sends the model. The model is asked to make the lesson clearer or more specific when the feedback shows the wording is unclear, and to keep it when the agent simply did not follow a correct rule. It may narrow a rule only when the feedback shows the rule is too broad, and must keep the behavior the rule requires. The result is a candidate like any other and goes through the same review or automatic-promotion gates. ```bash altimate-code learn status diff --git a/packages/opencode/src/altimate/learn/capture.ts b/packages/opencode/src/altimate/learn/capture.ts index 31e7d76d99..700d6fd69a 100644 --- a/packages/opencode/src/altimate/learn/capture.ts +++ b/packages/opencode/src/altimate/learn/capture.ts @@ -92,6 +92,39 @@ export interface MessageLike { time?: { created?: number; completed?: number } } +export type TurnMessage = MessageLike & { synthetic?: boolean; compaction?: boolean; replay?: boolean } + +type TurnPart = { type: string; synthetic?: boolean; metadata?: Record } + +/** + * Session history as turn boundaries. All-synthetic text continues a turn unless the message carries user input + * (an attachment or agent mention, whose text companions are synthetic). Compaction and its replay are marked. + */ +export function turnMessages(messages: readonly { info: MessageLike; parts: readonly TurnPart[] }[]): TurnMessage[] { + return messages.map(({ info, parts }) => { + const text = parts.filter((part) => part.type === "text") + return { + ...info, + synthetic: text.length > 0 && text.every((part) => part.synthetic) && !parts.some((part) => part.type === "file" || part.type === "agent"), + compaction: parts.some((part) => part.type === "compaction"), + replay: text.some((part) => typeof part.metadata?.learnOriginalMessage === "string"), + } + }) +} + +/** + * Start of the agent's response to the request the correction answers. Synthetic continuations are not + * requests. A request right after a compaction may be its replay, so its turn start is unknown. + */ +export function correctedTurnStart(messages: readonly TurnMessage[], messageID: string): number | undefined { + const end = messages.findIndex((message) => message.id === messageID && message.role === "user") + if (end < 0) return + const previous = messages.slice(0, end).findLastIndex((message) => message.role === "user" && !message.synthetic && !message.compaction) + if (previous < 0 || messages[previous].replay) return + if (messages.slice(0, previous).findLast((message) => message.role === "user")?.compaction) return + return messages.slice(previous + 1, end).find((message) => message.role === "assistant")?.time?.created +} + export interface PartLike { id: string sessionID: string @@ -107,6 +140,7 @@ export interface PartLike { export interface CaptureDeps { /** True when the session has a completed assistant message before the user message being classified. */ hasPriorAssistant: (sessionID: string, beforeMessageID: string) => Promise + turnStart?: (sessionID: string, messageID: string) => Promise record: (signal: NewSignal) => Promise } @@ -114,6 +148,7 @@ export class Capture { private messages = new Map() private pendingParts = new Map() private sessionsWithAssistant = new Map() + private corrected = new Set() private trackers = new Map() private pending = new Set>() @@ -170,16 +205,25 @@ export class Capture { if (message.role !== "user") return const reason = correctionReason(part.text) if (!reason) return + // Later updates of a recorded correction deduplicate to its first signal; skip their history reads. + if (this.corrected.has(part.messageID)) return const prior = this.sessionsWithAssistant.get(part.sessionID) const known = prior !== undefined && message.time?.created !== undefined && prior < message.time.created if (!known && !(await this.deps.hasPriorAssistant(part.sessionID, part.messageID))) return + const start = await this.deps.turnStart?.(part.sessionID, part.messageID).catch((error) => { + log.warn("learn turn start unavailable", { error }) + return undefined + }) await this.deps.record({ kind: "user_correction", + ...(start !== undefined && Number.isFinite(start) ? { turnStartedAt: new Date(start).toISOString() } : {}), + ...(message.time?.created !== undefined && Number.isFinite(message.time.created) ? { correctedAt: new Date(message.time.created).toISOString() } : {}), sessionID: part.sessionID, messageID: part.messageID, text: part.text, reason, }) + remember(this.corrected, part.messageID, RECENT_ID_CAP) } private async toolPart(part: PartLike) { @@ -239,6 +283,10 @@ export async function startCapture(ctx: { directory: string; worktree: string }) const { SessionID } = await import("../../session/schema") const capture = new Capture({ record: (signal) => appendSignal(root, signal), + turnStart: async (sessionID, messageID) => { + const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) + return correctedTurnStart(turnMessages(messages), messageID) + }, hasPriorAssistant: async (sessionID, beforeMessageID) => { const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) const before = messages.findIndex((message) => message.info.id === beforeMessageID) diff --git a/packages/opencode/src/altimate/learn/delivery.ts b/packages/opencode/src/altimate/learn/delivery.ts index 6778cd2aff..c246f91db9 100644 --- a/packages/opencode/src/altimate/learn/delivery.ts +++ b/packages/opencode/src/altimate/learn/delivery.ts @@ -79,6 +79,44 @@ async function read(root: string, file: string) { } } +export const lessonTextHash = (text: string) => Store.sha256(singleLine(text)) +export type ShownSnapshot = { + session: string + shown: { name: string; id: string; textHash: string; tier: Shown["tier"]; at: string }[] +} + +export async function readShownSnapshots(root: string, options: { limit?: number } = {}) { + const dir = path.join(root, ".altimate-code", "learn", ".sessions") + const coverage = { sessions_analyzed: 0, sessions_skipped_malformed: 0, truncated: false } + const snapshots: ShownSnapshot[] = [] + await SafeFS.assertSafePath(root, dir) + const entries = await fs.readdir(dir).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] + throw error + }) + const files = await Promise.all(entries.filter((name) => name.endsWith(".json")).map(async (name) => { + const file = path.join(dir, name) + return { file, mtime: await fs.lstat(file).then((stat) => stat.mtimeMs).catch(() => 0) } + })) + const limit = Math.max(0, Math.floor(options.limit ?? 200)) + coverage.truncated = files.length > limit + for (const { file } of files.sort((a, b) => b.mtime - a.mtime || a.file.localeCompare(b.file)).slice(0, limit)) { + try { + const state = State.parse(JSON.parse((await read(root, file)) ?? "")) + if (path.basename(file) !== Store.sha256(state.session) + ".json") throw new Error("Learn session state belongs to another session") + snapshots.push({ session: state.session, shown: state.shown.map(({ name, lesson, tier, at }) => ({ + name, id: lesson.id, textHash: lessonTextHash(lesson.text), tier, at, + })) }) + } catch (error) { + if (error instanceof SafeFS.UnsafeLearnPathError) throw error + coverage.sessions_skipped_malformed++ + log.warn("learn session snapshot skipped", { file, error }) + } + } + coverage.sessions_analyzed = snapshots.length + return { snapshots, coverage } +} + /** Persisted text snapshots keep the prefix stable even after edits, ranking changes or a restart. */ export class Delivery { private resolved?: Limits diff --git a/packages/opencode/src/altimate/learn/prompt.txt b/packages/opencode/src/altimate/learn/prompt.txt index 465c7700e3..25e922889f 100644 --- a/packages/opencode/src/altimate/learn/prompt.txt +++ b/packages/opencode/src/altimate/learn/prompt.txt @@ -16,8 +16,8 @@ How to work: 2. Infer general conventions. State one rule with the exact identifier it governs, without rationale or incident details. Keep the specific table, model, column, or file identifier when the correction depends on it; do not substitute a vague generality. 3. Learn only from the external feedback. Do not treat the agent's own claims of success as evidence. 4. For an existing bullet: - - HELPFUL when the feedback shows the agent followed it and the related checks passed. - - HARMFUL only with explicit evidence that the agent FOLLOWED the bullet and that doing so caused a failure or a reviewer objection. The agent not following a bullet, feedback that is unrelated to the bullet, or feedback that asks for exactly what the bullet says is never HARMFUL (the last is HELPFUL). If unsure, leave the bullet alone. + - HELPFUL only when the DIGEST shows the agent followed it and the feedback shows the related checks passed. When the feedback asks for exactly what the bullet says because the agent did not follow it, the bullet is right but was missed: it is neither HELPFUL nor HARMFUL. Leave its counters alone, and EDIT it only if its wording is unclear. + - HARMFUL only with explicit evidence that the agent FOLLOWED the bullet and that doing so caused a failure or a reviewer objection. The agent not following a bullet, feedback that is unrelated to the bullet, or feedback that asks for exactly what the bullet says is never HARMFUL. If unsure, leave the bullet alone. - When feedback states a different expected behaviour than a bullet, EDIT the bullet to that expected behaviour (or ADD with "supersedes"). HARMFUL alone loses the knowledge. Keep the corrected convention general. - REMOVE when the feedback shows it is plainly wrong or outdated. 5. Before ADD or EDIT, compare its code identifiers, naming prefixes and suffixes in backticks with every existing bullet. If another bullet touches the same convention, explicitly resolve their relationship: EDIT the existing rule, REMOVE the outdated rule, ADD a replacement with "supersedes": "", or ADD a compatible narrower rule with "coexists": ["", ...]. For example, a rule that analyses keep integer `*_cents` columns can coexist with a rule that staging models convert `*_cents` columns to dollars. EDIT also needs "coexists" for every other overlapping bullet it remains compatible with. Refer only to existing bullet ids. Two bullets that disagree must never both remain. When feedback contradicts a bullet, that bullet is outdated: supersede or remove it; do not merely ADD a conflicting rule or mark the old one HARMFUL. @@ -35,3 +35,5 @@ Bullet text rules (deterministic checks reject unsafe content and flag verificat Output: a JSON object {"deltas": [...]}. Each delta is {"op": "ADD"|"EDIT"|"REMOVE"|"HELPFUL"|"HARMFUL", "id": "", "text": "", "reason": ""}. Optional fields: "supersedes": "" for ADD, and "coexists": ["", ...] for ADD or EDIT. Omit these fields when they do not apply. + +r = sessions where this rule was shown and the user still corrected the same topic. If the feedback shows the wording is unclear or incomplete, EDIT it to be clearer or more specific. If the agent simply did not follow a correct rule, keep it. Narrow a rule only when the feedback shows it is too broad; never drop the behavior it requires. diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts new file mode 100644 index 0000000000..1efe985750 --- /dev/null +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -0,0 +1,161 @@ +// altimate_change - new file +import fs from "node:fs/promises" +import path from "node:path" +import { Log } from "@/util/log" +import { anchors } from "./anchors" +import { lessonTextHash, readShownSnapshots, type ShownSnapshot } from "./delivery" +import { parse, type Lesson } from "./lesson" +import { validateName } from "./playbook" +import { readSignalsSnapshot, type Signal } from "./signals" +import * as SafeFS from "./safe-fs" +import * as Store from "./store" + +const log = Log.create({ service: "learn.recurrence" }) +const STOP = new Set("the and for with that this from into when then than are was were have has had not use using should must always never only all any each your you our its their them they these those also but can will would could does do don't a an to of in on at as is be it or by if".split(" ")) +const identifiers = (text: string) => new Set(text.toLowerCase().match(/[a-z_][a-z0-9_]*/g) ?? []) +const tokens = (text: string) => new Set([...identifiers(text)].filter((word) => word.length >= 3 && !STOP.has(word))) +const key = (name: string, id: string, textHash: string) => JSON.stringify([name, id, textHash]) +export type NamedLesson = { name: string; lesson: Pick } +export type LessonRecurrence = { + name: string + id: string + textHash: string + sessions_shown: number + recurred: number + during: number + unknown: number + ambiguous: number + earlier_version: number +} + +export function attribute(signals: readonly Signal[], snapshots: readonly ShownSnapshot[], lessons: readonly NamedLesson[]) { + const prepared = lessons.map(({ name, lesson }) => ({ + name, id: lesson.id, textHash: lessonTextHash(lesson.text), + anchors: [...anchors(lesson.text)].filter((word) => !word.startsWith("_") && !word.endsWith("_")), + tokens: tokens(lesson.text), + })) + const sessions = new Map(snapshots.map((snapshot) => [snapshot.session, snapshot.shown])) + const rows = new Map> + }>() + for (const snapshot of snapshots) { + for (const shown of snapshot.shown) { + const current = prepared.find((lesson) => lesson.name === shown.name && lesson.id === shown.id) + const hash = current?.textHash ?? shown.textHash + const identity = key(shown.name, shown.id, hash) + if (!rows.has(identity)) rows.set(identity, { + row: { name: shown.name, id: shown.id, textHash: hash, sessions_shown: 0, recurred: 0, during: 0, unknown: 0, ambiguous: 0, earlier_version: 0 }, + counts: { sessions_shown: new Set(), recurred: new Set(), during: new Set(), unknown: new Set(), ambiguous: new Set(), earlier_version: new Set() }, + }) + const counts = rows.get(identity)!.counts + counts.sessions_shown.add(snapshot.session) + } + } + const unmatched = new Map; signals: Set }>() + for (const signal of signals) { + if (signal.kind !== "user_correction" || !sessions.has(signal.sessionID)) continue + const words = identifiers(signal.text) + const content = tokens(signal.text) + const matches = prepared.filter((lesson) => { + const overlap = [...lesson.tokens].filter((word) => content.has(word)) + return lesson.anchors.some((anchor) => words.has(anchor) && overlap.some((word) => word !== anchor)) || + (overlap.length >= 3 && overlap.length / lesson.tokens.size >= 0.6) + }) + if (!matches.length) { + const topics = new Set([...words].filter((word) => word.includes("_"))) + for (const anchor of anchors(signal.text)) topics.add(anchor) + for (const topic of topics) { + if (!unmatched.has(topic)) unmatched.set(topic, { sessions: new Set(), signals: new Set() }) + unmatched.get(topic)!.sessions.add(signal.sessionID) + unmatched.get(topic)!.signals.add(signal.id) + } + continue + } + // Only lessons this session saw in their current text before the correction compete; a lesson never + // shown, or first shown afterwards, cannot explain it. Unparseable times stay in and resolve to unknown. + // Persistence can lag the message; lessons selected for the correction itself must not count as earlier. + const correction = Date.parse(signal.correctedAt ?? signal.at) + const exposed = matches.flatMap((lesson) => { + const counts = rows.get(key(lesson.name, lesson.id, lesson.textHash))?.counts + const shown = sessions.get(signal.sessionID)!.filter((entry) => entry.name === lesson.name && entry.id === lesson.id && + !(Number.isFinite(correction) && Date.parse(entry.at) >= correction)) + const current = shown.filter((entry) => entry.textHash === lesson.textHash) + if (counts && !current.length && shown.length) counts.earlier_version.add(signal.sessionID) + return counts && current.length ? [{ counts, shown: current }] : [] + }) + for (const { counts, shown } of exposed) { + if (exposed.length > 1) { + counts.ambiguous.add(signal.sessionID) + continue + } + for (const entry of shown) { + const at = Date.parse(entry.at) + const start = Date.parse(signal.turnStartedAt ?? "") + if (!Number.isFinite(at) || !Number.isFinite(start) || !Number.isFinite(correction) || start >= correction) + counts.unknown.add(signal.sessionID) + else if (at < start) counts.recurred.add(signal.sessionID) + else counts.during.add(signal.sessionID) + } + } + } + return { + lessons: [...rows.values()].map(({ row, counts }) => { + for (const field of Object.keys(counts) as (keyof typeof counts)[]) row[field] = counts[field].size + return row + }).sort((a, b) => a.name.localeCompare(b.name) || a.id.localeCompare(b.id) || a.textHash.localeCompare(b.textHash)), + unmatched_topics: [...unmatched].filter(([, value]) => value.sessions.size >= 2) + .map(([identifier, value]) => ({ identifier, sessions: value.sessions.size, signal_ids: [...value.signals].sort() })) + .sort((a, b) => b.sessions - a.sessions || a.identifier.localeCompare(b.identifier)).slice(0, 5), + } +} + +/** With `name`, only that store's lessons, signals and exposures are attributed. */ +export async function analyzeRecurrence(root: string, options: { limit?: number; name?: string } = {}) { + const read = await readShownSnapshots(root, options) + const { coverage } = read + const snapshots = options.name === undefined ? read.snapshots + : read.snapshots.map((snapshot) => ({ ...snapshot, shown: snapshot.shown.filter((entry) => entry.name === options.name) })) + const dir = path.join(root, ".altimate-code", "learn") + await SafeFS.assertSafePath(root, dir) + const entries = await fs.readdir(dir, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] + throw error + }) + const lessons: NamedLesson[] = [] + const signals: Signal[] = [] + const sessions = new Set(snapshots.map((snapshot) => snapshot.session)) + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isDirectory()) continue + try { validateName(entry.name) } catch { continue } + if (options.name !== undefined && entry.name !== options.name) continue + if (sessions.size) signals.push(...(await readSignalsSnapshot(root, entry.name)).filter((signal) => sessions.has(signal.sessionID))) + const file = Store.paths(root, entry.name).approved + try { + await SafeFS.assertSafePath(root, file) + for (const lesson of parse(await fs.readFile(file, "utf8"))) lessons.push({ name: entry.name, lesson }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") log.warn("learn recurrence approved store skipped", { name: entry.name, error }) + } + } + return { ...attribute(signals, snapshots, lessons), coverage } +} + +export function recurrenceHint(result: Pick, "lessons">, name: string, bullet: { id: string; text: string }) { + const evidence = result.lessons.find((lesson) => lesson.name === name && lesson.id === bullet.id && lesson.textHash === lessonTextHash(bullet.text)) + return evidence && evidence.recurred >= 2 ? evidence.recurred : undefined +} + +// Uncorrected lessons are summarized; a store can hold up to 1,000. +const corrected = (lesson: LessonRecurrence) => lesson.recurred + lesson.during + lesson.unknown + lesson.ambiguous + lesson.earlier_version > 0 + +export function formatRecurrence(result: Awaited>) { + return [ + "Recurrence (observed counts need capture on):", + ` Coverage: ${result.coverage.sessions_analyzed} sessions; ${result.coverage.sessions_skipped_malformed} malformed skipped; truncated: ${result.coverage.truncated ? "yes" : "no"}`, + ...result.lessons.filter(corrected).map((lesson) => ` ${lesson.name}/${lesson.id}: ${lesson.sessions_shown} sessions shown; ${lesson.recurred} matching correction after exposure; ${lesson.during} during turn; ${lesson.unknown} unknown order; ${lesson.ambiguous} ambiguous; ${lesson.earlier_version} earlier version`), + ` ${result.lessons.filter((lesson) => !corrected(lesson)).length} other shown lessons: no observed correction`, + " Topics with no matching current lesson:", + ...result.unmatched_topics.map((topic) => ` ${topic.identifier}: ${topic.sessions} sessions (${topic.signal_ids.join(", ")})`), + ].join("\n") +} diff --git a/packages/opencode/src/altimate/learn/reflect.ts b/packages/opencode/src/altimate/learn/reflect.ts index f004e5f364..31ee390cbe 100644 --- a/packages/opencode/src/altimate/learn/reflect.ts +++ b/packages/opencode/src/altimate/learn/reflect.ts @@ -41,7 +41,7 @@ export interface ReflectInput { digest: string feedback: string kind: FeedbackKind - bullets: Bullet[] + bullets: (Bullet & { recurred?: number })[] } /** The model call, injectable so tests need no provider. Returns the raw object. */ @@ -79,7 +79,7 @@ function escapePromptSection(text: string): string { export function buildPrompt(input: ReflectInput): { system: string; prompt: string } { const playbook = input.bullets.length - ? input.bullets.map((b) => `[${b.id}] (h:${b.helpful} x:${b.harmful}${b.coexists?.length ? ` c:${b.coexists.join(",")}` : ""}) ${redactSecrets(b.text)}`).join("\n") + ? input.bullets.map((b) => `[${b.id}] (h:${b.helpful} x:${b.harmful}${(b.recurred ?? 0) >= 2 ? ` r:${b.recurred}` : ""}${b.coexists?.length ? ` c:${b.coexists.join(",")}` : ""}) ${redactSecrets(b.text)}`).join("\n") : "(empty)" const prompt = [ "", diff --git a/packages/opencode/src/altimate/learn/session-reflect.ts b/packages/opencode/src/altimate/learn/session-reflect.ts index 09e262bb76..9edd775d2c 100644 --- a/packages/opencode/src/altimate/learn/session-reflect.ts +++ b/packages/opencode/src/altimate/learn/session-reflect.ts @@ -14,6 +14,7 @@ import * as Signals from "./signals" import { curate, flagSuspiciousFeedback, type CurateResult } from "./curator" import { buildDigest, redactSecrets, sourceFromMessages, type DigestSource } from "./digest" import { FEEDBACK_CAP, feedbackText, reflect, replace, type FeedbackKind, type Generate } from "./reflect" +import { analyzeRecurrence, recurrenceHint } from "./recurrence" import { sharedAnchors } from "./anchors" import { processClaims } from "./claims" import { createUsageTracker, type UsageSummary } from "./usage" @@ -107,8 +108,15 @@ export async function reflectCore(input: ReflectCoreInput): Promise [b.id, b.text])) + const recurrence = await analyzeRecurrence(root, { name }).catch((error) => { + log.warn("learn recurrence unavailable", { error }) + return undefined + }) + const bullets = Playbook.bullets(snapshot).map((bullet) => ({ + ...bullet, ...(recurrence ? { recurred: recurrenceHint(recurrence, name, bullet) } : {}), + })) const deltas = await reflect( - { digest, feedback: input.feedback, kind: input.kind, bullets: Playbook.bullets(snapshot) }, + { digest, feedback: input.feedback, kind: input.kind, bullets }, generate, ).catch((e) => { throw new Error(`Model call failed (${input.modelLabel ?? "the default model"}): ${errText(e)}`) diff --git a/packages/opencode/src/altimate/learn/signals.ts b/packages/opencode/src/altimate/learn/signals.ts index 004923ebcb..1d8d4ac233 100644 --- a/packages/opencode/src/altimate/learn/signals.ts +++ b/packages/opencode/src/altimate/learn/signals.ts @@ -36,11 +36,14 @@ export interface Signal { text: string reason: string at: string + turnStartedAt?: string + /** Creation time of the correcting user message; lessons selected for it come after this. */ + correctedAt?: string status: "open" | "consumed" consumedBy?: string } -export type NewSignal = Pick +export type NewSignal = Pick export function signalsFile(root: string, name = DEFAULT_NAME): string { validateName(name) @@ -92,6 +95,8 @@ function parse(raw: string): { signals: Signal[]; malformed: boolean } { SIGNAL_KINDS.includes(s.kind) && typeof s.reason === "string" && typeof s.at === "string" && (s.status === "open" || s.status === "consumed") && (s.messageID === undefined || typeof s.messageID === "string") && + (s.turnStartedAt === undefined || (typeof s.turnStartedAt === "string" && Number.isFinite(Date.parse(s.turnStartedAt)))) && + (s.correctedAt === undefined || (typeof s.correctedAt === "string" && Number.isFinite(Date.parse(s.correctedAt)))) && (s.partID === undefined || typeof s.partID === "string") && (s.source === undefined || s.source === "bootstrap" || s.source === "import-reviews") && (s.provenance === undefined || typeof s.provenance === "string") && @@ -228,6 +233,8 @@ export function appendSignals(root: string, inputs: readonly NewSignal[], name = sessionID: input.sessionID, messageID: input.messageID, partID: input.partID, + turnStartedAt: input.turnStartedAt, + correctedAt: input.correctedAt, source: input.source, provenance: input.provenance === undefined ? undefined : redactProvenance(input.provenance), resolved: input.resolved, diff --git a/packages/opencode/src/cli/cmd/learn.ts b/packages/opencode/src/cli/cmd/learn.ts index 6ea9c74cd2..8fbe0621dc 100644 --- a/packages/opencode/src/cli/cmd/learn.ts +++ b/packages/opencode/src/cli/cmd/learn.ts @@ -27,6 +27,7 @@ import * as Store from "../../altimate/learn/store" import { MAX_TEXT, summarize, describeApplied, describeRejected } from "../../altimate/learn/curator" import { sourceFromTrajectory, redactSecrets, type DigestSource } from "../../altimate/learn/digest" import { DEFAULT_TIMEOUT_MS, FEEDBACK_KINDS, providerGenerate, type FeedbackKind, type Generate } from "../../altimate/learn/reflect" +import { analyzeRecurrence, formatRecurrence } from "../../altimate/learn/recurrence" import * as Signals from "../../altimate/learn/signals" import { learnMaxStored, learnModel } from "../../altimate/learn/auto" import { autoReflectEnabled, captureEnabled } from "../../altimate/learn/capture" @@ -387,7 +388,11 @@ const StatusCommand = effectCmd({ limits: { ...resolveLimits(learn), max_stored: learnMaxStored(learn?.max_stored), ...resolveRecoveryLimits(learn), ...resolveAutoPromoteLimits(learn) }, // altimate_change end } - }) + }).then(async (base) => ({ + ...base, + // Read-only and advisory: runs after the lock is released; its failure must not hide the rest of the status. + ...await analyzeRecurrence(root, { name }).then((recurrence) => ({ recurrence }), (error) => ({ recurrence_error: error instanceof Error ? error.message : String(error) })), + })) if (args.json) return out(JSON.stringify(status, null, 2)) out(`Learning enabled: ${status.enabled ? "yes" : "no"}`) // altimate_change start — surface the learning kill switch in human-readable status @@ -408,6 +413,7 @@ const StatusCommand = effectCmd({ out(`Lessons: ${status.approved} approved, ${status.candidate} candidate, ${status.retired} retired`) out(`Pinned lessons: ${status.pinned.length}${status.pinned.length ? ` (${status.pinned.join(", ")})` : ""}`) out(`Open signals: ${status.open_signals}`) + out("recurrence" in status ? formatRecurrence(status.recurrence) : `Recurrence unavailable: ${status.recurrence_error}`) out(`Pending recoveries: ${status.pending_recoveries} session(s); ${status.backoff_sessions} in backoff`) out(`Pending replacements: ${status.pending_replacements}`) out(status.last_reflection diff --git a/packages/opencode/test/altimate/learn/capture.test.ts b/packages/opencode/test/altimate/learn/capture.test.ts index a0f5a73d71..3f17381642 100644 --- a/packages/opencode/test/altimate/learn/capture.test.ts +++ b/packages/opencode/test/altimate/learn/capture.test.ts @@ -3,7 +3,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" -import { Capture, RETRY_THRESHOLD, ToolRetryTracker, type PartLike } from "../../../src/altimate/learn/capture" +import { correctedTurnStart, turnMessages, Capture, RETRY_THRESHOLD, ToolRetryTracker, type PartLike } from "../../../src/altimate/learn/capture" import * as Signals from "../../../src/altimate/learn/signals" const call = (id: string, tool: string, status: "error" | "completed" | "running", error = "boom", messageID = "m1") => ({ @@ -334,3 +334,116 @@ describe("Capture: tool retries", () => { expect(await Signals.readSignals(root)).toHaveLength(1) }) }) + +describe("correction turn start", () => { + const history = [ + { id: "u1", sessionID: "s", role: "user", time: { created: 1000 } }, + { id: "tool-step", sessionID: "s", role: "assistant", time: { created: 2000, completed: 2500 } }, + { id: "final", sessionID: "s", role: "assistant", time: { created: 4000, completed: 4500 } }, + { id: "correction", sessionID: "s", role: "user", time: { created: 5000 } }, + ] + test("uses the first assistant after the previous user, not the final step", () => { + expect(correctedTurnStart(history, "correction")).toBe(2000) + expect(correctedTurnStart(history, "missing")).toBeUndefined() + expect(correctedTurnStart(history, "u1")).toBeUndefined() + expect(correctedTurnStart([history[0], history[3]], "correction")).toBeUndefined() + expect(correctedTurnStart(history.slice(1), "correction")).toBeUndefined() + }) + test("a synthetic continuation is not the corrected request's boundary", () => { + const continued = [ + history[0], history[1], + { id: "continue", sessionID: "s", role: "user", synthetic: true, time: { created: 3000 } }, + history[2], history[3], + ] + expect(correctedTurnStart(continued, "correction")).toBe(2000) + expect(correctedTurnStart(continued.map((m) => ({ ...m, synthetic: false })), "correction")).toBe(4000) + }) + test("history parts mark synthetic continuations and compaction markers", () => { + const info = (id: string, role: string, created: number) => ({ id, sessionID: "s", role, time: { created, completed: created + 1 } }) + const mapped = turnMessages([ + { info: info("u1", "user", 1000), parts: [{ type: "text" }, { type: "file" }] }, + { info: info("a1", "assistant", 2000), parts: [{ type: "text" }] }, + { info: info("continue", "user", 3000), parts: [{ type: "text", synthetic: true }] }, + { info: info("marker", "user", 3500), parts: [{ type: "compaction" }] }, + { info: info("files", "user", 3600), parts: [{ type: "file" }] }, + ]) + expect(mapped.map(({ synthetic, compaction }) => [synthetic, compaction])).toEqual([ + [false, false], [false, false], [true, false], [false, true], [false, false], + ]) + const [attachment, replay] = turnMessages([ + { info: info("attach", "user", 4000), parts: [{ type: "text", synthetic: true }, { type: "file" }] }, + { info: info("replay", "user", 4100), parts: [{ type: "text", metadata: { learnOriginalMessage: "u1" } }] }, + ]) + expect([attachment.synthetic, replay.synthetic, replay.replay]).toEqual([false, false, true]) + }) + test("a replayed request after compaction has no known turn start", () => { + const at = (id: string, role: string, created: number, extra = {}) => ({ id, sessionID: "s", role, time: { created, completed: created + 1 }, ...extra }) + const compacted = [ + at("u1", "user", 1000), at("step", "assistant", 2000), + at("marker", "user", 3000, { compaction: true }), at("summary", "assistant", 3100), + at("replay", "user", 3200), at("final", "assistant", 4000), at("correction", "user", 5000), + ] + expect(correctedTurnStart(compacted, "correction")).toBeUndefined() + // A replay marked by its text metadata is unknown even when no marker precedes it. + const marked = compacted.filter((m) => m.id !== "marker").map((m) => m.id === "replay" ? { ...m, replay: true } : m) + expect(correctedTurnStart(marked, "correction")).toBeUndefined() + // A manual compaction without a replay leaves the original request as the boundary. + expect(correctedTurnStart(compacted.filter((m) => m.id !== "replay"), "correction")).toBe(2000) + }) + test("a failed record is retried by a later update", async () => { + let attempts = 0 + const capture = new Capture({ + hasPriorAssistant: async () => true, + turnStart: async (_, id) => correctedTurnStart(history, id), + record: async () => { if (attempts++ === 0) throw new Error("lock timeout"); return undefined }, + }) + for (const info of history) capture.onMessage(info) + for (const text of ["No, use UTC", "No, use UTC instead."]) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text }) + await capture.flush() + expect(attempts).toBe(2) + }) + test("repeated updates of one correction look up history once", async () => { + let lookups = 0 + const capture = new Capture({ + hasPriorAssistant: async () => true, + turnStart: async (_, id) => { lookups++; return correctedTurnStart(history, id) }, + record: async () => undefined, + }) + for (const info of history) capture.onMessage(info) + for (const text of ["No, use UTC", "No, use UTC instead."]) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text }) + await capture.flush() + expect(lookups).toBe(1) + }) + test("records the resolved turn start while preserving the eligibility shortcut", async () => { + const recorded: Signals.NewSignal[] = [] + let lookups = 0 + const capture = new Capture({ + hasPriorAssistant: async () => { throw new Error("shortcut should handle eligibility") }, + turnStart: async (_, id) => { lookups++; return correctedTurnStart(history, id) }, + record: async (signal) => { recorded.push(signal); return undefined }, + }) + for (const info of history) capture.onMessage(info) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text: "No, use UTC instead." }) + await capture.flush() + expect(lookups).toBe(1) + expect(recorded[0].turnStartedAt).toBe(new Date(2000).toISOString()) + expect(recorded[0].correctedAt).toBe(new Date(5000).toISOString()) + }) + test("uncertain history omits the field and lookup failure preserves the signal", async () => { + for (const fail of [false, true]) { + const recorded: Signals.NewSignal[] = [] + const capture = new Capture({ + hasPriorAssistant: async () => true, + turnStart: async () => { if (fail) throw new Error("unavailable"); return undefined }, + record: async (signal) => { recorded.push(signal); return undefined }, + }) + capture.onMessage(history[3]) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text: "No, use UTC instead." }) + await capture.flush() + expect(recorded).toHaveLength(1) + expect(recorded[0]).not.toHaveProperty("turnStartedAt") + } + }) +}) diff --git a/packages/opencode/test/altimate/learn/cli.test.ts b/packages/opencode/test/altimate/learn/cli.test.ts index a635acdaf9..59219f06e2 100644 --- a/packages/opencode/test/altimate/learn/cli.test.ts +++ b/packages/opencode/test/altimate/learn/cli.test.ts @@ -168,6 +168,34 @@ describe("learn pin and unpin", () => { }) describe("learn opt-in and status", () => { + test("status includes recurrence coverage even with learning disabled", async () => { + await using dir = await tmpdir({ git: true }) + const result = await runLearn(dir.path, ["status", "--json"], undefined, { ALTIMATE_LEARN: "0" }) + expect(result.code).toBe(0) + expect(JSON.parse(result.stdout).recurrence).toEqual({ + lessons: [], unmatched_topics: [], + coverage: { sessions_analyzed: 0, sessions_skipped_malformed: 0, truncated: false }, + }) + const shown = await runLearn(dir.path, ["status"], undefined, { ALTIMATE_LEARN: "0" }) + expect(shown.stdout).toContain("Recurrence (observed counts need capture on)") + expect(shown.stdout).toContain("no matching current lesson") + }) + + test("status still reports when recurrence analysis fails", async () => { + await using dir = await tmpdir({ git: true }) + await using outside = await tmpdir() + await fs.mkdir(path.join(dir.path, ".altimate-code", "learn"), { recursive: true }) + await fs.symlink(outside.path, path.join(dir.path, ".altimate-code", "learn", ".sessions")) + const result = await runLearn(dir.path, ["status", "--json"]) + expect(result.code).toBe(0) + const status = JSON.parse(result.stdout) + expect(status.recurrence).toBeUndefined() + expect(typeof status.recurrence_error).toBe("string") + expect(status.open_signals).toBe(0) + const shown = await learn(dir.path, "status") + expect(shown.stdout).toContain("Recurrence unavailable:") + }) + test.each(["config", "env"])("status reports the learning kill switch disabled by %s", async (source) => { await using dir = await tmpdir({ git: true }) await fs.writeFile(path.join(dir.path, "opencode.json"), JSON.stringify({ diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts new file mode 100644 index 0000000000..9056fa84e7 --- /dev/null +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -0,0 +1,241 @@ +// altimate_change - new file +import { describe, expect, test } from "bun:test" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { correctedTurnStart } from "../../../src/altimate/learn/capture" +import { attribute, analyzeRecurrence, formatRecurrence, recurrenceHint, type NamedLesson } from "../../../src/altimate/learn/recurrence" +import { lessonTextHash, readShownSnapshots, type ShownSnapshot } from "../../../src/altimate/learn/delivery" +import type { Signal } from "../../../src/altimate/learn/signals" +import * as Store from "../../../src/altimate/learn/store" + +const time = (n: number) => new Date(n * 1000).toISOString() +const lesson: NamedLesson = { name: "team-playbook", lesson: { id: "L-0001", text: "Convert `event_at` timestamps with `to_utc` before storage." } } +const signal = (sessionID = "s1", extra: Partial = {}): Signal => ({ + id: `sig_${sessionID}`, kind: "user_correction", sessionID, text: "No, convert event_at timestamps with to_utc.", + reason: "correction", at: time(5), turnStartedAt: time(2), status: "open", ...extra, +}) +const snapshot = (session = "s1", extra: Partial = {}): ShownSnapshot => ({ + session, shown: [{ name: lesson.name, id: lesson.lesson.id, textHash: lessonTextHash(lesson.lesson.text), tier: "core", at: time(1), ...extra }], +}) +const run = (signals = [signal()], snapshots = [snapshot()], lessons = [lesson]) => attribute(signals, snapshots, lessons) + +describe("recurrence attribution", () => { + test("matching correction after exposure counts, regardless of consumption", () => { + expect(run().lessons[0]).toMatchObject({ sessions_shown: 1, recurred: 1, unknown: 0 }) + expect(run([signal("s1", { status: "consumed", consumedBy: "reflect" })]).lessons[0].recurred).toBe(1) + expect(run([signal("s1", { kind: "review" })]).lessons[0].recurred).toBe(0) + }) + + test("mid-turn file exposure, including equality, is during rather than recurrence", () => { + for (const at of [time(2), time(3)]) { + expect(run([signal()], [snapshot("s1", { at, tier: "file" })]).lessons[0]).toMatchObject({ recurred: 0, during: 1 }) + } + }) + + test("tool step then file lesson then final assistant resolves to during", () => { + const history = [ + { id: "user", sessionID: "s1", role: "user", time: { created: 1000 } }, + { id: "tool-step", sessionID: "s1", role: "assistant", time: { created: 2000, completed: 2500 } }, + { id: "final", sessionID: "s1", role: "assistant", time: { created: 4000, completed: 4500 } }, + { id: "correction", sessionID: "s1", role: "user", time: { created: 5000 } }, + ] + const turnStartedAt = new Date(correctedTurnStart(history, "correction")!).toISOString() + expect(run([signal("s1", { turnStartedAt })], [snapshot("s1", { tier: "file", at: time(3) })]).lessons[0]) + .toMatchObject({ recurred: 0, during: 1 }) + }) + + test("missing, invalid and impossible ordering stays unknown", () => { + for (const turnStartedAt of [undefined, "bad", time(6)]) + expect(run([signal("s1", { turnStartedAt })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 1 }) + // Shown at the correction's own instant cannot have shaped the corrected response. + expect(run([signal()], [snapshot("s1", { at: time(5) })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 0 }) + expect(run([signal()], [snapshot("s1", { at: "bad" })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 1 }) + }) + + test("a new convention or a shared identifier about a different property does not match", () => { + for (const text of ["No, pin dependency versions.", "No, index `event_at`", "No, index `other_event_at`", "No, index `to_utc`"]) { + expect(run([signal("s1", { text })]).lessons[0].recurred).toBe(0) + } + const affix = { ...lesson, lesson: { ...lesson.lesson, text: "Convert `*_at` timestamps with `to_utc`" } } + expect(run([signal("s1", { text: "No, index `event_at`" })], [snapshot()], [affix]).lessons[0].recurred).toBe(0) + expect(run([signal("s1", { text: "No, timestamps `event_at`" })], [snapshot()], [affix]).lessons[0].recurred).toBe(0) + const prefix = { ...lesson, lesson: { ...lesson.lesson, text: "Name `stg_` models after their source" } } + expect(run([signal("s1", { text: "No, rename stg_orders models" })], [snapshot()], [prefix]).lessons[0].recurred).toBe(0) + }) + + test("full anchor needs another content token; backticks on correction are optional", () => { + expect(run([signal("s1", { text: "No, event_at timestamps" })]).lessons[0].recurred).toBe(1) + expect(run([signal("s1", { text: "No, event_at" })]).lessons[0].recurred).toBe(0) + }) + + test("lexical fallback requires three distinct tokens and sixty percent", () => { + const prose = { ...lesson, lesson: { ...lesson.lesson, text: "Preserve original timezone during timestamp conversion" } } + const shown = snapshot("s1", { textHash: lessonTextHash(prose.lesson.text) }) + expect(run([signal("s1", { text: "No, preserve original timezone during conversion" })], [shown], [prose]).lessons[0].recurred).toBe(1) + expect(run([signal("s1", { text: "No, timezone timezone conversion" })], [shown], [prose]).lessons[0].recurred).toBe(0) + expect(run([signal("s1", { text: "No, original timezone conversion" })], [shown], [prose]).lessons[0].recurred).toBe(0) + }) + + test("multiple matching lessons are ambiguous, including same ids in different stores", () => { + const other = { ...lesson, name: "other-store" } + const shown = snapshot() + shown.shown.push({ ...shown.shown[0], name: other.name }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons).toHaveLength(2) + for (const row of result.lessons) expect(row).toMatchObject({ recurred: 0, ambiguous: 1 }) + }) + + test("store-local ids retain independent evidence", () => { + const other = { name: "other-store", lesson: { id: lesson.lesson.id, text: "Index `customer_id` for efficient lookups." } } + const shown = snapshot() + shown.shown.push({ ...shown.shown[0], name: other.name, textHash: lessonTextHash(other.lesson.text) }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons.find((row) => row.name === lesson.name)?.recurred).toBe(1) + expect(result.lessons.find((row) => row.name === other.name)?.recurred).toBe(0) + }) + + test("an earlier version shown with another matching lesson is not ambiguity evidence", () => { + const other = { ...lesson, name: "other-store" } + const shown = snapshot("s1", { textHash: lessonTextHash("Earlier text") }) + shown.shown.push({ ...snapshot().shown[0], name: other.name }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons.find((row) => row.name === lesson.name)).toMatchObject({ ambiguous: 0, earlier_version: 1 }) + expect(result.lessons.find((row) => row.name === other.name)).toMatchObject({ ambiguous: 0, recurred: 1 }) + }) + + test("a matching lesson the session never saw does not hide recurrence", () => { + const added: NamedLesson = { name: "team-playbook", lesson: { id: "L-0003", text: "Always convert `event_at` timestamps with `to_utc` first." } } + const result = run([signal("s1"), signal("s2")], [snapshot("s1"), snapshot("s2")], [lesson, added]) + expect(result.lessons.find((row) => row.id === lesson.lesson.id)).toMatchObject({ recurred: 2, ambiguous: 0 }) + expect(recurrenceHint(result, lesson.name, lesson.lesson)).toBe(2) + }) + + test("an earlier version counts only with a matching correction", () => { + const old = snapshot("s1", { textHash: lessonTextHash("Earlier text") }) + expect(run([], [old]).lessons[0]).toMatchObject({ earlier_version: 0, sessions_shown: 1 }) + const text = formatRecurrence({ ...run([signal()], [old]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + expect(text).toContain("team-playbook/L-0001:") + expect(text).toContain("1 earlier version") + }) + + test("a lesson selected for the correction itself is not an earlier exposure, even if the signal is written later", () => { + // Correction message at t=4, request lesson selected at t=4.5, signal persisted at t=5. + const late = signal("s1", { correctedAt: time(4) }) + expect(run([late], [snapshot("s1", { at: time(4.5) })]).lessons[0]).toMatchObject({ during: 0, unknown: 0, recurred: 0 }) + expect(run([late], [snapshot("s1", { at: time(3) })]).lessons[0]).toMatchObject({ during: 1 }) + expect(run([late]).lessons[0]).toMatchObject({ recurred: 1 }) + }) + + test("a lesson first shown after the correction is ignored and does not compete", () => { + expect(run([signal()], [snapshot("s1", { at: time(6) })]).lessons[0]).toMatchObject({ unknown: 0, during: 0, recurred: 0 }) + const other = { ...lesson, name: "other-store" } + const shown = snapshot() + shown.shown.push({ ...shown.shown[0], name: other.name, at: time(6) }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons.find((row) => row.name === lesson.name)).toMatchObject({ recurred: 1, ambiguous: 0 }) + expect(result.lessons.find((row) => row.name === other.name)).toMatchObject({ recurred: 0, ambiguous: 0, unknown: 0 }) + }) + + test("earlier lesson version cannot supply current evidence", () => { + expect(run([signal()], [snapshot("s1", { textHash: lessonTextHash("Earlier text") })]).lessons[0]) + .toMatchObject({ earlier_version: 1, recurred: 0 }) + }) + + test("sessions count once; hints require two sessions and the bullet's own normalized text", () => { + const result = run([signal(), signal("s1", { id: "second" }), signal("s2")], [snapshot(), snapshot("s2")]) + expect(result.lessons[0].recurred).toBe(2) + expect(recurrenceHint(result, lesson.name, lesson.lesson)).toBe(2) + expect(recurrenceHint(result, lesson.name, { ...lesson.lesson, text: "Edited candidate" })).toBeUndefined() + expect(recurrenceHint(result, "another-store", lesson.lesson)).toBeUndefined() + expect(recurrenceHint(result, lesson.name, { ...lesson.lesson, text: " " + lesson.lesson.text.replaceAll(" ", "\n ") })).toBe(2) + expect(recurrenceHint(run(), lesson.name, lesson.lesson)).toBeUndefined() + expect(run([signal(), signal("s1", { id: "second" })]).lessons[0].recurred).toBe(1) + }) + + test("unmatched topics need distinct analyzed sessions and expose no correction text", () => { + const text = "No, index `customer_id` and `_is_deleted` immediately, private discussion." + const result = run([signal("s1", { text }), signal("s2", { text }), signal("outside", { text })], [snapshot(), snapshot("s2")]) + expect(result.unmatched_topics).toEqual(["_is_deleted", "customer_id"].map((identifier) => ({ identifier, sessions: 2, signal_ids: ["sig_s1", "sig_s2"] }))) + expect(run([signal("s1", { text }), signal("s1", { id: "duplicate", text })]).unmatched_topics).toEqual([]) + expect(JSON.stringify(result)).not.toContain("private discussion") + }) + + test("unmatched topics are capped at five with deterministic ties", () => { + const text = "No, a_id b_id c_id d_id e_id f_id" + expect(run([signal("s1", { text }), signal("s2", { text })], [snapshot(), snapshot("s2")]).unmatched_topics.map((topic) => topic.identifier)) + .toEqual(["a_id", "b_id", "c_id", "d_id", "e_id"]) + }) + + test("a row with only mid-turn evidence does not claim no correction", () => { + const text = formatRecurrence({ ...run([signal()], [snapshot("s1", { at: time(3) })]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + const row = text.split("\n").find((line) => line.includes("team-playbook/L-0001"))! + expect(row).toContain("0 matching correction after exposure; 1 during turn") + expect(row).not.toContain("no observed correction") + }) + + test("human status lists corrected lessons and summarizes the rest", () => { + const quiet: NamedLesson = { name: "team-playbook", lesson: { id: "L-0002", text: "Name staging models `stg___`." } } + const shown = snapshot() + shown.shown.push({ name: quiet.name, id: quiet.lesson.id, textHash: lessonTextHash(quiet.lesson.text), tier: "core", at: time(1) }) + const text = formatRecurrence({ ...run([signal()], [shown], [lesson, quiet]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + expect(text).toContain("team-playbook/L-0001: 1 sessions shown; 1 matching correction after exposure") + expect(text).not.toContain("L-0002") + expect(text).toContain("1 other shown lessons: no observed correction") + }) + + test("human status reports coverage, uncertainty and no matching current lesson", () => { + const unmatched = (session: string) => signal(session, { id: `sig_u_${session}`, text: "No, filter out _is_deleted rows" }) + const text = formatRecurrence({ ...run([unmatched("s1"), unmatched("s2")], [snapshot("s1"), snapshot("s2")]), coverage: { sessions_analyzed: 2, sessions_skipped_malformed: 0, truncated: false } }) + expect(text).toContain(" _is_deleted: 2 sessions (sig_u_s1, sig_u_s2)") + expect(text).toContain("Recurrence") + expect(text).toContain("no observed correction") + expect(text).toContain("counts need capture on") + expect(text).toContain("no matching current lesson") + expect(text).not.toContain("followed") + }) +}) + +test("snapshot analysis is bounded, skips malformed files and never repairs any input", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "learn-recurrence-")) + try { + const dir = path.join(root, ".altimate-code", "learn", ".sessions") + await fs.mkdir(dir, { recursive: true }) + const state = (session: string) => ({ version: 1, session, firstMessage: "m1", section: "", shown: [{ + name: lesson.name, lesson: { ...lesson.lesson, tags: [], scope: "project", helpful: 0, harmful: 0, applied: 0, created: time(0), updated: time(0) }, + tier: "file", at: time(1), queryHash: "hash", + }], requests: [], compactions: [], counted: [] }) + const old = path.join(dir, Store.sha256("old") + ".json") + const good = path.join(dir, Store.sha256("s1") + ".json") + const bad = path.join(dir, "bad.json") + await fs.writeFile(old, JSON.stringify(state("old"))) + await fs.writeFile(good, JSON.stringify(state("s1"))) + await fs.writeFile(bad, "broken") + await fs.utimes(old, 1, 1) + await fs.utimes(good, 2, 2) + await fs.utimes(bad, 3, 3) + const p = Store.paths(root, lesson.name) + await fs.mkdir(p.learnDir, { recursive: true }) + await fs.writeFile(p.approved, JSON.stringify([state("s1").shown[0].lesson])) + const signals = path.join(p.learnDir, "signals.jsonl") + const raw = JSON.stringify(signal()) + "\nmalformed\n" + JSON.stringify(signal("old")) + "\n" + await fs.writeFile(signals, raw) + const result = await analyzeRecurrence(root, { limit: 2 }) + expect(result.coverage).toEqual({ sessions_analyzed: 1, sessions_skipped_malformed: 1, truncated: true }) + expect(result.lessons[0]).toMatchObject({ sessions_shown: 1, recurred: 1 }) + expect(await fs.readFile(bad, "utf8")).toBe("broken") + expect(await fs.readFile(signals, "utf8")).toBe(raw) + expect((await fs.readdir(p.learnDir)).sort()).toEqual([path.basename(p.approved), "signals.jsonl"].sort()) + expect((await readShownSnapshots(root, { limit: 0 })).coverage).toMatchObject({ sessions_analyzed: 0, truncated: true }) + // A copied snapshot under another session's file name is not that session's evidence. + await fs.copyFile(good, path.join(dir, Store.sha256("copy") + ".json")) + expect((await readShownSnapshots(root)).coverage).toMatchObject({ sessions_analyzed: 2, sessions_skipped_malformed: 2 }) + // Another store's lessons and signals stay out of a named analysis. + expect((await analyzeRecurrence(root, { name: "other-store" })).lessons).toEqual([]) + // Unbounded, both valid sessions ("old" and "s1") carry a matching correction after exposure. + expect((await analyzeRecurrence(root, { name: lesson.name })).lessons[0]).toMatchObject({ recurred: 2 }) + // An unsafe snapshot path fails closed instead of counting as malformed. + await fs.symlink(good, path.join(dir, Store.sha256("link") + ".json")) + await expect(readShownSnapshots(root)).rejects.toThrow() + } finally { await fs.rm(root, { recursive: true, force: true }) } +}) diff --git a/packages/opencode/test/altimate/learn/reflect.test.ts b/packages/opencode/test/altimate/learn/reflect.test.ts index 2fd68cab8a..fd4bb1f575 100644 --- a/packages/opencode/test/altimate/learn/reflect.test.ts +++ b/packages/opencode/test/altimate/learn/reflect.test.ts @@ -11,6 +11,21 @@ import { accountUsage, type UsageSummary } from "../../../src/altimate/learn/usa const bullets = [{ id: "L-0001", text: "Staging models are prefixed stg_.", helpful: 2, harmful: 0 }] describe("buildPrompt", () => { + test("recurrence hints appear only at two distinct sessions", () => { + for (const recurred of [undefined, 0, 1, 2, 3]) { + const { prompt, system } = buildPrompt({ digest: "d", feedback: "f", kind: "user", bullets: [{ ...bullets[0], recurred }] }) + expect(prompt.includes(` r:${recurred}`)).toBe((recurred ?? 0) >= 2) + expect(system).toContain("If the agent simply did not follow a correct rule, keep it.") + } + }) + + test("a missed but correct bullet earns no helpful vote", () => { + const { system } = buildPrompt({ digest: "d", feedback: "f", kind: "user", bullets }) + expect(system).toContain("HELPFUL only when the DIGEST shows the agent followed it") + expect(system).toContain("it is neither HELPFUL nor HARMFUL") + expect(system).not.toContain("(the last is HELPFUL)") + }) + test("requires backticks around generated identifiers and naming patterns", () => { const { system } = buildPrompt({ digest: "d", feedback: "f", kind: "ci", bullets }) expect(system).toContain("Wrap every code identifier and naming pattern (including prefixes and suffixes) in backticks.") diff --git a/packages/opencode/test/altimate/learn/session-reflect.test.ts b/packages/opencode/test/altimate/learn/session-reflect.test.ts index c36722b142..cc1eb06988 100644 --- a/packages/opencode/test/altimate/learn/session-reflect.test.ts +++ b/packages/opencode/test/altimate/learn/session-reflect.test.ts @@ -1,10 +1,12 @@ // altimate_change - new file // // `reflect --session` from captured signals, with the model call and the session source stubbed. +import { analyzeRecurrence } from "../../../src/altimate/learn/recurrence" import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" +import { Delivery } from "../../../src/altimate/learn/delivery" import * as Signals from "../../../src/altimate/learn/signals" import * as Store from "../../../src/altimate/learn/store" import * as Playbook from "../../../src/altimate/learn/playbook" @@ -788,3 +790,48 @@ describe("opt-in switches", () => { expect(() => learnMaxStored(75, { ALTIMATE_LEARN_MAX_STORED: value })).toThrow("positive integer") }) }) + +describe("reflection recurrence evidence", () => { + test("adds repeated evidence only to the same store, id and current text", async () => { + const bullet = { id: "L-0001", text: "Convert `event_at` timestamps with `to_utc`.", helpful: 0, harmful: 0 } + await Store.saveCandidate(root, NAME, Playbook.withBullets(Playbook.create({ name: NAME }), [bullet])) + await Store.promote(root, NAME) + const delivery = new Delivery(root) + for (const sessionID of ["ses_one", "ses_two"]) { + await delivery.prepare(sessionID, "m1", "Convert event_at timestamps to_utc") + await Signals.appendSignal(root, { + kind: "user_correction", sessionID, text: "No, convert event_at timestamps with to_utc.", reason: "correction", + turnStartedAt: new Date(Date.now() + 1000).toISOString(), + }) + } + // Signal capture is later than the corrected turn; make that ordering deterministic. + const file = Signals.signalsFile(root) + const signals = await Signals.readSignalsSnapshot(root) + await fs.writeFile(file, signals.map((signal) => JSON.stringify({ ...signal, at: new Date(Date.now() + 2000).toISOString() }) + "\n").join("")) + const prompts: string[] = [] + const reflect = () => reflectCore({ + root, name: NAME, source: { prompts: [], calls: [] }, feedback: "No, convert timestamps.", kind: "user", origin: "test", + generate: async ({ prompt }) => { prompts.push(prompt); return { deltas: [] } }, + }) + await reflect() + expect(prompts[0]).toContain("[L-0001] (h:0 x:0 r:2)") + await Store.saveCandidate(root, NAME, Playbook.withBullets(Playbook.create({ name: NAME }), [{ ...bullet, text: "Convert `event_at` timestamps before storage." }])) + await reflect() + expect(prompts[1]).not.toContain(" r:") + }) + + test("analyzer failure does not prevent reflection", async () => { + const dir = path.join(root, ".altimate-code", "learn", ".sessions") + await fs.mkdir(dir, { recursive: true }) + const target = path.join(root, "snapshots") + await fs.rename(dir, target) + await fs.symlink(target, dir) + await expect(analyzeRecurrence(root, { name: NAME })).rejects.toThrow() + let called = false + await reflectCore({ + root, name: NAME, source: { prompts: [], calls: [] }, feedback: "No, use UTC.", kind: "user", origin: "test", + generate: async ({ prompt }) => { called = true; expect(prompt).not.toContain(" r:"); return { deltas: [] } }, + }) + expect(called).toBe(true) + }) +}) diff --git a/packages/opencode/test/altimate/learn/signals.test.ts b/packages/opencode/test/altimate/learn/signals.test.ts index ee02f85b07..8667a52359 100644 --- a/packages/opencode/test/altimate/learn/signals.test.ts +++ b/packages/opencode/test/altimate/learn/signals.test.ts @@ -14,6 +14,21 @@ afterEach(() => fs.rm(root, { recursive: true, force: true })) const base = { kind: "user_correction", sessionID: "ses_1", reason: "r" } as const describe("signal store", () => { + test("old signals parse and correction turn starts round-trip read-only", async () => { + await Signals.appendSignal(root, { ...base, text: "No, keep original", messageID: "old" }) + const turnStartedAt = "2026-10-18T10:00:00.000Z" + const correctedAt = "2026-10-18T10:00:05.000Z" + await Signals.appendSignal(root, { ...base, text: "No, keep timestamps", messageID: "new", turnStartedAt, correctedAt }) + const records = await Signals.readSignalsSnapshot(root) + expect(records[0].turnStartedAt).toBeUndefined() + expect(records[0].correctedAt).toBeUndefined() + expect(records[1]).toMatchObject({ turnStartedAt, correctedAt }) + const file = Signals.signalsFile(root) + await fs.appendFile(file, JSON.stringify({ ...records[1], id: "invalid", turnStartedAt: 42 }) + "\n") + await fs.appendFile(file, JSON.stringify({ ...records[1], id: "invalid2", correctedAt: "never" }) + "\n") + expect(await Signals.readSignalsSnapshot(root)).toHaveLength(2) + }) + test("records the documented fields under the default named lesson store", async () => { const s = await Signals.appendSignal(root, { ...base, messageID: "msg_1", text: "no, use ref()" }) expect(s).toMatchObject({ kind: "user_correction", sessionID: "ses_1", messageID: "msg_1", text: "no, use ref()", status: "open" })