From 43f6b8b9284b70e0da027f8fb8c758c5c317aa5e Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 14:24:55 -0700 Subject: [PATCH 1/9] feat(learn): attribute lesson recurrence from session corrections Learning delivered lessons but never checked whether a lesson that was shown still drew a correction on the same topic. Add a deterministic, read-only attribution pass (no model calls): - `learn/recurrence.ts`: match `user_correction` signals to approved lessons (full anchor plus another content token, or >=3 tokens at 60% coverage; affix anchors never match alone; several matches are ambiguous) and classify exposure as before the corrected turn, during it, or unknown order. Evidence is keyed by store, lesson id and normalized text hash, so edited lessons and same ids in other stores never borrow it. Bounded to the newest 200 session snapshots and reports coverage. - Capture records `turnStartedAt` on corrections: the first assistant message after the previous user message, resolved from session history, omitted when uncertain. The capture eligibility shortcut is unchanged. - `learn status` (text and `--json`) reports per-lesson recurrence and topics corrected in >=2 sessions with no matching current lesson. Text output lists only lessons with correction evidence. An analysis failure reports `recurrence_error` instead of failing status. - Reflection marks a bullet `r:N` when its current text was shown before a matching correction in N>=2 sessions; the prompt asks for a clearer EDIT, never a weaker one. Candidates still go through the existing approval gates. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/learn/capture.ts | 18 ++ .../opencode/src/altimate/learn/delivery.ts | 36 ++++ .../opencode/src/altimate/learn/prompt.txt | 2 + .../opencode/src/altimate/learn/recurrence.ts | 150 +++++++++++++++ .../opencode/src/altimate/learn/reflect.ts | 4 +- .../src/altimate/learn/session-reflect.ts | 10 +- .../opencode/src/altimate/learn/signals.ts | 5 +- packages/opencode/src/cli/cmd/learn.ts | 4 + .../test/altimate/learn/capture.test.ts | 47 ++++- .../opencode/test/altimate/learn/cli.test.ts | 28 +++ .../test/altimate/learn/recurrence.test.ts | 178 ++++++++++++++++++ .../test/altimate/learn/reflect.test.ts | 8 + .../altimate/learn/session-reflect.test.ts | 45 +++++ .../test/altimate/learn/signals.test.ts | 12 ++ 14 files changed, 542 insertions(+), 5 deletions(-) create mode 100644 packages/opencode/src/altimate/learn/recurrence.ts create mode 100644 packages/opencode/test/altimate/learn/recurrence.test.ts diff --git a/packages/opencode/src/altimate/learn/capture.ts b/packages/opencode/src/altimate/learn/capture.ts index 31e7d76d99..9ac51dea0f 100644 --- a/packages/opencode/src/altimate/learn/capture.ts +++ b/packages/opencode/src/altimate/learn/capture.ts @@ -92,6 +92,14 @@ export interface MessageLike { time?: { created?: number; completed?: number } } +export function correctedTurnStart(messages: readonly MessageLike[], messageID: string): number | undefined { + const end = messages.findIndex((message) => message.id === messageID && message.role === "user") + if (end < 0) return + const previous = messages.slice(0, end).findLastIndex((message) => message.role === "user") + if (previous < 0) return + return messages.slice(previous + 1, end).find((message) => message.role === "assistant")?.time?.created +} + export interface PartLike { id: string sessionID: string @@ -107,6 +115,7 @@ export interface PartLike { export interface CaptureDeps { /** True when the session has a completed assistant message before the user message being classified. */ hasPriorAssistant: (sessionID: string, beforeMessageID: string) => Promise + turnStart?: (sessionID: string, messageID: string) => Promise record: (signal: NewSignal) => Promise } @@ -173,8 +182,13 @@ export class Capture { const prior = this.sessionsWithAssistant.get(part.sessionID) const known = prior !== undefined && message.time?.created !== undefined && prior < message.time.created if (!known && !(await this.deps.hasPriorAssistant(part.sessionID, part.messageID))) return + const start = await this.deps.turnStart?.(part.sessionID, part.messageID).catch((error) => { + log.warn("learn turn start unavailable", { error }) + return undefined + }) await this.deps.record({ kind: "user_correction", + ...(start !== undefined && Number.isFinite(start) ? { turnStartedAt: new Date(start).toISOString() } : {}), sessionID: part.sessionID, messageID: part.messageID, text: part.text, @@ -239,6 +253,10 @@ export async function startCapture(ctx: { directory: string; worktree: string }) const { SessionID } = await import("../../session/schema") const capture = new Capture({ record: (signal) => appendSignal(root, signal), + turnStart: async (sessionID, messageID) => { + const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) + return correctedTurnStart(messages.map((message) => message.info), messageID) + }, hasPriorAssistant: async (sessionID, beforeMessageID) => { const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) const before = messages.findIndex((message) => message.info.id === beforeMessageID) diff --git a/packages/opencode/src/altimate/learn/delivery.ts b/packages/opencode/src/altimate/learn/delivery.ts index 6778cd2aff..7bdcf9eeae 100644 --- a/packages/opencode/src/altimate/learn/delivery.ts +++ b/packages/opencode/src/altimate/learn/delivery.ts @@ -79,6 +79,42 @@ async function read(root: string, file: string) { } } +export const lessonTextHash = (text: string) => Store.sha256(singleLine(text)) +export type ShownSnapshot = { + session: string + shown: { name: string; id: string; textHash: string; tier: Shown["tier"]; at: string }[] +} + +export async function readShownSnapshots(root: string, options: { limit?: number } = {}) { + const dir = path.join(root, ".altimate-code", "learn", ".sessions") + const coverage = { sessions_analyzed: 0, sessions_skipped_malformed: 0, truncated: false } + const snapshots: ShownSnapshot[] = [] + await SafeFS.assertSafePath(root, dir) + const entries = await fs.readdir(dir).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] + throw error + }) + const files = await Promise.all(entries.filter((name) => name.endsWith(".json")).map(async (name) => { + const file = path.join(dir, name) + return { file, mtime: await fs.lstat(file).then((stat) => stat.mtimeMs).catch(() => 0) } + })) + const limit = Math.max(0, Math.floor(options.limit ?? 200)) + coverage.truncated = files.length > limit + for (const { file } of files.sort((a, b) => b.mtime - a.mtime || a.file.localeCompare(b.file)).slice(0, limit)) { + try { + const state = State.parse(JSON.parse((await read(root, file)) ?? "")) + snapshots.push({ session: state.session, shown: state.shown.map(({ name, lesson, tier, at }) => ({ + name, id: lesson.id, textHash: lessonTextHash(lesson.text), tier, at, + })) }) + } catch (error) { + coverage.sessions_skipped_malformed++ + log.warn("learn session snapshot skipped", { file, error }) + } + } + coverage.sessions_analyzed = snapshots.length + return { snapshots, coverage } +} + /** Persisted text snapshots keep the prefix stable even after edits, ranking changes or a restart. */ export class Delivery { private resolved?: Limits diff --git a/packages/opencode/src/altimate/learn/prompt.txt b/packages/opencode/src/altimate/learn/prompt.txt index 465c7700e3..b7e6027ed5 100644 --- a/packages/opencode/src/altimate/learn/prompt.txt +++ b/packages/opencode/src/altimate/learn/prompt.txt @@ -35,3 +35,5 @@ Bullet text rules (deterministic checks reject unsafe content and flag verificat Output: a JSON object {"deltas": [...]}. Each delta is {"op": "ADD"|"EDIT"|"REMOVE"|"HELPFUL"|"HARMFUL", "id": "", "text": "", "reason": ""}. Optional fields: "supersedes": "" for ADD, and "coexists": ["", ...] for ADD or EDIT. Omit these fields when they do not apply. + +r = sessions where this rule was shown and the user still corrected the same topic; consider making it more specific or clearer (EDIT), never weaker diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts new file mode 100644 index 0000000000..2708edb9ed --- /dev/null +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -0,0 +1,150 @@ +// altimate_change - new file +import fs from "node:fs/promises" +import path from "node:path" +import { Log } from "@/util/log" +import { anchors } from "./anchors" +import { lessonTextHash, readShownSnapshots, type ShownSnapshot } from "./delivery" +import { parse, type Lesson } from "./lesson" +import { validateName } from "./playbook" +import { readSignalsSnapshot, type Signal } from "./signals" +import * as SafeFS from "./safe-fs" +import * as Store from "./store" + +const log = Log.create({ service: "learn.recurrence" }) +const STOP = new Set("the and for with that this from into when then than are was were have has had not use using should must always never only all any each your you our its their them they these those also but can will would could does do don't a an to of in on at as is be it or by if".split(" ")) +const identifiers = (text: string) => new Set(text.toLowerCase().match(/[a-z_][a-z0-9_]*/g) ?? []) +const tokens = (text: string) => new Set([...identifiers(text)].filter((word) => word.length >= 3 && !STOP.has(word))) +const key = (name: string, id: string, textHash: string) => JSON.stringify([name, id, textHash]) +export type NamedLesson = { name: string; lesson: Pick } +export type LessonRecurrence = { + name: string + id: string + textHash: string + sessions_shown: number + recurred: number + during: number + unknown: number + ambiguous: number + earlier_version: number +} + +export function attribute(signals: readonly Signal[], snapshots: readonly ShownSnapshot[], lessons: readonly NamedLesson[]) { + const prepared = lessons.map(({ name, lesson }) => ({ + name, id: lesson.id, textHash: lessonTextHash(lesson.text), + anchors: [...anchors(lesson.text)].filter((word) => !word.startsWith("_") && !word.endsWith("_")), + tokens: tokens(lesson.text), + })) + const sessions = new Map(snapshots.map((snapshot) => [snapshot.session, snapshot.shown])) + const rows = new Map> + }>() + for (const snapshot of snapshots) { + for (const shown of snapshot.shown) { + const current = prepared.find((lesson) => lesson.name === shown.name && lesson.id === shown.id) + const hash = current?.textHash ?? shown.textHash + const identity = key(shown.name, shown.id, hash) + if (!rows.has(identity)) rows.set(identity, { + row: { name: shown.name, id: shown.id, textHash: hash, sessions_shown: 0, recurred: 0, during: 0, unknown: 0, ambiguous: 0, earlier_version: 0 }, + counts: { sessions_shown: new Set(), recurred: new Set(), during: new Set(), unknown: new Set(), ambiguous: new Set(), earlier_version: new Set() }, + }) + const counts = rows.get(identity)!.counts + counts.sessions_shown.add(snapshot.session) + if (current && shown.textHash !== current.textHash) counts.earlier_version.add(snapshot.session) + } + } + const unmatched = new Map; signals: Set }>() + for (const signal of signals) { + if (signal.kind !== "user_correction" || !sessions.has(signal.sessionID)) continue + const words = identifiers(signal.text) + const content = tokens(signal.text) + const matches = prepared.filter((lesson) => { + const overlap = [...lesson.tokens].filter((word) => content.has(word)) + return lesson.anchors.some((anchor) => words.has(anchor) && overlap.some((word) => word !== anchor)) || + (overlap.length >= 3 && overlap.length / lesson.tokens.size >= 0.6) + }) + if (!matches.length) { + const topics = new Set([...words].filter((word) => word.includes("_"))) + for (const anchor of anchors(signal.text)) topics.add(anchor) + for (const topic of topics) { + if (!unmatched.has(topic)) unmatched.set(topic, { sessions: new Set(), signals: new Set() }) + unmatched.get(topic)!.sessions.add(signal.sessionID) + unmatched.get(topic)!.signals.add(signal.id) + } + continue + } + for (const lesson of matches) { + const counts = rows.get(key(lesson.name, lesson.id, lesson.textHash))?.counts + if (!counts) continue + const shown = sessions.get(signal.sessionID)!.filter((entry) => entry.name === lesson.name && entry.id === lesson.id) + if (!shown.length) continue + if (matches.length > 1) { + counts.ambiguous.add(signal.sessionID) + continue + } + for (const entry of shown) { + if (entry.textHash !== lesson.textHash) continue + const at = Date.parse(entry.at) + const start = Date.parse(signal.turnStartedAt ?? "") + const correction = Date.parse(signal.at) + if (!Number.isFinite(at) || !Number.isFinite(start) || !Number.isFinite(correction) || start >= correction || at >= correction) + counts.unknown.add(signal.sessionID) + else if (at < start) counts.recurred.add(signal.sessionID) + else counts.during.add(signal.sessionID) + } + } + } + return { + lessons: [...rows.values()].map(({ row, counts }) => { + for (const field of Object.keys(counts) as (keyof typeof counts)[]) row[field] = counts[field].size + return row + }).sort((a, b) => a.name.localeCompare(b.name) || a.id.localeCompare(b.id) || a.textHash.localeCompare(b.textHash)), + unmatched_topics: [...unmatched].filter(([, value]) => value.sessions.size >= 2) + .map(([identifier, value]) => ({ identifier, sessions: value.sessions.size, signal_ids: [...value.signals].sort() })) + .sort((a, b) => b.sessions - a.sessions || a.identifier.localeCompare(b.identifier)).slice(0, 5), + } +} + +export async function analyzeRecurrence(root: string, options: { limit?: number } = {}) { + const { snapshots, coverage } = await readShownSnapshots(root, options) + const dir = path.join(root, ".altimate-code", "learn") + await SafeFS.assertSafePath(root, dir) + const entries = await fs.readdir(dir, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] + throw error + }) + const lessons: NamedLesson[] = [] + const signals: Signal[] = [] + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isDirectory()) continue + try { validateName(entry.name) } catch { continue } + signals.push(...await readSignalsSnapshot(root, entry.name)) + const file = Store.paths(root, entry.name).approved + try { + await SafeFS.assertSafePath(root, file) + for (const lesson of parse(await fs.readFile(file, "utf8"))) lessons.push({ name: entry.name, lesson }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") log.warn("learn recurrence approved store skipped", { name: entry.name, error }) + } + } + return { ...attribute(signals, snapshots, lessons), coverage } +} + +export function recurrenceHint(result: Pick, "lessons">, name: string, bullet: { id: string; text: string }) { + const evidence = result.lessons.find((lesson) => lesson.name === name && lesson.id === bullet.id && lesson.textHash === lessonTextHash(bullet.text)) + return evidence && evidence.recurred >= 2 ? evidence.recurred : undefined +} + +// Uncorrected lessons are summarized; a store can hold up to 1,000. +const corrected = (lesson: LessonRecurrence) => lesson.recurred + lesson.during + lesson.unknown + lesson.ambiguous > 0 + +export function formatRecurrence(result: Awaited>) { + return [ + "Recurrence (observed counts need capture on):", + ` Coverage: ${result.coverage.sessions_analyzed} sessions; ${result.coverage.sessions_skipped_malformed} malformed skipped; truncated: ${result.coverage.truncated ? "yes" : "no"}`, + ...result.lessons.filter(corrected).map((lesson) => ` ${lesson.name}/${lesson.id}: ${lesson.sessions_shown} sessions shown; ${lesson.recurred ? `${lesson.recurred} matching correction after exposure` : "no observed correction"}; ${lesson.during} during turn; ${lesson.unknown} unknown order; ${lesson.ambiguous} ambiguous; ${lesson.earlier_version} earlier version`), + ` ${result.lessons.filter((lesson) => !corrected(lesson)).length} other shown lessons: no observed correction`, + " Topics with no matching current lesson:", + ...result.unmatched_topics.map((topic) => ` ${topic.identifier}: ${topic.sessions} sessions (${topic.signal_ids.join(", ")})`), + ].join("\n") +} diff --git a/packages/opencode/src/altimate/learn/reflect.ts b/packages/opencode/src/altimate/learn/reflect.ts index f004e5f364..31ee390cbe 100644 --- a/packages/opencode/src/altimate/learn/reflect.ts +++ b/packages/opencode/src/altimate/learn/reflect.ts @@ -41,7 +41,7 @@ export interface ReflectInput { digest: string feedback: string kind: FeedbackKind - bullets: Bullet[] + bullets: (Bullet & { recurred?: number })[] } /** The model call, injectable so tests need no provider. Returns the raw object. */ @@ -79,7 +79,7 @@ function escapePromptSection(text: string): string { export function buildPrompt(input: ReflectInput): { system: string; prompt: string } { const playbook = input.bullets.length - ? input.bullets.map((b) => `[${b.id}] (h:${b.helpful} x:${b.harmful}${b.coexists?.length ? ` c:${b.coexists.join(",")}` : ""}) ${redactSecrets(b.text)}`).join("\n") + ? input.bullets.map((b) => `[${b.id}] (h:${b.helpful} x:${b.harmful}${(b.recurred ?? 0) >= 2 ? ` r:${b.recurred}` : ""}${b.coexists?.length ? ` c:${b.coexists.join(",")}` : ""}) ${redactSecrets(b.text)}`).join("\n") : "(empty)" const prompt = [ "", diff --git a/packages/opencode/src/altimate/learn/session-reflect.ts b/packages/opencode/src/altimate/learn/session-reflect.ts index 09e262bb76..b6ed72004e 100644 --- a/packages/opencode/src/altimate/learn/session-reflect.ts +++ b/packages/opencode/src/altimate/learn/session-reflect.ts @@ -14,6 +14,7 @@ import * as Signals from "./signals" import { curate, flagSuspiciousFeedback, type CurateResult } from "./curator" import { buildDigest, redactSecrets, sourceFromMessages, type DigestSource } from "./digest" import { FEEDBACK_CAP, feedbackText, reflect, replace, type FeedbackKind, type Generate } from "./reflect" +import { analyzeRecurrence, recurrenceHint } from "./recurrence" import { sharedAnchors } from "./anchors" import { processClaims } from "./claims" import { createUsageTracker, type UsageSummary } from "./usage" @@ -107,8 +108,15 @@ export async function reflectCore(input: ReflectCoreInput): Promise [b.id, b.text])) + const recurrence = await analyzeRecurrence(root).catch((error) => { + log.warn("learn recurrence unavailable", { error }) + return undefined + }) + const bullets = Playbook.bullets(snapshot).map((bullet) => ({ + ...bullet, ...(recurrence ? { recurred: recurrenceHint(recurrence, name, bullet) } : {}), + })) const deltas = await reflect( - { digest, feedback: input.feedback, kind: input.kind, bullets: Playbook.bullets(snapshot) }, + { digest, feedback: input.feedback, kind: input.kind, bullets }, generate, ).catch((e) => { throw new Error(`Model call failed (${input.modelLabel ?? "the default model"}): ${errText(e)}`) diff --git a/packages/opencode/src/altimate/learn/signals.ts b/packages/opencode/src/altimate/learn/signals.ts index 004923ebcb..6eff4768d2 100644 --- a/packages/opencode/src/altimate/learn/signals.ts +++ b/packages/opencode/src/altimate/learn/signals.ts @@ -36,11 +36,12 @@ export interface Signal { text: string reason: string at: string + turnStartedAt?: string status: "open" | "consumed" consumedBy?: string } -export type NewSignal = Pick +export type NewSignal = Pick export function signalsFile(root: string, name = DEFAULT_NAME): string { validateName(name) @@ -92,6 +93,7 @@ function parse(raw: string): { signals: Signal[]; malformed: boolean } { SIGNAL_KINDS.includes(s.kind) && typeof s.reason === "string" && typeof s.at === "string" && (s.status === "open" || s.status === "consumed") && (s.messageID === undefined || typeof s.messageID === "string") && + (s.turnStartedAt === undefined || (typeof s.turnStartedAt === "string" && Number.isFinite(Date.parse(s.turnStartedAt)))) && (s.partID === undefined || typeof s.partID === "string") && (s.source === undefined || s.source === "bootstrap" || s.source === "import-reviews") && (s.provenance === undefined || typeof s.provenance === "string") && @@ -228,6 +230,7 @@ export function appendSignals(root: string, inputs: readonly NewSignal[], name = sessionID: input.sessionID, messageID: input.messageID, partID: input.partID, + turnStartedAt: input.turnStartedAt, source: input.source, provenance: input.provenance === undefined ? undefined : redactProvenance(input.provenance), resolved: input.resolved, diff --git a/packages/opencode/src/cli/cmd/learn.ts b/packages/opencode/src/cli/cmd/learn.ts index 6ea9c74cd2..2a7bb0420d 100644 --- a/packages/opencode/src/cli/cmd/learn.ts +++ b/packages/opencode/src/cli/cmd/learn.ts @@ -27,6 +27,7 @@ import * as Store from "../../altimate/learn/store" import { MAX_TEXT, summarize, describeApplied, describeRejected } from "../../altimate/learn/curator" import { sourceFromTrajectory, redactSecrets, type DigestSource } from "../../altimate/learn/digest" import { DEFAULT_TIMEOUT_MS, FEEDBACK_KINDS, providerGenerate, type FeedbackKind, type Generate } from "../../altimate/learn/reflect" +import { analyzeRecurrence, formatRecurrence } from "../../altimate/learn/recurrence" import * as Signals from "../../altimate/learn/signals" import { learnMaxStored, learnModel } from "../../altimate/learn/auto" import { autoReflectEnabled, captureEnabled } from "../../altimate/learn/capture" @@ -379,6 +380,8 @@ const StatusCommand = effectCmd({ candidate: (await Store.loadCandidateLessons(root, name))?.length ?? 0, retired: (await Store.loadRetired(root, name)).length, open_signals: signals.length, + // Attribution is advisory; its failure must not hide the rest of the status. + ...await analyzeRecurrence(root).then((recurrence) => ({ recurrence }), (error) => ({ recurrence_error: error instanceof Error ? error.message : String(error) })), pending_recoveries: sessions.length, pending_replacements: (await Store.readPendingReplacements(root, name)).length, backoff_sessions: sessions.filter((session) => (state.recoveries[session]?.retryAt ?? 0) > Date.now()).length, @@ -408,6 +411,7 @@ const StatusCommand = effectCmd({ out(`Lessons: ${status.approved} approved, ${status.candidate} candidate, ${status.retired} retired`) out(`Pinned lessons: ${status.pinned.length}${status.pinned.length ? ` (${status.pinned.join(", ")})` : ""}`) out(`Open signals: ${status.open_signals}`) + out("recurrence" in status ? formatRecurrence(status.recurrence) : `Recurrence unavailable: ${status.recurrence_error}`) out(`Pending recoveries: ${status.pending_recoveries} session(s); ${status.backoff_sessions} in backoff`) out(`Pending replacements: ${status.pending_replacements}`) out(status.last_reflection diff --git a/packages/opencode/test/altimate/learn/capture.test.ts b/packages/opencode/test/altimate/learn/capture.test.ts index a0f5a73d71..d3c6fb40d4 100644 --- a/packages/opencode/test/altimate/learn/capture.test.ts +++ b/packages/opencode/test/altimate/learn/capture.test.ts @@ -3,7 +3,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" -import { Capture, RETRY_THRESHOLD, ToolRetryTracker, type PartLike } from "../../../src/altimate/learn/capture" +import { correctedTurnStart, Capture, RETRY_THRESHOLD, ToolRetryTracker, type PartLike } from "../../../src/altimate/learn/capture" import * as Signals from "../../../src/altimate/learn/signals" const call = (id: string, tool: string, status: "error" | "completed" | "running", error = "boom", messageID = "m1") => ({ @@ -334,3 +334,48 @@ describe("Capture: tool retries", () => { expect(await Signals.readSignals(root)).toHaveLength(1) }) }) + +describe("correction turn start", () => { + const history = [ + { id: "u1", sessionID: "s", role: "user", time: { created: 1000 } }, + { id: "tool-step", sessionID: "s", role: "assistant", time: { created: 2000, completed: 2500 } }, + { id: "final", sessionID: "s", role: "assistant", time: { created: 4000, completed: 4500 } }, + { id: "correction", sessionID: "s", role: "user", time: { created: 5000 } }, + ] + test("uses the first assistant after the previous user, not the final step", () => { + expect(correctedTurnStart(history, "correction")).toBe(2000) + expect(correctedTurnStart(history, "missing")).toBeUndefined() + expect(correctedTurnStart(history, "u1")).toBeUndefined() + expect(correctedTurnStart([history[0], history[3]], "correction")).toBeUndefined() + expect(correctedTurnStart(history.slice(1), "correction")).toBeUndefined() + }) + test("records the resolved turn start while preserving the eligibility shortcut", async () => { + const recorded: Signals.NewSignal[] = [] + let lookups = 0 + const capture = new Capture({ + hasPriorAssistant: async () => { throw new Error("shortcut should handle eligibility") }, + turnStart: async (_, id) => { lookups++; return correctedTurnStart(history, id) }, + record: async (signal) => { recorded.push(signal); return undefined }, + }) + for (const info of history) capture.onMessage(info) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text: "No, use UTC instead." }) + await capture.flush() + expect(lookups).toBe(1) + expect(recorded[0].turnStartedAt).toBe(new Date(2000).toISOString()) + }) + test("uncertain history omits the field and lookup failure preserves the signal", async () => { + for (const fail of [false, true]) { + const recorded: Signals.NewSignal[] = [] + const capture = new Capture({ + hasPriorAssistant: async () => true, + turnStart: async () => { if (fail) throw new Error("unavailable"); return undefined }, + record: async (signal) => { recorded.push(signal); return undefined }, + }) + capture.onMessage(history[3]) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text: "No, use UTC instead." }) + await capture.flush() + expect(recorded).toHaveLength(1) + expect(recorded[0]).not.toHaveProperty("turnStartedAt") + } + }) +}) diff --git a/packages/opencode/test/altimate/learn/cli.test.ts b/packages/opencode/test/altimate/learn/cli.test.ts index a635acdaf9..a53a6a66d2 100644 --- a/packages/opencode/test/altimate/learn/cli.test.ts +++ b/packages/opencode/test/altimate/learn/cli.test.ts @@ -168,6 +168,34 @@ describe("learn pin and unpin", () => { }) describe("learn opt-in and status", () => { + test("status includes recurrence coverage even with learning disabled", async () => { + await using dir = await tmpdir({ git: true }) + const result = await runLearn(dir.path, ["status", "--json"], undefined, { ALTIMATE_LEARN: "0" }) + expect(result.code).toBe(0) + expect(JSON.parse(result.stdout).recurrence).toEqual({ + lessons: [], unmatched_topics: [], + coverage: { sessions_analyzed: 0, sessions_skipped_malformed: 0, truncated: false }, + }) + const shown = await learn(dir.path, "status") + expect(shown.stdout).toContain("Recurrence (observed counts need capture on)") + expect(shown.stdout).toContain("no matching current lesson") + }) + + test("status still reports when recurrence analysis fails", async () => { + await using dir = await tmpdir({ git: true }) + await using outside = await tmpdir() + await fs.mkdir(path.join(dir.path, ".altimate-code", "learn"), { recursive: true }) + await fs.symlink(outside.path, path.join(dir.path, ".altimate-code", "learn", ".sessions")) + const result = await runLearn(dir.path, ["status", "--json"]) + expect(result.code).toBe(0) + const status = JSON.parse(result.stdout) + expect(status.recurrence).toBeUndefined() + expect(typeof status.recurrence_error).toBe("string") + expect(status.open_signals).toBe(0) + const shown = await learn(dir.path, "status") + expect(shown.stdout).toContain("Recurrence unavailable:") + }) + test.each(["config", "env"])("status reports the learning kill switch disabled by %s", async (source) => { await using dir = await tmpdir({ git: true }) await fs.writeFile(path.join(dir.path, "opencode.json"), JSON.stringify({ diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts new file mode 100644 index 0000000000..183676b237 --- /dev/null +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -0,0 +1,178 @@ +// altimate_change - new file +import { describe, expect, test } from "bun:test" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { correctedTurnStart } from "../../../src/altimate/learn/capture" +import { attribute, analyzeRecurrence, formatRecurrence, recurrenceHint, type NamedLesson } from "../../../src/altimate/learn/recurrence" +import { lessonTextHash, readShownSnapshots, type ShownSnapshot } from "../../../src/altimate/learn/delivery" +import type { Signal } from "../../../src/altimate/learn/signals" +import * as Store from "../../../src/altimate/learn/store" + +const time = (n: number) => new Date(n * 1000).toISOString() +const lesson: NamedLesson = { name: "team-playbook", lesson: { id: "L-0001", text: "Convert `event_at` timestamps with `to_utc` before storage." } } +const signal = (sessionID = "s1", extra: Partial = {}): Signal => ({ + id: `sig_${sessionID}`, kind: "user_correction", sessionID, text: "No, convert event_at timestamps with to_utc.", + reason: "correction", at: time(5), turnStartedAt: time(2), status: "open", ...extra, +}) +const snapshot = (session = "s1", extra: Partial = {}): ShownSnapshot => ({ + session, shown: [{ name: lesson.name, id: lesson.lesson.id, textHash: lessonTextHash(lesson.lesson.text), tier: "core", at: time(1), ...extra }], +}) +const run = (signals = [signal()], snapshots = [snapshot()], lessons = [lesson]) => attribute(signals, snapshots, lessons) + +describe("recurrence attribution", () => { + test("matching correction after exposure counts, regardless of consumption", () => { + expect(run().lessons[0]).toMatchObject({ sessions_shown: 1, recurred: 1, unknown: 0 }) + expect(run([signal("s1", { status: "consumed", consumedBy: "reflect" })]).lessons[0].recurred).toBe(1) + expect(run([signal("s1", { kind: "review" })]).lessons[0].recurred).toBe(0) + }) + + test("mid-turn file exposure, including equality, is during rather than recurrence", () => { + for (const at of [time(2), time(3)]) { + expect(run([signal()], [snapshot("s1", { at, tier: "file" })]).lessons[0]).toMatchObject({ recurred: 0, during: 1 }) + } + }) + + test("tool step then file lesson then final assistant resolves to during", () => { + const history = [ + { id: "user", sessionID: "s1", role: "user", time: { created: 1000 } }, + { id: "tool-step", sessionID: "s1", role: "assistant", time: { created: 2000, completed: 2500 } }, + { id: "final", sessionID: "s1", role: "assistant", time: { created: 4000, completed: 4500 } }, + { id: "correction", sessionID: "s1", role: "user", time: { created: 5000 } }, + ] + const turnStartedAt = new Date(correctedTurnStart(history, "correction")!).toISOString() + expect(run([signal("s1", { turnStartedAt })], [snapshot("s1", { tier: "file", at: time(3) })]).lessons[0]) + .toMatchObject({ recurred: 0, during: 1 }) + }) + + test("missing, invalid and impossible ordering stays unknown", () => { + for (const turnStartedAt of [undefined, "bad", time(6)]) + expect(run([signal("s1", { turnStartedAt })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 1 }) + expect(run([signal()], [snapshot("s1", { at: time(5) })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 1 }) + }) + + test("a new convention or a shared identifier about a different property does not match", () => { + for (const text of ["No, pin dependency versions.", "No, index `event_at`", "No, index `other_event_at`", "No, index `to_utc`"]) { + expect(run([signal("s1", { text })]).lessons[0].recurred).toBe(0) + } + const affix = { ...lesson, lesson: { ...lesson.lesson, text: "Convert `*_at` timestamps with `to_utc`" } } + expect(run([signal("s1", { text: "No, index `event_at`" })], [snapshot()], [affix]).lessons[0].recurred).toBe(0) + expect(run([signal("s1", { text: "No, timestamps `event_at`" })], [snapshot()], [affix]).lessons[0].recurred).toBe(0) + const prefix = { ...lesson, lesson: { ...lesson.lesson, text: "Name `stg_` models after their source" } } + expect(run([signal("s1", { text: "No, rename stg_orders models" })], [snapshot()], [prefix]).lessons[0].recurred).toBe(0) + }) + + test("full anchor needs another content token; backticks on correction are optional", () => { + expect(run([signal("s1", { text: "No, event_at timestamps" })]).lessons[0].recurred).toBe(1) + expect(run([signal("s1", { text: "No, event_at" })]).lessons[0].recurred).toBe(0) + }) + + test("lexical fallback requires three distinct tokens and sixty percent", () => { + const prose = { ...lesson, lesson: { ...lesson.lesson, text: "Preserve original timezone during timestamp conversion" } } + const shown = snapshot("s1", { textHash: lessonTextHash(prose.lesson.text) }) + expect(run([signal("s1", { text: "No, preserve original timezone during conversion" })], [shown], [prose]).lessons[0].recurred).toBe(1) + expect(run([signal("s1", { text: "No, timezone timezone conversion" })], [shown], [prose]).lessons[0].recurred).toBe(0) + expect(run([signal("s1", { text: "No, original timezone conversion" })], [shown], [prose]).lessons[0].recurred).toBe(0) + }) + + test("multiple matching lessons are ambiguous, including same ids in different stores", () => { + const other = { ...lesson, name: "other-store" } + const shown = snapshot() + shown.shown.push({ ...shown.shown[0], name: other.name }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons).toHaveLength(2) + for (const row of result.lessons) expect(row).toMatchObject({ recurred: 0, ambiguous: 1 }) + }) + + test("store-local ids retain independent evidence", () => { + const other = { name: "other-store", lesson: { id: lesson.lesson.id, text: "Index `customer_id` for efficient lookups." } } + const shown = snapshot() + shown.shown.push({ ...shown.shown[0], name: other.name, textHash: lessonTextHash(other.lesson.text) }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons.find((row) => row.name === lesson.name)?.recurred).toBe(1) + expect(result.lessons.find((row) => row.name === other.name)?.recurred).toBe(0) + }) + + test("earlier lesson version cannot supply current evidence", () => { + expect(run([signal()], [snapshot("s1", { textHash: lessonTextHash("Earlier text") })]).lessons[0]) + .toMatchObject({ earlier_version: 1, recurred: 0 }) + }) + + test("sessions count once; hints require two sessions and the bullet's own normalized text", () => { + const result = run([signal(), signal("s1", { id: "second" }), signal("s2")], [snapshot(), snapshot("s2")]) + expect(result.lessons[0].recurred).toBe(2) + expect(recurrenceHint(result, lesson.name, lesson.lesson)).toBe(2) + expect(recurrenceHint(result, lesson.name, { ...lesson.lesson, text: "Edited candidate" })).toBeUndefined() + expect(recurrenceHint(result, "another-store", lesson.lesson)).toBeUndefined() + expect(recurrenceHint(result, lesson.name, { ...lesson.lesson, text: " " + lesson.lesson.text.replaceAll(" ", "\n ") })).toBe(2) + expect(recurrenceHint(run(), lesson.name, lesson.lesson)).toBeUndefined() + expect(run([signal(), signal("s1", { id: "second" })]).lessons[0].recurred).toBe(1) + }) + + test("unmatched topics need distinct analyzed sessions and expose no correction text", () => { + const text = "No, index `customer_id` and `_is_deleted` immediately, private discussion." + const result = run([signal("s1", { text }), signal("s2", { text }), signal("outside", { text })], [snapshot(), snapshot("s2")]) + expect(result.unmatched_topics).toEqual(["_is_deleted", "customer_id"].map((identifier) => ({ identifier, sessions: 2, signal_ids: ["sig_s1", "sig_s2"] }))) + expect(run([signal("s1", { text }), signal("s1", { id: "duplicate", text })]).unmatched_topics).toEqual([]) + expect(JSON.stringify(result)).not.toContain("private discussion") + }) + + test("unmatched topics are capped at five with deterministic ties", () => { + const text = "No, a_id b_id c_id d_id e_id f_id" + expect(run([signal("s1", { text }), signal("s2", { text })], [snapshot(), snapshot("s2")]).unmatched_topics.map((topic) => topic.identifier)) + .toEqual(["a_id", "b_id", "c_id", "d_id", "e_id"]) + }) + + test("human status lists corrected lessons and summarizes the rest", () => { + const quiet: NamedLesson = { name: "team-playbook", lesson: { id: "L-0002", text: "Name staging models `stg___`." } } + const shown = snapshot() + shown.shown.push({ name: quiet.name, id: quiet.lesson.id, textHash: lessonTextHash(quiet.lesson.text), tier: "core", at: time(1) }) + const text = formatRecurrence({ ...run([signal()], [shown], [lesson, quiet]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + expect(text).toContain("team-playbook/L-0001: 1 sessions shown; 1 matching correction after exposure") + expect(text).not.toContain("L-0002") + expect(text).toContain("1 other shown lessons: no observed correction") + }) + + test("human status reports coverage, uncertainty and no matching current lesson", () => { + const text = formatRecurrence({ ...run([]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + expect(text).toContain("Recurrence") + expect(text).toContain("no observed correction") + expect(text).toContain("counts need capture on") + expect(text).toContain("no matching current lesson") + expect(text).not.toContain("followed") + }) +}) + +test("snapshot analysis is bounded, skips malformed files and never repairs any input", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "learn-recurrence-")) + try { + const dir = path.join(root, ".altimate-code", "learn", ".sessions") + await fs.mkdir(dir, { recursive: true }) + const state = (session: string) => ({ version: 1, session, firstMessage: "m1", section: "", shown: [{ + name: lesson.name, lesson: { ...lesson.lesson, tags: [], scope: "project", helpful: 0, harmful: 0, applied: 0, created: time(0), updated: time(0) }, + tier: "file", at: time(1), queryHash: "hash", + }], requests: [], compactions: [], counted: [] }) + const old = path.join(dir, "old.json") + const good = path.join(dir, "good.json") + const bad = path.join(dir, "bad.json") + await fs.writeFile(old, JSON.stringify(state("old"))) + await fs.writeFile(good, JSON.stringify(state("s1"))) + await fs.writeFile(bad, "broken") + await fs.utimes(old, 1, 1) + await fs.utimes(good, 2, 2) + await fs.utimes(bad, 3, 3) + const p = Store.paths(root, lesson.name) + await fs.mkdir(p.learnDir, { recursive: true }) + await fs.writeFile(p.approved, JSON.stringify([state("s1").shown[0].lesson])) + const signals = path.join(p.learnDir, "signals.jsonl") + const raw = JSON.stringify(signal()) + "\nmalformed\n" + JSON.stringify(signal("old")) + "\n" + await fs.writeFile(signals, raw) + const result = await analyzeRecurrence(root, { limit: 2 }) + expect(result.coverage).toEqual({ sessions_analyzed: 1, sessions_skipped_malformed: 1, truncated: true }) + expect(result.lessons[0]).toMatchObject({ sessions_shown: 1, recurred: 1 }) + expect(await fs.readFile(bad, "utf8")).toBe("broken") + expect(await fs.readFile(signals, "utf8")).toBe(raw) + expect((await fs.readdir(p.learnDir)).sort()).toEqual([path.basename(p.approved), "signals.jsonl"].sort()) + expect((await readShownSnapshots(root, { limit: 0 })).coverage).toMatchObject({ sessions_analyzed: 0, truncated: true }) + } finally { await fs.rm(root, { recursive: true, force: true }) } +}) diff --git a/packages/opencode/test/altimate/learn/reflect.test.ts b/packages/opencode/test/altimate/learn/reflect.test.ts index 2fd68cab8a..0ce7b9aef9 100644 --- a/packages/opencode/test/altimate/learn/reflect.test.ts +++ b/packages/opencode/test/altimate/learn/reflect.test.ts @@ -11,6 +11,14 @@ import { accountUsage, type UsageSummary } from "../../../src/altimate/learn/usa const bullets = [{ id: "L-0001", text: "Staging models are prefixed stg_.", helpful: 2, harmful: 0 }] describe("buildPrompt", () => { + test("recurrence hints appear only at two distinct sessions", () => { + for (const recurred of [undefined, 0, 1, 2, 3]) { + const { prompt, system } = buildPrompt({ digest: "d", feedback: "f", kind: "user", bullets: [{ ...bullets[0], recurred }] }) + expect(prompt.includes(` r:${recurred}`)).toBe((recurred ?? 0) >= 2) + expect(system).toContain("never weaker") + } + }) + test("requires backticks around generated identifiers and naming patterns", () => { const { system } = buildPrompt({ digest: "d", feedback: "f", kind: "ci", bullets }) expect(system).toContain("Wrap every code identifier and naming pattern (including prefixes and suffixes) in backticks.") diff --git a/packages/opencode/test/altimate/learn/session-reflect.test.ts b/packages/opencode/test/altimate/learn/session-reflect.test.ts index c36722b142..8fefb58928 100644 --- a/packages/opencode/test/altimate/learn/session-reflect.test.ts +++ b/packages/opencode/test/altimate/learn/session-reflect.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" +import { Delivery } from "../../../src/altimate/learn/delivery" import * as Signals from "../../../src/altimate/learn/signals" import * as Store from "../../../src/altimate/learn/store" import * as Playbook from "../../../src/altimate/learn/playbook" @@ -788,3 +789,47 @@ describe("opt-in switches", () => { expect(() => learnMaxStored(75, { ALTIMATE_LEARN_MAX_STORED: value })).toThrow("positive integer") }) }) + +describe("reflection recurrence evidence", () => { + test("adds repeated evidence only to the same store, id and current text", async () => { + const bullet = { id: "L-0001", text: "Convert `event_at` timestamps with `to_utc`.", helpful: 0, harmful: 0 } + await Store.saveCandidate(root, NAME, Playbook.withBullets(Playbook.create({ name: NAME }), [bullet])) + await Store.promote(root, NAME) + const delivery = new Delivery(root) + for (const sessionID of ["ses_one", "ses_two"]) { + await delivery.prepare(sessionID, "m1", "Convert event_at timestamps to_utc") + await Signals.appendSignal(root, { + kind: "user_correction", sessionID, text: "No, convert event_at timestamps with to_utc.", reason: "correction", + turnStartedAt: new Date(Date.now() + 1000).toISOString(), + }) + } + // Signal capture is later than the corrected turn; make that ordering deterministic. + const file = Signals.signalsFile(root) + const signals = await Signals.readSignalsSnapshot(root) + await fs.writeFile(file, signals.map((signal) => JSON.stringify({ ...signal, at: new Date(Date.now() + 2000).toISOString() }) + "\n").join("")) + const prompts: string[] = [] + const reflect = () => reflectCore({ + root, name: NAME, source: { prompts: [], calls: [] }, feedback: "No, convert timestamps.", kind: "user", origin: "test", + generate: async ({ prompt }) => { prompts.push(prompt); return { deltas: [] } }, + }) + await reflect() + expect(prompts[0]).toContain("[L-0001] (h:0 x:0 r:2)") + await Store.saveCandidate(root, NAME, Playbook.withBullets(Playbook.create({ name: NAME }), [{ ...bullet, text: "Convert `event_at` timestamps before storage." }])) + await reflect() + expect(prompts[1]).not.toContain(" r:") + }) + + test("analyzer failure does not prevent reflection", async () => { + const dir = path.join(root, ".altimate-code", "learn", ".sessions") + await fs.mkdir(dir, { recursive: true }) + const target = path.join(root, "snapshots") + await fs.rename(dir, target) + await fs.symlink(target, dir) + let called = false + await reflectCore({ + root, name: NAME, source: { prompts: [], calls: [] }, feedback: "No, use UTC.", kind: "user", origin: "test", + generate: async ({ prompt }) => { called = true; expect(prompt).not.toContain(" r:"); return { deltas: [] } }, + }) + expect(called).toBe(true) + }) +}) diff --git a/packages/opencode/test/altimate/learn/signals.test.ts b/packages/opencode/test/altimate/learn/signals.test.ts index ee02f85b07..2149c7e006 100644 --- a/packages/opencode/test/altimate/learn/signals.test.ts +++ b/packages/opencode/test/altimate/learn/signals.test.ts @@ -14,6 +14,18 @@ afterEach(() => fs.rm(root, { recursive: true, force: true })) const base = { kind: "user_correction", sessionID: "ses_1", reason: "r" } as const describe("signal store", () => { + test("old signals parse and correction turn starts round-trip read-only", async () => { + await Signals.appendSignal(root, { ...base, text: "No, keep original", messageID: "old" }) + const turnStartedAt = "2026-10-18T10:00:00.000Z" + await Signals.appendSignal(root, { ...base, text: "No, keep timestamps", messageID: "new", turnStartedAt }) + const records = await Signals.readSignalsSnapshot(root) + expect(records[0].turnStartedAt).toBeUndefined() + expect(records[1].turnStartedAt).toBe(turnStartedAt) + const file = Signals.signalsFile(root) + await fs.appendFile(file, JSON.stringify({ ...records[1], id: "invalid", turnStartedAt: 42 }) + "\n") + expect(await Signals.readSignalsSnapshot(root)).toHaveLength(2) + }) + test("records the documented fields under the default named lesson store", async () => { const s = await Signals.appendSignal(root, { ...base, messageID: "msg_1", text: "no, use ref()" }) expect(s).toMatchObject({ kind: "user_correction", sessionID: "ses_1", messageID: "msg_1", text: "no, use ref()", status: "open" }) From 1a22075d8300b016822233bafccfec99454c9f71 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 14:31:20 -0700 Subject: [PATCH 2/9] fix(learn): tighten recurrence attribution boundaries - Synthetic continuation user messages no longer count as the corrected request boundary. - An earlier lesson version shown alongside another matching lesson no longer counts as ambiguity evidence for the current text. - Skip signal reads when no session snapshots exist and keep only signals from analyzed sessions. - Look up session history once per correction message. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/learn/capture.ts | 14 +++++++++--- .../opencode/src/altimate/learn/recurrence.ts | 7 +++--- .../test/altimate/learn/capture.test.ts | 22 +++++++++++++++++++ .../test/altimate/learn/recurrence.test.ts | 9 ++++++++ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/learn/capture.ts b/packages/opencode/src/altimate/learn/capture.ts index 9ac51dea0f..2cb41d773f 100644 --- a/packages/opencode/src/altimate/learn/capture.ts +++ b/packages/opencode/src/altimate/learn/capture.ts @@ -92,10 +92,11 @@ export interface MessageLike { time?: { created?: number; completed?: number } } -export function correctedTurnStart(messages: readonly MessageLike[], messageID: string): number | undefined { +/** Start of the agent's response to the request the correction answers; synthetic continuations are not requests. */ +export function correctedTurnStart(messages: readonly (MessageLike & { synthetic?: boolean })[], messageID: string): number | undefined { const end = messages.findIndex((message) => message.id === messageID && message.role === "user") if (end < 0) return - const previous = messages.slice(0, end).findLastIndex((message) => message.role === "user") + const previous = messages.slice(0, end).findLastIndex((message) => message.role === "user" && !message.synthetic) if (previous < 0) return return messages.slice(previous + 1, end).find((message) => message.role === "assistant")?.time?.created } @@ -123,6 +124,7 @@ export class Capture { private messages = new Map() private pendingParts = new Map() private sessionsWithAssistant = new Map() + private corrected = new Set() private trackers = new Map() private pending = new Set>() @@ -182,6 +184,9 @@ export class Capture { const prior = this.sessionsWithAssistant.get(part.sessionID) const known = prior !== undefined && message.time?.created !== undefined && prior < message.time.created if (!known && !(await this.deps.hasPriorAssistant(part.sessionID, part.messageID))) return + // Later updates of the same message deduplicate to the first signal; skip their history reads. + if (this.corrected.has(part.messageID)) return + remember(this.corrected, part.messageID, RECENT_ID_CAP) const start = await this.deps.turnStart?.(part.sessionID, part.messageID).catch((error) => { log.warn("learn turn start unavailable", { error }) return undefined @@ -255,7 +260,10 @@ export async function startCapture(ctx: { directory: string; worktree: string }) record: (signal) => appendSignal(root, signal), turnStart: async (sessionID, messageID) => { const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) - return correctedTurnStart(messages.map((message) => message.info), messageID) + return correctedTurnStart(messages.map((message) => { + const text = message.parts.filter((part) => part.type === "text") + return { ...message.info, synthetic: text.length > 0 && text.every((part) => part.synthetic) } + }), messageID) }, hasPriorAssistant: async (sessionID, beforeMessageID) => { const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts index 2708edb9ed..300182df18 100644 --- a/packages/opencode/src/altimate/learn/recurrence.ts +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -76,14 +76,14 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS for (const lesson of matches) { const counts = rows.get(key(lesson.name, lesson.id, lesson.textHash))?.counts if (!counts) continue - const shown = sessions.get(signal.sessionID)!.filter((entry) => entry.name === lesson.name && entry.id === lesson.id) + const shown = sessions.get(signal.sessionID)!.filter((entry) => + entry.name === lesson.name && entry.id === lesson.id && entry.textHash === lesson.textHash) if (!shown.length) continue if (matches.length > 1) { counts.ambiguous.add(signal.sessionID) continue } for (const entry of shown) { - if (entry.textHash !== lesson.textHash) continue const at = Date.parse(entry.at) const start = Date.parse(signal.turnStartedAt ?? "") const correction = Date.parse(signal.at) @@ -115,10 +115,11 @@ export async function analyzeRecurrence(root: string, options: { limit?: number }) const lessons: NamedLesson[] = [] const signals: Signal[] = [] + const sessions = new Set(snapshots.map((snapshot) => snapshot.session)) for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { if (!entry.isDirectory()) continue try { validateName(entry.name) } catch { continue } - signals.push(...await readSignalsSnapshot(root, entry.name)) + if (sessions.size) signals.push(...(await readSignalsSnapshot(root, entry.name)).filter((signal) => sessions.has(signal.sessionID))) const file = Store.paths(root, entry.name).approved try { await SafeFS.assertSafePath(root, file) diff --git a/packages/opencode/test/altimate/learn/capture.test.ts b/packages/opencode/test/altimate/learn/capture.test.ts index d3c6fb40d4..790b1e6dcd 100644 --- a/packages/opencode/test/altimate/learn/capture.test.ts +++ b/packages/opencode/test/altimate/learn/capture.test.ts @@ -349,6 +349,28 @@ describe("correction turn start", () => { expect(correctedTurnStart([history[0], history[3]], "correction")).toBeUndefined() expect(correctedTurnStart(history.slice(1), "correction")).toBeUndefined() }) + test("a synthetic continuation is not the corrected request's boundary", () => { + const continued = [ + history[0], history[1], + { id: "continue", sessionID: "s", role: "user", synthetic: true, time: { created: 3000 } }, + history[2], history[3], + ] + expect(correctedTurnStart(continued, "correction")).toBe(2000) + expect(correctedTurnStart(continued.map((m) => ({ ...m, synthetic: false })), "correction")).toBe(4000) + }) + test("repeated updates of one correction look up history once", async () => { + let lookups = 0 + const capture = new Capture({ + hasPriorAssistant: async () => true, + turnStart: async (_, id) => { lookups++; return correctedTurnStart(history, id) }, + record: async () => undefined, + }) + for (const info of history) capture.onMessage(info) + for (const text of ["No, use UTC", "No, use UTC instead."]) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text }) + await capture.flush() + expect(lookups).toBe(1) + }) test("records the resolved turn start while preserving the eligibility shortcut", async () => { const recorded: Signals.NewSignal[] = [] let lookups = 0 diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts index 183676b237..2c0655e559 100644 --- a/packages/opencode/test/altimate/learn/recurrence.test.ts +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -93,6 +93,15 @@ describe("recurrence attribution", () => { expect(result.lessons.find((row) => row.name === other.name)?.recurred).toBe(0) }) + test("an earlier version shown with another matching lesson is not ambiguity evidence", () => { + const other = { ...lesson, name: "other-store" } + const shown = snapshot("s1", { textHash: lessonTextHash("Earlier text") }) + shown.shown.push({ ...snapshot().shown[0], name: other.name }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons.find((row) => row.name === lesson.name)).toMatchObject({ ambiguous: 0, earlier_version: 1 }) + expect(result.lessons.find((row) => row.name === other.name)).toMatchObject({ ambiguous: 1 }) + }) + test("earlier lesson version cannot supply current evidence", () => { expect(run([signal()], [snapshot("s1", { textHash: lessonTextHash("Earlier text") })]).lessons[0]) .toMatchObject({ earlier_version: 1, recurred: 0 }) From cd539a8677eb23cfeb5e6139e4d1a7d8e2c8d058 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 14:44:10 -0700 Subject: [PATCH 3/9] fix(learn): compaction-safe turn starts and exposure-scoped ambiguity - A request right after a compaction marker may be its replay, so its turn start is unknown; compaction markers are never request boundaries. - Extract `turnMessages` (synthetic and compaction flags from real parts) and test it. - Ambiguity counts only lessons the session was shown in their current text; a later lesson created from the correction no longer hides recurrence. - Mark a correction message handled only after its signal is recorded, so a failed write is retried by a later update. - Document recurrence in `learn status` and the reflection `r:N` hint. Co-Authored-By: Claude Opus 5.5 --- docs/docs/configure/learn.md | 16 +++++++- .../opencode/src/altimate/learn/capture.ts | 35 ++++++++++++----- .../opencode/src/altimate/learn/recurrence.ts | 10 +++-- .../test/altimate/learn/capture.test.ts | 39 ++++++++++++++++++- .../test/altimate/learn/recurrence.test.ts | 9 ++++- 5 files changed, 92 insertions(+), 17 deletions(-) diff --git a/docs/docs/configure/learn.md b/docs/docs/configure/learn.md index 31ef056111..37a9bb2f19 100644 --- a/docs/docs/configure/learn.md +++ b/docs/docs/configure/learn.md @@ -222,7 +222,21 @@ altimate-code learn disable ### status -Shows whether learning is on, whether automatic promotion is on with its last promotion and last held-back reason, lesson counts (approved, candidate, retired), open signals, pending recoveries, the last reflection and its summary, and the resolved limits. `--json` also includes the last reflection's token use and estimated cost. +Shows whether learning is on, whether automatic promotion is on with its last promotion and last held-back reason, lesson counts (approved, candidate, retired), open signals, pending recoveries, the last reflection and its summary, the resolved limits, and lesson recurrence. `--json` also includes the last reflection's token use and estimated cost. + +**Recurrence** answers "was this lesson shown, and did the user still correct the same thing?". It reads the newest 200 session snapshots and the recorded corrections. It makes no model calls and writes nothing. For each shown lesson it counts sessions where a matching correction came: + +| Count | Meaning | +|---|---| +| `recurred` | after the lesson was shown, at the start of the corrected request or earlier | +| `during` | while the agent was answering, for example a file-hook lesson added mid-turn; the lesson may have arrived too late | +| `unknown` | in an order that cannot be told, for example older corrections or a request replayed after compaction | +| `ambiguous` | when the correction matched more than one lesson the session was shown | +| `earlier_version` | to a different version of the lesson text; it never counts for the current text | + +A correction matches a lesson when it names one of the lesson's backtick identifiers plus another word from the lesson, or contains most of the lesson's words. The text output lists only lessons with a matching correction and summarizes the rest as "no observed correction". That is not proof the agent followed them. Corrections are recorded only while capture is on. Status also lists up to 5 identifiers that were corrected in 2 or more sessions with no matching current lesson. They are taken from your local corrections and never leave the project. If the analysis fails, status shows `Recurrence unavailable` (`recurrence_error` in JSON), and the rest of the status still prints. + +When the current text of a lesson has `recurred` in 2 or more sessions, reflection marks it `r:N` in the playbook it sends the model. The model is asked to make the lesson clearer or more specific, never weaker. The result is a candidate like any other and goes through the same review or automatic-promotion gates. ```bash altimate-code learn status diff --git a/packages/opencode/src/altimate/learn/capture.ts b/packages/opencode/src/altimate/learn/capture.ts index 2cb41d773f..9e0380e609 100644 --- a/packages/opencode/src/altimate/learn/capture.ts +++ b/packages/opencode/src/altimate/learn/capture.ts @@ -92,12 +92,30 @@ export interface MessageLike { time?: { created?: number; completed?: number } } -/** Start of the agent's response to the request the correction answers; synthetic continuations are not requests. */ -export function correctedTurnStart(messages: readonly (MessageLike & { synthetic?: boolean })[], messageID: string): number | undefined { +export type TurnMessage = MessageLike & { synthetic?: boolean; compaction?: boolean } + +/** Session history as turn boundaries: all-synthetic text continues a turn; a compaction part marks compaction. */ +export function turnMessages(messages: readonly { info: MessageLike; parts: readonly { type: string; synthetic?: boolean }[] }[]): TurnMessage[] { + return messages.map(({ info, parts }) => { + const text = parts.filter((part) => part.type === "text") + return { + ...info, + synthetic: text.length > 0 && text.every((part) => part.synthetic), + compaction: parts.some((part) => part.type === "compaction"), + } + }) +} + +/** + * Start of the agent's response to the request the correction answers. Synthetic continuations are not + * requests. A request right after a compaction may be its replay, so its turn start is unknown. + */ +export function correctedTurnStart(messages: readonly TurnMessage[], messageID: string): number | undefined { const end = messages.findIndex((message) => message.id === messageID && message.role === "user") if (end < 0) return - const previous = messages.slice(0, end).findLastIndex((message) => message.role === "user" && !message.synthetic) + const previous = messages.slice(0, end).findLastIndex((message) => message.role === "user" && !message.synthetic && !message.compaction) if (previous < 0) return + if (messages.slice(0, previous).findLast((message) => message.role === "user")?.compaction) return return messages.slice(previous + 1, end).find((message) => message.role === "assistant")?.time?.created } @@ -181,12 +199,11 @@ export class Capture { if (message.role !== "user") return const reason = correctionReason(part.text) if (!reason) return + // Later updates of a recorded correction deduplicate to its first signal; skip their history reads. + if (this.corrected.has(part.messageID)) return const prior = this.sessionsWithAssistant.get(part.sessionID) const known = prior !== undefined && message.time?.created !== undefined && prior < message.time.created if (!known && !(await this.deps.hasPriorAssistant(part.sessionID, part.messageID))) return - // Later updates of the same message deduplicate to the first signal; skip their history reads. - if (this.corrected.has(part.messageID)) return - remember(this.corrected, part.messageID, RECENT_ID_CAP) const start = await this.deps.turnStart?.(part.sessionID, part.messageID).catch((error) => { log.warn("learn turn start unavailable", { error }) return undefined @@ -199,6 +216,7 @@ export class Capture { text: part.text, reason, }) + remember(this.corrected, part.messageID, RECENT_ID_CAP) } private async toolPart(part: PartLike) { @@ -260,10 +278,7 @@ export async function startCapture(ctx: { directory: string; worktree: string }) record: (signal) => appendSignal(root, signal), turnStart: async (sessionID, messageID) => { const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) - return correctedTurnStart(messages.map((message) => { - const text = message.parts.filter((part) => part.type === "text") - return { ...message.info, synthetic: text.length > 0 && text.every((part) => part.synthetic) } - }), messageID) + return correctedTurnStart(turnMessages(messages), messageID) }, hasPriorAssistant: async (sessionID, beforeMessageID) => { const messages = await Session.messages({ sessionID: SessionID.make(sessionID) }) diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts index 300182df18..029ab6b0a1 100644 --- a/packages/opencode/src/altimate/learn/recurrence.ts +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -73,13 +73,15 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS } continue } - for (const lesson of matches) { + // Only lessons this session saw in their current text compete; one never shown cannot explain the correction. + const exposed = matches.flatMap((lesson) => { const counts = rows.get(key(lesson.name, lesson.id, lesson.textHash))?.counts - if (!counts) continue const shown = sessions.get(signal.sessionID)!.filter((entry) => entry.name === lesson.name && entry.id === lesson.id && entry.textHash === lesson.textHash) - if (!shown.length) continue - if (matches.length > 1) { + return counts && shown.length ? [{ counts, shown }] : [] + }) + for (const { counts, shown } of exposed) { + if (exposed.length > 1) { counts.ambiguous.add(signal.sessionID) continue } diff --git a/packages/opencode/test/altimate/learn/capture.test.ts b/packages/opencode/test/altimate/learn/capture.test.ts index 790b1e6dcd..75a6951cf4 100644 --- a/packages/opencode/test/altimate/learn/capture.test.ts +++ b/packages/opencode/test/altimate/learn/capture.test.ts @@ -3,7 +3,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" -import { correctedTurnStart, Capture, RETRY_THRESHOLD, ToolRetryTracker, type PartLike } from "../../../src/altimate/learn/capture" +import { correctedTurnStart, turnMessages, Capture, RETRY_THRESHOLD, ToolRetryTracker, type PartLike } from "../../../src/altimate/learn/capture" import * as Signals from "../../../src/altimate/learn/signals" const call = (id: string, tool: string, status: "error" | "completed" | "running", error = "boom", messageID = "m1") => ({ @@ -358,6 +358,43 @@ describe("correction turn start", () => { expect(correctedTurnStart(continued, "correction")).toBe(2000) expect(correctedTurnStart(continued.map((m) => ({ ...m, synthetic: false })), "correction")).toBe(4000) }) + test("history parts mark synthetic continuations and compaction markers", () => { + const info = (id: string, role: string, created: number) => ({ id, sessionID: "s", role, time: { created, completed: created + 1 } }) + const mapped = turnMessages([ + { info: info("u1", "user", 1000), parts: [{ type: "text" }, { type: "file" }] }, + { info: info("a1", "assistant", 2000), parts: [{ type: "text" }] }, + { info: info("continue", "user", 3000), parts: [{ type: "text", synthetic: true }] }, + { info: info("marker", "user", 3500), parts: [{ type: "compaction" }] }, + { info: info("files", "user", 3600), parts: [{ type: "file" }] }, + ]) + expect(mapped.map(({ synthetic, compaction }) => [synthetic, compaction])).toEqual([ + [false, false], [false, false], [true, false], [false, true], [false, false], + ]) + }) + test("a replayed request after compaction has no known turn start", () => { + const at = (id: string, role: string, created: number, extra = {}) => ({ id, sessionID: "s", role, time: { created, completed: created + 1 }, ...extra }) + const compacted = [ + at("u1", "user", 1000), at("step", "assistant", 2000), + at("marker", "user", 3000, { compaction: true }), at("summary", "assistant", 3100), + at("replay", "user", 3200), at("final", "assistant", 4000), at("correction", "user", 5000), + ] + expect(correctedTurnStart(compacted, "correction")).toBeUndefined() + // A manual compaction without a replay leaves the original request as the boundary. + expect(correctedTurnStart(compacted.filter((m) => m.id !== "replay"), "correction")).toBe(2000) + }) + test("a failed record is retried by a later update", async () => { + let attempts = 0 + const capture = new Capture({ + hasPriorAssistant: async () => true, + turnStart: async (_, id) => correctedTurnStart(history, id), + record: async () => { if (attempts++ === 0) throw new Error("lock timeout"); return undefined }, + }) + for (const info of history) capture.onMessage(info) + for (const text of ["No, use UTC", "No, use UTC instead."]) + await capture.onPart({ id: "part", messageID: "correction", sessionID: "s", type: "text", text }) + await capture.flush() + expect(attempts).toBe(2) + }) test("repeated updates of one correction look up history once", async () => { let lookups = 0 const capture = new Capture({ diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts index 2c0655e559..6e58fff67c 100644 --- a/packages/opencode/test/altimate/learn/recurrence.test.ts +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -99,7 +99,14 @@ describe("recurrence attribution", () => { shown.shown.push({ ...snapshot().shown[0], name: other.name }) const result = run([signal()], [shown], [lesson, other]) expect(result.lessons.find((row) => row.name === lesson.name)).toMatchObject({ ambiguous: 0, earlier_version: 1 }) - expect(result.lessons.find((row) => row.name === other.name)).toMatchObject({ ambiguous: 1 }) + expect(result.lessons.find((row) => row.name === other.name)).toMatchObject({ ambiguous: 0, recurred: 1 }) + }) + + test("a matching lesson the session never saw does not hide recurrence", () => { + const added: NamedLesson = { name: "team-playbook", lesson: { id: "L-0003", text: "Always convert `event_at` timestamps with `to_utc` first." } } + const result = run([signal("s1"), signal("s2")], [snapshot("s1"), snapshot("s2")], [lesson, added]) + expect(result.lessons.find((row) => row.id === lesson.lesson.id)).toMatchObject({ recurred: 2, ambiguous: 0 }) + expect(recurrenceHint(result, lesson.name, lesson.lesson)).toBe(2) }) test("earlier lesson version cannot supply current evidence", () => { From 873cff01072fb1da181bea2b67ff577b8e0e9bba Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 14:56:45 -0700 Subject: [PATCH 4/9] fix(learn): scope recurrence to the named store and harden snapshot reads - `learn status --name` and reflection attribute only that store. - Compaction replays (text metadata `learnOriginalMessage`) have unknown turn starts; attachment-only and agent-mention turns stay request boundaries even though their text companions are synthetic. - Snapshot reads fail closed on unsafe paths and skip a file whose name does not match its session. - The `r` prompt line asks for an EDIT only when the wording is unclear and to keep a correct rule the agent did not follow. - Tests: assert the analyzer really fails in the reflection fallback test, check status text with learning disabled, assert a real unmatched-topic line. Co-Authored-By: Claude Opus 5.5 --- docs/docs/configure/learn.md | 2 +- .../opencode/src/altimate/learn/capture.ts | 16 +++++++++++----- .../opencode/src/altimate/learn/delivery.ts | 2 ++ .../opencode/src/altimate/learn/prompt.txt | 2 +- .../opencode/src/altimate/learn/recurrence.ts | 4 +++- .../src/altimate/learn/session-reflect.ts | 2 +- packages/opencode/src/cli/cmd/learn.ts | 2 +- .../test/altimate/learn/capture.test.ts | 8 ++++++++ .../opencode/test/altimate/learn/cli.test.ts | 2 +- .../test/altimate/learn/recurrence.test.ts | 18 +++++++++++++++--- .../test/altimate/learn/reflect.test.ts | 2 +- .../altimate/learn/session-reflect.test.ts | 2 ++ 12 files changed, 47 insertions(+), 15 deletions(-) diff --git a/docs/docs/configure/learn.md b/docs/docs/configure/learn.md index 37a9bb2f19..d95a280e52 100644 --- a/docs/docs/configure/learn.md +++ b/docs/docs/configure/learn.md @@ -236,7 +236,7 @@ Shows whether learning is on, whether automatic promotion is on with its last pr A correction matches a lesson when it names one of the lesson's backtick identifiers plus another word from the lesson, or contains most of the lesson's words. The text output lists only lessons with a matching correction and summarizes the rest as "no observed correction". That is not proof the agent followed them. Corrections are recorded only while capture is on. Status also lists up to 5 identifiers that were corrected in 2 or more sessions with no matching current lesson. They are taken from your local corrections and never leave the project. If the analysis fails, status shows `Recurrence unavailable` (`recurrence_error` in JSON), and the rest of the status still prints. -When the current text of a lesson has `recurred` in 2 or more sessions, reflection marks it `r:N` in the playbook it sends the model. The model is asked to make the lesson clearer or more specific, never weaker. The result is a candidate like any other and goes through the same review or automatic-promotion gates. +When the current text of a lesson has `recurred` in 2 or more sessions, reflection marks it `r:N` in the playbook it sends the model. The model is asked to make the lesson clearer or more specific when the feedback shows the wording is unclear, and to keep it when the agent simply did not follow a correct rule. The result is a candidate like any other and goes through the same review or automatic-promotion gates. ```bash altimate-code learn status diff --git a/packages/opencode/src/altimate/learn/capture.ts b/packages/opencode/src/altimate/learn/capture.ts index 9e0380e609..cf16d22bd2 100644 --- a/packages/opencode/src/altimate/learn/capture.ts +++ b/packages/opencode/src/altimate/learn/capture.ts @@ -92,16 +92,22 @@ export interface MessageLike { time?: { created?: number; completed?: number } } -export type TurnMessage = MessageLike & { synthetic?: boolean; compaction?: boolean } +export type TurnMessage = MessageLike & { synthetic?: boolean; compaction?: boolean; replay?: boolean } -/** Session history as turn boundaries: all-synthetic text continues a turn; a compaction part marks compaction. */ -export function turnMessages(messages: readonly { info: MessageLike; parts: readonly { type: string; synthetic?: boolean }[] }[]): TurnMessage[] { +type TurnPart = { type: string; synthetic?: boolean; metadata?: Record } + +/** + * Session history as turn boundaries. All-synthetic text continues a turn unless the message carries user input + * (an attachment or agent mention, whose text companions are synthetic). Compaction and its replay are marked. + */ +export function turnMessages(messages: readonly { info: MessageLike; parts: readonly TurnPart[] }[]): TurnMessage[] { return messages.map(({ info, parts }) => { const text = parts.filter((part) => part.type === "text") return { ...info, - synthetic: text.length > 0 && text.every((part) => part.synthetic), + synthetic: text.length > 0 && text.every((part) => part.synthetic) && !parts.some((part) => part.type === "file" || part.type === "agent"), compaction: parts.some((part) => part.type === "compaction"), + replay: text.some((part) => typeof part.metadata?.learnOriginalMessage === "string"), } }) } @@ -114,7 +120,7 @@ export function correctedTurnStart(messages: readonly TurnMessage[], messageID: const end = messages.findIndex((message) => message.id === messageID && message.role === "user") if (end < 0) return const previous = messages.slice(0, end).findLastIndex((message) => message.role === "user" && !message.synthetic && !message.compaction) - if (previous < 0) return + if (previous < 0 || messages[previous].replay) return if (messages.slice(0, previous).findLast((message) => message.role === "user")?.compaction) return return messages.slice(previous + 1, end).find((message) => message.role === "assistant")?.time?.created } diff --git a/packages/opencode/src/altimate/learn/delivery.ts b/packages/opencode/src/altimate/learn/delivery.ts index 7bdcf9eeae..c246f91db9 100644 --- a/packages/opencode/src/altimate/learn/delivery.ts +++ b/packages/opencode/src/altimate/learn/delivery.ts @@ -103,10 +103,12 @@ export async function readShownSnapshots(root: string, options: { limit?: number for (const { file } of files.sort((a, b) => b.mtime - a.mtime || a.file.localeCompare(b.file)).slice(0, limit)) { try { const state = State.parse(JSON.parse((await read(root, file)) ?? "")) + if (path.basename(file) !== Store.sha256(state.session) + ".json") throw new Error("Learn session state belongs to another session") snapshots.push({ session: state.session, shown: state.shown.map(({ name, lesson, tier, at }) => ({ name, id: lesson.id, textHash: lessonTextHash(lesson.text), tier, at, })) }) } catch (error) { + if (error instanceof SafeFS.UnsafeLearnPathError) throw error coverage.sessions_skipped_malformed++ log.warn("learn session snapshot skipped", { file, error }) } diff --git a/packages/opencode/src/altimate/learn/prompt.txt b/packages/opencode/src/altimate/learn/prompt.txt index b7e6027ed5..68275d4fbf 100644 --- a/packages/opencode/src/altimate/learn/prompt.txt +++ b/packages/opencode/src/altimate/learn/prompt.txt @@ -36,4 +36,4 @@ Bullet text rules (deterministic checks reject unsafe content and flag verificat Output: a JSON object {"deltas": [...]}. Each delta is {"op": "ADD"|"EDIT"|"REMOVE"|"HELPFUL"|"HARMFUL", "id": "", "text": "", "reason": ""}. Optional fields: "supersedes": "" for ADD, and "coexists": ["", ...] for ADD or EDIT. Omit these fields when they do not apply. -r = sessions where this rule was shown and the user still corrected the same topic; consider making it more specific or clearer (EDIT), never weaker +r = sessions where this rule was shown and the user still corrected the same topic. If the feedback shows the wording is unclear or incomplete, EDIT it to be clearer or more specific. If the agent simply did not follow a correct rule, keep it. Narrow a rule only when the feedback shows it is too broad; never drop the behavior it requires. diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts index 029ab6b0a1..ae28855941 100644 --- a/packages/opencode/src/altimate/learn/recurrence.ts +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -107,7 +107,8 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS } } -export async function analyzeRecurrence(root: string, options: { limit?: number } = {}) { +/** With `name`, only that store's lessons and signals are attributed. */ +export async function analyzeRecurrence(root: string, options: { limit?: number; name?: string } = {}) { const { snapshots, coverage } = await readShownSnapshots(root, options) const dir = path.join(root, ".altimate-code", "learn") await SafeFS.assertSafePath(root, dir) @@ -121,6 +122,7 @@ export async function analyzeRecurrence(root: string, options: { limit?: number for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { if (!entry.isDirectory()) continue try { validateName(entry.name) } catch { continue } + if (options.name !== undefined && entry.name !== options.name) continue if (sessions.size) signals.push(...(await readSignalsSnapshot(root, entry.name)).filter((signal) => sessions.has(signal.sessionID))) const file = Store.paths(root, entry.name).approved try { diff --git a/packages/opencode/src/altimate/learn/session-reflect.ts b/packages/opencode/src/altimate/learn/session-reflect.ts index b6ed72004e..9edd775d2c 100644 --- a/packages/opencode/src/altimate/learn/session-reflect.ts +++ b/packages/opencode/src/altimate/learn/session-reflect.ts @@ -108,7 +108,7 @@ export async function reflectCore(input: ReflectCoreInput): Promise [b.id, b.text])) - const recurrence = await analyzeRecurrence(root).catch((error) => { + const recurrence = await analyzeRecurrence(root, { name }).catch((error) => { log.warn("learn recurrence unavailable", { error }) return undefined }) diff --git a/packages/opencode/src/cli/cmd/learn.ts b/packages/opencode/src/cli/cmd/learn.ts index 2a7bb0420d..e54f2a47f6 100644 --- a/packages/opencode/src/cli/cmd/learn.ts +++ b/packages/opencode/src/cli/cmd/learn.ts @@ -381,7 +381,7 @@ const StatusCommand = effectCmd({ retired: (await Store.loadRetired(root, name)).length, open_signals: signals.length, // Attribution is advisory; its failure must not hide the rest of the status. - ...await analyzeRecurrence(root).then((recurrence) => ({ recurrence }), (error) => ({ recurrence_error: error instanceof Error ? error.message : String(error) })), + ...await analyzeRecurrence(root, { name }).then((recurrence) => ({ recurrence }), (error) => ({ recurrence_error: error instanceof Error ? error.message : String(error) })), pending_recoveries: sessions.length, pending_replacements: (await Store.readPendingReplacements(root, name)).length, backoff_sessions: sessions.filter((session) => (state.recoveries[session]?.retryAt ?? 0) > Date.now()).length, diff --git a/packages/opencode/test/altimate/learn/capture.test.ts b/packages/opencode/test/altimate/learn/capture.test.ts index 75a6951cf4..800af173d0 100644 --- a/packages/opencode/test/altimate/learn/capture.test.ts +++ b/packages/opencode/test/altimate/learn/capture.test.ts @@ -370,6 +370,11 @@ describe("correction turn start", () => { expect(mapped.map(({ synthetic, compaction }) => [synthetic, compaction])).toEqual([ [false, false], [false, false], [true, false], [false, true], [false, false], ]) + const [attachment, replay] = turnMessages([ + { info: info("attach", "user", 4000), parts: [{ type: "text", synthetic: true }, { type: "file" }] }, + { info: info("replay", "user", 4100), parts: [{ type: "text", metadata: { learnOriginalMessage: "u1" } }] }, + ]) + expect([attachment.synthetic, replay.synthetic, replay.replay]).toEqual([false, false, true]) }) test("a replayed request after compaction has no known turn start", () => { const at = (id: string, role: string, created: number, extra = {}) => ({ id, sessionID: "s", role, time: { created, completed: created + 1 }, ...extra }) @@ -379,6 +384,9 @@ describe("correction turn start", () => { at("replay", "user", 3200), at("final", "assistant", 4000), at("correction", "user", 5000), ] expect(correctedTurnStart(compacted, "correction")).toBeUndefined() + // A replay marked by its text metadata is unknown even when no marker precedes it. + const marked = compacted.filter((m) => m.id !== "marker").map((m) => m.id === "replay" ? { ...m, replay: true } : m) + expect(correctedTurnStart(marked, "correction")).toBeUndefined() // A manual compaction without a replay leaves the original request as the boundary. expect(correctedTurnStart(compacted.filter((m) => m.id !== "replay"), "correction")).toBe(2000) }) diff --git a/packages/opencode/test/altimate/learn/cli.test.ts b/packages/opencode/test/altimate/learn/cli.test.ts index a53a6a66d2..59219f06e2 100644 --- a/packages/opencode/test/altimate/learn/cli.test.ts +++ b/packages/opencode/test/altimate/learn/cli.test.ts @@ -176,7 +176,7 @@ describe("learn opt-in and status", () => { lessons: [], unmatched_topics: [], coverage: { sessions_analyzed: 0, sessions_skipped_malformed: 0, truncated: false }, }) - const shown = await learn(dir.path, "status") + const shown = await runLearn(dir.path, ["status"], undefined, { ALTIMATE_LEARN: "0" }) expect(shown.stdout).toContain("Recurrence (observed counts need capture on)") expect(shown.stdout).toContain("no matching current lesson") }) diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts index 6e58fff67c..ae1c25eaf6 100644 --- a/packages/opencode/test/altimate/learn/recurrence.test.ts +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -150,7 +150,9 @@ describe("recurrence attribution", () => { }) test("human status reports coverage, uncertainty and no matching current lesson", () => { - const text = formatRecurrence({ ...run([]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + const unmatched = (session: string) => signal(session, { id: `sig_u_${session}`, text: "No, filter out _is_deleted rows" }) + const text = formatRecurrence({ ...run([unmatched("s1"), unmatched("s2")], [snapshot("s1"), snapshot("s2")]), coverage: { sessions_analyzed: 2, sessions_skipped_malformed: 0, truncated: false } }) + expect(text).toContain(" _is_deleted: 2 sessions (sig_u_s1, sig_u_s2)") expect(text).toContain("Recurrence") expect(text).toContain("no observed correction") expect(text).toContain("counts need capture on") @@ -168,8 +170,8 @@ test("snapshot analysis is bounded, skips malformed files and never repairs any name: lesson.name, lesson: { ...lesson.lesson, tags: [], scope: "project", helpful: 0, harmful: 0, applied: 0, created: time(0), updated: time(0) }, tier: "file", at: time(1), queryHash: "hash", }], requests: [], compactions: [], counted: [] }) - const old = path.join(dir, "old.json") - const good = path.join(dir, "good.json") + const old = path.join(dir, Store.sha256("old") + ".json") + const good = path.join(dir, Store.sha256("s1") + ".json") const bad = path.join(dir, "bad.json") await fs.writeFile(old, JSON.stringify(state("old"))) await fs.writeFile(good, JSON.stringify(state("s1"))) @@ -190,5 +192,15 @@ test("snapshot analysis is bounded, skips malformed files and never repairs any expect(await fs.readFile(signals, "utf8")).toBe(raw) expect((await fs.readdir(p.learnDir)).sort()).toEqual([path.basename(p.approved), "signals.jsonl"].sort()) expect((await readShownSnapshots(root, { limit: 0 })).coverage).toMatchObject({ sessions_analyzed: 0, truncated: true }) + // A copied snapshot under another session's file name is not that session's evidence. + await fs.copyFile(good, path.join(dir, Store.sha256("copy") + ".json")) + expect((await readShownSnapshots(root)).coverage).toMatchObject({ sessions_analyzed: 2, sessions_skipped_malformed: 2 }) + // Another store's lessons and signals stay out of a named analysis. + expect((await analyzeRecurrence(root, { name: "other-store" })).lessons.every((row) => row.recurred === 0)).toBe(true) + // Unbounded, both valid sessions ("old" and "s1") carry a matching correction after exposure. + expect((await analyzeRecurrence(root, { name: lesson.name })).lessons[0]).toMatchObject({ recurred: 2 }) + // An unsafe snapshot path fails closed instead of counting as malformed. + await fs.symlink(good, path.join(dir, Store.sha256("link") + ".json")) + await expect(readShownSnapshots(root)).rejects.toThrow() } finally { await fs.rm(root, { recursive: true, force: true }) } }) diff --git a/packages/opencode/test/altimate/learn/reflect.test.ts b/packages/opencode/test/altimate/learn/reflect.test.ts index 0ce7b9aef9..05a13c3c85 100644 --- a/packages/opencode/test/altimate/learn/reflect.test.ts +++ b/packages/opencode/test/altimate/learn/reflect.test.ts @@ -15,7 +15,7 @@ describe("buildPrompt", () => { for (const recurred of [undefined, 0, 1, 2, 3]) { const { prompt, system } = buildPrompt({ digest: "d", feedback: "f", kind: "user", bullets: [{ ...bullets[0], recurred }] }) expect(prompt.includes(` r:${recurred}`)).toBe((recurred ?? 0) >= 2) - expect(system).toContain("never weaker") + expect(system).toContain("If the agent simply did not follow a correct rule, keep it.") } }) diff --git a/packages/opencode/test/altimate/learn/session-reflect.test.ts b/packages/opencode/test/altimate/learn/session-reflect.test.ts index 8fefb58928..cc1eb06988 100644 --- a/packages/opencode/test/altimate/learn/session-reflect.test.ts +++ b/packages/opencode/test/altimate/learn/session-reflect.test.ts @@ -1,6 +1,7 @@ // altimate_change - new file // // `reflect --session` from captured signals, with the model call and the session source stubbed. +import { analyzeRecurrence } from "../../../src/altimate/learn/recurrence" import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import fs from "node:fs/promises" import os from "node:os" @@ -825,6 +826,7 @@ describe("reflection recurrence evidence", () => { const target = path.join(root, "snapshots") await fs.rename(dir, target) await fs.symlink(target, dir) + await expect(analyzeRecurrence(root, { name: NAME })).rejects.toThrow() let called = false await reflectCore({ root, name: NAME, source: { prompts: [], calls: [] }, feedback: "No, use UTC.", kind: "user", origin: "test", From 3bcde3d56af9388b02353cd410560491de402ef3 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 16:09:00 -0700 Subject: [PATCH 5/9] fix(learn): ignore exposures after the correction; earlier versions need a correction - A lesson first shown at or after a correction neither competes for it nor counts as unknown; unparseable times still resolve to unknown. - `earlier_version` counts a session only when a matching correction found only an earlier text shown, and text status lists those lessons. - Docs: exact matching thresholds, strict ordering, current-text scope. Co-Authored-By: Claude Opus 5.5 --- docs/docs/configure/learn.md | 8 +++---- .../opencode/src/altimate/learn/recurrence.ts | 18 ++++++++------- .../test/altimate/learn/recurrence.test.ts | 22 ++++++++++++++++++- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/docs/docs/configure/learn.md b/docs/docs/configure/learn.md index d95a280e52..e582cc9667 100644 --- a/docs/docs/configure/learn.md +++ b/docs/docs/configure/learn.md @@ -228,13 +228,13 @@ Shows whether learning is on, whether automatic promotion is on with its last pr | Count | Meaning | |---|---| -| `recurred` | after the lesson was shown, at the start of the corrected request or earlier | +| `recurred` | after the lesson was shown, before the agent started answering the corrected request | | `during` | while the agent was answering, for example a file-hook lesson added mid-turn; the lesson may have arrived too late | | `unknown` | in an order that cannot be told, for example older corrections or a request replayed after compaction | -| `ambiguous` | when the correction matched more than one lesson the session was shown | -| `earlier_version` | to a different version of the lesson text; it never counts for the current text | +| `ambiguous` | when the correction matched more than one lesson the session was shown in its current text | +| `earlier_version` | when the session was shown only an earlier version of the lesson text; it never counts for the current text | -A correction matches a lesson when it names one of the lesson's backtick identifiers plus another word from the lesson, or contains most of the lesson's words. The text output lists only lessons with a matching correction and summarizes the rest as "no observed correction". That is not proof the agent followed them. Corrections are recorded only while capture is on. Status also lists up to 5 identifiers that were corrected in 2 or more sessions with no matching current lesson. They are taken from your local corrections and never leave the project. If the analysis fails, status shows `Recurrence unavailable` (`recurrence_error` in JSON), and the rest of the status still prints. +A correction matches a lesson when it names one of the lesson's full backtick identifiers plus another word from the lesson, or contains at least 3 of the lesson's words and 60% of them. A prefix or suffix identifier such as `_at` or `stg_` never matches on its own. Lessons first shown after the correction are ignored for it. The text output lists only lessons with a matching correction and summarizes the rest as "no observed correction". That is not proof the agent followed them. Corrections are recorded only while capture is on. Status also lists up to 5 identifiers that were corrected in 2 or more sessions with no matching current lesson. They are taken from your local corrections and never leave the project. If the analysis fails, status shows `Recurrence unavailable` (`recurrence_error` in JSON), and the rest of the status still prints. When the current text of a lesson has `recurred` in 2 or more sessions, reflection marks it `r:N` in the playbook it sends the model. The model is asked to make the lesson clearer or more specific when the feedback shows the wording is unclear, and to keep it when the agent simply did not follow a correct rule. The result is a candidate like any other and goes through the same review or automatic-promotion gates. diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts index ae28855941..3059f9315e 100644 --- a/packages/opencode/src/altimate/learn/recurrence.ts +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -50,7 +50,6 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS }) const counts = rows.get(identity)!.counts counts.sessions_shown.add(snapshot.session) - if (current && shown.textHash !== current.textHash) counts.earlier_version.add(snapshot.session) } } const unmatched = new Map; signals: Set }>() @@ -73,12 +72,16 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS } continue } - // Only lessons this session saw in their current text compete; one never shown cannot explain the correction. + // Only lessons this session saw in their current text before the correction compete; a lesson never + // shown, or first shown afterwards, cannot explain it. Unparseable times stay in and resolve to unknown. + const correction = Date.parse(signal.at) const exposed = matches.flatMap((lesson) => { const counts = rows.get(key(lesson.name, lesson.id, lesson.textHash))?.counts - const shown = sessions.get(signal.sessionID)!.filter((entry) => - entry.name === lesson.name && entry.id === lesson.id && entry.textHash === lesson.textHash) - return counts && shown.length ? [{ counts, shown }] : [] + const shown = sessions.get(signal.sessionID)!.filter((entry) => entry.name === lesson.name && entry.id === lesson.id && + !(Number.isFinite(correction) && Date.parse(entry.at) >= correction)) + const current = shown.filter((entry) => entry.textHash === lesson.textHash) + if (counts && !current.length && shown.length) counts.earlier_version.add(signal.sessionID) + return counts && current.length ? [{ counts, shown: current }] : [] }) for (const { counts, shown } of exposed) { if (exposed.length > 1) { @@ -88,8 +91,7 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS for (const entry of shown) { const at = Date.parse(entry.at) const start = Date.parse(signal.turnStartedAt ?? "") - const correction = Date.parse(signal.at) - if (!Number.isFinite(at) || !Number.isFinite(start) || !Number.isFinite(correction) || start >= correction || at >= correction) + if (!Number.isFinite(at) || !Number.isFinite(start) || !Number.isFinite(correction) || start >= correction) counts.unknown.add(signal.sessionID) else if (at < start) counts.recurred.add(signal.sessionID) else counts.during.add(signal.sessionID) @@ -141,7 +143,7 @@ export function recurrenceHint(result: Pick, "lesso } // Uncorrected lessons are summarized; a store can hold up to 1,000. -const corrected = (lesson: LessonRecurrence) => lesson.recurred + lesson.during + lesson.unknown + lesson.ambiguous > 0 +const corrected = (lesson: LessonRecurrence) => lesson.recurred + lesson.during + lesson.unknown + lesson.ambiguous + lesson.earlier_version > 0 export function formatRecurrence(result: Awaited>) { return [ diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts index ae1c25eaf6..ba280e3934 100644 --- a/packages/opencode/test/altimate/learn/recurrence.test.ts +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -48,7 +48,9 @@ describe("recurrence attribution", () => { test("missing, invalid and impossible ordering stays unknown", () => { for (const turnStartedAt of [undefined, "bad", time(6)]) expect(run([signal("s1", { turnStartedAt })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 1 }) - expect(run([signal()], [snapshot("s1", { at: time(5) })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 1 }) + // Shown at the correction's own instant cannot have shaped the corrected response. + expect(run([signal()], [snapshot("s1", { at: time(5) })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 0 }) + expect(run([signal()], [snapshot("s1", { at: "bad" })]).lessons[0]).toMatchObject({ recurred: 0, unknown: 1 }) }) test("a new convention or a shared identifier about a different property does not match", () => { @@ -109,6 +111,24 @@ describe("recurrence attribution", () => { expect(recurrenceHint(result, lesson.name, lesson.lesson)).toBe(2) }) + test("an earlier version counts only with a matching correction", () => { + const old = snapshot("s1", { textHash: lessonTextHash("Earlier text") }) + expect(run([], [old]).lessons[0]).toMatchObject({ earlier_version: 0, sessions_shown: 1 }) + const text = formatRecurrence({ ...run([signal()], [old]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + expect(text).toContain("team-playbook/L-0001:") + expect(text).toContain("1 earlier version") + }) + + test("a lesson first shown after the correction is ignored and does not compete", () => { + expect(run([signal()], [snapshot("s1", { at: time(6) })]).lessons[0]).toMatchObject({ unknown: 0, during: 0, recurred: 0 }) + const other = { ...lesson, name: "other-store" } + const shown = snapshot() + shown.shown.push({ ...shown.shown[0], name: other.name, at: time(6) }) + const result = run([signal()], [shown], [lesson, other]) + expect(result.lessons.find((row) => row.name === lesson.name)).toMatchObject({ recurred: 1, ambiguous: 0 }) + expect(result.lessons.find((row) => row.name === other.name)).toMatchObject({ recurred: 0, ambiguous: 0, unknown: 0 }) + }) + test("earlier lesson version cannot supply current evidence", () => { expect(run([signal()], [snapshot("s1", { textHash: lessonTextHash("Earlier text") })]).lessons[0]) .toMatchObject({ earlier_version: 1, recurred: 0 }) From 6d8eb89e67a98ec52c9ea44059159552c74c7645 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 16:15:15 -0700 Subject: [PATCH 6/9] fix(learn): scope exposures to the named store; analyze outside the learn lock - `analyzeRecurrence({ name })` also filters session exposures, so named status lists no other store's lessons. - `learn status` runs recurrence after releasing the learn lock. - Docs: reflection may narrow a too-broad rule but keeps its required behavior. Co-Authored-By: Claude Opus 5.5 --- docs/docs/configure/learn.md | 2 +- packages/opencode/src/altimate/learn/recurrence.ts | 7 +++++-- packages/opencode/src/cli/cmd/learn.ts | 8 +++++--- packages/opencode/test/altimate/learn/recurrence.test.ts | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/docs/configure/learn.md b/docs/docs/configure/learn.md index e582cc9667..67a80b0f0c 100644 --- a/docs/docs/configure/learn.md +++ b/docs/docs/configure/learn.md @@ -236,7 +236,7 @@ Shows whether learning is on, whether automatic promotion is on with its last pr A correction matches a lesson when it names one of the lesson's full backtick identifiers plus another word from the lesson, or contains at least 3 of the lesson's words and 60% of them. A prefix or suffix identifier such as `_at` or `stg_` never matches on its own. Lessons first shown after the correction are ignored for it. The text output lists only lessons with a matching correction and summarizes the rest as "no observed correction". That is not proof the agent followed them. Corrections are recorded only while capture is on. Status also lists up to 5 identifiers that were corrected in 2 or more sessions with no matching current lesson. They are taken from your local corrections and never leave the project. If the analysis fails, status shows `Recurrence unavailable` (`recurrence_error` in JSON), and the rest of the status still prints. -When the current text of a lesson has `recurred` in 2 or more sessions, reflection marks it `r:N` in the playbook it sends the model. The model is asked to make the lesson clearer or more specific when the feedback shows the wording is unclear, and to keep it when the agent simply did not follow a correct rule. The result is a candidate like any other and goes through the same review or automatic-promotion gates. +When the current text of a lesson has `recurred` in 2 or more sessions, reflection marks it `r:N` in the playbook it sends the model. The model is asked to make the lesson clearer or more specific when the feedback shows the wording is unclear, and to keep it when the agent simply did not follow a correct rule. It may narrow a rule only when the feedback shows the rule is too broad, and must keep the behavior the rule requires. The result is a candidate like any other and goes through the same review or automatic-promotion gates. ```bash altimate-code learn status diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts index 3059f9315e..2f6fcdf9b7 100644 --- a/packages/opencode/src/altimate/learn/recurrence.ts +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -109,9 +109,12 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS } } -/** With `name`, only that store's lessons and signals are attributed. */ +/** With `name`, only that store's lessons, signals and exposures are attributed. */ export async function analyzeRecurrence(root: string, options: { limit?: number; name?: string } = {}) { - const { snapshots, coverage } = await readShownSnapshots(root, options) + const read = await readShownSnapshots(root, options) + const { coverage } = read + const snapshots = options.name === undefined ? read.snapshots + : read.snapshots.map((snapshot) => ({ ...snapshot, shown: snapshot.shown.filter((entry) => entry.name === options.name) })) const dir = path.join(root, ".altimate-code", "learn") await SafeFS.assertSafePath(root, dir) const entries = await fs.readdir(dir, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { diff --git a/packages/opencode/src/cli/cmd/learn.ts b/packages/opencode/src/cli/cmd/learn.ts index e54f2a47f6..8fbe0621dc 100644 --- a/packages/opencode/src/cli/cmd/learn.ts +++ b/packages/opencode/src/cli/cmd/learn.ts @@ -380,8 +380,6 @@ const StatusCommand = effectCmd({ candidate: (await Store.loadCandidateLessons(root, name))?.length ?? 0, retired: (await Store.loadRetired(root, name)).length, open_signals: signals.length, - // Attribution is advisory; its failure must not hide the rest of the status. - ...await analyzeRecurrence(root, { name }).then((recurrence) => ({ recurrence }), (error) => ({ recurrence_error: error instanceof Error ? error.message : String(error) })), pending_recoveries: sessions.length, pending_replacements: (await Store.readPendingReplacements(root, name)).length, backoff_sessions: sessions.filter((session) => (state.recoveries[session]?.retryAt ?? 0) > Date.now()).length, @@ -390,7 +388,11 @@ const StatusCommand = effectCmd({ limits: { ...resolveLimits(learn), max_stored: learnMaxStored(learn?.max_stored), ...resolveRecoveryLimits(learn), ...resolveAutoPromoteLimits(learn) }, // altimate_change end } - }) + }).then(async (base) => ({ + ...base, + // Read-only and advisory: runs after the lock is released; its failure must not hide the rest of the status. + ...await analyzeRecurrence(root, { name }).then((recurrence) => ({ recurrence }), (error) => ({ recurrence_error: error instanceof Error ? error.message : String(error) })), + })) if (args.json) return out(JSON.stringify(status, null, 2)) out(`Learning enabled: ${status.enabled ? "yes" : "no"}`) // altimate_change start — surface the learning kill switch in human-readable status diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts index ba280e3934..49e3ec2d8c 100644 --- a/packages/opencode/test/altimate/learn/recurrence.test.ts +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -216,7 +216,7 @@ test("snapshot analysis is bounded, skips malformed files and never repairs any await fs.copyFile(good, path.join(dir, Store.sha256("copy") + ".json")) expect((await readShownSnapshots(root)).coverage).toMatchObject({ sessions_analyzed: 2, sessions_skipped_malformed: 2 }) // Another store's lessons and signals stay out of a named analysis. - expect((await analyzeRecurrence(root, { name: "other-store" })).lessons.every((row) => row.recurred === 0)).toBe(true) + expect((await analyzeRecurrence(root, { name: "other-store" })).lessons).toEqual([]) // Unbounded, both valid sessions ("old" and "s1") carry a matching correction after exposure. expect((await analyzeRecurrence(root, { name: lesson.name })).lessons[0]).toMatchObject({ recurred: 2 }) // An unsafe snapshot path fails closed instead of counting as malformed. From 86cc88fdc0c1c19b503b846af5aac64b2c787aaa Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 16:22:10 -0700 Subject: [PATCH 7/9] fix(learn): order exposures against the correction message time Capture records `correctedAt`, the creation time of the correcting user message. Attribution compares exposures with it instead of the later signal-persistence time, so lessons selected for the correction request itself never count as exposures for the response being corrected. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/learn/capture.ts | 1 + packages/opencode/src/altimate/learn/recurrence.ts | 3 ++- packages/opencode/src/altimate/learn/signals.ts | 6 +++++- packages/opencode/test/altimate/learn/capture.test.ts | 1 + packages/opencode/test/altimate/learn/recurrence.test.ts | 8 ++++++++ packages/opencode/test/altimate/learn/signals.test.ts | 7 +++++-- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/learn/capture.ts b/packages/opencode/src/altimate/learn/capture.ts index cf16d22bd2..700d6fd69a 100644 --- a/packages/opencode/src/altimate/learn/capture.ts +++ b/packages/opencode/src/altimate/learn/capture.ts @@ -217,6 +217,7 @@ export class Capture { await this.deps.record({ kind: "user_correction", ...(start !== undefined && Number.isFinite(start) ? { turnStartedAt: new Date(start).toISOString() } : {}), + ...(message.time?.created !== undefined && Number.isFinite(message.time.created) ? { correctedAt: new Date(message.time.created).toISOString() } : {}), sessionID: part.sessionID, messageID: part.messageID, text: part.text, diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts index 2f6fcdf9b7..6c5dad2d70 100644 --- a/packages/opencode/src/altimate/learn/recurrence.ts +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -74,7 +74,8 @@ export function attribute(signals: readonly Signal[], snapshots: readonly ShownS } // Only lessons this session saw in their current text before the correction compete; a lesson never // shown, or first shown afterwards, cannot explain it. Unparseable times stay in and resolve to unknown. - const correction = Date.parse(signal.at) + // Persistence can lag the message; lessons selected for the correction itself must not count as earlier. + const correction = Date.parse(signal.correctedAt ?? signal.at) const exposed = matches.flatMap((lesson) => { const counts = rows.get(key(lesson.name, lesson.id, lesson.textHash))?.counts const shown = sessions.get(signal.sessionID)!.filter((entry) => entry.name === lesson.name && entry.id === lesson.id && diff --git a/packages/opencode/src/altimate/learn/signals.ts b/packages/opencode/src/altimate/learn/signals.ts index 6eff4768d2..1d8d4ac233 100644 --- a/packages/opencode/src/altimate/learn/signals.ts +++ b/packages/opencode/src/altimate/learn/signals.ts @@ -37,11 +37,13 @@ export interface Signal { reason: string at: string turnStartedAt?: string + /** Creation time of the correcting user message; lessons selected for it come after this. */ + correctedAt?: string status: "open" | "consumed" consumedBy?: string } -export type NewSignal = Pick +export type NewSignal = Pick export function signalsFile(root: string, name = DEFAULT_NAME): string { validateName(name) @@ -94,6 +96,7 @@ function parse(raw: string): { signals: Signal[]; malformed: boolean } { (s.status === "open" || s.status === "consumed") && (s.messageID === undefined || typeof s.messageID === "string") && (s.turnStartedAt === undefined || (typeof s.turnStartedAt === "string" && Number.isFinite(Date.parse(s.turnStartedAt)))) && + (s.correctedAt === undefined || (typeof s.correctedAt === "string" && Number.isFinite(Date.parse(s.correctedAt)))) && (s.partID === undefined || typeof s.partID === "string") && (s.source === undefined || s.source === "bootstrap" || s.source === "import-reviews") && (s.provenance === undefined || typeof s.provenance === "string") && @@ -231,6 +234,7 @@ export function appendSignals(root: string, inputs: readonly NewSignal[], name = messageID: input.messageID, partID: input.partID, turnStartedAt: input.turnStartedAt, + correctedAt: input.correctedAt, source: input.source, provenance: input.provenance === undefined ? undefined : redactProvenance(input.provenance), resolved: input.resolved, diff --git a/packages/opencode/test/altimate/learn/capture.test.ts b/packages/opencode/test/altimate/learn/capture.test.ts index 800af173d0..3f17381642 100644 --- a/packages/opencode/test/altimate/learn/capture.test.ts +++ b/packages/opencode/test/altimate/learn/capture.test.ts @@ -429,6 +429,7 @@ describe("correction turn start", () => { await capture.flush() expect(lookups).toBe(1) expect(recorded[0].turnStartedAt).toBe(new Date(2000).toISOString()) + expect(recorded[0].correctedAt).toBe(new Date(5000).toISOString()) }) test("uncertain history omits the field and lookup failure preserves the signal", async () => { for (const fail of [false, true]) { diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts index 49e3ec2d8c..e3949a976e 100644 --- a/packages/opencode/test/altimate/learn/recurrence.test.ts +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -119,6 +119,14 @@ describe("recurrence attribution", () => { expect(text).toContain("1 earlier version") }) + test("a lesson selected for the correction itself is not an earlier exposure, even if the signal is written later", () => { + // Correction message at t=4, request lesson selected at t=4.5, signal persisted at t=5. + const late = signal("s1", { correctedAt: time(4) }) + expect(run([late], [snapshot("s1", { at: time(4.5) })]).lessons[0]).toMatchObject({ during: 0, unknown: 0, recurred: 0 }) + expect(run([late], [snapshot("s1", { at: time(3) })]).lessons[0]).toMatchObject({ during: 1 }) + expect(run([late]).lessons[0]).toMatchObject({ recurred: 1 }) + }) + test("a lesson first shown after the correction is ignored and does not compete", () => { expect(run([signal()], [snapshot("s1", { at: time(6) })]).lessons[0]).toMatchObject({ unknown: 0, during: 0, recurred: 0 }) const other = { ...lesson, name: "other-store" } diff --git a/packages/opencode/test/altimate/learn/signals.test.ts b/packages/opencode/test/altimate/learn/signals.test.ts index 2149c7e006..8667a52359 100644 --- a/packages/opencode/test/altimate/learn/signals.test.ts +++ b/packages/opencode/test/altimate/learn/signals.test.ts @@ -17,12 +17,15 @@ describe("signal store", () => { test("old signals parse and correction turn starts round-trip read-only", async () => { await Signals.appendSignal(root, { ...base, text: "No, keep original", messageID: "old" }) const turnStartedAt = "2026-10-18T10:00:00.000Z" - await Signals.appendSignal(root, { ...base, text: "No, keep timestamps", messageID: "new", turnStartedAt }) + const correctedAt = "2026-10-18T10:00:05.000Z" + await Signals.appendSignal(root, { ...base, text: "No, keep timestamps", messageID: "new", turnStartedAt, correctedAt }) const records = await Signals.readSignalsSnapshot(root) expect(records[0].turnStartedAt).toBeUndefined() - expect(records[1].turnStartedAt).toBe(turnStartedAt) + expect(records[0].correctedAt).toBeUndefined() + expect(records[1]).toMatchObject({ turnStartedAt, correctedAt }) const file = Signals.signalsFile(root) await fs.appendFile(file, JSON.stringify({ ...records[1], id: "invalid", turnStartedAt: 42 }) + "\n") + await fs.appendFile(file, JSON.stringify({ ...records[1], id: "invalid2", correctedAt: "never" }) + "\n") expect(await Signals.readSignalsSnapshot(root)).toHaveLength(2) }) From 28031380597608190a86fa6d9a2a97b4d35da521 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 20:07:16 -0700 Subject: [PATCH 8/9] fix(learn): no helpful vote for a correct lesson the agent missed The reflect prompt told the model that feedback asking for exactly what a lesson says is HELPFUL, even when the agent had ignored that lesson. An ignored lesson gained helpful votes, which raise its core ranking and offset harmful votes, and it contradicted the new `r:N` hint. HELPFUL now requires the digest to show the agent followed the lesson; a correct but missed lesson keeps its counters and is edited only if its wording is unclear. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/learn/prompt.txt | 4 ++-- packages/opencode/test/altimate/learn/reflect.test.ts | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/learn/prompt.txt b/packages/opencode/src/altimate/learn/prompt.txt index 68275d4fbf..25e922889f 100644 --- a/packages/opencode/src/altimate/learn/prompt.txt +++ b/packages/opencode/src/altimate/learn/prompt.txt @@ -16,8 +16,8 @@ How to work: 2. Infer general conventions. State one rule with the exact identifier it governs, without rationale or incident details. Keep the specific table, model, column, or file identifier when the correction depends on it; do not substitute a vague generality. 3. Learn only from the external feedback. Do not treat the agent's own claims of success as evidence. 4. For an existing bullet: - - HELPFUL when the feedback shows the agent followed it and the related checks passed. - - HARMFUL only with explicit evidence that the agent FOLLOWED the bullet and that doing so caused a failure or a reviewer objection. The agent not following a bullet, feedback that is unrelated to the bullet, or feedback that asks for exactly what the bullet says is never HARMFUL (the last is HELPFUL). If unsure, leave the bullet alone. + - HELPFUL only when the DIGEST shows the agent followed it and the feedback shows the related checks passed. When the feedback asks for exactly what the bullet says because the agent did not follow it, the bullet is right but was missed: it is neither HELPFUL nor HARMFUL. Leave its counters alone, and EDIT it only if its wording is unclear. + - HARMFUL only with explicit evidence that the agent FOLLOWED the bullet and that doing so caused a failure or a reviewer objection. The agent not following a bullet, feedback that is unrelated to the bullet, or feedback that asks for exactly what the bullet says is never HARMFUL. If unsure, leave the bullet alone. - When feedback states a different expected behaviour than a bullet, EDIT the bullet to that expected behaviour (or ADD with "supersedes"). HARMFUL alone loses the knowledge. Keep the corrected convention general. - REMOVE when the feedback shows it is plainly wrong or outdated. 5. Before ADD or EDIT, compare its code identifiers, naming prefixes and suffixes in backticks with every existing bullet. If another bullet touches the same convention, explicitly resolve their relationship: EDIT the existing rule, REMOVE the outdated rule, ADD a replacement with "supersedes": "", or ADD a compatible narrower rule with "coexists": ["", ...]. For example, a rule that analyses keep integer `*_cents` columns can coexist with a rule that staging models convert `*_cents` columns to dollars. EDIT also needs "coexists" for every other overlapping bullet it remains compatible with. Refer only to existing bullet ids. Two bullets that disagree must never both remain. When feedback contradicts a bullet, that bullet is outdated: supersede or remove it; do not merely ADD a conflicting rule or mark the old one HARMFUL. diff --git a/packages/opencode/test/altimate/learn/reflect.test.ts b/packages/opencode/test/altimate/learn/reflect.test.ts index 05a13c3c85..fd4bb1f575 100644 --- a/packages/opencode/test/altimate/learn/reflect.test.ts +++ b/packages/opencode/test/altimate/learn/reflect.test.ts @@ -19,6 +19,13 @@ describe("buildPrompt", () => { } }) + test("a missed but correct bullet earns no helpful vote", () => { + const { system } = buildPrompt({ digest: "d", feedback: "f", kind: "user", bullets }) + expect(system).toContain("HELPFUL only when the DIGEST shows the agent followed it") + expect(system).toContain("it is neither HELPFUL nor HARMFUL") + expect(system).not.toContain("(the last is HELPFUL)") + }) + test("requires backticks around generated identifiers and naming patterns", () => { const { system } = buildPrompt({ digest: "d", feedback: "f", kind: "ci", bullets }) expect(system).toContain("Wrap every code identifier and naming pattern (including prefixes and suffixes) in backticks.") From e0b04c3d6d3cce6699517da5359596c4e8d6c993 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 18 Oct 2026 20:49:45 -0700 Subject: [PATCH 9/9] fix(learn): status rows report zero recurrences instead of "no observed correction" A lesson listed for mid-turn, unknown, ambiguous or earlier-version evidence said "no observed correction" next to that evidence. The row now reports "0 matching correction after exposure". Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/learn/recurrence.ts | 2 +- packages/opencode/test/altimate/learn/recurrence.test.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/learn/recurrence.ts b/packages/opencode/src/altimate/learn/recurrence.ts index 6c5dad2d70..1efe985750 100644 --- a/packages/opencode/src/altimate/learn/recurrence.ts +++ b/packages/opencode/src/altimate/learn/recurrence.ts @@ -153,7 +153,7 @@ export function formatRecurrence(result: Awaited ` ${lesson.name}/${lesson.id}: ${lesson.sessions_shown} sessions shown; ${lesson.recurred ? `${lesson.recurred} matching correction after exposure` : "no observed correction"}; ${lesson.during} during turn; ${lesson.unknown} unknown order; ${lesson.ambiguous} ambiguous; ${lesson.earlier_version} earlier version`), + ...result.lessons.filter(corrected).map((lesson) => ` ${lesson.name}/${lesson.id}: ${lesson.sessions_shown} sessions shown; ${lesson.recurred} matching correction after exposure; ${lesson.during} during turn; ${lesson.unknown} unknown order; ${lesson.ambiguous} ambiguous; ${lesson.earlier_version} earlier version`), ` ${result.lessons.filter((lesson) => !corrected(lesson)).length} other shown lessons: no observed correction`, " Topics with no matching current lesson:", ...result.unmatched_topics.map((topic) => ` ${topic.identifier}: ${topic.sessions} sessions (${topic.signal_ids.join(", ")})`), diff --git a/packages/opencode/test/altimate/learn/recurrence.test.ts b/packages/opencode/test/altimate/learn/recurrence.test.ts index e3949a976e..9056fa84e7 100644 --- a/packages/opencode/test/altimate/learn/recurrence.test.ts +++ b/packages/opencode/test/altimate/learn/recurrence.test.ts @@ -167,6 +167,13 @@ describe("recurrence attribution", () => { .toEqual(["a_id", "b_id", "c_id", "d_id", "e_id"]) }) + test("a row with only mid-turn evidence does not claim no correction", () => { + const text = formatRecurrence({ ...run([signal()], [snapshot("s1", { at: time(3) })]), coverage: { sessions_analyzed: 1, sessions_skipped_malformed: 0, truncated: false } }) + const row = text.split("\n").find((line) => line.includes("team-playbook/L-0001"))! + expect(row).toContain("0 matching correction after exposure; 1 during turn") + expect(row).not.toContain("no observed correction") + }) + test("human status lists corrected lessons and summarizes the rest", () => { const quiet: NamedLesson = { name: "team-playbook", lesson: { id: "L-0002", text: "Name staging models `stg___`." } } const shown = snapshot()