From 971c4efea0bcc0e90d6dfd8af5641b5d97e3aac1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 09:35:59 +0000 Subject: [PATCH 1/3] feat(learn): add continuous knowledge consolidation layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `smriti consolidate` and `smriti learnings`, applying Progressive Summarization: cheap Stage-1 segmentation runs broadly over dense sessions into a new smriti_knowledge_units table, and expensive Stage-2 polish (existing segmentSession/generateDocument pipeline) only runs once a unit proves reuse via recall or scored high relevance at extraction time. - src/db.ts: smriti_knowledge_units table + CRUD helpers (insertKnowledgeUnit, findUnsegmentedDenseSessions, findPromotableUnits, incrementRetrievalCount, promoteKnowledgeUnit, listKnowledgeUnits) - src/search/recall.ts: track retrieval_count on every recall() path - src/learn/consolidate.ts: segment + promote phases, reusing the existing 3-stage segmentation pipeline from src/team/segment.ts and document.ts - src/index.ts, src/format.ts: CLI wiring for `consolidate` and `learnings` CLI-only, not wired into the daemon — consolidation runs two LLM stages, which the daemon's flush path deliberately avoids (see the enrichOnIngest comment in src/daemon/index.ts). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PnYgLUDwWALu1bUAgrFQDG --- src/db.ts | 209 ++++++++++++++++++++++++ src/format.ts | 57 +++++++ src/index.ts | 43 ++++- src/learn/consolidate.ts | 196 +++++++++++++++++++++++ src/search/recall.ts | 22 +++ src/team/share.ts | 2 +- test/learn-consolidate.test.ts | 283 +++++++++++++++++++++++++++++++++ 7 files changed, 810 insertions(+), 2 deletions(-) create mode 100644 src/learn/consolidate.ts create mode 100644 test/learn-consolidate.test.ts diff --git a/src/db.ts b/src/db.ts index 59d8ba7..b04bd3e 100644 --- a/src/db.ts +++ b/src/db.ts @@ -14,6 +14,7 @@ import { QMD_DB_PATH, SMRITI_SESSIONS_DIR } from "./config"; import { initializeMemoryTables } from "./qmd"; import { createStore } from "../qmd/src/index"; import { setQmdStore, closeQmdStore } from "./store"; +import type { KnowledgeUnit } from "./team/types"; // ============================================================================= // Connection @@ -204,6 +205,33 @@ export function initializeSmritiTables(db: Database): void { entities TEXT ); + -- Knowledge consolidation: raw Stage-1 extracts, promoted to canonical on reuse + CREATE TABLE IF NOT EXISTS smriti_knowledge_units ( + id TEXT PRIMARY KEY, -- KnowledgeUnit.id (uuid) + session_id TEXT NOT NULL, + project_id TEXT, + topic TEXT NOT NULL, + category TEXT NOT NULL, + relevance REAL NOT NULL DEFAULT 0, -- 0-10, from Stage 1 + entities TEXT, -- JSON array + files TEXT, -- JSON array + plain_text TEXT NOT NULL, -- raw Stage-1 extract + line_ranges TEXT, -- JSON array of {start,end} + content_hash TEXT NOT NULL, -- hashContent({topic,category,plainText}) — Stage-1 dedup key + tier TEXT NOT NULL DEFAULT 'segmented', -- 'segmented' | 'canonical' + retrieval_count INTEGER NOT NULL DEFAULT 0, + last_recalled_at TEXT, + promoted_at TEXT, + canonical_doc_path TEXT, -- relative path under .smriti/knowledge/, set on promotion + share_id TEXT, -- points at the smriti_shares row created on promotion + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_session ON smriti_knowledge_units(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_hash ON smriti_knowledge_units(content_hash); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_tier ON smriti_knowledge_units(tier); + -- Tool usage tracking CREATE TABLE IF NOT EXISTS smriti_tool_usage ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -1246,6 +1274,187 @@ export function getDensityScore(db: Database, sessionId: string): number { return row?.density_score ?? 0; } +// ============================================================================= +// Knowledge Consolidation (Progressive Summarization) +// ============================================================================= + +export interface StoredKnowledgeUnit { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string[]; + files: string[]; + plain_text: string; + line_ranges: Array<{ start: number; end: number }>; + content_hash: string; + tier: "segmented" | "canonical"; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; +} + +type KnowledgeUnitRow = { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string | null; + files: string | null; + plain_text: string; + line_ranges: string | null; + content_hash: string; + tier: string; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; +}; + +function deserializeKnowledgeUnit(row: KnowledgeUnitRow): StoredKnowledgeUnit { + return { + ...row, + entities: row.entities ? JSON.parse(row.entities) : [], + files: row.files ? JSON.parse(row.files) : [], + line_ranges: row.line_ranges ? JSON.parse(row.line_ranges) : [], + tier: row.tier as "segmented" | "canonical", + }; +} + +/** + * Insert a Stage-1 knowledge unit if its content hash isn't already stored. + * Returns true if inserted, false if it was a duplicate (caller distinguishes + * "stored" from "skipped" the same way shareSegmentedKnowledge does for shares). + */ +export function insertKnowledgeUnit( + db: Database, + unit: KnowledgeUnit, + sessionId: string, + projectId: string | null, + contentHash: string +): boolean { + const exists = db + .prepare(`SELECT 1 FROM smriti_knowledge_units WHERE content_hash = ?`) + .get(contentHash); + if (exists) return false; + + db.prepare( + `INSERT INTO smriti_knowledge_units + (id, session_id, project_id, topic, category, relevance, entities, files, plain_text, line_ranges, content_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + unit.id, + sessionId, + projectId, + unit.topic, + unit.category, + unit.relevance, + JSON.stringify(unit.entities || []), + JSON.stringify(unit.files || []), + unit.plainText, + JSON.stringify(unit.lineRanges || []), + contentHash + ); + return true; +} + +/** Dense sessions (by density_score) that haven't been segmented into knowledge units yet. */ +export function findUnsegmentedDenseSessions( + db: Database, + minDensity: number, + limit?: number +): Array<{ session_id: string; project_id: string | null; density_score: number }> { + const query = ` + SELECT sm.session_id, sm.project_id, sm.density_score + FROM smriti_session_meta sm + WHERE sm.density_score >= ? + AND NOT EXISTS (SELECT 1 FROM smriti_knowledge_units ku WHERE ku.session_id = sm.session_id) + ORDER BY sm.density_score DESC + ${limit ? "LIMIT ?" : ""} + `; + const rows = limit + ? db.prepare(query).all(minDensity, limit) + : db.prepare(query).all(minDensity); + return rows as Array<{ session_id: string; project_id: string | null; density_score: number }>; +} + +/** Segmented units that have proven reuse (via recall) or scored high relevance at extraction time. */ +export function findPromotableUnits( + db: Database, + minRetrievals: number, + minRelevance: number +): StoredKnowledgeUnit[] { + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units + WHERE tier = 'segmented' AND (retrieval_count >= ? OR relevance >= ?)` + ) + .all(minRetrievals, minRelevance) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Bump retrieval_count for any knowledge units belonging to a recalled session. No-op if none exist yet. */ +export function incrementRetrievalCount(db: Database, sessionId: string): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET retrieval_count = retrieval_count + 1, + last_recalled_at = datetime('now'), + updated_at = datetime('now') + WHERE session_id = ?` + ).run(sessionId); +} + +export function promoteKnowledgeUnit( + db: Database, + unitId: string, + canonicalDocPath: string, + shareId: string +): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET tier = 'canonical', promoted_at = datetime('now'), + canonical_doc_path = ?, share_id = ?, updated_at = datetime('now') + WHERE id = ?` + ).run(canonicalDocPath, shareId, unitId); +} + +export function listKnowledgeUnits( + db: Database, + options: { tier?: "segmented" | "canonical"; minRetrievals?: number; limit?: number } = {} +): StoredKnowledgeUnit[] { + const conditions: string[] = []; + const params: any[] = []; + + if (options.tier) { + conditions.push("tier = ?"); + params.push(options.tier); + } + if (options.minRetrievals !== undefined) { + conditions.push("retrieval_count >= ?"); + params.push(options.minRetrievals); + } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + const limitClause = options.limit ? "LIMIT ?" : ""; + if (options.limit) params.push(options.limit); + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units ${where} + ORDER BY retrieval_count DESC, relevance DESC + ${limitClause}` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + // ============================================================================= // Session Query Labels (#60) // ============================================================================= diff --git a/src/format.ts b/src/format.ts index 023b0c3..e414186 100644 --- a/src/format.ts +++ b/src/format.ts @@ -268,6 +268,63 @@ export function formatShareResult(result: { return lines.join("\n"); } +// ============================================================================= +// Consolidate Result Formatting +// ============================================================================= + +export function formatConsolidateResult(result: { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + errors: string[]; +}): string { + const lines = [ + `Sessions segmented: ${result.sessionsSegmented}`, + `Units stored: ${result.unitsStored}`, + `Units skipped (dedup): ${result.unitsSkipped}`, + `Units promoted: ${result.unitsPromoted}`, + ]; + + if (result.errors.length > 0) { + lines.push(`Errors: ${result.errors.length}`); + for (const err of result.errors.slice(0, 5)) { + lines.push(` - ${err}`); + } + } + + return lines.join("\n"); +} + +// ============================================================================= +// Knowledge Units (Learnings) Formatting +// ============================================================================= + +export function formatLearnings( + units: Array<{ + tier: string; + topic: string; + category: string; + retrieval_count: number; + relevance: number; + canonical_doc_path: string | null; + }> +): string { + if (units.length === 0) return "No knowledge units found."; + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance", "Doc Path"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + u.canonical_doc_path || "-", + ]); + + return table(headers, rows, [14, 40, 20, 10, 9, 40]); +} + // ============================================================================= // Sync Result Formatting // ============================================================================= diff --git a/src/index.ts b/src/index.ts index bb186d5..ecea67b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ * schema-based categorization, and team knowledge sharing. */ -import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, getTagUsage, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds } from "./db"; +import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, getTagUsage, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds, listKnowledgeUnits } from "./db"; import { getMessages, getSession, getMemoryStatus, embedMemoryMessages } from "./qmd"; import { ingest, ingestAll } from "./ingest/index"; import { categorizeUncategorized } from "./categorize/classifier"; @@ -16,6 +16,7 @@ import { searchFiltered, listSessions } from "./search/index"; import { recall } from "./search/recall"; import { shareKnowledge } from "./team/share"; import { syncTeamKnowledge, listTeamContributions } from "./team/sync"; +import { consolidateKnowledge } from "./learn/consolidate"; import { generateContext, compareSessions, @@ -53,6 +54,8 @@ import { formatTagUsage, formatDensityBreakdown, formatDigest, + formatConsolidateResult, + formatLearnings, json, } from "./format"; import { generateDigest } from "./digest"; @@ -213,6 +216,8 @@ Commands: compare Compare two sessions (tokens, tools, files) compare --last Compare last 2 sessions for current project share [filters] Export knowledge to .smriti/ + consolidate [options] Segment dense sessions, promote reused knowledge units + learnings [options] List extracted knowledge units (tier, retrievals, relevance) sync Import team knowledge from .smriti/ team View team contributions list [filters] List sessions @@ -812,6 +817,42 @@ async function main() { break; } + // ===================================================================== + // CONSOLIDATE + // ===================================================================== + case "consolidate": { + const result = await consolidateKnowledge(db, { + minDensity: Number(getArg(args, "--min-density")) || undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + minRelevance: Number(getArg(args, "--min-relevance")) || undefined, + model: getArg(args, "--model"), + outputDir: getArg(args, "--output"), + sessionLimit: Number(getArg(args, "--session-limit")) || undefined, + onProgress: (msg) => console.log(` ${msg}`), + }); + + console.log(formatConsolidateResult(result)); + break; + } + + // ===================================================================== + // LEARNINGS + // ===================================================================== + case "learnings": { + const units = listKnowledgeUnits(db, { + tier: getArg(args, "--tier") as "segmented" | "canonical" | undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + limit: Number(getArg(args, "--limit")) || 50, + }); + + if (hasFlag(args, "--json")) { + console.log(json(units)); + } else { + console.log(formatLearnings(units)); + } + break; + } + // ===================================================================== // SYNC // ===================================================================== diff --git a/src/learn/consolidate.ts b/src/learn/consolidate.ts new file mode 100644 index 0000000..b6d973b --- /dev/null +++ b/src/learn/consolidate.ts @@ -0,0 +1,196 @@ +/** + * learn/consolidate.ts - Continuous knowledge consolidation + * + * Progressive Summarization: cheap Stage-1 extraction runs broadly over dense + * sessions; expensive Stage-2 polish only runs once a unit proves it's reused + * (recalled repeatedly) or scored high relevance at extraction time. + * + * Two independent phases, run sequentially: + * - Segment: dense, not-yet-segmented sessions -> segmentSession() -> smriti_knowledge_units + * - Promote: knowledge units that cleared the reuse/relevance bar -> generateDocument() + * -> written to .smriti/knowledge/ + recorded in smriti_shares + * + * CLI-only, like `categorize`/`share` — never wired into the daemon (see + * src/daemon/index.ts's enrichOnIngest comment for why LLM work per-flush is unsafe). + */ + +import type { Database } from "bun:sqlite"; +import { mkdirSync } from "fs"; +import { join } from "path"; +import { SMRITI_DIR, AUTHOR } from "../config"; +import { hashContent } from "../qmd"; +import { + findUnsegmentedDenseSessions, + insertKnowledgeUnit, + findPromotableUnits, + promoteKnowledgeUnit, +} from "../db"; +import { getSessionMessages } from "../team/share"; +import { segmentSession } from "../team/segment"; +import { generateDocument, generateFrontmatter } from "../team/document"; +import { isSessionWorthSharing } from "../team/formatter"; +import type { RawMessage } from "../team/formatter"; +import type { KnowledgeUnit } from "../team/types"; + +// ============================================================================= +// Types +// ============================================================================= + +export type ConsolidateOptions = { + minDensity?: number; + minRetrievals?: number; + minRelevance?: number; + model?: string; + outputDir?: string; + author?: string; + sessionLimit?: number; + onProgress?: (msg: string) => void; +}; + +export type ConsolidateResult = { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + errors: string[]; +}; + +// ============================================================================= +// Consolidation +// ============================================================================= + +export async function consolidateKnowledge( + db: Database, + options: ConsolidateOptions = {} +): Promise { + const author = options.author || AUTHOR; + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + + const result: ConsolidateResult = { + sessionsSegmented: 0, + unitsStored: 0, + unitsSkipped: 0, + unitsPromoted: 0, + errors: [], + }; + + // =========================================================================== + // Segment phase: cheap Stage-1 extraction over dense, unsegmented sessions + // =========================================================================== + + const sessions = findUnsegmentedDenseSessions( + db, + options.minDensity ?? 0.5, + options.sessionLimit ?? 20 + ); + + for (const s of sessions) { + try { + const messages = getSessionMessages(db, s.session_id); + if (messages.length === 0) continue; + + const rawMessages: RawMessage[] = messages.map((m) => ({ + role: m.role, + content: m.content, + })); + + if (!isSessionWorthSharing(rawMessages)) continue; + + const segmentationResult = await segmentSession(db, s.session_id, rawMessages, { + model: options.model, + }); + result.sessionsSegmented++; + + for (const unit of segmentationResult.units) { + const contentHash = await hashContent( + JSON.stringify({ topic: unit.topic, category: unit.category, plainText: unit.plainText }) + ); + const inserted = insertKnowledgeUnit(db, unit, s.session_id, s.project_id, contentHash); + inserted ? result.unitsStored++ : result.unitsSkipped++; + } + } catch (err: any) { + result.errors.push(`segment ${s.session_id}: ${err.message}`); + } + } + + options.onProgress?.( + `segment phase: ${result.sessionsSegmented} sessions, ${result.unitsStored} units stored, ${result.unitsSkipped} skipped` + ); + + // =========================================================================== + // Promote phase: expensive Stage-2 polish for units that proved reuse + // =========================================================================== + + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + + const promotable = findPromotableUnits( + db, + options.minRetrievals ?? 3, + options.minRelevance ?? 8 + ); + + for (const stored of promotable) { + try { + const unit: KnowledgeUnit = { + id: stored.id, + topic: stored.topic, + category: stored.category, + relevance: stored.relevance, + entities: stored.entities, + files: stored.files, + plainText: stored.plain_text, + lineRanges: stored.line_ranges, + }; + + const doc = await generateDocument(unit, stored.topic, { + model: options.model, + projectSmritiDir: outputDir, + author, + }); + + const categoryDir = join(knowledgeDir, doc.category.replaceAll("/", "-")); + mkdirSync(categoryDir, { recursive: true }); + const filePath = join(categoryDir, doc.filename); + + const fm = generateFrontmatter( + stored.session_id, + doc.unitId, + { ...doc.frontmatter, pipeline: "consolidated" }, + author, + stored.project_id || undefined + ); + await Bun.write(filePath, fm + "\n\n" + doc.markdown); + + const shareId = crypto.randomUUID().slice(0, 8); + const shareHash = await hashContent( + JSON.stringify({ content: doc.markdown, category: doc.category, entities: doc.frontmatter.entities }) + ); + + db.prepare( + `INSERT INTO smriti_shares (id, session_id, category_id, project_id, author, content_hash, unit_id, relevance_score, entities) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + shareId, + stored.session_id, + doc.category, + stored.project_id, + author, + shareHash, + doc.unitId, + stored.relevance, + JSON.stringify(stored.entities) + ); + + const relPath = `knowledge/${doc.category.replaceAll("/", "-")}/${doc.filename}`; + promoteKnowledgeUnit(db, stored.id, relPath, shareId); + result.unitsPromoted++; + } catch (err: any) { + result.errors.push(`promote ${stored.id}: ${err.message}`); + } + } + + options.onProgress?.(`promote phase: ${result.unitsPromoted} units promoted`); + + return result; +} diff --git a/src/search/recall.ts b/src/search/recall.ts index 1e219a4..f94fe7b 100644 --- a/src/search/recall.ts +++ b/src/search/recall.ts @@ -9,6 +9,7 @@ import { DEFAULT_RECALL_LIMIT, OLLAMA_HOST, OLLAMA_MODEL } from "../config"; import { recallMemories, ollamaRecall } from "../qmd"; import { searchFiltered, type SearchFilters, type SearchResult } from "./index"; import { getQmdStore } from "../store"; +import { incrementRetrievalCount } from "../db"; // ============================================================================= // Types @@ -27,6 +28,24 @@ export type RecallResult = { synthesis?: string; }; +// ============================================================================= +// Retrieval Tracking +// ============================================================================= + +/** + * Best-effort bump of retrieval_count for any consolidated knowledge units + * belonging to the recalled sessions. Never lets a tracking failure break recall. + */ +function trackRetrieval(db: Database, results: SearchResult[]): void { + try { + for (const sessionId of new Set(results.map((r) => r.session_id).filter(Boolean))) { + incrementRetrievalCount(db, sessionId); + } + } catch { + // Never let this break recall. + } +} + // ============================================================================= // Filtered Recall // ============================================================================= @@ -58,6 +77,7 @@ export async function recall( if (options.synthesize && storeResults.length > 0) { synthesis = await synthesizeResults(query, storeResults, options); } + trackRetrieval(db, storeResults); return { results: storeResults, synthesis }; } @@ -70,6 +90,7 @@ export async function recall( fast: options.fast, intent: rerankIntent, }); + trackRetrieval(db, qmdResult.results); return { results: qmdResult.results, synthesis: qmdResult.synthesis, @@ -102,6 +123,7 @@ export async function recall( synthesis = await synthesizeResults(query, deduped, options); } + trackRetrieval(db, deduped); return { results: deduped, synthesis }; } diff --git a/src/team/share.ts b/src/team/share.ts index 06c44e5..0d61733 100644 --- a/src/team/share.ts +++ b/src/team/share.ts @@ -140,7 +140,7 @@ function querySessions( } /** Get messages for a session */ -function getSessionMessages( +export function getSessionMessages( db: Database, sessionId: string ): Array<{ diff --git a/test/learn-consolidate.test.ts b/test/learn-consolidate.test.ts new file mode 100644 index 0000000..8748d8c --- /dev/null +++ b/test/learn-consolidate.test.ts @@ -0,0 +1,283 @@ +/** + * test/learn-consolidate.test.ts - Tests for continuous knowledge consolidation + * + * Mirrors test/team-segmented.test.ts's style: initSmriti(":memory:"), a + * mocked global.fetch standing in for Ollama, and a scratch tmpDir for + * filesystem output (never process.cwd() — consolidateKnowledge writes real + * files). + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync, writeFileSync, existsSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + listKnowledgeUnits, +} from "../src/db"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import { recall } from "../src/search/recall"; +import type { KnowledgeUnit } from "../src/team/types"; + +// ============================================================================= +// Setup +// ============================================================================= + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-consolidate-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +/** Insert a session with real message rows so getSessionMessages() finds content. */ +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +const DENSE_CONVERSATION = [ + { role: "user", content: "I'm getting a JWT token expiry issue. Sessions timeout after 1 hour but tests expect 24 hours." }, + { role: "assistant", content: "Let me look at the auth middleware to understand the token expiry logic." }, + { role: "user", content: "Found it — src/auth.ts hardcodes 3600 seconds instead of reading JWT_TTL from the environment." }, + { role: "assistant", content: "Updated it to use process.env.JWT_TTL || 3600. Tests pass now." }, +]; + +/** Mock fetch that distinguishes Stage 1 (segmentation) vs Stage 2 (document) calls by prompt content. */ +function mockOllamaFetch(stage1Response: () => object) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify(stage1Response()) + "\n```" }), + { status: 200 } + ); + } + return new Response( + JSON.stringify({ response: "# Consolidated Doc\n\nPolished content." }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Segment-phase dedup +// ============================================================================= + +test("consolidate dedups knowledge units with identical content across sessions", async () => { + seedSession("dedup-s1", "dedupproj", DENSE_CONVERSATION); + seedSession("dedup-s2", "dedupproj", DENSE_CONVERSATION); + updateDensityScore(db, "dedup-s1", 0.9); + updateDensityScore(db, "dedup-s2", 0.9); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ + units: [{ topic: "JWT token expiry bug", category: "bug/fix", relevance: 9, entities: ["JWT"] }], + })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + outputDir: join(tmpDir, "dedup-output"), + }); + + expect(result.sessionsSegmented).toBe(2); + expect(result.unitsStored).toBe(1); + expect(result.unitsSkipped).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Promotion threshold +// ============================================================================= + +test("promote phase only promotes units clearing the retrieval/relevance bar", async () => { + const belowThreshold: KnowledgeUnit = { + id: "unit-below", + topic: "Minor formatting note", + category: "code/pattern", + relevance: 3, + entities: [], + files: [], + plainText: "Use consistent indentation.", + lineRanges: [{ start: 0, end: 1 }], + }; + const aboveThreshold: KnowledgeUnit = { + id: "unit-above", + topic: "Redis caching decision", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use Redis with a 5-minute TTL for API responses.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, belowThreshold, "promote-s1", "promoteproj", "hash-below"); + insertKnowledgeUnit(db, aboveThreshold, "promote-s2", "promoteproj", "hash-above"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no sessions qualify for the segment phase — isolates promote phase + minRetrievals: 3, + minRelevance: 8, + outputDir: join(tmpDir, "promote-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-above"); + expect(canonical.map((u) => u.id)).not.toContain("unit-below"); + + const promoted = canonical.find((u) => u.id === "unit-above")!; + expect(promoted.canonical_doc_path).toContain("architecture-decision"); + expect(promoted.share_id).toBeTruthy(); + + const shareRow = db + .prepare(`SELECT * FROM smriti_shares WHERE unit_id = ?`) + .get("unit-above") as any; + expect(shareRow).toBeTruthy(); + expect(shareRow.session_id).toBe("promote-s2"); + + const writtenFile = join(tmpDir, "promote-output", promoted.canonical_doc_path!); + expect(existsSync(writtenFile)).toBe(true); + + const stillSegmented = listKnowledgeUnits(db, { tier: "segmented" }); + expect(stillSegmented.map((u) => u.id)).toContain("unit-below"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Graceful degradation +// ============================================================================= + +test("promote phase continues past a per-unit failure and records the error", async () => { + // Pre-create a *file* at the exact path the loop will try to mkdir for + // category "bug/fix" (slug "bug-fix"), forcing that unit's write to throw. + const outputDir = join(tmpDir, "degrade-output"); + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + writeFileSync(join(knowledgeDir, "bug-fix"), "occupied"); + + const willFail: KnowledgeUnit = { + id: "unit-fails", + topic: "Broken unit", + category: "bug/fix", + relevance: 9, + entities: [], + files: [], + plainText: "This unit's category dir collides with a file.", + lineRanges: [{ start: 0, end: 1 }], + }; + const willSucceed: KnowledgeUnit = { + id: "unit-succeeds", + topic: "Working unit", + category: "topic/learning", + relevance: 9, + entities: [], + files: [], + plainText: "This unit writes fine.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, willFail, "degrade-s1", "degradeproj", "hash-fails"); + insertKnowledgeUnit(db, willSucceed, "degrade-s2", "degradeproj", "hash-succeeds"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, // exclude leftover segmented units from earlier tests; only relevance-9 units below qualify + minRelevance: 8, + outputDir, + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors.length).toBe(1); + expect(result.errors[0]).toContain("unit-fails"); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-succeeds"); + expect(canonical.map((u) => u.id)).not.toContain("unit-fails"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Retrieval tracking (recall -> incrementRetrievalCount) +// ============================================================================= + +test("recall increments retrieval_count for knowledge units of the recalled session", async () => { + seedSession("track-s1", "trackproj", [ + { role: "user", content: "How do we configure the rate limiter for the public API?" }, + { role: "assistant", content: "Use a token bucket with 100 requests per minute per API key." }, + ]); + + const unit: KnowledgeUnit = { + id: "unit-tracked", + topic: "Rate limiter config", + category: "code/pattern", + relevance: 7, + entities: [], + files: [], + plainText: "Token bucket rate limiting for the public API.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, unit, "track-s1", "trackproj", "hash-tracked"); + + await recall(db, "rate limiter", { project: "trackproj" }); + + const tracked = db + .prepare(`SELECT retrieval_count FROM smriti_knowledge_units WHERE id = ?`) + .get("unit-tracked") as { retrieval_count: number }; + expect(tracked).toBeDefined(); + expect(tracked.retrieval_count).toBeGreaterThanOrEqual(1); +}); + +test("recall does not throw for sessions with no consolidated knowledge units", async () => { + seedSession("untracked-s1", "untrackedproj", [ + { role: "user", content: "What's our deploy process look like?" }, + { role: "assistant", content: "Push to main triggers the CI pipeline and auto-deploys." }, + ]); + + await expect(recall(db, "deploy process", { project: "untrackedproj" })).resolves.toBeDefined(); +}); From 933d53b7092ad170a1268dffa70c1994b990434f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 16:14:19 +0000 Subject: [PATCH 2/3] feat(learn): add RDF-inspired entities + relationships graph layer Extends the Continuous Knowledge Consolidation Layer with canonical entities and typed subject-predicate-object relationship triples, inspired by RDF's data model (adapted pragmatically: no URIs/SPARQL, just typed (subject_type, subject_id) pairs in SQLite). - src/db.ts: smriti_entities + smriti_relationships tables; new `minEntityReach` promotion criterion in findPromotableUnits (a unit becomes promotable once its entity is independently mentioned by K other units, even at 0 retrievals); seeds a "team" agent (fixes a latent FK bug in syncTeamKnowledge's existing agent fallback) - src/learn/entities.ts: resolveEntity (exact-normalize canonicalization via slugify), insertRelationship/getRelationships (triple store), findRelatedCandidates, findEntity, getUnitsForEntity - src/learn/consolidate.ts: segment phase turns Stage-1 entities into "mentions" edges for free; promote phase adds one bounded LLM call to infer relatesTo/supersedes/contradicts edges against entity-sharing candidates, persisting what ollamaCheckConflicts previously only computed ephemerally - src/team/config.ts, share.ts, sync.ts: entities propagate team/org-wide through the same .smriti/config.json round-trip custom categories already use (exportEntities/mergeEntities mirror exportCustomCategories/mergeCategories); unit-to-unit relationship edges propagate via frontmatter directly, needing no canonicalization since unit ids are already portable UUIDs. Also fixes sync.ts treating "consolidated" pipeline docs as raw conversation transcripts (only "segmented" was previously recognized as single-message). - src/index.ts, src/format.ts: `smriti graph ` command; `--min-entity-reach` flag on `smriti consolidate` Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PnYgLUDwWALu1bUAgrFQDG --- src/db.ts | 66 +++++- src/format.ts | 47 +++++ src/index.ts | 35 ++++ src/learn/consolidate.ts | 142 ++++++++++++- src/learn/entities.ts | 232 ++++++++++++++++++++ src/team/config.ts | 78 +++++++ src/team/share.ts | 8 +- src/team/sync.ts | 43 +++- test/learn-entities-sync.test.ts | 166 +++++++++++++++ test/learn-entities.test.ts | 349 +++++++++++++++++++++++++++++++ 10 files changed, 1152 insertions(+), 14 deletions(-) create mode 100644 src/learn/entities.ts create mode 100644 test/learn-entities-sync.test.ts create mode 100644 test/learn-entities.test.ts diff --git a/src/db.ts b/src/db.ts index b04bd3e..048cb55 100644 --- a/src/db.ts +++ b/src/db.ts @@ -232,6 +232,40 @@ export function initializeSmritiTables(db: Database): void { CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_hash ON smriti_knowledge_units(content_hash); CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_tier ON smriti_knowledge_units(tier); + -- Canonical entity registry: resolves free-text entity mentions (from Stage 1 + -- extraction) onto a stable node, so recurrence is detected regardless of wording. + -- Propagated team/org-wide via .smriti/config.json, same mechanism as custom categories. + CREATE TABLE IF NOT EXISTS smriti_entities ( + id TEXT PRIMARY KEY, -- slug, e.g. "jwt", "redis" + label TEXT NOT NULL, -- canonical display name + entity_type TEXT NOT NULL DEFAULT 'concept', -- 'technology' | 'concept' | 'file' | 'pattern' + aliases TEXT NOT NULL DEFAULT '[]', -- JSON array of raw strings seen + mention_count INTEGER NOT NULL DEFAULT 0, + first_seen_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_entities_label ON smriti_entities(label); + + -- Relationship triples (subject/object polymorphic via type+id, not literal RDF URIs). + -- knowledge_unit -mentions-> entity edges come free from Stage-1 extraction; + -- knowledge_unit -relatesTo/supersedes/contradicts-> knowledge_unit edges are LLM-gated, + -- only at promotion time (see src/learn/consolidate.ts), persisting what + -- ollamaCheckConflicts previously only computed ephemerally. + CREATE TABLE IF NOT EXISTS smriti_relationships ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + subject_type TEXT NOT NULL, -- 'knowledge_unit' | 'entity' | 'session' + subject_id TEXT NOT NULL, + predicate TEXT NOT NULL, -- 'mentions' | 'relatesTo' | 'supersedes' | 'contradicts' + object_type TEXT NOT NULL, + object_id TEXT NOT NULL, + confidence REAL DEFAULT 1.0, + source TEXT DEFAULT 'extraction', -- 'extraction' | 'derived' | 'llm' + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(subject_type, subject_id, predicate, object_type, object_id) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_subject ON smriti_relationships(subject_type, subject_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_object ON smriti_relationships(object_type, object_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_predicate ON smriti_relationships(predicate); + -- Tool usage tracking CREATE TABLE IF NOT EXISTS smriti_tool_usage ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -503,6 +537,12 @@ const DEFAULT_AGENTS = [ log_pattern: null, parser: "claude-web", }, + { + id: "team", + display_name: "Team Import", + log_pattern: null, + parser: "generic", + }, ] as const; /** Default category taxonomy */ @@ -1389,14 +1429,34 @@ export function findUnsegmentedDenseSessions( export function findPromotableUnits( db: Database, minRetrievals: number, - minRelevance: number + minRelevance: number, + minEntityReach?: number ): StoredKnowledgeUnit[] { + // minEntityReach: a unit is promotable if one of its entities is + // independently mentioned by >= minEntityReach OTHER units — a structural + // reuse signal (cross-session recurrence) that doesn't depend on recall() + // ever having been called on this particular unit. + const entityReachClause = minEntityReach + ? `OR id IN ( + SELECT r1.subject_id FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id AND r2.predicate = 'mentions' + AND r1.predicate = 'mentions' AND r1.subject_id != r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' + GROUP BY r1.subject_id + HAVING COUNT(DISTINCT r2.subject_id) >= ? + )` + : ""; + const params = minEntityReach + ? [minRetrievals, minRelevance, minEntityReach] + : [minRetrievals, minRelevance]; + const rows = db .prepare( `SELECT * FROM smriti_knowledge_units - WHERE tier = 'segmented' AND (retrieval_count >= ? OR relevance >= ?)` + WHERE tier = 'segmented' AND (retrieval_count >= ? OR relevance >= ? ${entityReachClause})` ) - .all(minRetrievals, minRelevance) as KnowledgeUnitRow[]; + .all(...params) as KnowledgeUnitRow[]; return rows.map(deserializeKnowledgeUnit); } diff --git a/src/format.ts b/src/format.ts index e414186..f78280f 100644 --- a/src/format.ts +++ b/src/format.ts @@ -325,6 +325,49 @@ export function formatLearnings( return table(headers, rows, [14, 40, 20, 10, 9, 40]); } +// ============================================================================= +// Entity Graph Formatting (smriti graph ) +// ============================================================================= + +export function formatEntityGraph( + entity: { id: string; label: string; entity_type: string; aliases: string[]; mention_count: number }, + units: Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }>, + edges: Array<{ subject_id: string; predicate: string; object_id: string }> +): string { + const lines = [ + `Entity: ${entity.label} (${entity.id})`, + `Type: ${entity.entity_type}`, + `Aliases: ${entity.aliases.join(", ") || "-"}`, + `Mentioned ${entity.mention_count} time(s) across ${units.length} unit(s)`, + "", + ]; + + if (units.length === 0) { + lines.push("No knowledge units mention this entity yet."); + return lines.join("\n"); + } + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + ]); + lines.push(table(headers, rows, [14, 40, 20, 10, 9])); + + const nonMentionEdges = edges.filter((e) => e.predicate !== "mentions"); + if (nonMentionEdges.length > 0) { + lines.push("", "Relationships between these units:"); + for (const e of nonMentionEdges) { + lines.push(` ${e.subject_id} --${e.predicate}--> ${e.object_id}`); + } + } + + return lines.join("\n"); +} + // ============================================================================= // Sync Result Formatting // ============================================================================= @@ -335,6 +378,7 @@ export function formatSyncResult(result: { skipped: number; errors: string[]; categoriesImported?: number; + entitiesImported?: number; }): string { const lines = [ `Files processed: ${result.filesProcessed}`, @@ -344,6 +388,9 @@ export function formatSyncResult(result: { if (result.categoriesImported && result.categoriesImported > 0) { lines.push(`Categories imported: ${result.categoriesImported}`); } + if (result.entitiesImported && result.entitiesImported > 0) { + lines.push(`Entities imported: ${result.entitiesImported}`); + } if (result.errors.length > 0) { lines.push(`Errors: ${result.errors.length}`); diff --git a/src/index.ts b/src/index.ts index ecea67b..f5c8956 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,7 @@ import { recall } from "./search/recall"; import { shareKnowledge } from "./team/share"; import { syncTeamKnowledge, listTeamContributions } from "./team/sync"; import { consolidateKnowledge } from "./learn/consolidate"; +import { findEntity, getUnitsForEntity, getRelationships } from "./learn/entities"; import { generateContext, compareSessions, @@ -56,6 +57,7 @@ import { formatDigest, formatConsolidateResult, formatLearnings, + formatEntityGraph, json, } from "./format"; import { generateDigest } from "./digest"; @@ -218,6 +220,7 @@ Commands: share [filters] Export knowledge to .smriti/ consolidate [options] Segment dense sessions, promote reused knowledge units learnings [options] List extracted knowledge units (tier, retrievals, relevance) + graph Show a canonical entity's mentions and relationship edges sync Import team knowledge from .smriti/ team View team contributions list [filters] List sessions @@ -825,6 +828,7 @@ async function main() { minDensity: Number(getArg(args, "--min-density")) || undefined, minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, minRelevance: Number(getArg(args, "--min-relevance")) || undefined, + minEntityReach: Number(getArg(args, "--min-entity-reach")) || undefined, model: getArg(args, "--model"), outputDir: getArg(args, "--output"), sessionLimit: Number(getArg(args, "--session-limit")) || undefined, @@ -853,6 +857,37 @@ async function main() { break; } + // ===================================================================== + // GRAPH + // ===================================================================== + case "graph": { + const query = getPositional(args, 1); + if (!query) { + console.error("Usage: smriti graph "); + process.exit(1); + } + + const entity = findEntity(db, query); + if (!entity) { + console.log(`No entity found matching "${query}".`); + break; + } + + const units = getUnitsForEntity(db, entity.id); + const unitIds = new Set(units.map((u) => u.id)); + const edges = units.flatMap((u) => + getRelationships(db, { subjectType: "knowledge_unit", subjectId: u.id }) + .filter((r) => r.predicate !== "mentions" && unitIds.has(r.object_id)) + ); + + if (hasFlag(args, "--json")) { + console.log(json({ entity, units, edges })); + } else { + console.log(formatEntityGraph(entity, units, edges)); + } + break; + } + // ===================================================================== // SYNC // ===================================================================== diff --git a/src/learn/consolidate.ts b/src/learn/consolidate.ts index b6d973b..38d9a94 100644 --- a/src/learn/consolidate.ts +++ b/src/learn/consolidate.ts @@ -29,8 +29,16 @@ import { getSessionMessages } from "../team/share"; import { segmentSession } from "../team/segment"; import { generateDocument, generateFrontmatter } from "../team/document"; import { isSessionWorthSharing } from "../team/formatter"; +import { callOllama } from "../team/ollama"; import type { RawMessage } from "../team/formatter"; import type { KnowledgeUnit } from "../team/types"; +import { + resolveEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + type RelationshipPredicate, +} from "./entities"; // ============================================================================= // Types @@ -40,6 +48,7 @@ export type ConsolidateOptions = { minDensity?: number; minRetrievals?: number; minRelevance?: number; + minEntityReach?: number; model?: string; outputDir?: string; author?: string; @@ -107,6 +116,19 @@ export async function consolidateKnowledge( ); const inserted = insertKnowledgeUnit(db, unit, s.session_id, s.project_id, contentHash); inserted ? result.unitsStored++ : result.unitsSkipped++; + + // Turn Stage 1's free-text entities into canonical "mentions" edges — + // pure post-processing of data already extracted, no extra LLM calls. + if (inserted) { + for (const rawEntity of unit.entities) { + const entityId = resolveEntity(db, rawEntity); + if (entityId) { + insertRelationship(db, "knowledge_unit", unit.id, "mentions", "entity", entityId, { + source: "extraction", + }); + } + } + } } } catch (err: any) { result.errors.push(`segment ${s.session_id}: ${err.message}`); @@ -127,7 +149,8 @@ export async function consolidateKnowledge( const promotable = findPromotableUnits( db, options.minRetrievals ?? 3, - options.minRelevance ?? 8 + options.minRelevance ?? 8, + options.minEntityReach ); for (const stored of promotable) { @@ -143,6 +166,15 @@ export async function consolidateKnowledge( lineRanges: stored.line_ranges, }; + // Bounded relationship inference: only runs if this unit shares a + // canonical entity with at least one other unit, and costs exactly one + // extra LLM call (same cost discipline as Stage 2) — persists what + // ollamaCheckConflicts previously only computed ephemerally. + const candidates = findRelatedCandidates(db, stored.id, 5); + if (candidates.length > 0) { + await inferRelationships(db, unit, candidates, options.model); + } + const doc = await generateDocument(unit, stored.topic, { model: options.model, projectSmritiDir: outputDir, @@ -153,10 +185,31 @@ export async function consolidateKnowledge( mkdirSync(categoryDir, { recursive: true }); const filePath = join(categoryDir, doc.filename); + // Carry canonical entity ids + unit-to-unit edges into shared + // frontmatter. Unlike entity ids, these edges need no team-level + // canonicalization step — unit.id is already a portable UUID once + // shared (see src/team/document.ts's frontmatter `id` field), so + // syncTeamKnowledge can re-create them on a teammate's machine as-is. + const entityIds = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + predicate: "mentions", + objectType: "entity", + }).map((r) => r.object_id); + const outgoingEdges = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + }).filter((r) => r.predicate !== "mentions"); + const fm = generateFrontmatter( stored.session_id, doc.unitId, - { ...doc.frontmatter, pipeline: "consolidated" }, + { + ...doc.frontmatter, + pipeline: "consolidated", + entity_ids: entityIds, + ...groupEdgesByPredicate(outgoingEdges), + }, author, stored.project_id || undefined ); @@ -194,3 +247,88 @@ export async function consolidateKnowledge( return result; } + +// ============================================================================= +// Relationship Inference (promote-time, LLM-gated) +// ============================================================================= + +const RELATION_LINE = /RELATION\s*\[(\d+)\]:\s*(relatesTo|supersedes|contradicts|none)/gi; +const MAX_EXCERPT_CHARS = 800; + +// Case-insensitive regex match -> canonical camelCase predicate (avoid a blind +// .toLowerCase() on the match, which would turn "relatesTo" into "relatesto"). +const PREDICATE_BY_LOWERCASE: Record = { + relatesto: "relatesTo", + supersedes: "supersedes", + contradicts: "contradicts", +}; + +function truncate(text: string, max: number): string { + return text.length > max ? text.slice(0, max) + "…" : text; +} + +/** + * Ask the LLM whether the unit being promoted relatesTo/supersedes/contradicts + * any of its entity-sharing candidates, and persist the answer as edges. + * Best-effort: a failure here (LLM down, unparseable response) is swallowed — + * it's enrichment on top of promotion, not a precondition for it. + */ +async function inferRelationships( + db: Database, + unit: KnowledgeUnit, + candidates: Array<{ id: string; topic: string; category: string; plain_text: string }>, + model?: string +): Promise { + try { + const candidateBlock = candidates + .map((c, i) => `[${i}] Topic: ${c.topic}\nCategory: ${c.category}\nContent: ${truncate(c.plain_text, MAX_EXCERPT_CHARS)}`) + .join("\n\n"); + + const prompt = `You are comparing a NEW knowledge unit against CANDIDATE units that already mention at least one of the same topics/entities. + +NEW UNIT +Topic: ${unit.topic} +Category: ${unit.category} +Content: ${truncate(unit.plainText, MAX_EXCERPT_CHARS)} + +CANDIDATES +${candidateBlock} + +For each candidate, decide the relationship of the NEW unit to it: +- relatesTo: related but neither replaces nor conflicts with the other +- supersedes: the NEW unit replaces/updates the candidate's guidance +- contradicts: the NEW unit conflicts with the candidate +- none: no meaningful relationship + +Respond with exactly one line per candidate, in this format: +RELATION [i]: relatesTo|supersedes|contradicts|none`; + + const response = await callOllama(prompt, { model }); + + for (const match of response.matchAll(RELATION_LINE)) { + const index = Number(match[1]); + const raw = match[2].toLowerCase(); + if (raw === "none") continue; + const predicate = PREDICATE_BY_LOWERCASE[raw]; + const candidate = candidates[index]; + if (!predicate || !candidate) continue; + + insertRelationship(db, "knowledge_unit", unit.id, predicate, "knowledge_unit", candidate.id, { + source: "llm", + }); + } + } catch { + // Enrichment only — never block promotion on a failed/unparseable relation call. + } +} + +/** Group outgoing relationship edges by predicate into frontmatter-ready arrays of object unit ids. */ +function groupEdgesByPredicate( + edges: Array<{ predicate: string; object_id: string }> +): Record { + const grouped: Record = {}; + for (const edge of edges) { + (grouped[edge.predicate] ??= []).push(edge.object_id); + } + return grouped; +} diff --git a/src/learn/entities.ts b/src/learn/entities.ts new file mode 100644 index 0000000..474c944 --- /dev/null +++ b/src/learn/entities.ts @@ -0,0 +1,232 @@ +/** + * learn/entities.ts - Canonical entity resolution + relationship triples + * + * RDF-inspired, not literal RDF: no URIs/Turtle/SPARQL. Subject/object are + * (type, id) pairs instead of global URIs, since Smriti is a local SQLite + * tool, not a web-facing linked-data endpoint. The useful ideas kept are: + * stable resource identity (so recurrence is detected across wording + * variance) and typed subject-predicate-object facts that survive team + * sharing (see src/team/config.ts's exportEntities/mergeEntities and + * src/team/document.ts's frontmatter for how these propagate org-wide). + * + * v1 entity resolution is exact-normalize only (case/whitespace/punctuation + * via slugify) — "JWT" and "jwt" merge, "JWT" and "JSON Web Token" do not. + * True synonym resolution needs semantic matching and is out of scope here. + */ + +import type { Database } from "bun:sqlite"; +import { slugify } from "../team/utils"; + +// ============================================================================= +// Types +// ============================================================================= + +export type EntityType = "technology" | "concept" | "file" | "pattern"; +export type RelationshipPredicate = "mentions" | "relatesTo" | "supersedes" | "contradicts"; +export type RelationshipSubjectType = "knowledge_unit" | "entity" | "session"; +export type RelationshipSource = "extraction" | "derived" | "llm"; + +export type StoredEntity = { + id: string; + label: string; + entity_type: EntityType; + aliases: string[]; + mention_count: number; + first_seen_at: string; +}; + +export type StoredRelationship = { + id: number; + subject_type: RelationshipSubjectType; + subject_id: string; + predicate: RelationshipPredicate; + object_type: RelationshipSubjectType; + object_id: string; + confidence: number; + source: RelationshipSource; + created_at: string; +}; + +type EntityRow = { + id: string; + label: string; + entity_type: string; + aliases: string; + mention_count: number; + first_seen_at: string; +}; + +function deserializeEntity(row: EntityRow): StoredEntity { + return { + ...row, + entity_type: row.entity_type as EntityType, + aliases: JSON.parse(row.aliases), + }; +} + +// ============================================================================= +// Entity Resolution +// ============================================================================= + +/** + * Resolve a raw, free-text entity label to a canonical entity id, creating + * the entity if it doesn't exist yet. Matching is exact-normalize (via + * slugify) — same case/whitespace variant collapses to one node; different + * wordings for the same concept do not (see module docstring). + */ +export function resolveEntity( + db: Database, + rawLabel: string, + entityType: EntityType = "concept" +): string | null { + const trimmed = rawLabel.trim(); + if (!trimmed) return null; + + const id = slugify(trimmed); + if (!id) return null; + + const existing = db + .prepare(`SELECT aliases FROM smriti_entities WHERE id = ?`) + .get(id) as { aliases: string } | null; + + if (existing) { + const aliases: string[] = JSON.parse(existing.aliases); + if (!aliases.includes(trimmed)) { + aliases.push(trimmed); + db.prepare( + `UPDATE smriti_entities SET aliases = ?, mention_count = mention_count + 1 WHERE id = ?` + ).run(JSON.stringify(aliases), id); + } else { + db.prepare(`UPDATE smriti_entities SET mention_count = mention_count + 1 WHERE id = ?`).run(id); + } + return id; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 1)` + ).run(id, trimmed, entityType, JSON.stringify([trimmed])); + return id; +} + +export function getEntity(db: Database, id: string): StoredEntity | null { + const row = db.prepare(`SELECT * FROM smriti_entities WHERE id = ?`).get(id) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Look up an entity by exact id, or by slugified/label match against a raw query string. */ +export function findEntity(db: Database, query: string): StoredEntity | null { + const bySlug = getEntity(db, slugify(query)); + if (bySlug) return bySlug; + + const row = db + .prepare(`SELECT * FROM smriti_entities WHERE LOWER(label) = LOWER(?)`) + .get(query.trim()) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Knowledge units that `mentions` a given canonical entity — the display side of `smriti graph `. */ +export function getUnitsForEntity( + db: Database, + entityId: string +): Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }> { + return db + .prepare( + `SELECT ku.id, ku.topic, ku.category, ku.relevance, ku.tier, ku.retrieval_count + FROM smriti_relationships r + JOIN smriti_knowledge_units ku ON ku.id = r.subject_id + WHERE r.subject_type = 'knowledge_unit' AND r.object_type = 'entity' + AND r.predicate = 'mentions' AND r.object_id = ? + ORDER BY ku.retrieval_count DESC, ku.relevance DESC` + ) + .all(entityId) as Array<{ + id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number; + }>; +} + +export function listEntities(db: Database, limit?: number): StoredEntity[] { + const rows = ( + limit + ? db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC LIMIT ?`).all(limit) + : db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC`).all() + ) as EntityRow[]; + return rows.map(deserializeEntity); +} + +// ============================================================================= +// Relationship Triples +// ============================================================================= + +/** Insert a (subject, predicate, object) triple. Deduped via the table's UNIQUE constraint. */ +export function insertRelationship( + db: Database, + subjectType: RelationshipSubjectType, + subjectId: string, + predicate: RelationshipPredicate, + objectType: RelationshipSubjectType, + objectId: string, + options: { confidence?: number; source?: RelationshipSource } = {} +): void { + db.prepare( + `INSERT OR IGNORE INTO smriti_relationships + (subject_type, subject_id, predicate, object_type, object_id, confidence, source) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run( + subjectType, + subjectId, + predicate, + objectType, + objectId, + options.confidence ?? 1.0, + options.source ?? "extraction" + ); +} + +export type TriplePattern = { + subjectType?: RelationshipSubjectType; + subjectId?: string; + predicate?: RelationshipPredicate; + objectType?: RelationshipSubjectType; + objectId?: string; +}; + +/** Single-pattern triple lookup — the basic-graph-pattern piece of SPARQL, simplified to one triple at a time. */ +export function getRelationships(db: Database, pattern: TriplePattern): StoredRelationship[] { + const conditions: string[] = []; + const params: any[] = []; + + if (pattern.subjectType) { conditions.push("subject_type = ?"); params.push(pattern.subjectType); } + if (pattern.subjectId) { conditions.push("subject_id = ?"); params.push(pattern.subjectId); } + if (pattern.predicate) { conditions.push("predicate = ?"); params.push(pattern.predicate); } + if (pattern.objectType) { conditions.push("object_type = ?"); params.push(pattern.objectType); } + if (pattern.objectId) { conditions.push("object_id = ?"); params.push(pattern.objectId); } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + return db.prepare(`SELECT * FROM smriti_relationships ${where}`).all(...params) as StoredRelationship[]; +} + +/** + * Find other knowledge units that `mentions` at least one of the same + * canonical entities as `unitId` — the candidate set for promote-time + * LLM relationship inference (bounded, so that call stays cheap). + */ +export function findRelatedCandidates( + db: Database, + unitId: string, + limit: number = 5 +): Array<{ id: string; topic: string; category: string; plain_text: string }> { + return db + .prepare( + `SELECT DISTINCT ku.id, ku.topic, ku.category, ku.plain_text + FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id + AND r2.object_type = 'entity' AND r2.predicate = 'mentions' + JOIN smriti_knowledge_units ku ON ku.id = r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' AND r1.subject_id = ? + AND r1.object_type = 'entity' AND r1.predicate = 'mentions' + AND r2.subject_id != r1.subject_id + LIMIT ?` + ) + .all(unitId, limit) as Array<{ id: string; topic: string; category: string; plain_text: string }>; +} diff --git a/src/team/config.ts b/src/team/config.ts index 8dddcbc..13c0efe 100644 --- a/src/team/config.ts +++ b/src/team/config.ts @@ -17,9 +17,17 @@ export type CustomCategoryDef = { description?: string; }; +export type CustomEntityDef = { + id: string; + label: string; + entity_type: string; + aliases: string[]; +}; + export type SmritiConfig = { version: number; categories?: CustomCategoryDef[]; + entities?: CustomEntityDef[]; allowedCategories?: string[]; autoSync?: boolean; }; @@ -115,3 +123,73 @@ export function exportCustomCategories(db: Database): CustomCategoryDef[] { ...(r.description ? { description: r.description } : {}), })); } + +// ============================================================================= +// Entity Merge — team/org propagation for smriti_entities +// +// Same reasoning as categories: an entity's canonical id is only meaningful +// if every teammate's machine agrees on it. .smriti/config.json is the +// git-committed source of truth each local smriti_entities table converges +// toward on every share/sync — the same role a published vocabulary plays +// for literal RDF, minus the URIs. +// ============================================================================= + +/** + * Upsert entities from config into the local DB. Matches first by id, then + * falls back to a normalized-label match (so two machines that independently + * minted different ids for the same concept still converge once either + * syncs the shared file). Unions aliases on conflict rather than overwriting. + * Returns count of newly created entities. + */ +export function mergeEntities(db: Database, entities: CustomEntityDef[]): number { + if (entities.length === 0) return 0; + + let created = 0; + for (const entity of entities) { + const existingById = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE id = ?`) + .get(entity.id) as { id: string; aliases: string } | null; + + if (existingById) { + const aliases = new Set(JSON.parse(existingById.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), entity.id); + continue; + } + + const normalizedLabel = entity.label.trim().toLowerCase(); + const existingByLabel = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE LOWER(label) = ?`) + .get(normalizedLabel) as { id: string; aliases: string } | null; + + if (existingByLabel) { + const aliases = new Set(JSON.parse(existingByLabel.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), existingByLabel.id); + continue; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 0)` + ).run(entity.id, entity.label, entity.entity_type, JSON.stringify(entity.aliases)); + created++; + } + return created; +} + +/** Query smriti_entities and return as config defs for export to .smriti/config.json. */ +export function exportEntities(db: Database): CustomEntityDef[] { + const rows = db + .prepare(`SELECT id, label, entity_type, aliases FROM smriti_entities`) + .all() as Array<{ id: string; label: string; entity_type: string; aliases: string }>; + + return rows.map((r) => ({ + id: r.id, + label: r.label, + entity_type: r.entity_type, + aliases: JSON.parse(r.aliases), + })); +} diff --git a/src/team/share.ts b/src/team/share.ts index 0d61733..22adef8 100644 --- a/src/team/share.ts +++ b/src/team/share.ts @@ -10,7 +10,7 @@ import { SMRITI_DIR, AUTHOR } from "../config"; import { hashContent } from "../qmd"; import { existsSync, mkdirSync } from "fs"; import { join } from "path"; -import { readConfig, writeConfig, exportCustomCategories } from "./config"; +import { readConfig, writeConfig, exportCustomCategories, exportEntities } from "./config"; import { formatSessionAsFallback, isSessionWorthSharing, @@ -178,15 +178,17 @@ async function writeManifest( const fullManifest = [...existingManifest, ...newEntries]; await Bun.write(indexPath, JSON.stringify(fullManifest, null, 2)); - // Write config — always update with latest custom categories + // Write config — always update with latest custom categories + canonical entities const existing = readConfig(outputDir); const customCategories = db ? exportCustomCategories(db) : []; + const entities = db ? exportEntities(db) : []; const config = { ...existing, - version: customCategories.length > 0 ? 2 : (existing.version ?? 1), + version: customCategories.length > 0 || entities.length > 0 ? 2 : (existing.version ?? 1), allowedCategories: existing.allowedCategories ?? ["*"], autoSync: existing.autoSync ?? false, ...(customCategories.length > 0 ? { categories: customCategories } : {}), + ...(entities.length > 0 ? { entities } : {}), }; await writeConfig(outputDir, config); diff --git a/src/team/sync.ts b/src/team/sync.ts index fdb9d5a..b4e51cc 100644 --- a/src/team/sync.ts +++ b/src/team/sync.ts @@ -9,7 +9,8 @@ import type { Database } from "bun:sqlite"; import { SMRITI_DIR } from "../config"; import { addMessage, hashContent } from "../qmd"; import { join } from "path"; -import { readConfig, mergeCategories } from "./config"; +import { readConfig, mergeCategories, mergeEntities } from "./config"; +import { insertRelationship, type RelationshipPredicate } from "../learn/entities"; // ============================================================================= // Types @@ -26,6 +27,7 @@ export type SyncResult = { skipped: number; errors: string[]; categoriesImported: number; + entitiesImported: number; }; // ============================================================================= @@ -116,6 +118,7 @@ export async function syncTeamKnowledge( skipped: 0, errors: [], categoriesImported: 0, + entitiesImported: 0, }; // Determine input directory @@ -144,11 +147,16 @@ export async function syncTeamKnowledge( ).map((r) => r.content_hash) ); - // Import custom categories from config.json (v2+) before scanning files + // Import custom categories + canonical entities from config.json (v2+) + // before scanning files — entities must exist locally before per-file + // "mentions"/relationship edges below can reference them. const config = readConfig(inputDir); if (config.categories && config.categories.length > 0) { result.categoriesImported = mergeCategories(db, config.categories); } + if (config.entities && config.entities.length > 0) { + result.entitiesImported = mergeEntities(db, config.entities); + } // Scan for markdown files const knowledgeDir = join(inputDir, "knowledge"); @@ -175,10 +183,11 @@ export async function syncTeamKnowledge( continue; } - // Segmented pipeline docs don't have **user**/**assistant** patterns; - // treat the whole body as a single assistant message. - const isSegmented = meta.pipeline === "segmented"; - const messages = isSegmented + // Segmented and consolidated pipeline docs don't have + // **user**/**assistant** patterns; treat the whole body as a single + // assistant message. + const isSingleMessageDoc = meta.pipeline === "segmented" || meta.pipeline === "consolidated"; + const messages = isSingleMessageDoc ? [{ role: "assistant", content: body.trim() }] : extractMessages(body); @@ -242,6 +251,28 @@ export async function syncTeamKnowledge( contentHash ); + // Re-create relationship edges from frontmatter. Unlike entities, + // these need no canonicalization: unit ids are portable UUIDs + // already shared as `sessionId` above, so edges reference the same + // node on every machine that imports this file. + const asArray = (v: string | string[] | undefined): string[] => + v === undefined ? [] : Array.isArray(v) ? v : [v]; + + for (const entityId of asArray(meta.entity_ids)) { + if (!entityId) continue; + insertRelationship(db, "knowledge_unit", sessionId, "mentions", "entity", entityId, { + source: "extraction", + }); + } + for (const predicate of ["relatesTo", "supersedes", "contradicts"] as RelationshipPredicate[]) { + for (const objectId of asArray(meta[predicate])) { + if (!objectId) continue; + insertRelationship(db, "knowledge_unit", sessionId, predicate, "knowledge_unit", objectId, { + source: "llm", + }); + } + } + result.imported++; } catch (err: any) { result.errors.push(`${match}: ${err.message}`); diff --git a/test/learn-entities-sync.test.ts b/test/learn-entities-sync.test.ts new file mode 100644 index 0000000..8b850c4 --- /dev/null +++ b/test/learn-entities-sync.test.ts @@ -0,0 +1,166 @@ +/** + * test/learn-entities-sync.test.ts - Team/org propagation of the entities + + * relationships layer via the existing share/sync git round-trip. + * + * This is the test that directly answers "how does this reach the org/team + * level": two independent in-memory DBs stand in for two teammates' machines, + * connected only through a shared tmp `.smriti/` directory (config.json + + * knowledge/*.md), exactly like real git-committed team knowledge. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { initSmriti, closeDb, insertKnowledgeUnit, upsertProject } from "../src/db"; +import { readConfig, writeConfig, exportEntities } from "../src/team/config"; +import { syncTeamKnowledge } from "../src/team/sync"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import { getEntity, resolveEntity, insertRelationship, getRelationships } from "../src/learn/entities"; +import type { KnowledgeUnit } from "../src/team/types"; + +// Two independent :memory: databases (via separate initSmriti calls) stand in +// for two teammates' machines. initSmriti's underlying store singleton is +// process-wide, but every call site below threads the returned `db` handle +// explicitly rather than going through the singleton getDb(), so the two +// stay genuinely isolated for everything this test touches. +let dbA: Database; +let dbB: Database; +let sharedDir: string; + +beforeAll(async () => { + dbA = await initSmriti(":memory:"); + dbB = await initSmriti(":memory:"); + sharedDir = join(tmpdir(), `smriti-propagation-test-${Date.now()}`); + mkdirSync(sharedDir, { recursive: true }); + + // Pre-existing FK requirement of syncTeamKnowledge/upsertSessionMeta, + // unrelated to the entities/relationships work: the importing machine + // must already have the project registered locally (smriti_session_meta + // FKs to smriti_projects). Both "teammates" in this test are in the same project. + upsertProject(dbA, "propproj"); + upsertProject(dbB, "propproj"); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(sharedDir, { recursive: true }); } catch {} +}); + +test("entity canonical id and unit-relationship edges converge across two machines via share -> sync", async () => { + // --- Machine A: create + promote a unit that mentions "Redis" --- + const unit: KnowledgeUnit = { + id: "prop-unit-a", + topic: "Redis TTL guidance", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use a 5-minute TTL for API response caching.", + lineRanges: [], + }; + insertKnowledgeUnit(dbA, unit, "prop-session-a", "propproj", "prop-hash-a"); + const redisIdOnA = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-unit-a", "mentions", "entity", redisIdOnA); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Redis TTL Guidance\n\nUse a 5-minute TTL." }), { status: 200 }) + ) as any; + + try { + const result = await consolidateKnowledge(dbA, { + minDensity: 999, // no segment-phase sessions on A — isolates the promote phase + minRetrievals: 999, + minRelevance: 8, // unit's relevance (9) qualifies + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } + + // --- Machine A "shares": publish its canonical entity registry to the shared config.json --- + // (this is the exact operation writeManifest performs in src/team/share.ts, just called + // directly here to isolate propagation from the rest of the share pipeline's session-querying) + const existingConfig = readConfig(sharedDir); + await writeConfig(sharedDir, { ...existingConfig, version: 2, entities: exportEntities(dbA) }); + + // --- Machine B has never seen "Redis" before --- + expect(getEntity(dbB, "redis")).toBeNull(); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + + expect(syncResult.errors).toEqual([]); + expect(syncResult.entitiesImported).toBeGreaterThanOrEqual(1); + expect(syncResult.imported).toBeGreaterThanOrEqual(1); + + // The entity converged onto the SAME canonical id — not an independently re-slugified duplicate. + const redisOnB = getEntity(dbB, "redis"); + expect(redisOnB).toBeTruthy(); + expect(redisOnB!.id).toBe(redisIdOnA); + expect(redisOnB!.aliases).toContain("Redis"); + + // The unit's "mentions" edge was re-created on B, referencing that same entity id. + const bMentions = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-unit-a", + predicate: "mentions", + objectType: "entity", + }); + expect(bMentions.map((r) => r.object_id)).toContain(redisIdOnA); +}); + +test("supersedes edges between shared units survive the round-trip with no canonicalization needed", async () => { + // Machine A: an existing unit, and a new one that supersedes it — both promoted. + const existing: KnowledgeUnit = { + id: "prop-superseded-unit", topic: "Old Redis TTL", category: "architecture/decision", relevance: 8, + entities: ["Redis"], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const superseding: KnowledgeUnit = { + id: "prop-superseding-unit", topic: "New Redis TTL", category: "architecture/decision", relevance: 9, + entities: ["Redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(dbA, existing, "prop-session-b1", "propproj", "prop-hash-existing"); + insertKnowledgeUnit(dbA, superseding, "prop-session-b2", "propproj", "prop-hash-superseding"); + const redisId = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-superseded-unit", "mentions", "entity", redisId); + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "mentions", "entity", redisId); + // Simulate what promote-phase LLM relationship inference would have discovered: + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "supersedes", "knowledge_unit", "prop-superseded-unit", { source: "llm" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }) + ) as any; + + try { + // Only "prop-superseding-unit" clears the bar this round (existing unit stays at relevance 8 + // < minRelevance 8.5, so we promote exactly the one whose frontmatter should carry the edge). + const result = await consolidateKnowledge(dbA, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8.5, + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } + + await writeConfig(sharedDir, { ...readConfig(sharedDir), version: 2, entities: exportEntities(dbA) }); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + expect(syncResult.errors).toEqual([]); + + const bEdges = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-superseding-unit", + predicate: "supersedes", + }); + // No entity-style canonicalization needed here: unit ids are portable UUIDs, + // so the edge lands referencing the exact same object id on both machines. + expect(bEdges.map((r) => r.object_id)).toContain("prop-superseded-unit"); +}); diff --git a/test/learn-entities.test.ts b/test/learn-entities.test.ts new file mode 100644 index 0000000..210e557 --- /dev/null +++ b/test/learn-entities.test.ts @@ -0,0 +1,349 @@ +/** + * test/learn-entities.test.ts - Tests for canonical entity resolution and + * relationship triples (RDF-inspired knowledge graph layer). + * + * Mirrors test/learn-consolidate.test.ts's style: initSmriti(":memory:"), + * mocked global.fetch standing in for Ollama, scratch tmpDir for output. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + findPromotableUnits, +} from "../src/db"; +import { + resolveEntity, + getEntity, + findEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + getUnitsForEntity, +} from "../src/learn/entities"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import type { KnowledgeUnit } from "../src/team/types"; + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-entities-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +function mockOllamaFetch(handlers: { stage1?: () => object; relation?: () => string; stage2?: () => string }) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const prompt = body.prompt as string; + if (prompt.includes("Knowledge Unit Segmentation")) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify((handlers.stage1 ?? (() => ({ units: [] })))()) + "\n```" }), + { status: 200 } + ); + } + if (prompt.includes("CANDIDATES")) { + return new Response(JSON.stringify({ response: (handlers.relation ?? (() => ""))() }), { status: 200 }); + } + return new Response( + JSON.stringify({ response: (handlers.stage2 ?? (() => "# Doc\n\nContent."))() }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Entity resolution +// ============================================================================= + +test("resolveEntity merges exact-normalize variants (case/whitespace) onto one canonical id", () => { + const id1 = resolveEntity(db, "JWT")!; + const id2 = resolveEntity(db, " jwt ")!; + const id3 = resolveEntity(db, "JWT."); + + expect(id1).toBe(id2); + expect(id1).toBe(id3); + + const entity = getEntity(db, id1)!; + expect(entity.label).toBe("JWT"); // first-seen label wins + expect(entity.aliases).toContain("JWT"); + expect(entity.aliases).toContain("jwt"); + expect(entity.mention_count).toBe(3); +}); + +test("resolveEntity does not merge genuinely different wordings for the same concept", () => { + const jwtId = resolveEntity(db, "distinct-jwt-test")!; + const fullFormId = resolveEntity(db, "distinct-json-web-token-test")!; + + expect(jwtId).not.toBe(fullFormId); +}); + +test("resolveEntity returns null for blank labels", () => { + expect(resolveEntity(db, " ")).toBeNull(); +}); + +test("findEntity looks up by exact id or by label", () => { + resolveEntity(db, "Redis"); + expect(findEntity(db, "Redis")?.id).toBe("redis"); + expect(findEntity(db, "redis")?.id).toBe("redis"); + expect(findEntity(db, "nonexistent-entity-xyz")).toBeNull(); +}); + +// ============================================================================= +// Relationship triples +// ============================================================================= + +test("insertRelationship dedups via the UNIQUE constraint", () => { + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + + const rows = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "unit-a", predicate: "mentions" }); + expect(rows.length).toBe(1); +}); + +test("getRelationships supports single-triple-pattern lookup", () => { + insertRelationship(db, "knowledge_unit", "unit-b", "supersedes", "knowledge_unit", "unit-a", { source: "llm" }); + + const bySubject = getRelationships(db, { subjectId: "unit-b" }); + expect(bySubject.some((r) => r.predicate === "supersedes" && r.object_id === "unit-a")).toBe(true); + + const byPredicate = getRelationships(db, { predicate: "supersedes" }); + expect(byPredicate.length).toBeGreaterThanOrEqual(1); +}); + +// ============================================================================= +// Segment phase: mentions edges created with no extra LLM calls +// ============================================================================= + +test("consolidate segment phase creates mentions edges for every stored entity, no extra LLM calls", async () => { + seedSession("ent-s1", "entproj", [ + { role: "user", content: "We need to decide on a caching strategy for the API. Considering Redis vs in-memory caching." }, + { role: "assistant", content: "Redis is better — it's external state, handles multi-instance, fast, and proven." }, + ]); + updateDensityScore(db, "ent-s1", 0.9); + + let fetchCallCount = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + fetchCallCount++; + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + // relevance kept low (1) deliberately: this unit stays in the shared + // test DB as tier='segmented' after this test, and must never satisfy + // a later test's minRelevance threshold (e.g. 8) via leftover state. + return new Response( + JSON.stringify({ + response: "```json\n" + JSON.stringify({ + units: [{ topic: "Redis caching decision", category: "architecture/decision", relevance: 1, entities: ["Redis", "Caching"] }], + }) + "\n```", + }), + { status: 200 } + ); + } + return new Response(JSON.stringify({ response: "unexpected call" }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + minRetrievals: 999, + minRelevance: 999, // nothing promotable — isolates the segment phase + outputDir: join(tmpDir, "ent-output"), + }); + + expect(result.unitsStored).toBe(1); + expect(fetchCallCount).toBe(1); // segmentation only, no promote-time LLM calls + + const unitRow = db + .prepare(`SELECT id FROM smriti_knowledge_units WHERE session_id = 'ent-s1'`) + .get() as { id: string }; + + const mentions = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: unitRow.id, + predicate: "mentions", + }); + expect(mentions.length).toBe(2); + const objectIds = mentions.map((m) => m.object_id).sort(); + expect(objectIds).toEqual(["caching", "redis"]); + + const unitsForRedis = getUnitsForEntity(db, "redis"); + expect(unitsForRedis.map((u) => u.id)).toContain(unitRow.id); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// minEntityReach promotion criterion +// ============================================================================= + +test("minEntityReach promotes a unit whose entity is shared by >= K other units, even at 0 retrievals", () => { + const shared: KnowledgeUnit = { + id: "reach-unit-1", topic: "Topic A", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content A", lineRanges: [], + }; + const sharedOther1: KnowledgeUnit = { + id: "reach-unit-2", topic: "Topic B", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content B", lineRanges: [], + }; + const sharedOther2: KnowledgeUnit = { + id: "reach-unit-3", topic: "Topic C", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content C", lineRanges: [], + }; + + insertKnowledgeUnit(db, shared, "reach-s1", "reachproj", "reach-hash-1"); + insertKnowledgeUnit(db, sharedOther1, "reach-s2", "reachproj", "reach-hash-2"); + insertKnowledgeUnit(db, sharedOther2, "reach-s3", "reachproj", "reach-hash-3"); + + const entityId = resolveEntity(db, "shared-webhook-retries")!; + insertRelationship(db, "knowledge_unit", "reach-unit-1", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-2", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-3", "mentions", "entity", entityId); + + // Below scalar thresholds (relevance=2, retrieval_count=0) but 2 OTHER units share the entity. + const promotableWithReach = findPromotableUnits(db, 999, 999, 2); + expect(promotableWithReach.map((u) => u.id)).toContain("reach-unit-1"); + + // Without minEntityReach, the same unit is not promotable. + const promotableWithoutReach = findPromotableUnits(db, 999, 999); + expect(promotableWithoutReach.map((u) => u.id)).not.toContain("reach-unit-1"); +}); + +// ============================================================================= +// Promote-phase relationship inference (LLM-gated, bounded) +// ============================================================================= + +test("promote phase persists LLM-inferred relatesTo/supersedes/contradicts edges", async () => { + const existing: KnowledgeUnit = { + id: "infer-existing", topic: "Old Redis TTL guidance", category: "architecture/decision", relevance: 5, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "infer-s1", "inferproj", "infer-hash-existing"); + const redisId = resolveEntity(db, "infer-redis")!; + insertRelationship(db, "knowledge_unit", "infer-existing", "mentions", "entity", redisId); + + const promoted: KnowledgeUnit = { + id: "infer-new", topic: "New Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: ["infer-redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "infer-s2", "inferproj", "infer-hash-new"); + insertRelationship(db, "knowledge_unit", "infer-new", "mentions", "entity", redisId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch({ relation: () => "RELATION [0]: supersedes" }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no segment-phase sessions + minRetrievals: 999, + minRelevance: 8, // only infer-new (relevance 9) qualifies + outputDir: join(tmpDir, "infer-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "infer-new", predicate: "supersedes" }); + expect(edges.length).toBe(1); + expect(edges[0].object_id).toBe("infer-existing"); + expect(edges[0].source).toBe("llm"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("promote phase relationship inference is best-effort: a broken LLM response doesn't block promotion", async () => { + const existing: KnowledgeUnit = { + id: "badllm-existing", topic: "Existing", category: "code/pattern", relevance: 5, + entities: [], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "badllm-s1", "badllmproj", "badllm-hash-existing"); + const entId = resolveEntity(db, "badllm-entity")!; + insertRelationship(db, "knowledge_unit", "badllm-existing", "mentions", "entity", entId); + + const promoted: KnowledgeUnit = { + id: "badllm-new", topic: "New", category: "code/pattern", relevance: 9, + entities: ["badllm-entity"], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "badllm-s2", "badllmproj", "badllm-hash-new"); + insertRelationship(db, "knowledge_unit", "badllm-new", "mentions", "entity", entId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const prompt = body.prompt as string; + if (prompt.includes("CANDIDATES")) throw new Error("connection refused"); + return new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, + outputDir: join(tmpDir, "badllm-output"), + }); + + // Promotion itself succeeds even though relationship inference failed. + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "badllm-new" }) + .filter((r) => r.predicate !== "mentions"); + expect(edges.length).toBe(0); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// findRelatedCandidates +// ============================================================================= + +test("findRelatedCandidates finds other units sharing a canonical entity, excluding self", () => { + const a: KnowledgeUnit = { id: "cand-a", topic: "A", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "a", lineRanges: [] }; + const b: KnowledgeUnit = { id: "cand-b", topic: "B", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "b", lineRanges: [] }; + insertKnowledgeUnit(db, a, "cand-s1", "candproj", "cand-hash-a"); + insertKnowledgeUnit(db, b, "cand-s2", "candproj", "cand-hash-b"); + + const sharedEntity = resolveEntity(db, "cand-shared-entity")!; + insertRelationship(db, "knowledge_unit", "cand-a", "mentions", "entity", sharedEntity); + insertRelationship(db, "knowledge_unit", "cand-b", "mentions", "entity", sharedEntity); + + const candidates = findRelatedCandidates(db, "cand-a", 5); + expect(candidates.map((c) => c.id)).toEqual(["cand-b"]); + expect(candidates.map((c) => c.id)).not.toContain("cand-a"); +}); From 0109d351cf0103ac79e65f3c048c0c37ef01e265 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 16:22:51 +0000 Subject: [PATCH 3/3] fix(learn): prevent bidirectional directional-predicate edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When two units sharing an entity are promoted in the same consolidate run, each independently asks the LLM "do I supersede/contradict the other" — found via a live demo that this can produce both directions asserted simultaneously (A supersedes B and B supersedes A), which is incoherent for a directional predicate. Skip inserting the reverse edge if the candidate already asserted it in the other direction. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PnYgLUDwWALu1bUAgrFQDG --- src/learn/consolidate.ts | 15 ++++++++++++ test/learn-entities.test.ts | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/src/learn/consolidate.ts b/src/learn/consolidate.ts index 38d9a94..5e35606 100644 --- a/src/learn/consolidate.ts +++ b/src/learn/consolidate.ts @@ -313,6 +313,21 @@ RELATION [i]: relatesTo|supersedes|contradicts|none`; const candidate = candidates[index]; if (!predicate || !candidate) continue; + // Directional predicates shouldn't hold in both directions for the same + // pair. When two entity-sharing units are promoted in the same run, + // each independently asks "do I relate to/supersede the other" — if the + // candidate already asserted the reverse relation (e.g. its own + // promotion ran first in this batch), keep that one and skip the + // contradictory reverse edge rather than storing both. + const reverseAlreadyAsserted = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: candidate.id, + predicate, + objectType: "knowledge_unit", + objectId: unit.id, + }).length > 0; + if (reverseAlreadyAsserted) continue; + insertRelationship(db, "knowledge_unit", unit.id, predicate, "knowledge_unit", candidate.id, { source: "llm", }); diff --git a/test/learn-entities.test.ts b/test/learn-entities.test.ts index 210e557..2c91e55 100644 --- a/test/learn-entities.test.ts +++ b/test/learn-entities.test.ts @@ -285,6 +285,54 @@ test("promote phase persists LLM-inferred relatesTo/supersedes/contradicts edges } }); +test("promote phase never asserts a directional predicate in both directions for the same pair", async () => { + // Two units sharing an entity, BOTH clearing the promotion bar in the same + // run — each independently asks "do I supersede the other" and (with a + // naive LLM/mock that doesn't reason about recency) can get "yes" from both + // sides. Only one direction should end up persisted. + const unitA: KnowledgeUnit = { + id: "bidir-unit-a", topic: "Redis TTL: 1 minute", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const unitB: KnowledgeUnit = { + id: "bidir-unit-b", topic: "Redis TTL: 15 minutes", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 15-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, unitA, "bidir-s1", "bidirproj", "bidir-hash-a"); + insertKnowledgeUnit(db, unitB, "bidir-s2", "bidirproj", "bidir-hash-b"); + const entityId = resolveEntity(db, "bidir-redis")!; + insertRelationship(db, "knowledge_unit", "bidir-unit-a", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "bidir-unit-b", "mentions", "entity", entityId); + + const originalFetch = globalThis.fetch; + // Always answers "supersedes" regardless of which side is asking — the + // worst case for this bug, and realistic for a small/local model given a + // prompt with no explicit recency signal. + globalThis.fetch = mockOllamaFetch({ relation: () => "RELATION [0]: supersedes" }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, // both unitA and unitB qualify + outputDir: join(tmpDir, "bidir-output"), + }); + + expect(result.unitsPromoted).toBe(2); + + const aToB = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-a", predicate: "supersedes", objectId: "bidir-unit-b", + }); + const bToA = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-b", predicate: "supersedes", objectId: "bidir-unit-a", + }); + // Exactly one direction persisted, never both. + expect(aToB.length + bToA.length).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } +}); + test("promote phase relationship inference is best-effort: a broken LLM response doesn't block promotion", async () => { const existing: KnowledgeUnit = { id: "badllm-existing", topic: "Existing", category: "code/pattern", relevance: 5,