diff --git a/src/db.ts b/src/db.ts index 59d8ba7..048cb55 100644 --- a/src/db.ts +++ b/src/db.ts @@ -14,6 +14,7 @@ import { QMD_DB_PATH, SMRITI_SESSIONS_DIR } from "./config"; import { initializeMemoryTables } from "./qmd"; import { createStore } from "../qmd/src/index"; import { setQmdStore, closeQmdStore } from "./store"; +import type { KnowledgeUnit } from "./team/types"; // ============================================================================= // Connection @@ -204,6 +205,67 @@ export function initializeSmritiTables(db: Database): void { entities TEXT ); + -- Knowledge consolidation: raw Stage-1 extracts, promoted to canonical on reuse + CREATE TABLE IF NOT EXISTS smriti_knowledge_units ( + id TEXT PRIMARY KEY, -- KnowledgeUnit.id (uuid) + session_id TEXT NOT NULL, + project_id TEXT, + topic TEXT NOT NULL, + category TEXT NOT NULL, + relevance REAL NOT NULL DEFAULT 0, -- 0-10, from Stage 1 + entities TEXT, -- JSON array + files TEXT, -- JSON array + plain_text TEXT NOT NULL, -- raw Stage-1 extract + line_ranges TEXT, -- JSON array of {start,end} + content_hash TEXT NOT NULL, -- hashContent({topic,category,plainText}) — Stage-1 dedup key + tier TEXT NOT NULL DEFAULT 'segmented', -- 'segmented' | 'canonical' + retrieval_count INTEGER NOT NULL DEFAULT 0, + last_recalled_at TEXT, + promoted_at TEXT, + canonical_doc_path TEXT, -- relative path under .smriti/knowledge/, set on promotion + share_id TEXT, -- points at the smriti_shares row created on promotion + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_session ON smriti_knowledge_units(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_hash ON smriti_knowledge_units(content_hash); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_tier ON smriti_knowledge_units(tier); + + -- Canonical entity registry: resolves free-text entity mentions (from Stage 1 + -- extraction) onto a stable node, so recurrence is detected regardless of wording. + -- Propagated team/org-wide via .smriti/config.json, same mechanism as custom categories. + CREATE TABLE IF NOT EXISTS smriti_entities ( + id TEXT PRIMARY KEY, -- slug, e.g. "jwt", "redis" + label TEXT NOT NULL, -- canonical display name + entity_type TEXT NOT NULL DEFAULT 'concept', -- 'technology' | 'concept' | 'file' | 'pattern' + aliases TEXT NOT NULL DEFAULT '[]', -- JSON array of raw strings seen + mention_count INTEGER NOT NULL DEFAULT 0, + first_seen_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_entities_label ON smriti_entities(label); + + -- Relationship triples (subject/object polymorphic via type+id, not literal RDF URIs). + -- knowledge_unit -mentions-> entity edges come free from Stage-1 extraction; + -- knowledge_unit -relatesTo/supersedes/contradicts-> knowledge_unit edges are LLM-gated, + -- only at promotion time (see src/learn/consolidate.ts), persisting what + -- ollamaCheckConflicts previously only computed ephemerally. + CREATE TABLE IF NOT EXISTS smriti_relationships ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + subject_type TEXT NOT NULL, -- 'knowledge_unit' | 'entity' | 'session' + subject_id TEXT NOT NULL, + predicate TEXT NOT NULL, -- 'mentions' | 'relatesTo' | 'supersedes' | 'contradicts' + object_type TEXT NOT NULL, + object_id TEXT NOT NULL, + confidence REAL DEFAULT 1.0, + source TEXT DEFAULT 'extraction', -- 'extraction' | 'derived' | 'llm' + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(subject_type, subject_id, predicate, object_type, object_id) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_subject ON smriti_relationships(subject_type, subject_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_object ON smriti_relationships(object_type, object_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_predicate ON smriti_relationships(predicate); + -- Tool usage tracking CREATE TABLE IF NOT EXISTS smriti_tool_usage ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -475,6 +537,12 @@ const DEFAULT_AGENTS = [ log_pattern: null, parser: "claude-web", }, + { + id: "team", + display_name: "Team Import", + log_pattern: null, + parser: "generic", + }, ] as const; /** Default category taxonomy */ @@ -1246,6 +1314,207 @@ export function getDensityScore(db: Database, sessionId: string): number { return row?.density_score ?? 0; } +// ============================================================================= +// Knowledge Consolidation (Progressive Summarization) +// ============================================================================= + +export interface StoredKnowledgeUnit { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string[]; + files: string[]; + plain_text: string; + line_ranges: Array<{ start: number; end: number }>; + content_hash: string; + tier: "segmented" | "canonical"; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; +} + +type KnowledgeUnitRow = { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string | null; + files: string | null; + plain_text: string; + line_ranges: string | null; + content_hash: string; + tier: string; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; +}; + +function deserializeKnowledgeUnit(row: KnowledgeUnitRow): StoredKnowledgeUnit { + return { + ...row, + entities: row.entities ? JSON.parse(row.entities) : [], + files: row.files ? JSON.parse(row.files) : [], + line_ranges: row.line_ranges ? JSON.parse(row.line_ranges) : [], + tier: row.tier as "segmented" | "canonical", + }; +} + +/** + * Insert a Stage-1 knowledge unit if its content hash isn't already stored. + * Returns true if inserted, false if it was a duplicate (caller distinguishes + * "stored" from "skipped" the same way shareSegmentedKnowledge does for shares). + */ +export function insertKnowledgeUnit( + db: Database, + unit: KnowledgeUnit, + sessionId: string, + projectId: string | null, + contentHash: string +): boolean { + const exists = db + .prepare(`SELECT 1 FROM smriti_knowledge_units WHERE content_hash = ?`) + .get(contentHash); + if (exists) return false; + + db.prepare( + `INSERT INTO smriti_knowledge_units + (id, session_id, project_id, topic, category, relevance, entities, files, plain_text, line_ranges, content_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + unit.id, + sessionId, + projectId, + unit.topic, + unit.category, + unit.relevance, + JSON.stringify(unit.entities || []), + JSON.stringify(unit.files || []), + unit.plainText, + JSON.stringify(unit.lineRanges || []), + contentHash + ); + return true; +} + +/** Dense sessions (by density_score) that haven't been segmented into knowledge units yet. */ +export function findUnsegmentedDenseSessions( + db: Database, + minDensity: number, + limit?: number +): Array<{ session_id: string; project_id: string | null; density_score: number }> { + const query = ` + SELECT sm.session_id, sm.project_id, sm.density_score + FROM smriti_session_meta sm + WHERE sm.density_score >= ? + AND NOT EXISTS (SELECT 1 FROM smriti_knowledge_units ku WHERE ku.session_id = sm.session_id) + ORDER BY sm.density_score DESC + ${limit ? "LIMIT ?" : ""} + `; + const rows = limit + ? db.prepare(query).all(minDensity, limit) + : db.prepare(query).all(minDensity); + return rows as Array<{ session_id: string; project_id: string | null; density_score: number }>; +} + +/** Segmented units that have proven reuse (via recall) or scored high relevance at extraction time. */ +export function findPromotableUnits( + db: Database, + minRetrievals: number, + minRelevance: number, + minEntityReach?: number +): StoredKnowledgeUnit[] { + // minEntityReach: a unit is promotable if one of its entities is + // independently mentioned by >= minEntityReach OTHER units — a structural + // reuse signal (cross-session recurrence) that doesn't depend on recall() + // ever having been called on this particular unit. + const entityReachClause = minEntityReach + ? `OR id IN ( + SELECT r1.subject_id FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id AND r2.predicate = 'mentions' + AND r1.predicate = 'mentions' AND r1.subject_id != r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' + GROUP BY r1.subject_id + HAVING COUNT(DISTINCT r2.subject_id) >= ? + )` + : ""; + const params = minEntityReach + ? [minRetrievals, minRelevance, minEntityReach] + : [minRetrievals, minRelevance]; + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units + WHERE tier = 'segmented' AND (retrieval_count >= ? OR relevance >= ? ${entityReachClause})` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Bump retrieval_count for any knowledge units belonging to a recalled session. No-op if none exist yet. */ +export function incrementRetrievalCount(db: Database, sessionId: string): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET retrieval_count = retrieval_count + 1, + last_recalled_at = datetime('now'), + updated_at = datetime('now') + WHERE session_id = ?` + ).run(sessionId); +} + +export function promoteKnowledgeUnit( + db: Database, + unitId: string, + canonicalDocPath: string, + shareId: string +): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET tier = 'canonical', promoted_at = datetime('now'), + canonical_doc_path = ?, share_id = ?, updated_at = datetime('now') + WHERE id = ?` + ).run(canonicalDocPath, shareId, unitId); +} + +export function listKnowledgeUnits( + db: Database, + options: { tier?: "segmented" | "canonical"; minRetrievals?: number; limit?: number } = {} +): StoredKnowledgeUnit[] { + const conditions: string[] = []; + const params: any[] = []; + + if (options.tier) { + conditions.push("tier = ?"); + params.push(options.tier); + } + if (options.minRetrievals !== undefined) { + conditions.push("retrieval_count >= ?"); + params.push(options.minRetrievals); + } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + const limitClause = options.limit ? "LIMIT ?" : ""; + if (options.limit) params.push(options.limit); + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units ${where} + ORDER BY retrieval_count DESC, relevance DESC + ${limitClause}` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + // ============================================================================= // Session Query Labels (#60) // ============================================================================= diff --git a/src/format.ts b/src/format.ts index 023b0c3..f78280f 100644 --- a/src/format.ts +++ b/src/format.ts @@ -268,6 +268,106 @@ export function formatShareResult(result: { return lines.join("\n"); } +// ============================================================================= +// Consolidate Result Formatting +// ============================================================================= + +export function formatConsolidateResult(result: { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + errors: string[]; +}): string { + const lines = [ + `Sessions segmented: ${result.sessionsSegmented}`, + `Units stored: ${result.unitsStored}`, + `Units skipped (dedup): ${result.unitsSkipped}`, + `Units promoted: ${result.unitsPromoted}`, + ]; + + if (result.errors.length > 0) { + lines.push(`Errors: ${result.errors.length}`); + for (const err of result.errors.slice(0, 5)) { + lines.push(` - ${err}`); + } + } + + return lines.join("\n"); +} + +// ============================================================================= +// Knowledge Units (Learnings) Formatting +// ============================================================================= + +export function formatLearnings( + units: Array<{ + tier: string; + topic: string; + category: string; + retrieval_count: number; + relevance: number; + canonical_doc_path: string | null; + }> +): string { + if (units.length === 0) return "No knowledge units found."; + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance", "Doc Path"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + u.canonical_doc_path || "-", + ]); + + return table(headers, rows, [14, 40, 20, 10, 9, 40]); +} + +// ============================================================================= +// Entity Graph Formatting (smriti graph ) +// ============================================================================= + +export function formatEntityGraph( + entity: { id: string; label: string; entity_type: string; aliases: string[]; mention_count: number }, + units: Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }>, + edges: Array<{ subject_id: string; predicate: string; object_id: string }> +): string { + const lines = [ + `Entity: ${entity.label} (${entity.id})`, + `Type: ${entity.entity_type}`, + `Aliases: ${entity.aliases.join(", ") || "-"}`, + `Mentioned ${entity.mention_count} time(s) across ${units.length} unit(s)`, + "", + ]; + + if (units.length === 0) { + lines.push("No knowledge units mention this entity yet."); + return lines.join("\n"); + } + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + ]); + lines.push(table(headers, rows, [14, 40, 20, 10, 9])); + + const nonMentionEdges = edges.filter((e) => e.predicate !== "mentions"); + if (nonMentionEdges.length > 0) { + lines.push("", "Relationships between these units:"); + for (const e of nonMentionEdges) { + lines.push(` ${e.subject_id} --${e.predicate}--> ${e.object_id}`); + } + } + + return lines.join("\n"); +} + // ============================================================================= // Sync Result Formatting // ============================================================================= @@ -278,6 +378,7 @@ export function formatSyncResult(result: { skipped: number; errors: string[]; categoriesImported?: number; + entitiesImported?: number; }): string { const lines = [ `Files processed: ${result.filesProcessed}`, @@ -287,6 +388,9 @@ export function formatSyncResult(result: { if (result.categoriesImported && result.categoriesImported > 0) { lines.push(`Categories imported: ${result.categoriesImported}`); } + if (result.entitiesImported && result.entitiesImported > 0) { + lines.push(`Entities imported: ${result.entitiesImported}`); + } if (result.errors.length > 0) { lines.push(`Errors: ${result.errors.length}`); diff --git a/src/index.ts b/src/index.ts index bb186d5..f5c8956 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ * schema-based categorization, and team knowledge sharing. */ -import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, getTagUsage, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds } from "./db"; +import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, getTagUsage, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds, listKnowledgeUnits } from "./db"; import { getMessages, getSession, getMemoryStatus, embedMemoryMessages } from "./qmd"; import { ingest, ingestAll } from "./ingest/index"; import { categorizeUncategorized } from "./categorize/classifier"; @@ -16,6 +16,8 @@ import { searchFiltered, listSessions } from "./search/index"; import { recall } from "./search/recall"; import { shareKnowledge } from "./team/share"; import { syncTeamKnowledge, listTeamContributions } from "./team/sync"; +import { consolidateKnowledge } from "./learn/consolidate"; +import { findEntity, getUnitsForEntity, getRelationships } from "./learn/entities"; import { generateContext, compareSessions, @@ -53,6 +55,9 @@ import { formatTagUsage, formatDensityBreakdown, formatDigest, + formatConsolidateResult, + formatLearnings, + formatEntityGraph, json, } from "./format"; import { generateDigest } from "./digest"; @@ -213,6 +218,9 @@ Commands: compare Compare two sessions (tokens, tools, files) compare --last Compare last 2 sessions for current project share [filters] Export knowledge to .smriti/ + consolidate [options] Segment dense sessions, promote reused knowledge units + learnings [options] List extracted knowledge units (tier, retrievals, relevance) + graph Show a canonical entity's mentions and relationship edges sync Import team knowledge from .smriti/ team View team contributions list [filters] List sessions @@ -812,6 +820,74 @@ async function main() { break; } + // ===================================================================== + // CONSOLIDATE + // ===================================================================== + case "consolidate": { + const result = await consolidateKnowledge(db, { + minDensity: Number(getArg(args, "--min-density")) || undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + minRelevance: Number(getArg(args, "--min-relevance")) || undefined, + minEntityReach: Number(getArg(args, "--min-entity-reach")) || undefined, + model: getArg(args, "--model"), + outputDir: getArg(args, "--output"), + sessionLimit: Number(getArg(args, "--session-limit")) || undefined, + onProgress: (msg) => console.log(` ${msg}`), + }); + + console.log(formatConsolidateResult(result)); + break; + } + + // ===================================================================== + // LEARNINGS + // ===================================================================== + case "learnings": { + const units = listKnowledgeUnits(db, { + tier: getArg(args, "--tier") as "segmented" | "canonical" | undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + limit: Number(getArg(args, "--limit")) || 50, + }); + + if (hasFlag(args, "--json")) { + console.log(json(units)); + } else { + console.log(formatLearnings(units)); + } + break; + } + + // ===================================================================== + // GRAPH + // ===================================================================== + case "graph": { + const query = getPositional(args, 1); + if (!query) { + console.error("Usage: smriti graph "); + process.exit(1); + } + + const entity = findEntity(db, query); + if (!entity) { + console.log(`No entity found matching "${query}".`); + break; + } + + const units = getUnitsForEntity(db, entity.id); + const unitIds = new Set(units.map((u) => u.id)); + const edges = units.flatMap((u) => + getRelationships(db, { subjectType: "knowledge_unit", subjectId: u.id }) + .filter((r) => r.predicate !== "mentions" && unitIds.has(r.object_id)) + ); + + if (hasFlag(args, "--json")) { + console.log(json({ entity, units, edges })); + } else { + console.log(formatEntityGraph(entity, units, edges)); + } + break; + } + // ===================================================================== // SYNC // ===================================================================== diff --git a/src/learn/consolidate.ts b/src/learn/consolidate.ts new file mode 100644 index 0000000..5e35606 --- /dev/null +++ b/src/learn/consolidate.ts @@ -0,0 +1,349 @@ +/** + * learn/consolidate.ts - Continuous knowledge consolidation + * + * Progressive Summarization: cheap Stage-1 extraction runs broadly over dense + * sessions; expensive Stage-2 polish only runs once a unit proves it's reused + * (recalled repeatedly) or scored high relevance at extraction time. + * + * Two independent phases, run sequentially: + * - Segment: dense, not-yet-segmented sessions -> segmentSession() -> smriti_knowledge_units + * - Promote: knowledge units that cleared the reuse/relevance bar -> generateDocument() + * -> written to .smriti/knowledge/ + recorded in smriti_shares + * + * CLI-only, like `categorize`/`share` — never wired into the daemon (see + * src/daemon/index.ts's enrichOnIngest comment for why LLM work per-flush is unsafe). + */ + +import type { Database } from "bun:sqlite"; +import { mkdirSync } from "fs"; +import { join } from "path"; +import { SMRITI_DIR, AUTHOR } from "../config"; +import { hashContent } from "../qmd"; +import { + findUnsegmentedDenseSessions, + insertKnowledgeUnit, + findPromotableUnits, + promoteKnowledgeUnit, +} from "../db"; +import { getSessionMessages } from "../team/share"; +import { segmentSession } from "../team/segment"; +import { generateDocument, generateFrontmatter } from "../team/document"; +import { isSessionWorthSharing } from "../team/formatter"; +import { callOllama } from "../team/ollama"; +import type { RawMessage } from "../team/formatter"; +import type { KnowledgeUnit } from "../team/types"; +import { + resolveEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + type RelationshipPredicate, +} from "./entities"; + +// ============================================================================= +// Types +// ============================================================================= + +export type ConsolidateOptions = { + minDensity?: number; + minRetrievals?: number; + minRelevance?: number; + minEntityReach?: number; + model?: string; + outputDir?: string; + author?: string; + sessionLimit?: number; + onProgress?: (msg: string) => void; +}; + +export type ConsolidateResult = { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + errors: string[]; +}; + +// ============================================================================= +// Consolidation +// ============================================================================= + +export async function consolidateKnowledge( + db: Database, + options: ConsolidateOptions = {} +): Promise { + const author = options.author || AUTHOR; + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + + const result: ConsolidateResult = { + sessionsSegmented: 0, + unitsStored: 0, + unitsSkipped: 0, + unitsPromoted: 0, + errors: [], + }; + + // =========================================================================== + // Segment phase: cheap Stage-1 extraction over dense, unsegmented sessions + // =========================================================================== + + const sessions = findUnsegmentedDenseSessions( + db, + options.minDensity ?? 0.5, + options.sessionLimit ?? 20 + ); + + for (const s of sessions) { + try { + const messages = getSessionMessages(db, s.session_id); + if (messages.length === 0) continue; + + const rawMessages: RawMessage[] = messages.map((m) => ({ + role: m.role, + content: m.content, + })); + + if (!isSessionWorthSharing(rawMessages)) continue; + + const segmentationResult = await segmentSession(db, s.session_id, rawMessages, { + model: options.model, + }); + result.sessionsSegmented++; + + for (const unit of segmentationResult.units) { + const contentHash = await hashContent( + JSON.stringify({ topic: unit.topic, category: unit.category, plainText: unit.plainText }) + ); + const inserted = insertKnowledgeUnit(db, unit, s.session_id, s.project_id, contentHash); + inserted ? result.unitsStored++ : result.unitsSkipped++; + + // Turn Stage 1's free-text entities into canonical "mentions" edges — + // pure post-processing of data already extracted, no extra LLM calls. + if (inserted) { + for (const rawEntity of unit.entities) { + const entityId = resolveEntity(db, rawEntity); + if (entityId) { + insertRelationship(db, "knowledge_unit", unit.id, "mentions", "entity", entityId, { + source: "extraction", + }); + } + } + } + } + } catch (err: any) { + result.errors.push(`segment ${s.session_id}: ${err.message}`); + } + } + + options.onProgress?.( + `segment phase: ${result.sessionsSegmented} sessions, ${result.unitsStored} units stored, ${result.unitsSkipped} skipped` + ); + + // =========================================================================== + // Promote phase: expensive Stage-2 polish for units that proved reuse + // =========================================================================== + + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + + const promotable = findPromotableUnits( + db, + options.minRetrievals ?? 3, + options.minRelevance ?? 8, + options.minEntityReach + ); + + for (const stored of promotable) { + try { + const unit: KnowledgeUnit = { + id: stored.id, + topic: stored.topic, + category: stored.category, + relevance: stored.relevance, + entities: stored.entities, + files: stored.files, + plainText: stored.plain_text, + lineRanges: stored.line_ranges, + }; + + // Bounded relationship inference: only runs if this unit shares a + // canonical entity with at least one other unit, and costs exactly one + // extra LLM call (same cost discipline as Stage 2) — persists what + // ollamaCheckConflicts previously only computed ephemerally. + const candidates = findRelatedCandidates(db, stored.id, 5); + if (candidates.length > 0) { + await inferRelationships(db, unit, candidates, options.model); + } + + const doc = await generateDocument(unit, stored.topic, { + model: options.model, + projectSmritiDir: outputDir, + author, + }); + + const categoryDir = join(knowledgeDir, doc.category.replaceAll("/", "-")); + mkdirSync(categoryDir, { recursive: true }); + const filePath = join(categoryDir, doc.filename); + + // Carry canonical entity ids + unit-to-unit edges into shared + // frontmatter. Unlike entity ids, these edges need no team-level + // canonicalization step — unit.id is already a portable UUID once + // shared (see src/team/document.ts's frontmatter `id` field), so + // syncTeamKnowledge can re-create them on a teammate's machine as-is. + const entityIds = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + predicate: "mentions", + objectType: "entity", + }).map((r) => r.object_id); + const outgoingEdges = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + }).filter((r) => r.predicate !== "mentions"); + + const fm = generateFrontmatter( + stored.session_id, + doc.unitId, + { + ...doc.frontmatter, + pipeline: "consolidated", + entity_ids: entityIds, + ...groupEdgesByPredicate(outgoingEdges), + }, + author, + stored.project_id || undefined + ); + await Bun.write(filePath, fm + "\n\n" + doc.markdown); + + const shareId = crypto.randomUUID().slice(0, 8); + const shareHash = await hashContent( + JSON.stringify({ content: doc.markdown, category: doc.category, entities: doc.frontmatter.entities }) + ); + + db.prepare( + `INSERT INTO smriti_shares (id, session_id, category_id, project_id, author, content_hash, unit_id, relevance_score, entities) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + shareId, + stored.session_id, + doc.category, + stored.project_id, + author, + shareHash, + doc.unitId, + stored.relevance, + JSON.stringify(stored.entities) + ); + + const relPath = `knowledge/${doc.category.replaceAll("/", "-")}/${doc.filename}`; + promoteKnowledgeUnit(db, stored.id, relPath, shareId); + result.unitsPromoted++; + } catch (err: any) { + result.errors.push(`promote ${stored.id}: ${err.message}`); + } + } + + options.onProgress?.(`promote phase: ${result.unitsPromoted} units promoted`); + + return result; +} + +// ============================================================================= +// Relationship Inference (promote-time, LLM-gated) +// ============================================================================= + +const RELATION_LINE = /RELATION\s*\[(\d+)\]:\s*(relatesTo|supersedes|contradicts|none)/gi; +const MAX_EXCERPT_CHARS = 800; + +// Case-insensitive regex match -> canonical camelCase predicate (avoid a blind +// .toLowerCase() on the match, which would turn "relatesTo" into "relatesto"). +const PREDICATE_BY_LOWERCASE: Record = { + relatesto: "relatesTo", + supersedes: "supersedes", + contradicts: "contradicts", +}; + +function truncate(text: string, max: number): string { + return text.length > max ? text.slice(0, max) + "…" : text; +} + +/** + * Ask the LLM whether the unit being promoted relatesTo/supersedes/contradicts + * any of its entity-sharing candidates, and persist the answer as edges. + * Best-effort: a failure here (LLM down, unparseable response) is swallowed — + * it's enrichment on top of promotion, not a precondition for it. + */ +async function inferRelationships( + db: Database, + unit: KnowledgeUnit, + candidates: Array<{ id: string; topic: string; category: string; plain_text: string }>, + model?: string +): Promise { + try { + const candidateBlock = candidates + .map((c, i) => `[${i}] Topic: ${c.topic}\nCategory: ${c.category}\nContent: ${truncate(c.plain_text, MAX_EXCERPT_CHARS)}`) + .join("\n\n"); + + const prompt = `You are comparing a NEW knowledge unit against CANDIDATE units that already mention at least one of the same topics/entities. + +NEW UNIT +Topic: ${unit.topic} +Category: ${unit.category} +Content: ${truncate(unit.plainText, MAX_EXCERPT_CHARS)} + +CANDIDATES +${candidateBlock} + +For each candidate, decide the relationship of the NEW unit to it: +- relatesTo: related but neither replaces nor conflicts with the other +- supersedes: the NEW unit replaces/updates the candidate's guidance +- contradicts: the NEW unit conflicts with the candidate +- none: no meaningful relationship + +Respond with exactly one line per candidate, in this format: +RELATION [i]: relatesTo|supersedes|contradicts|none`; + + const response = await callOllama(prompt, { model }); + + for (const match of response.matchAll(RELATION_LINE)) { + const index = Number(match[1]); + const raw = match[2].toLowerCase(); + if (raw === "none") continue; + const predicate = PREDICATE_BY_LOWERCASE[raw]; + const candidate = candidates[index]; + if (!predicate || !candidate) continue; + + // Directional predicates shouldn't hold in both directions for the same + // pair. When two entity-sharing units are promoted in the same run, + // each independently asks "do I relate to/supersede the other" — if the + // candidate already asserted the reverse relation (e.g. its own + // promotion ran first in this batch), keep that one and skip the + // contradictory reverse edge rather than storing both. + const reverseAlreadyAsserted = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: candidate.id, + predicate, + objectType: "knowledge_unit", + objectId: unit.id, + }).length > 0; + if (reverseAlreadyAsserted) continue; + + insertRelationship(db, "knowledge_unit", unit.id, predicate, "knowledge_unit", candidate.id, { + source: "llm", + }); + } + } catch { + // Enrichment only — never block promotion on a failed/unparseable relation call. + } +} + +/** Group outgoing relationship edges by predicate into frontmatter-ready arrays of object unit ids. */ +function groupEdgesByPredicate( + edges: Array<{ predicate: string; object_id: string }> +): Record { + const grouped: Record = {}; + for (const edge of edges) { + (grouped[edge.predicate] ??= []).push(edge.object_id); + } + return grouped; +} diff --git a/src/learn/entities.ts b/src/learn/entities.ts new file mode 100644 index 0000000..474c944 --- /dev/null +++ b/src/learn/entities.ts @@ -0,0 +1,232 @@ +/** + * learn/entities.ts - Canonical entity resolution + relationship triples + * + * RDF-inspired, not literal RDF: no URIs/Turtle/SPARQL. Subject/object are + * (type, id) pairs instead of global URIs, since Smriti is a local SQLite + * tool, not a web-facing linked-data endpoint. The useful ideas kept are: + * stable resource identity (so recurrence is detected across wording + * variance) and typed subject-predicate-object facts that survive team + * sharing (see src/team/config.ts's exportEntities/mergeEntities and + * src/team/document.ts's frontmatter for how these propagate org-wide). + * + * v1 entity resolution is exact-normalize only (case/whitespace/punctuation + * via slugify) — "JWT" and "jwt" merge, "JWT" and "JSON Web Token" do not. + * True synonym resolution needs semantic matching and is out of scope here. + */ + +import type { Database } from "bun:sqlite"; +import { slugify } from "../team/utils"; + +// ============================================================================= +// Types +// ============================================================================= + +export type EntityType = "technology" | "concept" | "file" | "pattern"; +export type RelationshipPredicate = "mentions" | "relatesTo" | "supersedes" | "contradicts"; +export type RelationshipSubjectType = "knowledge_unit" | "entity" | "session"; +export type RelationshipSource = "extraction" | "derived" | "llm"; + +export type StoredEntity = { + id: string; + label: string; + entity_type: EntityType; + aliases: string[]; + mention_count: number; + first_seen_at: string; +}; + +export type StoredRelationship = { + id: number; + subject_type: RelationshipSubjectType; + subject_id: string; + predicate: RelationshipPredicate; + object_type: RelationshipSubjectType; + object_id: string; + confidence: number; + source: RelationshipSource; + created_at: string; +}; + +type EntityRow = { + id: string; + label: string; + entity_type: string; + aliases: string; + mention_count: number; + first_seen_at: string; +}; + +function deserializeEntity(row: EntityRow): StoredEntity { + return { + ...row, + entity_type: row.entity_type as EntityType, + aliases: JSON.parse(row.aliases), + }; +} + +// ============================================================================= +// Entity Resolution +// ============================================================================= + +/** + * Resolve a raw, free-text entity label to a canonical entity id, creating + * the entity if it doesn't exist yet. Matching is exact-normalize (via + * slugify) — same case/whitespace variant collapses to one node; different + * wordings for the same concept do not (see module docstring). + */ +export function resolveEntity( + db: Database, + rawLabel: string, + entityType: EntityType = "concept" +): string | null { + const trimmed = rawLabel.trim(); + if (!trimmed) return null; + + const id = slugify(trimmed); + if (!id) return null; + + const existing = db + .prepare(`SELECT aliases FROM smriti_entities WHERE id = ?`) + .get(id) as { aliases: string } | null; + + if (existing) { + const aliases: string[] = JSON.parse(existing.aliases); + if (!aliases.includes(trimmed)) { + aliases.push(trimmed); + db.prepare( + `UPDATE smriti_entities SET aliases = ?, mention_count = mention_count + 1 WHERE id = ?` + ).run(JSON.stringify(aliases), id); + } else { + db.prepare(`UPDATE smriti_entities SET mention_count = mention_count + 1 WHERE id = ?`).run(id); + } + return id; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 1)` + ).run(id, trimmed, entityType, JSON.stringify([trimmed])); + return id; +} + +export function getEntity(db: Database, id: string): StoredEntity | null { + const row = db.prepare(`SELECT * FROM smriti_entities WHERE id = ?`).get(id) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Look up an entity by exact id, or by slugified/label match against a raw query string. */ +export function findEntity(db: Database, query: string): StoredEntity | null { + const bySlug = getEntity(db, slugify(query)); + if (bySlug) return bySlug; + + const row = db + .prepare(`SELECT * FROM smriti_entities WHERE LOWER(label) = LOWER(?)`) + .get(query.trim()) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Knowledge units that `mentions` a given canonical entity — the display side of `smriti graph `. */ +export function getUnitsForEntity( + db: Database, + entityId: string +): Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }> { + return db + .prepare( + `SELECT ku.id, ku.topic, ku.category, ku.relevance, ku.tier, ku.retrieval_count + FROM smriti_relationships r + JOIN smriti_knowledge_units ku ON ku.id = r.subject_id + WHERE r.subject_type = 'knowledge_unit' AND r.object_type = 'entity' + AND r.predicate = 'mentions' AND r.object_id = ? + ORDER BY ku.retrieval_count DESC, ku.relevance DESC` + ) + .all(entityId) as Array<{ + id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number; + }>; +} + +export function listEntities(db: Database, limit?: number): StoredEntity[] { + const rows = ( + limit + ? db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC LIMIT ?`).all(limit) + : db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC`).all() + ) as EntityRow[]; + return rows.map(deserializeEntity); +} + +// ============================================================================= +// Relationship Triples +// ============================================================================= + +/** Insert a (subject, predicate, object) triple. Deduped via the table's UNIQUE constraint. */ +export function insertRelationship( + db: Database, + subjectType: RelationshipSubjectType, + subjectId: string, + predicate: RelationshipPredicate, + objectType: RelationshipSubjectType, + objectId: string, + options: { confidence?: number; source?: RelationshipSource } = {} +): void { + db.prepare( + `INSERT OR IGNORE INTO smriti_relationships + (subject_type, subject_id, predicate, object_type, object_id, confidence, source) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run( + subjectType, + subjectId, + predicate, + objectType, + objectId, + options.confidence ?? 1.0, + options.source ?? "extraction" + ); +} + +export type TriplePattern = { + subjectType?: RelationshipSubjectType; + subjectId?: string; + predicate?: RelationshipPredicate; + objectType?: RelationshipSubjectType; + objectId?: string; +}; + +/** Single-pattern triple lookup — the basic-graph-pattern piece of SPARQL, simplified to one triple at a time. */ +export function getRelationships(db: Database, pattern: TriplePattern): StoredRelationship[] { + const conditions: string[] = []; + const params: any[] = []; + + if (pattern.subjectType) { conditions.push("subject_type = ?"); params.push(pattern.subjectType); } + if (pattern.subjectId) { conditions.push("subject_id = ?"); params.push(pattern.subjectId); } + if (pattern.predicate) { conditions.push("predicate = ?"); params.push(pattern.predicate); } + if (pattern.objectType) { conditions.push("object_type = ?"); params.push(pattern.objectType); } + if (pattern.objectId) { conditions.push("object_id = ?"); params.push(pattern.objectId); } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + return db.prepare(`SELECT * FROM smriti_relationships ${where}`).all(...params) as StoredRelationship[]; +} + +/** + * Find other knowledge units that `mentions` at least one of the same + * canonical entities as `unitId` — the candidate set for promote-time + * LLM relationship inference (bounded, so that call stays cheap). + */ +export function findRelatedCandidates( + db: Database, + unitId: string, + limit: number = 5 +): Array<{ id: string; topic: string; category: string; plain_text: string }> { + return db + .prepare( + `SELECT DISTINCT ku.id, ku.topic, ku.category, ku.plain_text + FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id + AND r2.object_type = 'entity' AND r2.predicate = 'mentions' + JOIN smriti_knowledge_units ku ON ku.id = r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' AND r1.subject_id = ? + AND r1.object_type = 'entity' AND r1.predicate = 'mentions' + AND r2.subject_id != r1.subject_id + LIMIT ?` + ) + .all(unitId, limit) as Array<{ id: string; topic: string; category: string; plain_text: string }>; +} diff --git a/src/search/recall.ts b/src/search/recall.ts index 1e219a4..f94fe7b 100644 --- a/src/search/recall.ts +++ b/src/search/recall.ts @@ -9,6 +9,7 @@ import { DEFAULT_RECALL_LIMIT, OLLAMA_HOST, OLLAMA_MODEL } from "../config"; import { recallMemories, ollamaRecall } from "../qmd"; import { searchFiltered, type SearchFilters, type SearchResult } from "./index"; import { getQmdStore } from "../store"; +import { incrementRetrievalCount } from "../db"; // ============================================================================= // Types @@ -27,6 +28,24 @@ export type RecallResult = { synthesis?: string; }; +// ============================================================================= +// Retrieval Tracking +// ============================================================================= + +/** + * Best-effort bump of retrieval_count for any consolidated knowledge units + * belonging to the recalled sessions. Never lets a tracking failure break recall. + */ +function trackRetrieval(db: Database, results: SearchResult[]): void { + try { + for (const sessionId of new Set(results.map((r) => r.session_id).filter(Boolean))) { + incrementRetrievalCount(db, sessionId); + } + } catch { + // Never let this break recall. + } +} + // ============================================================================= // Filtered Recall // ============================================================================= @@ -58,6 +77,7 @@ export async function recall( if (options.synthesize && storeResults.length > 0) { synthesis = await synthesizeResults(query, storeResults, options); } + trackRetrieval(db, storeResults); return { results: storeResults, synthesis }; } @@ -70,6 +90,7 @@ export async function recall( fast: options.fast, intent: rerankIntent, }); + trackRetrieval(db, qmdResult.results); return { results: qmdResult.results, synthesis: qmdResult.synthesis, @@ -102,6 +123,7 @@ export async function recall( synthesis = await synthesizeResults(query, deduped, options); } + trackRetrieval(db, deduped); return { results: deduped, synthesis }; } diff --git a/src/team/config.ts b/src/team/config.ts index 8dddcbc..13c0efe 100644 --- a/src/team/config.ts +++ b/src/team/config.ts @@ -17,9 +17,17 @@ export type CustomCategoryDef = { description?: string; }; +export type CustomEntityDef = { + id: string; + label: string; + entity_type: string; + aliases: string[]; +}; + export type SmritiConfig = { version: number; categories?: CustomCategoryDef[]; + entities?: CustomEntityDef[]; allowedCategories?: string[]; autoSync?: boolean; }; @@ -115,3 +123,73 @@ export function exportCustomCategories(db: Database): CustomCategoryDef[] { ...(r.description ? { description: r.description } : {}), })); } + +// ============================================================================= +// Entity Merge — team/org propagation for smriti_entities +// +// Same reasoning as categories: an entity's canonical id is only meaningful +// if every teammate's machine agrees on it. .smriti/config.json is the +// git-committed source of truth each local smriti_entities table converges +// toward on every share/sync — the same role a published vocabulary plays +// for literal RDF, minus the URIs. +// ============================================================================= + +/** + * Upsert entities from config into the local DB. Matches first by id, then + * falls back to a normalized-label match (so two machines that independently + * minted different ids for the same concept still converge once either + * syncs the shared file). Unions aliases on conflict rather than overwriting. + * Returns count of newly created entities. + */ +export function mergeEntities(db: Database, entities: CustomEntityDef[]): number { + if (entities.length === 0) return 0; + + let created = 0; + for (const entity of entities) { + const existingById = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE id = ?`) + .get(entity.id) as { id: string; aliases: string } | null; + + if (existingById) { + const aliases = new Set(JSON.parse(existingById.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), entity.id); + continue; + } + + const normalizedLabel = entity.label.trim().toLowerCase(); + const existingByLabel = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE LOWER(label) = ?`) + .get(normalizedLabel) as { id: string; aliases: string } | null; + + if (existingByLabel) { + const aliases = new Set(JSON.parse(existingByLabel.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), existingByLabel.id); + continue; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 0)` + ).run(entity.id, entity.label, entity.entity_type, JSON.stringify(entity.aliases)); + created++; + } + return created; +} + +/** Query smriti_entities and return as config defs for export to .smriti/config.json. */ +export function exportEntities(db: Database): CustomEntityDef[] { + const rows = db + .prepare(`SELECT id, label, entity_type, aliases FROM smriti_entities`) + .all() as Array<{ id: string; label: string; entity_type: string; aliases: string }>; + + return rows.map((r) => ({ + id: r.id, + label: r.label, + entity_type: r.entity_type, + aliases: JSON.parse(r.aliases), + })); +} diff --git a/src/team/share.ts b/src/team/share.ts index 06c44e5..22adef8 100644 --- a/src/team/share.ts +++ b/src/team/share.ts @@ -10,7 +10,7 @@ import { SMRITI_DIR, AUTHOR } from "../config"; import { hashContent } from "../qmd"; import { existsSync, mkdirSync } from "fs"; import { join } from "path"; -import { readConfig, writeConfig, exportCustomCategories } from "./config"; +import { readConfig, writeConfig, exportCustomCategories, exportEntities } from "./config"; import { formatSessionAsFallback, isSessionWorthSharing, @@ -140,7 +140,7 @@ function querySessions( } /** Get messages for a session */ -function getSessionMessages( +export function getSessionMessages( db: Database, sessionId: string ): Array<{ @@ -178,15 +178,17 @@ async function writeManifest( const fullManifest = [...existingManifest, ...newEntries]; await Bun.write(indexPath, JSON.stringify(fullManifest, null, 2)); - // Write config — always update with latest custom categories + // Write config — always update with latest custom categories + canonical entities const existing = readConfig(outputDir); const customCategories = db ? exportCustomCategories(db) : []; + const entities = db ? exportEntities(db) : []; const config = { ...existing, - version: customCategories.length > 0 ? 2 : (existing.version ?? 1), + version: customCategories.length > 0 || entities.length > 0 ? 2 : (existing.version ?? 1), allowedCategories: existing.allowedCategories ?? ["*"], autoSync: existing.autoSync ?? false, ...(customCategories.length > 0 ? { categories: customCategories } : {}), + ...(entities.length > 0 ? { entities } : {}), }; await writeConfig(outputDir, config); diff --git a/src/team/sync.ts b/src/team/sync.ts index fdb9d5a..b4e51cc 100644 --- a/src/team/sync.ts +++ b/src/team/sync.ts @@ -9,7 +9,8 @@ import type { Database } from "bun:sqlite"; import { SMRITI_DIR } from "../config"; import { addMessage, hashContent } from "../qmd"; import { join } from "path"; -import { readConfig, mergeCategories } from "./config"; +import { readConfig, mergeCategories, mergeEntities } from "./config"; +import { insertRelationship, type RelationshipPredicate } from "../learn/entities"; // ============================================================================= // Types @@ -26,6 +27,7 @@ export type SyncResult = { skipped: number; errors: string[]; categoriesImported: number; + entitiesImported: number; }; // ============================================================================= @@ -116,6 +118,7 @@ export async function syncTeamKnowledge( skipped: 0, errors: [], categoriesImported: 0, + entitiesImported: 0, }; // Determine input directory @@ -144,11 +147,16 @@ export async function syncTeamKnowledge( ).map((r) => r.content_hash) ); - // Import custom categories from config.json (v2+) before scanning files + // Import custom categories + canonical entities from config.json (v2+) + // before scanning files — entities must exist locally before per-file + // "mentions"/relationship edges below can reference them. const config = readConfig(inputDir); if (config.categories && config.categories.length > 0) { result.categoriesImported = mergeCategories(db, config.categories); } + if (config.entities && config.entities.length > 0) { + result.entitiesImported = mergeEntities(db, config.entities); + } // Scan for markdown files const knowledgeDir = join(inputDir, "knowledge"); @@ -175,10 +183,11 @@ export async function syncTeamKnowledge( continue; } - // Segmented pipeline docs don't have **user**/**assistant** patterns; - // treat the whole body as a single assistant message. - const isSegmented = meta.pipeline === "segmented"; - const messages = isSegmented + // Segmented and consolidated pipeline docs don't have + // **user**/**assistant** patterns; treat the whole body as a single + // assistant message. + const isSingleMessageDoc = meta.pipeline === "segmented" || meta.pipeline === "consolidated"; + const messages = isSingleMessageDoc ? [{ role: "assistant", content: body.trim() }] : extractMessages(body); @@ -242,6 +251,28 @@ export async function syncTeamKnowledge( contentHash ); + // Re-create relationship edges from frontmatter. Unlike entities, + // these need no canonicalization: unit ids are portable UUIDs + // already shared as `sessionId` above, so edges reference the same + // node on every machine that imports this file. + const asArray = (v: string | string[] | undefined): string[] => + v === undefined ? [] : Array.isArray(v) ? v : [v]; + + for (const entityId of asArray(meta.entity_ids)) { + if (!entityId) continue; + insertRelationship(db, "knowledge_unit", sessionId, "mentions", "entity", entityId, { + source: "extraction", + }); + } + for (const predicate of ["relatesTo", "supersedes", "contradicts"] as RelationshipPredicate[]) { + for (const objectId of asArray(meta[predicate])) { + if (!objectId) continue; + insertRelationship(db, "knowledge_unit", sessionId, predicate, "knowledge_unit", objectId, { + source: "llm", + }); + } + } + result.imported++; } catch (err: any) { result.errors.push(`${match}: ${err.message}`); diff --git a/test/learn-consolidate.test.ts b/test/learn-consolidate.test.ts new file mode 100644 index 0000000..8748d8c --- /dev/null +++ b/test/learn-consolidate.test.ts @@ -0,0 +1,283 @@ +/** + * test/learn-consolidate.test.ts - Tests for continuous knowledge consolidation + * + * Mirrors test/team-segmented.test.ts's style: initSmriti(":memory:"), a + * mocked global.fetch standing in for Ollama, and a scratch tmpDir for + * filesystem output (never process.cwd() — consolidateKnowledge writes real + * files). + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync, writeFileSync, existsSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + listKnowledgeUnits, +} from "../src/db"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import { recall } from "../src/search/recall"; +import type { KnowledgeUnit } from "../src/team/types"; + +// ============================================================================= +// Setup +// ============================================================================= + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-consolidate-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +/** Insert a session with real message rows so getSessionMessages() finds content. */ +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +const DENSE_CONVERSATION = [ + { role: "user", content: "I'm getting a JWT token expiry issue. Sessions timeout after 1 hour but tests expect 24 hours." }, + { role: "assistant", content: "Let me look at the auth middleware to understand the token expiry logic." }, + { role: "user", content: "Found it — src/auth.ts hardcodes 3600 seconds instead of reading JWT_TTL from the environment." }, + { role: "assistant", content: "Updated it to use process.env.JWT_TTL || 3600. Tests pass now." }, +]; + +/** Mock fetch that distinguishes Stage 1 (segmentation) vs Stage 2 (document) calls by prompt content. */ +function mockOllamaFetch(stage1Response: () => object) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify(stage1Response()) + "\n```" }), + { status: 200 } + ); + } + return new Response( + JSON.stringify({ response: "# Consolidated Doc\n\nPolished content." }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Segment-phase dedup +// ============================================================================= + +test("consolidate dedups knowledge units with identical content across sessions", async () => { + seedSession("dedup-s1", "dedupproj", DENSE_CONVERSATION); + seedSession("dedup-s2", "dedupproj", DENSE_CONVERSATION); + updateDensityScore(db, "dedup-s1", 0.9); + updateDensityScore(db, "dedup-s2", 0.9); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ + units: [{ topic: "JWT token expiry bug", category: "bug/fix", relevance: 9, entities: ["JWT"] }], + })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + outputDir: join(tmpDir, "dedup-output"), + }); + + expect(result.sessionsSegmented).toBe(2); + expect(result.unitsStored).toBe(1); + expect(result.unitsSkipped).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Promotion threshold +// ============================================================================= + +test("promote phase only promotes units clearing the retrieval/relevance bar", async () => { + const belowThreshold: KnowledgeUnit = { + id: "unit-below", + topic: "Minor formatting note", + category: "code/pattern", + relevance: 3, + entities: [], + files: [], + plainText: "Use consistent indentation.", + lineRanges: [{ start: 0, end: 1 }], + }; + const aboveThreshold: KnowledgeUnit = { + id: "unit-above", + topic: "Redis caching decision", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use Redis with a 5-minute TTL for API responses.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, belowThreshold, "promote-s1", "promoteproj", "hash-below"); + insertKnowledgeUnit(db, aboveThreshold, "promote-s2", "promoteproj", "hash-above"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no sessions qualify for the segment phase — isolates promote phase + minRetrievals: 3, + minRelevance: 8, + outputDir: join(tmpDir, "promote-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-above"); + expect(canonical.map((u) => u.id)).not.toContain("unit-below"); + + const promoted = canonical.find((u) => u.id === "unit-above")!; + expect(promoted.canonical_doc_path).toContain("architecture-decision"); + expect(promoted.share_id).toBeTruthy(); + + const shareRow = db + .prepare(`SELECT * FROM smriti_shares WHERE unit_id = ?`) + .get("unit-above") as any; + expect(shareRow).toBeTruthy(); + expect(shareRow.session_id).toBe("promote-s2"); + + const writtenFile = join(tmpDir, "promote-output", promoted.canonical_doc_path!); + expect(existsSync(writtenFile)).toBe(true); + + const stillSegmented = listKnowledgeUnits(db, { tier: "segmented" }); + expect(stillSegmented.map((u) => u.id)).toContain("unit-below"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Graceful degradation +// ============================================================================= + +test("promote phase continues past a per-unit failure and records the error", async () => { + // Pre-create a *file* at the exact path the loop will try to mkdir for + // category "bug/fix" (slug "bug-fix"), forcing that unit's write to throw. + const outputDir = join(tmpDir, "degrade-output"); + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + writeFileSync(join(knowledgeDir, "bug-fix"), "occupied"); + + const willFail: KnowledgeUnit = { + id: "unit-fails", + topic: "Broken unit", + category: "bug/fix", + relevance: 9, + entities: [], + files: [], + plainText: "This unit's category dir collides with a file.", + lineRanges: [{ start: 0, end: 1 }], + }; + const willSucceed: KnowledgeUnit = { + id: "unit-succeeds", + topic: "Working unit", + category: "topic/learning", + relevance: 9, + entities: [], + files: [], + plainText: "This unit writes fine.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, willFail, "degrade-s1", "degradeproj", "hash-fails"); + insertKnowledgeUnit(db, willSucceed, "degrade-s2", "degradeproj", "hash-succeeds"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, // exclude leftover segmented units from earlier tests; only relevance-9 units below qualify + minRelevance: 8, + outputDir, + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors.length).toBe(1); + expect(result.errors[0]).toContain("unit-fails"); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-succeeds"); + expect(canonical.map((u) => u.id)).not.toContain("unit-fails"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Retrieval tracking (recall -> incrementRetrievalCount) +// ============================================================================= + +test("recall increments retrieval_count for knowledge units of the recalled session", async () => { + seedSession("track-s1", "trackproj", [ + { role: "user", content: "How do we configure the rate limiter for the public API?" }, + { role: "assistant", content: "Use a token bucket with 100 requests per minute per API key." }, + ]); + + const unit: KnowledgeUnit = { + id: "unit-tracked", + topic: "Rate limiter config", + category: "code/pattern", + relevance: 7, + entities: [], + files: [], + plainText: "Token bucket rate limiting for the public API.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, unit, "track-s1", "trackproj", "hash-tracked"); + + await recall(db, "rate limiter", { project: "trackproj" }); + + const tracked = db + .prepare(`SELECT retrieval_count FROM smriti_knowledge_units WHERE id = ?`) + .get("unit-tracked") as { retrieval_count: number }; + expect(tracked).toBeDefined(); + expect(tracked.retrieval_count).toBeGreaterThanOrEqual(1); +}); + +test("recall does not throw for sessions with no consolidated knowledge units", async () => { + seedSession("untracked-s1", "untrackedproj", [ + { role: "user", content: "What's our deploy process look like?" }, + { role: "assistant", content: "Push to main triggers the CI pipeline and auto-deploys." }, + ]); + + await expect(recall(db, "deploy process", { project: "untrackedproj" })).resolves.toBeDefined(); +}); diff --git a/test/learn-entities-sync.test.ts b/test/learn-entities-sync.test.ts new file mode 100644 index 0000000..8b850c4 --- /dev/null +++ b/test/learn-entities-sync.test.ts @@ -0,0 +1,166 @@ +/** + * test/learn-entities-sync.test.ts - Team/org propagation of the entities + + * relationships layer via the existing share/sync git round-trip. + * + * This is the test that directly answers "how does this reach the org/team + * level": two independent in-memory DBs stand in for two teammates' machines, + * connected only through a shared tmp `.smriti/` directory (config.json + + * knowledge/*.md), exactly like real git-committed team knowledge. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { initSmriti, closeDb, insertKnowledgeUnit, upsertProject } from "../src/db"; +import { readConfig, writeConfig, exportEntities } from "../src/team/config"; +import { syncTeamKnowledge } from "../src/team/sync"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import { getEntity, resolveEntity, insertRelationship, getRelationships } from "../src/learn/entities"; +import type { KnowledgeUnit } from "../src/team/types"; + +// Two independent :memory: databases (via separate initSmriti calls) stand in +// for two teammates' machines. initSmriti's underlying store singleton is +// process-wide, but every call site below threads the returned `db` handle +// explicitly rather than going through the singleton getDb(), so the two +// stay genuinely isolated for everything this test touches. +let dbA: Database; +let dbB: Database; +let sharedDir: string; + +beforeAll(async () => { + dbA = await initSmriti(":memory:"); + dbB = await initSmriti(":memory:"); + sharedDir = join(tmpdir(), `smriti-propagation-test-${Date.now()}`); + mkdirSync(sharedDir, { recursive: true }); + + // Pre-existing FK requirement of syncTeamKnowledge/upsertSessionMeta, + // unrelated to the entities/relationships work: the importing machine + // must already have the project registered locally (smriti_session_meta + // FKs to smriti_projects). Both "teammates" in this test are in the same project. + upsertProject(dbA, "propproj"); + upsertProject(dbB, "propproj"); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(sharedDir, { recursive: true }); } catch {} +}); + +test("entity canonical id and unit-relationship edges converge across two machines via share -> sync", async () => { + // --- Machine A: create + promote a unit that mentions "Redis" --- + const unit: KnowledgeUnit = { + id: "prop-unit-a", + topic: "Redis TTL guidance", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use a 5-minute TTL for API response caching.", + lineRanges: [], + }; + insertKnowledgeUnit(dbA, unit, "prop-session-a", "propproj", "prop-hash-a"); + const redisIdOnA = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-unit-a", "mentions", "entity", redisIdOnA); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Redis TTL Guidance\n\nUse a 5-minute TTL." }), { status: 200 }) + ) as any; + + try { + const result = await consolidateKnowledge(dbA, { + minDensity: 999, // no segment-phase sessions on A — isolates the promote phase + minRetrievals: 999, + minRelevance: 8, // unit's relevance (9) qualifies + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } + + // --- Machine A "shares": publish its canonical entity registry to the shared config.json --- + // (this is the exact operation writeManifest performs in src/team/share.ts, just called + // directly here to isolate propagation from the rest of the share pipeline's session-querying) + const existingConfig = readConfig(sharedDir); + await writeConfig(sharedDir, { ...existingConfig, version: 2, entities: exportEntities(dbA) }); + + // --- Machine B has never seen "Redis" before --- + expect(getEntity(dbB, "redis")).toBeNull(); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + + expect(syncResult.errors).toEqual([]); + expect(syncResult.entitiesImported).toBeGreaterThanOrEqual(1); + expect(syncResult.imported).toBeGreaterThanOrEqual(1); + + // The entity converged onto the SAME canonical id — not an independently re-slugified duplicate. + const redisOnB = getEntity(dbB, "redis"); + expect(redisOnB).toBeTruthy(); + expect(redisOnB!.id).toBe(redisIdOnA); + expect(redisOnB!.aliases).toContain("Redis"); + + // The unit's "mentions" edge was re-created on B, referencing that same entity id. + const bMentions = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-unit-a", + predicate: "mentions", + objectType: "entity", + }); + expect(bMentions.map((r) => r.object_id)).toContain(redisIdOnA); +}); + +test("supersedes edges between shared units survive the round-trip with no canonicalization needed", async () => { + // Machine A: an existing unit, and a new one that supersedes it — both promoted. + const existing: KnowledgeUnit = { + id: "prop-superseded-unit", topic: "Old Redis TTL", category: "architecture/decision", relevance: 8, + entities: ["Redis"], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const superseding: KnowledgeUnit = { + id: "prop-superseding-unit", topic: "New Redis TTL", category: "architecture/decision", relevance: 9, + entities: ["Redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(dbA, existing, "prop-session-b1", "propproj", "prop-hash-existing"); + insertKnowledgeUnit(dbA, superseding, "prop-session-b2", "propproj", "prop-hash-superseding"); + const redisId = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-superseded-unit", "mentions", "entity", redisId); + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "mentions", "entity", redisId); + // Simulate what promote-phase LLM relationship inference would have discovered: + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "supersedes", "knowledge_unit", "prop-superseded-unit", { source: "llm" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }) + ) as any; + + try { + // Only "prop-superseding-unit" clears the bar this round (existing unit stays at relevance 8 + // < minRelevance 8.5, so we promote exactly the one whose frontmatter should carry the edge). + const result = await consolidateKnowledge(dbA, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8.5, + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } + + await writeConfig(sharedDir, { ...readConfig(sharedDir), version: 2, entities: exportEntities(dbA) }); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + expect(syncResult.errors).toEqual([]); + + const bEdges = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-superseding-unit", + predicate: "supersedes", + }); + // No entity-style canonicalization needed here: unit ids are portable UUIDs, + // so the edge lands referencing the exact same object id on both machines. + expect(bEdges.map((r) => r.object_id)).toContain("prop-superseded-unit"); +}); diff --git a/test/learn-entities.test.ts b/test/learn-entities.test.ts new file mode 100644 index 0000000..2c91e55 --- /dev/null +++ b/test/learn-entities.test.ts @@ -0,0 +1,397 @@ +/** + * test/learn-entities.test.ts - Tests for canonical entity resolution and + * relationship triples (RDF-inspired knowledge graph layer). + * + * Mirrors test/learn-consolidate.test.ts's style: initSmriti(":memory:"), + * mocked global.fetch standing in for Ollama, scratch tmpDir for output. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + findPromotableUnits, +} from "../src/db"; +import { + resolveEntity, + getEntity, + findEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + getUnitsForEntity, +} from "../src/learn/entities"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import type { KnowledgeUnit } from "../src/team/types"; + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-entities-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +function mockOllamaFetch(handlers: { stage1?: () => object; relation?: () => string; stage2?: () => string }) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const prompt = body.prompt as string; + if (prompt.includes("Knowledge Unit Segmentation")) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify((handlers.stage1 ?? (() => ({ units: [] })))()) + "\n```" }), + { status: 200 } + ); + } + if (prompt.includes("CANDIDATES")) { + return new Response(JSON.stringify({ response: (handlers.relation ?? (() => ""))() }), { status: 200 }); + } + return new Response( + JSON.stringify({ response: (handlers.stage2 ?? (() => "# Doc\n\nContent."))() }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Entity resolution +// ============================================================================= + +test("resolveEntity merges exact-normalize variants (case/whitespace) onto one canonical id", () => { + const id1 = resolveEntity(db, "JWT")!; + const id2 = resolveEntity(db, " jwt ")!; + const id3 = resolveEntity(db, "JWT."); + + expect(id1).toBe(id2); + expect(id1).toBe(id3); + + const entity = getEntity(db, id1)!; + expect(entity.label).toBe("JWT"); // first-seen label wins + expect(entity.aliases).toContain("JWT"); + expect(entity.aliases).toContain("jwt"); + expect(entity.mention_count).toBe(3); +}); + +test("resolveEntity does not merge genuinely different wordings for the same concept", () => { + const jwtId = resolveEntity(db, "distinct-jwt-test")!; + const fullFormId = resolveEntity(db, "distinct-json-web-token-test")!; + + expect(jwtId).not.toBe(fullFormId); +}); + +test("resolveEntity returns null for blank labels", () => { + expect(resolveEntity(db, " ")).toBeNull(); +}); + +test("findEntity looks up by exact id or by label", () => { + resolveEntity(db, "Redis"); + expect(findEntity(db, "Redis")?.id).toBe("redis"); + expect(findEntity(db, "redis")?.id).toBe("redis"); + expect(findEntity(db, "nonexistent-entity-xyz")).toBeNull(); +}); + +// ============================================================================= +// Relationship triples +// ============================================================================= + +test("insertRelationship dedups via the UNIQUE constraint", () => { + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + + const rows = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "unit-a", predicate: "mentions" }); + expect(rows.length).toBe(1); +}); + +test("getRelationships supports single-triple-pattern lookup", () => { + insertRelationship(db, "knowledge_unit", "unit-b", "supersedes", "knowledge_unit", "unit-a", { source: "llm" }); + + const bySubject = getRelationships(db, { subjectId: "unit-b" }); + expect(bySubject.some((r) => r.predicate === "supersedes" && r.object_id === "unit-a")).toBe(true); + + const byPredicate = getRelationships(db, { predicate: "supersedes" }); + expect(byPredicate.length).toBeGreaterThanOrEqual(1); +}); + +// ============================================================================= +// Segment phase: mentions edges created with no extra LLM calls +// ============================================================================= + +test("consolidate segment phase creates mentions edges for every stored entity, no extra LLM calls", async () => { + seedSession("ent-s1", "entproj", [ + { role: "user", content: "We need to decide on a caching strategy for the API. Considering Redis vs in-memory caching." }, + { role: "assistant", content: "Redis is better — it's external state, handles multi-instance, fast, and proven." }, + ]); + updateDensityScore(db, "ent-s1", 0.9); + + let fetchCallCount = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + fetchCallCount++; + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + // relevance kept low (1) deliberately: this unit stays in the shared + // test DB as tier='segmented' after this test, and must never satisfy + // a later test's minRelevance threshold (e.g. 8) via leftover state. + return new Response( + JSON.stringify({ + response: "```json\n" + JSON.stringify({ + units: [{ topic: "Redis caching decision", category: "architecture/decision", relevance: 1, entities: ["Redis", "Caching"] }], + }) + "\n```", + }), + { status: 200 } + ); + } + return new Response(JSON.stringify({ response: "unexpected call" }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + minRetrievals: 999, + minRelevance: 999, // nothing promotable — isolates the segment phase + outputDir: join(tmpDir, "ent-output"), + }); + + expect(result.unitsStored).toBe(1); + expect(fetchCallCount).toBe(1); // segmentation only, no promote-time LLM calls + + const unitRow = db + .prepare(`SELECT id FROM smriti_knowledge_units WHERE session_id = 'ent-s1'`) + .get() as { id: string }; + + const mentions = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: unitRow.id, + predicate: "mentions", + }); + expect(mentions.length).toBe(2); + const objectIds = mentions.map((m) => m.object_id).sort(); + expect(objectIds).toEqual(["caching", "redis"]); + + const unitsForRedis = getUnitsForEntity(db, "redis"); + expect(unitsForRedis.map((u) => u.id)).toContain(unitRow.id); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// minEntityReach promotion criterion +// ============================================================================= + +test("minEntityReach promotes a unit whose entity is shared by >= K other units, even at 0 retrievals", () => { + const shared: KnowledgeUnit = { + id: "reach-unit-1", topic: "Topic A", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content A", lineRanges: [], + }; + const sharedOther1: KnowledgeUnit = { + id: "reach-unit-2", topic: "Topic B", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content B", lineRanges: [], + }; + const sharedOther2: KnowledgeUnit = { + id: "reach-unit-3", topic: "Topic C", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content C", lineRanges: [], + }; + + insertKnowledgeUnit(db, shared, "reach-s1", "reachproj", "reach-hash-1"); + insertKnowledgeUnit(db, sharedOther1, "reach-s2", "reachproj", "reach-hash-2"); + insertKnowledgeUnit(db, sharedOther2, "reach-s3", "reachproj", "reach-hash-3"); + + const entityId = resolveEntity(db, "shared-webhook-retries")!; + insertRelationship(db, "knowledge_unit", "reach-unit-1", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-2", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-3", "mentions", "entity", entityId); + + // Below scalar thresholds (relevance=2, retrieval_count=0) but 2 OTHER units share the entity. + const promotableWithReach = findPromotableUnits(db, 999, 999, 2); + expect(promotableWithReach.map((u) => u.id)).toContain("reach-unit-1"); + + // Without minEntityReach, the same unit is not promotable. + const promotableWithoutReach = findPromotableUnits(db, 999, 999); + expect(promotableWithoutReach.map((u) => u.id)).not.toContain("reach-unit-1"); +}); + +// ============================================================================= +// Promote-phase relationship inference (LLM-gated, bounded) +// ============================================================================= + +test("promote phase persists LLM-inferred relatesTo/supersedes/contradicts edges", async () => { + const existing: KnowledgeUnit = { + id: "infer-existing", topic: "Old Redis TTL guidance", category: "architecture/decision", relevance: 5, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "infer-s1", "inferproj", "infer-hash-existing"); + const redisId = resolveEntity(db, "infer-redis")!; + insertRelationship(db, "knowledge_unit", "infer-existing", "mentions", "entity", redisId); + + const promoted: KnowledgeUnit = { + id: "infer-new", topic: "New Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: ["infer-redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "infer-s2", "inferproj", "infer-hash-new"); + insertRelationship(db, "knowledge_unit", "infer-new", "mentions", "entity", redisId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch({ relation: () => "RELATION [0]: supersedes" }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no segment-phase sessions + minRetrievals: 999, + minRelevance: 8, // only infer-new (relevance 9) qualifies + outputDir: join(tmpDir, "infer-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "infer-new", predicate: "supersedes" }); + expect(edges.length).toBe(1); + expect(edges[0].object_id).toBe("infer-existing"); + expect(edges[0].source).toBe("llm"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("promote phase never asserts a directional predicate in both directions for the same pair", async () => { + // Two units sharing an entity, BOTH clearing the promotion bar in the same + // run — each independently asks "do I supersede the other" and (with a + // naive LLM/mock that doesn't reason about recency) can get "yes" from both + // sides. Only one direction should end up persisted. + const unitA: KnowledgeUnit = { + id: "bidir-unit-a", topic: "Redis TTL: 1 minute", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const unitB: KnowledgeUnit = { + id: "bidir-unit-b", topic: "Redis TTL: 15 minutes", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 15-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, unitA, "bidir-s1", "bidirproj", "bidir-hash-a"); + insertKnowledgeUnit(db, unitB, "bidir-s2", "bidirproj", "bidir-hash-b"); + const entityId = resolveEntity(db, "bidir-redis")!; + insertRelationship(db, "knowledge_unit", "bidir-unit-a", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "bidir-unit-b", "mentions", "entity", entityId); + + const originalFetch = globalThis.fetch; + // Always answers "supersedes" regardless of which side is asking — the + // worst case for this bug, and realistic for a small/local model given a + // prompt with no explicit recency signal. + globalThis.fetch = mockOllamaFetch({ relation: () => "RELATION [0]: supersedes" }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, // both unitA and unitB qualify + outputDir: join(tmpDir, "bidir-output"), + }); + + expect(result.unitsPromoted).toBe(2); + + const aToB = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-a", predicate: "supersedes", objectId: "bidir-unit-b", + }); + const bToA = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-b", predicate: "supersedes", objectId: "bidir-unit-a", + }); + // Exactly one direction persisted, never both. + expect(aToB.length + bToA.length).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("promote phase relationship inference is best-effort: a broken LLM response doesn't block promotion", async () => { + const existing: KnowledgeUnit = { + id: "badllm-existing", topic: "Existing", category: "code/pattern", relevance: 5, + entities: [], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "badllm-s1", "badllmproj", "badllm-hash-existing"); + const entId = resolveEntity(db, "badllm-entity")!; + insertRelationship(db, "knowledge_unit", "badllm-existing", "mentions", "entity", entId); + + const promoted: KnowledgeUnit = { + id: "badllm-new", topic: "New", category: "code/pattern", relevance: 9, + entities: ["badllm-entity"], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "badllm-s2", "badllmproj", "badllm-hash-new"); + insertRelationship(db, "knowledge_unit", "badllm-new", "mentions", "entity", entId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const prompt = body.prompt as string; + if (prompt.includes("CANDIDATES")) throw new Error("connection refused"); + return new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, + outputDir: join(tmpDir, "badllm-output"), + }); + + // Promotion itself succeeds even though relationship inference failed. + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "badllm-new" }) + .filter((r) => r.predicate !== "mentions"); + expect(edges.length).toBe(0); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// findRelatedCandidates +// ============================================================================= + +test("findRelatedCandidates finds other units sharing a canonical entity, excluding self", () => { + const a: KnowledgeUnit = { id: "cand-a", topic: "A", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "a", lineRanges: [] }; + const b: KnowledgeUnit = { id: "cand-b", topic: "B", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "b", lineRanges: [] }; + insertKnowledgeUnit(db, a, "cand-s1", "candproj", "cand-hash-a"); + insertKnowledgeUnit(db, b, "cand-s2", "candproj", "cand-hash-b"); + + const sharedEntity = resolveEntity(db, "cand-shared-entity")!; + insertRelationship(db, "knowledge_unit", "cand-a", "mentions", "entity", sharedEntity); + insertRelationship(db, "knowledge_unit", "cand-b", "mentions", "entity", sharedEntity); + + const candidates = findRelatedCandidates(db, "cand-a", 5); + expect(candidates.map((c) => c.id)).toEqual(["cand-b"]); + expect(candidates.map((c) => c.id)).not.toContain("cand-a"); +});