From dc3e24bc91074f0f859252e634ab8f1d9e78bd38 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 06:02:50 +0000 Subject: [PATCH 1/2] Trim verbose comments in b2-core, the UI and the eval examples Cut restated code, history and repeated rationale; shorten essay-style doc comments to their intent plus the one non-obvious reason. Spec, invariant and issue citations stay with the reasons they back. Comment-only: no code changes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ArrZkgfz7HGyUuTadEUMKy --- crates/b2-core/Cargo.toml | 9 +- crates/b2-core/src/add.rs | 63 +- crates/b2-core/src/chat.rs | 156 +- crates/b2-core/src/chunk.rs | 184 +- crates/b2-core/src/db.rs | 703 ++---- crates/b2-core/src/dirs.rs | 27 +- crates/b2-core/src/discover.rs | 235 +- crates/b2-core/src/embed.rs | 91 +- crates/b2-core/src/error.rs | 117 +- crates/b2-core/src/graph.rs | 56 +- crates/b2-core/src/import.rs | 105 +- crates/b2-core/src/ingest.rs | 415 +--- crates/b2-core/src/link.rs | 137 +- crates/b2-core/src/llm.rs | 165 +- crates/b2-core/src/mv.rs | 307 +-- crates/b2-core/src/note.rs | 160 +- crates/b2-core/src/pathspec.rs | 88 +- crates/b2-core/src/relation.rs | 24 +- crates/b2-core/src/resource.rs | 30 +- crates/b2-core/src/rm.rs | 77 +- crates/b2-core/src/search.rs | 310 +-- crates/b2-core/src/snippet.rs | 15 +- crates/b2-core/src/vault.rs | 607 ++--- crates/b2-core/src/vault/chat.rs | 136 +- crates/b2-core/src/vault/views.rs | 239 +- crates/b2-embed/Cargo.toml | 47 +- crates/b2-embed/examples/calibrate.rs | 360 +-- crates/b2-embed/examples/common/mod.rs | 95 +- crates/b2-embed/examples/eval/ablation.rs | 55 +- crates/b2-embed/examples/eval/dense.rs | 135 +- crates/b2-embed/examples/eval/discovery.rs | 219 +- crates/b2-embed/examples/eval/evidence.rs | 215 +- crates/b2-embed/examples/eval/fold.rs | 223 +- crates/b2-embed/examples/eval/gate.rs | 110 +- crates/b2-embed/examples/eval/instrument.rs | 48 +- crates/b2-embed/examples/eval/labels.rs | 51 +- crates/b2-embed/examples/eval/main.rs | 130 +- crates/b2-embed/examples/eval/metrics.rs | 16 +- crates/b2-embed/examples/eval/report.rs | 186 +- crates/b2-embed/examples/eval/retrieval.rs | 25 +- crates/b2-embed/examples/eval/tail.rs | 171 +- crates/b2-embed/examples/stability.rs | 154 +- crates/b2-llm/examples/groundedness.rs | 85 +- ui/scripts/gen-icons.ts | 85 +- ui/src/api.ts | 257 +-- ui/src/bindings.test.ts | 154 +- ui/src/bindings.ts | 365 +-- ui/src/chat.test.ts | 45 +- ui/src/chat.ts | 188 +- ui/src/chatview.ts | 79 +- ui/src/cmdhold.test.ts | 68 +- ui/src/cmdhold.ts | 84 +- ui/src/copypath.test.ts | 32 +- ui/src/copypath.ts | 29 +- ui/src/coverage.test.ts | 9 +- ui/src/coverage.ts | 18 +- ui/src/droplink.test.ts | 61 +- ui/src/droplink.ts | 161 +- ui/src/editorcmds.test.ts | 6 +- ui/src/editorcmds.ts | 81 +- ui/src/editorkeys.test.ts | 91 +- ui/src/editorkeys.ts | 85 +- ui/src/embedlink.test.ts | 32 +- ui/src/embedreminder.test.ts | 9 +- ui/src/embedreminder.ts | 44 +- ui/src/embeds.test.ts | 19 +- ui/src/embeds.ts | 123 +- ui/src/escape.ts | 7 +- ui/src/explain.test.ts | 2 +- ui/src/explain.ts | 46 +- ui/src/explainview.ts | 10 +- ui/src/findbar.ts | 28 +- ui/src/findfield.test.ts | 6 +- ui/src/findfield.ts | 10 +- ui/src/format.test.ts | 4 +- ui/src/format.ts | 51 +- ui/src/graph.test.ts | 16 +- ui/src/graph.ts | 60 +- ui/src/graphview.ts | 72 +- ui/src/highlight.test.ts | 57 +- ui/src/highlight.ts | 90 +- ui/src/hints.test.ts | 21 +- ui/src/hints.ts | 26 +- ui/src/icons.test.ts | 69 +- ui/src/icons.ts | 86 +- ui/src/importfiles.test.ts | 11 +- ui/src/importfiles.ts | 51 +- ui/src/keymap.test.ts | 79 +- ui/src/keymap.ts | 114 +- ui/src/keysview.ts | 80 +- ui/src/linebreaks.test.ts | 15 +- ui/src/links.test.ts | 29 +- ui/src/links.ts | 58 +- ui/src/list.test.ts | 69 +- ui/src/list.ts | 193 +- ui/src/livepreview.test.ts | 153 +- ui/src/livepreview.ts | 256 +-- ui/src/main.ts | 2303 +++++-------------- ui/src/markdown.ts | 91 +- ui/src/menukeys.test.ts | 65 +- ui/src/menukeys.ts | 69 +- ui/src/move.test.ts | 5 +- ui/src/move.ts | 65 +- ui/src/newentry.test.ts | 4 +- ui/src/newentry.ts | 20 +- ui/src/panes.test.ts | 12 +- ui/src/panes.ts | 95 +- ui/src/paste.test.ts | 8 +- ui/src/paste.ts | 67 +- ui/src/reconcile.test.ts | 19 +- ui/src/reconcile.ts | 69 +- ui/src/recorder.test.ts | 56 +- ui/src/recorder.ts | 66 +- ui/src/render.test.ts | 261 +-- ui/src/render.ts | 372 +-- ui/src/resourceview.test.ts | 11 +- ui/src/sanitize.test.ts | 44 +- ui/src/sanitize.ts | 65 +- ui/src/settingstabs.test.ts | 21 +- ui/src/settingstabs.ts | 59 +- ui/src/settingsview.ts | 235 +- ui/src/shortcuts.test.ts | 57 +- ui/src/shortcuts.ts | 108 +- ui/src/sidenav.test.ts | 23 +- ui/src/sidenav.ts | 125 +- ui/src/state.ts | 266 +-- ui/src/strength.test.ts | 15 +- ui/src/strength.ts | 69 +- ui/src/theme.test.ts | 5 +- ui/src/theme.ts | 15 +- ui/src/treenav.test.ts | 33 +- ui/src/treenav.ts | 83 +- ui/src/turndown-plugin-gfm.d.ts | 5 +- ui/src/types.ts | 254 +- ui/src/widgets.ts | 33 +- ui/src/wikicomplete.test.ts | 5 +- ui/src/wikicomplete.ts | 38 +- ui/src/zoom.test.ts | 24 +- ui/src/zoom.ts | 92 +- ui/style.css | 627 ++--- ui/vite.config.ts | 8 +- 141 files changed, 4487 insertions(+), 12155 deletions(-) diff --git a/crates/b2-core/Cargo.toml b/crates/b2-core/Cargo.toml index f29dded..422aca6 100644 --- a/crates/b2-core/Cargo.toml +++ b/crates/b2-core/Cargo.toml @@ -7,14 +7,12 @@ description = "B2 index engine — the SQLite-backed disposable projection of (M [dependencies] blake3 = "1.8.5" -# "trace" exposes SQLite's own sqlite3_trace_v2 profiler — per-statement SQL + -# execution time, the source of the `b2::sqlite` query-timing events (db.rs). +# "trace": per-statement timing for the `b2::sqlite` events (db.rs). rusqlite = { version = "0.40.1", features = ["bundled", "trace"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" thiserror = "2.0.18" -# Structured debug logging (facade only — emitting events costs nothing until an -# adapter installs a subscriber; the CLI's is opt-in via B2_LOG/B2_DEBUG). +# Facade only; adapters install the subscriber. tracing = "0.1.41" yaml-rust2 = "0.11.0" @@ -22,6 +20,5 @@ yaml-rust2 = "0.11.0" proptest = "1.11.0" rusqlite = "0.40.1" tempfile = "3.27.0" -# The logging integration test installs a real JSON subscriber to assert the -# emitted events are parseable, plottable JSONL (tests/logging.rs). +# For tests/logging.rs. tracing-subscriber = { version = "0.3.20", features = ["json"] } diff --git a/crates/b2-core/src/add.rs b/crates/b2-core/src/add.rs index 3637679..9cd873f 100644 --- a/crates/b2-core/src/add.rs +++ b/crates/b2-core/src/add.rs @@ -1,13 +1,7 @@ -//! Create a new note (the `b2 add` kernel op — CRUD's *create*). -//! -//! B2 authors a *new* file here, which is not the same as authoring a human's body: the -//! whole document is B2-minted on request, and its frontmatter is B2's managed zone. What -//! B2 still never does is inject structure into an *existing* note (ADR-0004). -//! -//! **Markdown-first**: write the `.md`, then project it from that source of truth. The -//! note is fully reconstructible from Markdown — a file at a path, and that path is its -//! identity (ADR-0003) — so `add` records nothing durable of its own. The `created` date -//! is passed in, keeping `b2-core` wall-clock-free. +//! Create a new note (`b2 add`). B2 authors a new file on request; it never injects +//! structure into an existing note (ADR-0004). Markdown first, then project; the path is +//! the identity (ADR-0003), so nothing else is recorded. `created` is passed in, keeping +//! the core clock-free. use crate::error::{Error, Result}; use crate::ingest::{self, EmbedCtx, ProjectionCtx}; @@ -16,20 +10,14 @@ use serde::Serialize; use std::io::Write; use std::path::Path; -/// What [`add_note`] did: the created note's vault-relative path -/// (`.md`-normalized), which is its identity (L1). +/// The created note's `.md`-normalized path, its identity (L1). #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct AddReport { pub path: String, } -/// Create a new note at `path_input` (a `.md` suffix is optional and added if missing) -/// with a minimal, valid frontmatter and `content` as its body, then project it. -/// -/// Refuses to clobber: [`Error::AddTargetExists`] if anything already sits there, -/// [`Error::AddDestination`] for an empty/absolute/vault-escaping path. Missing parent -/// directories are created, mirroring `mv`. Projection **embeds** the new note's chunks, -/// so the caller must open with the embedder the index was built with. +/// Create a note at `path_input` (`.md` optional) with minimal frontmatter and `content`, +/// then project and embed it. Refuses to clobber; creates missing parent folders. pub fn add_note( ctx: EmbedCtx, path_input: &str, @@ -39,17 +27,12 @@ pub fn add_note( ) -> Result { let rel = write_new_note(ctx.proj.root, path_input, title, content, created)?; - // 2. Project from that Markdown: chunk + embed the body, and derive any edges - // its content authors. ingest::ingest_file(ctx, &rel)?; Ok(AddReport { path: rel }) } -/// The **model-free** sibling of [`add_note`] — the desktop's New-note action: same file -/// write, but projected through [`ingest::project_file`] with no embedder, the same pass -/// `Vault::write` runs after a save. The new chunks join the pending set for any later -/// embed, and a body-less note has nothing to embed anyway. Same refusals as [`add_note`]. -/// Taking a [`ProjectionCtx`] is what makes that posture the type system's to keep. +/// The model-free [`add_note`] (the desktop's New note): projected without embedding, +/// as `Vault::write` does. Same refusals. pub fn create_note( ctx: ProjectionCtx, path_input: &str, @@ -62,11 +45,7 @@ pub fn create_note( Ok(AddReport { path: rel }) } -/// The shared create step: validate `path_input`, refuse to clobber, render the -/// minimal frontmatter + body, and write the new file (creating missing parent -/// dirs). Markdown first (step 1 of both entry points). Returns the vault-relative -/// `.md` path — which is also the note's identity, so there is nothing further to -/// mint or record. +/// Validate, render and write the new file, refusing to clobber. Returns its path. fn write_new_note( vault_root: &Path, path_input: &str, @@ -76,8 +55,7 @@ fn write_new_note( ) -> Result { let rel = crate::pathspec::normalize_rel_md(path_input).map_err(Error::AddDestination)?; let doc = render_note(title, content, created); - // The refusal is the create itself, never a check before it: a file appearing at - // `rel` in between is refused rather than clobbered. + // The create itself is the refusal (see `place_new`). crate::import::place_new( &vault_root.join(&rel), || Error::AddTargetExists(rel.clone()), @@ -86,14 +64,9 @@ fn write_new_note( Ok(rel) } -/// Render a new note's text: a minimal valid frontmatter block followed by the body. -/// `title` is YAML-quoted and omitted entirely when `None`; `content` is trimmed of -/// trailing newlines and placed after one blank line. -/// -/// The template seeds only what can't be reconstructed later: `created` (lost forever if -/// not stamped now) and an optional `title`. `type:` is deliberately not seeded — ingest -/// defaults it to `"note"`, so stamping it would be redundancy (GH #80). No key is -/// `b2`-namespaced: these are seeded courtesies the human owns the moment they exist. +/// Render a new note: minimal frontmatter, then the body after a blank line. Seeds only +/// what can't be reconstructed later (`created`, optional `title`); not `type:` (GH #80). +/// The human owns these keys once written. fn render_note(title: Option<&str>, content: Option<&str>, created: &str) -> String { let mut s = String::from("---\n"); if let Some(t) = title { @@ -128,15 +101,13 @@ mod tests { fn omits_title_when_absent_and_body_when_empty() { let out = render_note(None, None, "2026-07-03"); assert_eq!(out, "---\ncreated: 2026-07-03\n---\n"); - // An explicitly-empty content string is treated like no body. let blank = render_note(None, Some("\n\n"), "2026-07-03"); assert_eq!(blank, "---\ncreated: 2026-07-03\n---\n"); } #[test] fn does_not_seed_type_ingest_defaults_it() { - // `type:` is not seeded (GH #80) — the template stamps only what can't be - // reconstructed later; ingest defaults an absent type to "note". + // GH #80. let out = render_note(None, None, "2026-07-03"); assert!(!out.contains("type:"), "{out}"); } @@ -149,15 +120,13 @@ mod tests { #[test] fn the_rendered_note_round_trips_and_parses_its_fields() { - // A note `add` writes must parse back with exactly the fields it set. let out = render_note(Some("Spaced repetition"), Some("Body."), "2026-07-03"); let parsed = crate::note::parse(&out); assert_eq!(parsed.as_str(), out, "renders round-trip losslessly"); let f = parsed.fields(); assert!(f.r#type.is_none(), "type is not seeded (GH #80)"); assert_eq!(f.created.as_deref(), Some("2026-07-03")); - // `title:` is inert (the display title is the filename), so it is checked as - // the frontmatter bytes it was written as. + // `title:` is inert, so check the bytes. assert_eq!( parsed.frontmatter(), Some("title: \"Spaced repetition\"\ncreated: 2026-07-03\n") diff --git a/crates/b2-core/src/chat.rs b/crates/b2-core/src/chat.rs index 13c5dcb..215538e 100644 --- a/crates/b2-core/src/chat.rs +++ b/crates/b2-core/src/chat.rs @@ -1,12 +1,9 @@ -//! Flow ④ — grounded chat: the core logic of `Vault::ask`, condense -> retrieve -> -//! assemble -> stream -> cite. This module owns what must be deterministic and testable -//! against [`FakeLlm`] — prompt assembly, the condensation degrade, citation-marker -//! parsing — while the façade owns retrieval and resolving markers into display views, -//! mirroring the `discover.rs`/`vault.rs` split. +//! Flow ④, grounded chat: the deterministic parts testable against [`FakeLlm`] (prompt +//! assembly, the condense degrade, citation parsing). The façade owns retrieval and +//! display views. //! -//! Chat is a **reader** whose output is never stored: no response caching, no `meta` rows, -//! session-only history. Model output is untrusted content (ADR-0016), enforced where it -//! renders. +//! Chat output is never stored, and history is session-only. Model output is untrusted +//! (ADR-0016), enforced where it renders. //! //! [`FakeLlm`]: crate::llm::FakeLlm @@ -16,37 +13,26 @@ use crate::llm::{ use std::collections::BTreeSet; use std::ops::ControlFlow; -/// How many passages retrieval hands the model — the "~8–12" of the spec -/// (GH #153). Retrieval goes through `Vault::search_chunks`, so the passage -/// view's pool discipline applies unchanged: a 10-ask reaches -/// `chunk_candidate_pool(10) = 60` candidates per signal (GH #141/#142) — -/// chat introduces no new candidate width. +/// How many passages retrieval hands the model (GH #153). Uses `search_chunks`'s pool, +/// so chat adds no new candidate width (GH #141/#142). pub const ASK_PASSAGES: usize = 10; -/// The grounded system prompt (flow ④ step 2). Prompt assembly is core logic: -/// authored here, rendered with the numbered passages by -/// [`ChatRequest::system_message`], asserted by the engine suite. The quoted -/// no-evidence sentence is [`crate::llm::NO_EVIDENCE_ANSWER`] verbatim (the -/// suite pins the containment), so the instruction and the fake that obeys it -/// can never drift apart. +/// The grounded system prompt (flow ④ step 2). Its no-evidence sentence must stay +/// [`crate::llm::NO_EVIDENCE_ANSWER`] verbatim; the suite pins it. pub const GROUNDED_SYSTEM_PROMPT: &str = "You are B2, answering questions about the user's own \ notes. Answer ONLY from the numbered passages below. Cite each claim with the supporting \ passage's [n] marker. If the passages do not support an answer, say \"I don't find that in \ your notes.\" — never answer from general knowledge."; -/// The condensation prompt (flow ④ step 0): rewrite a follow-up into a -/// standalone retrieval query ("tell me more about that" → a query that names -/// *that*). The reply is used as search input, never shown to the human. +/// The condensation prompt (flow ④ step 0): a follow-up rewritten as a standalone +/// search query, never shown to the human. pub const CONDENSE_SYSTEM_PROMPT: &str = "Rewrite the user's latest message as one standalone \ search query that names its subject explicitly, using the earlier conversation to fill in \ any pronouns or references. Reply with the query text only — no preamble, no quotes."; -/// Step 0 — condense a follow-up into a standalone retrieval query. Multi-turn -/// only; the caller skips this entirely for a single-turn ask. **This step can -/// never break chat**: on provider failure, a cancelled stream, or a blank -/// reply, it degrades to the raw question text — a worse retrieval query, not -/// an error. Tokens are consumed internally (the condensed query is search -/// input, not answer text), so nothing streams up from this call. +/// Step 0: condense a follow-up into a standalone retrieval query (multi-turn only). +/// Never breaks chat: any failure or blank reply degrades to the raw question. Nothing +/// streams up. pub fn condense_query(llm: &dyn LlmProvider, question: &str, history: &[ChatTurn]) -> String { let req = ChatRequest { kind: RequestKind::Condense, @@ -76,8 +62,7 @@ pub fn condense_query(llm: &dyn LlmProvider, question: &str, history: &[ChatTurn } } -/// Step 2 — assemble the grounded chat request: the system prompt, the -/// conversation ending in the current question, and the numbered passages. +/// Step 2: assemble the grounded chat request. pub fn build_request( question: &str, history: &[ChatTurn], @@ -93,22 +78,16 @@ pub fn build_request( } } -/// How many passage pairs a "why was this suggested?" explanation is grounded in. The -/// first is the pair the card's score came from; the next two show whether the -/// resemblance is one passage or a pattern. Up to `2 × WHY_PAIRS` passages, well under -/// [`ASK_PASSAGES`], so the explanation costs less context than an ordinary ask. +/// Passage pairs a Why? explanation is grounded in: the card's scoring pair, plus two to +/// show whether the resemblance is one passage or a pattern. pub const WHY_PAIRS: usize = 3; /// How many shared neighbours the facts block names before it counts the rest. const WHY_SHARED_NAMED: usize = 5; -/// The system prompt for explaining one *Similar & unlinked* row (`Vault::why_similar`). -/// Grounded chat with a narrower subject, so it keeps grounded chat's two rules — cite by -/// `[n]`, never reach for general knowledge — and the same no-evidence sentence -/// ([`crate::llm::NO_EVIDENCE_ANSWER`], pinned by the suite). The facts block -/// [`build_why_request`] appends is what B2's own discovery reads found for the pair; -/// the model's job is the part no statistic can do, saying what the matched passages -/// have in common in the human's terms. +/// The system prompt for explaining one Similar & unlinked row. Keeps grounded chat's +/// rules and no-evidence sentence ([`crate::llm::NO_EVIDENCE_ANSWER`], pinned by the +/// suite). pub const WHY_SYSTEM_PROMPT: &str = "You are B2, explaining why one of the user's notes was \ suggested as similar to the note they are reading. B2 suggests a note when one of its passages \ sits close to a passage of the open note in embedding space and the two notes are not linked \ @@ -126,42 +105,32 @@ answer from general knowledge."; /// `(anchor marker, candidate marker, score)`. pub type MarkedPair = (usize, usize, f64); -/// What B2's discovery reads found for one (anchor, suggested note) pair — the facts half -/// of a "why was this suggested?" request, gathered by the façade and rendered here so -/// the wording is core logic the suite can assert. Owned: built once per click and handed -/// straight to [`build_why_request`]. +/// What B2's discovery reads found for one (anchor, suggested note) pair: the facts half +/// of a Why? request. #[derive(Debug, Clone, PartialEq)] pub struct WhyFacts { - /// The note being read — the list's anchor. + /// The note being read: the list's anchor. pub anchor_path: String, pub anchor_title: Option, /// The suggested note the human asked about. pub candidate_path: String, pub candidate_title: Option, - /// `(rank, served)`: the 1-based position `similar` served the note at, out of how - /// many rows it served. `None` when the note is not in that list (already linked, or - /// beyond the list's length) — no rank is claimed for a row that would not be shown. + /// `(rank, served)`: 1-based position in `similar`'s list. `None` when not in it. pub rank: Option<(usize, usize)>, /// The row's strength z, when the list was graded (`SimilarView::z`). pub z: Option, /// Whether the two notes are already directly linked. pub linked: bool, - /// Notes directly linked to **both** — the triadic-closure signal discovery keeps in - /// the pool but does not rank on. + /// Notes directly linked to both. pub shared_neighbors: Vec, - /// The matched pairs, nearest first, as `(anchor marker, candidate marker, score)` - /// over the request's 1-based passage numbering. + /// The matched pairs, nearest first. pub pairs: Vec, /// Whether both notes had stored vectors to compare at all. pub embedded: bool, } -/// Assemble the "why was this suggested?" request: [`WHY_SYSTEM_PROMPT`], the rendered -/// facts, one user turn naming both notes, and the matched passages. A -/// [`RequestKind::Chat`] request — it is answered from numbered passages exactly as a -/// grounded ask is, so every provider (and the fake) serves it unchanged. It carries no -/// history: the evidence for one row is self-contained, and a follow-up question goes -/// through the ordinary `ask` with the explanation as its context. +/// Assemble the plain Why? request: prompt, facts, one user turn and the matched +/// passages. A [`RequestKind::Chat`] request, so every provider serves it unchanged. pub fn build_why_request(facts: &WhyFacts, passages: Vec) -> ChatRequest { ChatRequest { kind: RequestKind::Chat, @@ -173,8 +142,7 @@ pub fn build_why_request(facts: &WhyFacts, passages: Vec) -> Cha } } -/// The one user turn of a why-request, naming both notes by path (the identity the -/// tools take). +/// The one user turn of a why-request, naming both notes by path. fn why_turn(facts: &WhyFacts) -> ChatTurn { ChatTurn::user(&format!( "Why was {} suggested as similar to {}?", @@ -182,16 +150,9 @@ fn why_turn(facts: &WhyFacts) -> ChatTurn { )) } -// --- the tool-using turn --------------------------------------------------------- -// -// `Vault::why_similar` is an *agent* turn: the model is offered B2's read-only tools and -// makes the lookups itself. Three things keep that honest on a small local model. The -// first round is the lookup round: its text is never shown, a model that calls nothing -// there is handed the evidence instead, and one that skips the pair lookup has it made for -// it. The loop is bounded, and its last round offers no tools. And a model with no tool -// support at all degrades to [`build_why_request`], the same evidence in one plain request. +// --- the tool-using turn (see `Vault::why_similar`) ------------------------------- -/// The matched passage pairs between two notes — what `b2 similar` ranked the row on. +/// The matched passage pairs between two notes. pub const TOOL_PASSAGE_PAIRS: &str = "b2_passage_pairs"; /// The ranked *Similar & unlinked* list for a note. pub const TOOL_SIMILAR: &str = "b2_similar"; @@ -200,22 +161,17 @@ pub const TOOL_NEIGHBORS: &str = "b2_neighbors"; /// A note's passages, in order. pub const TOOL_READ: &str = "b2_read"; -/// Model calls per why-turn, the final answer included. The last round offers no tools, -/// so a model that would look things up forever still has to answer. +/// Model calls per why-turn, the final answer included. pub const MAX_TOOL_ROUNDS: usize = 4; -/// Tool calls honoured per round; any beyond this are dropped unanswered-for. A bound on -/// what one malformed or runaway completion can make B2 do. +/// Tool calls honoured per round; the rest are dropped. Bounds a runaway completion. pub const MAX_CALLS_PER_ROUND: usize = 4; -/// Passages one `b2_read` call returns. Enough to read a note's opening sections without -/// one call filling a small model's context. +/// Passages one `b2_read` call returns, small enough for a small model's context. pub const READ_PASSAGES: usize = 4; -/// The system prompt of the tool-using why-turn. [`WHY_SYSTEM_PROMPT`]'s rules — cite by -/// `[n]`, nothing from general knowledge, the same no-evidence sentence — plus what the -/// tools are for. The first lookup's result is already in the conversation when the model -/// first sees it. +/// The system prompt of the tool-using why-turn: [`WHY_SYSTEM_PROMPT`]'s rules plus what +/// the tools are for. pub const WHY_AGENT_SYSTEM_PROMPT: &str = "You are B2, explaining why one of the user's notes \ was suggested as similar to the note they are reading. B2 suggests a note when one of its \ passages sits close to a passage of the open note in embedding space and the two notes are not \ @@ -232,10 +188,8 @@ written exactly as the bracketed number — [1], [2] — never as \"passage 1\". suggestion, not a verdict: if the passages share little, say so plainly. If no tool returned \ any passage, say \"I don't find that in your notes.\" — never answer from general knowledge."; -/// The tools a why-turn offers — each one read-only `Vault` op, described for the model. -/// Every argument is optional and defaults to this turn's pair — `note` to the open note, -/// `candidate` (and `b2_read`'s `note`, since the open note is already on screen) to the -/// suggested one — so `{}` is always a valid call, which is what a small model sends. +/// The tools a why-turn offers, each a read-only `Vault` op. Every argument defaults to +/// this turn's pair, so `{}` (what a small model sends) is always valid. pub fn why_tools() -> Vec { let note = |what: &str| { serde_json::json!({ @@ -303,8 +257,7 @@ pub fn why_tools() -> Vec { } /// Assemble one round of the tool-using why-turn. `passages` is the citation ledger so -/// far; the pair lines are left out of the facts because the seeded `b2_passage_pairs` -/// result carries them, beside the passages they number. +/// far; pair lines come from the `b2_passage_pairs` result, not the facts. pub fn build_why_agent_request( facts: &WhyFacts, tools: Vec, @@ -324,8 +277,7 @@ pub fn build_why_agent_request( } } -/// One matched pair as a line of text — shared by the handoff's facts block and the -/// `b2_passage_pairs` tool result, so the two read the same. +/// One matched pair as a line of text, shared by the facts block and the tool result. pub fn pair_line( ordinal: usize, anchor_marker: usize, @@ -343,15 +295,13 @@ pub fn pair_line( ) } -/// One numbered passage as tool-result text — [`ContextPassage::block`], the layout -/// [`ChatRequest::system_message`] uses too, so a passage looks the same whichever way -/// it reached the model. +/// One numbered passage as tool-result text, in the same layout as +/// [`ChatRequest::system_message`]. pub fn passage_block(marker: usize, p: &ContextPassage) -> String { p.block(marker) } -/// The facts block, one line per finding, each naming the B2 read it came from so the -/// model (and a human reading a debug log) can tell a measured fact from an inference. +/// The facts block, one line per finding, each naming the B2 read it came from. fn why_facts_block(f: &WhyFacts, handoff: bool) -> String { let named = |path: &str, title: &Option| match title { Some(t) => format!("{path} (\"{t}\")"), @@ -376,8 +326,7 @@ fn why_facts_block(f: &WhyFacts, handoff: bool) -> String { "- `b2 similar`: ranked #{rank} of {served} unlinked candidates{strength}." )); } - // The graph facts are the handoff's only: a tool-using turn reads them itself, with - // `b2_neighbors`, and a fact the model looked up is one it is likelier to use. + // Graph facts are the handoff's only; a tool-using turn looks them up itself. if !handoff { return lines.join("\n"); } @@ -393,8 +342,7 @@ fn why_facts_block(f: &WhyFacts, handoff: bool) -> String { lines.push(if f.shared_neighbors.is_empty() { "- `b2 neighbors`: they share no linked neighbours.".to_string() } else { - // Named up to a handful: a hub-heavy vault can share dozens, and a long list is - // what a small model parrots back instead of answering. + // Capped: a small model parrots a long list back instead of answering. let named: Vec<&str> = f .shared_neighbors .iter() @@ -426,27 +374,21 @@ fn why_facts_block(f: &WhyFacts, handoff: bool) -> String { lines.join("\n") } -/// The conversation as the provider sees it: the adapter-held history, oldest -/// first, with the current question appended as the final user turn. +/// The history, oldest first, with the question appended as the final user turn. fn turns_with_question(history: &[ChatTurn], question: &str) -> Vec { let mut turns = history.to_vec(); turns.push(ChatTurn::user(question)); turns } -/// Step 4 (the parse half) — the distinct `[n]` markers in an answer that -/// actually name a passage: 1-based, `n <= passage_count`, ascending. A -/// hallucinated marker (`[0]`, `[99]` over ten passages, digits too long to -/// parse) simply doesn't resolve — it stays in the answer text verbatim -/// (model output is never rewritten) and contributes no citation. The façade -/// maps each surviving marker onto its passage for the display view. +/// Step 4: the distinct `[n]` markers naming a real passage (1-based, `n <= +/// passage_count`), ascending. A hallucinated marker just yields no citation. pub fn cited_markers(text: &str, passage_count: usize) -> Vec { let mut found = BTreeSet::new(); let bytes = text.as_bytes(); let mut i = 0; while i < bytes.len() { - // A byte-wise scan is UTF-8-safe: '[', ']' and ASCII digits never occur - // as continuation bytes, so a multi-byte character can't fake a marker. + // UTF-8-safe: '[', ']' and ASCII digits never occur as continuation bytes. if bytes[i] == b'[' { let mut j = i + 1; while j < bytes.len() && bytes[j].is_ascii_digit() { diff --git a/crates/b2-core/src/chunk.rs b/crates/b2-core/src/chunk.rs index ba9d95a..b6a1f24 100644 --- a/crates/b2-core/src/chunk.rs +++ b/crates/b2-core/src/chunk.rs @@ -1,55 +1,33 @@ -//! Body chunker — the qmd heuristic as adapted (docs/index-engine.md §1, GH #19). +//! Body chunker, the [tobi/qmd](https://github.com/tobi/qmd) heuristic adapted +//! (index-engine.md §1, GH #19): size-targeted, overlapping, Markdown-aware chunks, each +//! with a `heading_path` breadcrumb. At the target size it scans back for the best +//! structural break under a quadratic distance decay, and never cuts inside a code fence +//! or table (#41). //! -//! Splits a note body into **size-targeted, overlapping, Markdown-aware** chunks that -//! each carry a `heading_path` breadcrumb. The shape, borrowed wholesale from -//! [tobi/qmd](https://github.com/tobi/qmd): -//! -//! - accumulate toward `cfg.target_tokens` (default 450 — under bge's 512-token -//! truncation); -//! - at the target, scan **backward** over a `cfg.backscan_tokens` window and cut at the -//! best-scoring structural break (`cfg.weights`), weighted by a **quadratic distance -//! decay** so the boundary is the cleanest *near* the target, not an arbitrary slice; -//! - carry `cfg.overlap_frac` of the tail forward, so the `char_start..char_end` ranges -//! overlap rather than partition the body; -//! - never cut *inside* a fenced code block or a Markdown table — a forced cut with no -//! clean break is pushed past the block's end (#41), trading a slightly oversized chunk -//! for a coherent one (`protected_regions`); -//! - track a running heading stack and stamp each chunk's `heading_path`. -//! -//! The core is **model-free** (ADR-0005): there is no tokenizer here, so chunks are sized -//! by a cheap deterministic proxy (`chars / cfg.chars_per_token`) with the embedder's own -//! 512-token truncation as the hard backstop — a proxy under-estimate merely clips the -//! tail of one unusually dense chunk. `chunk_body` is a **pure function** of -//! `(body, ChunkConfig)`, so swapping chunkers is a pure re-projection. -//! -//! `char_start..char_end` always addresses the body slice that produced `text`, **except** -//! under `cfg.prepend_heading_path` — an eval knob (default off) that prepends the -//! breadcrumb into the embedded text, which the range then does not cover. - -/// The tuning surface for [`chunk_body`] (index-engine.md §1). Every lever that shapes a -/// cut lives here; `Default` reproduces the shipped values, so adapters pass -/// `&ChunkConfig::default()` and the eval harness (`make eval-sweep`) sweeps parameters -/// in one process (a loop over configs) rather than one recompile per cell. Kept a plain -/// params struct — no async/generics/traits — so it stays pure, deterministic, -/// model-free. +//! Model-free (ADR-0005): sized by a `chars / chars_per_token` proxy, with the embedder's +//! 512-token truncation as the backstop. A pure function of `(body, ChunkConfig)`. +//! `char_start..char_end` addresses the source slice, except that +//! `cfg.prepend_heading_path` adds text outside it. + +/// The tuning surface for [`chunk_body`] (index-engine.md §1). `Default` is the shipped +/// config; `make eval-sweep` varies it in one process. #[derive(Debug, Clone, PartialEq)] pub struct ChunkConfig { - /// Target chunk size in *estimated* tokens. Default 450 — headroom under bge's - /// 512-token truncation for the token proxy's error and any prepended breadcrumb. + /// Target chunk size in estimated tokens. 450 leaves headroom under bge's 512 for the + /// proxy's error and a prepended breadcrumb. pub target_tokens: usize, /// Fraction of a chunk re-shared with the next one. Default 0.15. pub overlap_frac: f32, - /// The model-free token proxy: `tokens ≈ chars / chars_per_token`. Default - /// 4.0 (English ≈ 4 chars/token; code and tables run denser — a lever, not a law). + /// The token proxy: `tokens ≈ chars / chars_per_token`. Default 4.0 (English; code + /// and tables run denser). pub chars_per_token: f32, /// How far back (in estimated tokens) the boundary search looks from the target. /// Default 200. pub backscan_tokens: usize, /// The Markdown break-point scorer (qmd's H1=100 … list=5). pub weights: BreakWeights, - /// Prepend `heading_path` into the *embedded* `text` (contextual chunk headers). - /// An eval-gated retrieval knob, measured rank-neutral; **default off**. Storing `heading_path` is - /// unconditional — this only controls whether it also seeds the vector. + /// Prepend `heading_path` into the embedded `text`. An eval knob, measured + /// rank-neutral; default off. `heading_path` is stored either way. pub prepend_heading_path: bool, } @@ -66,9 +44,7 @@ impl Default for ChunkConfig { } } -/// Markdown break-point weights (qmd's boundary scorer, index-engine.md §1). Higher = a cleaner -/// place to end a chunk. `heading[i]` is the weight of an H{i+1}; `word` is the -/// lowest-value fallback that lets a giant single-line paragraph still split. +/// Markdown break-point weights (index-engine.md §1). Higher = a cleaner place to cut. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BreakWeights { /// H1..H6 (index 0 = H1). qmd: H1=100, H2=90, then a gentle gradient. @@ -81,7 +57,7 @@ pub struct BreakWeights { pub list_item: u32, /// A plain paragraph/text line start. pub paragraph: u32, - /// A word boundary within a line — the finest fallback. + /// A word boundary within a line, so a giant one-line paragraph can still split. pub word: u32, } @@ -104,17 +80,14 @@ pub struct Chunk { pub seq: usize, pub char_start: usize, pub char_end: usize, - /// The token *estimate* (`chars / chars_per_token`) that sized this chunk — - /// not an exact token count (it was a whitespace word count under the old splitter). + /// The token estimate (`chars / chars_per_token`), not an exact count. pub token_count: usize, - /// The H1 › H2 › H3 breadcrumb the chunk falls under (`"A > B"`), or `None` - /// when the chunk starts before any heading. + /// The heading breadcrumb (`"A > B"`), or `None` before any heading. pub heading_path: Option, pub text: String, } -/// Chunk `body` into size-targeted, overlapping, Markdown-aware chunks (see the -/// module docs). Returns an empty vec for an empty/all-blank body. +/// Chunk `body` (see the module docs). Empty for an all-blank body. pub fn chunk_body(body: &str, cfg: &ChunkConfig) -> Vec { let body_len = body.len(); if body.trim().is_empty() { @@ -123,26 +96,21 @@ pub fn chunk_body(body: &str, cfg: &ChunkConfig) -> Vec { let (breaks, line_paths) = scan(body, &cfg.weights); - // Token params → char thresholds once (offsets are bytes; the proxy is uniform, - // so a token position is just `offset / chars_per_token`). + // Token params → byte thresholds once. let cpt = (cfg.chars_per_token as f64).max(0.1); let target_chars = ((cfg.target_tokens as f64) * cpt).round().max(1.0) as usize; let backscan_chars = ((cfg.backscan_tokens as f64) * cpt).round() as usize; let overlap_chars = ((cfg.target_tokens as f64) * (cfg.overlap_frac as f64) * cpt).round() as usize; - // #41: byte ranges (balanced code fences, GFM tables) a boundary must not cut - // *inside* — a forced cut with no clean break in the window is exactly where a - // bisected block would otherwise happen (this dense content is what the chars/4 - // token proxy mis-sizes). + // #41: fences and tables a boundary must not cut inside. let regions = protected_regions(body); let mut chunks: Vec = Vec::new(); let mut start = 0usize; loop { - // The remainder fits in one chunk → emit it and stop (never split off a - // final sliver). + // The remainder fits: emit it rather than split off a sliver. if body_len - start <= target_chars { push_chunk(&mut chunks, body, &line_paths, start, body_len, cfg); break; @@ -156,39 +124,32 @@ pub fn chunk_body(body: &str, cfg: &ChunkConfig) -> Vec { cfg.backscan_tokens, cpt, ); - // If that cut lands inside a fence/table, push it past the block's end so the - // whole block stays in one chunk. An oversized chunk (the embedder truncates - // at 512 tokens) is fine; a half-fence / header-less table is not. + // Push a cut inside a fence/table past its end: oversized beats a half-fence. let end = snap_past_region(end, ®ions); push_chunk(&mut chunks, body, &line_paths, start, end, cfg); if end >= body_len { break; } - // The next chunk's overlap tail must not start inside a block either — skip - // such a start to just past the block (no overlap into it, rather than a chunk - // that opens mid-fence). + // Nor may the next chunk start inside a block. let overlap_start = choose_overlap_start(&breaks, start, end, overlap_chars); start = snap_past_region(overlap_start, ®ions); } - // Re-number after the fact: an all-whitespace span is skipped by `push_chunk`, - // so `seq` is assigned here to stay contiguous and gap-free. + // `push_chunk` skips blank spans, so number contiguously here. for (i, c) in chunks.iter_mut().enumerate() { c.seq = i; } chunks } -/// A candidate chunk boundary at a byte `offset`, with the structural `score` of the -/// line/word that starts there (qmd's scorer — higher = a cleaner cut). +/// A candidate chunk boundary at a byte `offset`, with its structural `score`. struct Break { offset: usize, score: u32, } -/// Single pass over the body: the ordered break candidates (line starts scored -/// structurally, plus word starts as the fallback) and, per line start, the heading -/// breadcrumb in effect there. Both vecs come out sorted by offset. +/// One pass over the body: the break candidates, and the breadcrumb at each line start. +/// Both sorted by offset. fn scan(body: &str, w: &BreakWeights) -> (Vec, Vec<(usize, Option)>) { let mut breaks = Vec::new(); let mut line_paths = Vec::new(); @@ -203,8 +164,7 @@ fn scan(body: &str, w: &BreakWeights) -> (Vec, Vec<(usize, Option let trimmed = content.trim_start(); let is_fence = is_fence_line(trimmed); - // Classify the line for its break score, and detect a heading (never inside - // a fence — a `# comment` in code must not corrupt the breadcrumb). + // Never a heading inside a fence: a `# comment` in code isn't one. let (score, heading) = if content.trim().is_empty() { (w.blank_line, None) } else if is_fence { @@ -222,8 +182,7 @@ fn scan(body: &str, w: &BreakWeights) -> (Vec, Vec<(usize, Option (w.paragraph, None) }; - // Update the stack AFTER classifying, so a chunk starting on `## X` records a - // path that includes X (the section it opens). + // After classifying, so a chunk starting on `## X` includes X in its path. if let Some((level, text)) = heading { while stack.last().is_some_and(|(l, _)| *l >= level) { stack.pop(); @@ -245,19 +204,16 @@ fn scan(body: &str, w: &BreakWeights) -> (Vec, Vec<(usize, Option (breaks, line_paths) } -/// One `split_inclusive('\n')` line without its `\n` / `\r\n` ending. fn line_content(line: &str) -> &str { line.trim_end_matches('\n').trim_end_matches('\r') } -/// Whether a left-trimmed line opens or closes a fenced code block (```` ``` ```` or -/// `~~~`) — the one fence rule [`scan`] and [`protected_regions`] both track. +/// Whether a left-trimmed line is a code fence; shared by [`scan`] and [`protected_regions`]. fn is_fence_line(trimmed: &str) -> bool { trimmed.starts_with("```") || trimmed.starts_with("~~~") } -/// The ATX heading level (1..=6) if `trimmed` is a heading, else `None`. The `#` run -/// must be followed by a space or end-of-line, so `#tag` is not a heading. +/// The ATX heading level (1..=6), if any. `#tag` is not a heading. fn heading_level(trimmed: &str) -> Option { let hashes = trimmed.bytes().take_while(|&b| b == b'#').count(); if (1..=6).contains(&hashes) { @@ -269,7 +225,6 @@ fn heading_level(trimmed: &str) -> Option { None } -/// The heading's display text: the `#`s and any ATX closing `#`s stripped, trimmed. fn heading_text(trimmed: &str, level: u8) -> String { trimmed[level as usize..] .trim() @@ -278,7 +233,6 @@ fn heading_text(trimmed: &str, level: u8) -> String { .to_string() } -/// Whether `trimmed` opens a Markdown list item (`- `/`* `/`+ ` or `1. `/`1) `). fn is_list_item(trimmed: &str) -> bool { if let Some(rest) = trimmed.strip_prefix(['-', '*', '+']) { return rest.starts_with(' '); @@ -291,8 +245,7 @@ fn is_list_item(trimmed: &str) -> bool { false } -/// Append a low-weight break at each word start within a line (skipping the line -/// start, already a structural break), so a giant single-line paragraph can split. +/// A low-weight break at each word start after the line start. fn push_word_breaks(breaks: &mut Vec, content: &str, line_start: usize, word: u32) { let mut prev_ws = true; for (i, c) in content.char_indices() { @@ -307,7 +260,6 @@ fn push_word_breaks(breaks: &mut Vec, content: &str, line_start: usize, w } } -/// The heading breadcrumb (`"A > B"`) for a stack, or `None` when empty. fn join_path(stack: &[(u8, String)]) -> Option { if stack.is_empty() { None @@ -322,16 +274,13 @@ fn join_path(stack: &[(u8, String)]) -> Option { } } -/// The offset of the first break at or after `off`, or `None` if none exists. fn first_break_at_or_after(breaks: &[Break], off: usize) -> Option { let idx = breaks.partition_point(|b| b.offset < off); breaks.get(idx).map(|b| b.offset) } -/// Pick the chunk end: the break in `(start, anchor]` within the backscan window -/// maximizing `score · decay²`, where `decay = 1 - dist/backscan` falls off -/// quadratically with token distance from the target. Falls back to `anchor` when -/// the window holds no usable break. +/// The chunk end: the break in the backscan window maximizing `score · decay²`, with +/// `decay = 1 - dist/backscan`. Falls back to `anchor`. fn choose_end( breaks: &[Break], start: usize, @@ -358,8 +307,7 @@ fn choose_end( }; let decay = (1.0 - frac).max(0.0); let score = b.score as f64 * decay * decay; - // `>=` so that on a tie the larger offset (nearer the target, less content - // dropped) wins, since offsets are walked ascending. + // `>=`: on a tie the later offset, nearer the target, wins. if best.map(|(bs, _)| score >= bs).unwrap_or(true) { best = Some((score, b.offset)); } @@ -367,10 +315,8 @@ fn choose_end( best.map(|(_, o)| o).unwrap_or(anchor) } -/// The next chunk's start: the break inside `(start, end)` nearest to -/// `end - overlap`, preferring a cleaner (higher-scoring) boundary on a tie. Always -/// `> start` (so the walk makes progress); falls back to `end` (no overlap) when the -/// chunk has no interior break. +/// The next chunk's start: the interior break nearest `end - overlap`, cleaner on a tie. +/// Always `> start`, so the walk progresses; falls back to `end` (no overlap). fn choose_overlap_start(breaks: &[Break], start: usize, end: usize, overlap_chars: usize) -> usize { if overlap_chars == 0 { return end; @@ -380,7 +326,6 @@ fn choose_overlap_start(breaks: &[Break], start: usize, end: usize, overlap_char let hi = breaks.partition_point(|b| b.offset < end); let mut best: Option<((usize, u32), usize)> = None; for b in &breaks[lo..hi] { - // Order by (distance to desired, then higher score). let key = (b.offset.abs_diff(desired), u32::MAX - b.score); if best.map(|(bk, _)| key < bk).unwrap_or(true) { best = Some((key, b.offset)); @@ -389,11 +334,8 @@ fn choose_overlap_start(breaks: &[Break], start: usize, end: usize, overlap_char best.map(|(_, o)| o).unwrap_or(end) } -/// If `off` falls **strictly inside** one of the protected `regions` (a balanced -/// fenced code block or a GFM table, from [`protected_regions`]), return that region's -/// end so the boundary clears the block; otherwise return `off` unchanged. A cut -/// exactly at a region edge is already clean and left as-is. `regions` is sorted by -/// start and non-overlapping, so the first region reaching past `off` decides it. +/// `off`, or the end of the protected region it falls strictly inside. `regions` is sorted +/// and non-overlapping. fn snap_past_region(off: usize, regions: &[(usize, usize)]) -> usize { for &(rs, re) in regions { if rs >= off { @@ -406,22 +348,16 @@ fn snap_past_region(off: usize, regions: &[(usize, usize)]) -> usize { off } -/// Byte ranges `[start, end)` a chunk boundary must not fall **strictly inside** (#41): a -/// balanced fenced code block and a GFM table. Cutting at either edge is clean; cutting -/// the interior would embed an unbalanced fence, or orphan a table's header from its rows. -/// -/// Fence tracking mirrors [`scan`]'s; an **unterminated** fence is left unprotected rather -/// than swallowing the whole tail into one chunk. A table is a header row directly above a -/// delimiter row plus the contiguous rows beneath. Regions come out sorted and never -/// overlap (table scanning is suppressed inside a fence). +/// Byte ranges `[start, end)` a boundary must not fall strictly inside (#41): balanced code +/// fences and GFM tables. An unterminated fence is left unprotected rather than swallow the +/// tail. Sorted, non-overlapping. fn protected_regions(body: &str) -> Vec<(usize, usize)> { let mut regions: Vec<(usize, usize)> = Vec::new(); let mut in_fence = false; let mut fence_start = 0usize; - // Table accumulation across consecutive rows. - let mut prev_row_start: Option = None; // last row candidate — a header-in-waiting + let mut prev_row_start: Option = None; // a header-in-waiting let mut table_start: Option = None; // set once a delimiter row confirms a table - let mut table_end = 0usize; // end offset (past the newline) of the table's last row + let mut table_end = 0usize; let mut offset = 0usize; for line in body.split_inclusive('\n') { @@ -431,7 +367,7 @@ fn protected_regions(body: &str) -> Vec<(usize, usize)> { let is_fence = is_fence_line(trimmed); if is_fence { - // A fence edge ends any table in progress, then toggles the fence. + // A fence edge ends any table in progress. if let Some(ts) = table_start.take() { regions.push((ts, table_end)); } @@ -450,7 +386,7 @@ fn protected_regions(body: &str) -> Vec<(usize, usize)> { if is_table_row(trimmed) { if table_start.is_some() { - table_end = offset; // extend the active table by this row + table_end = offset; } else if is_table_delim(trimmed) { if let Some(header_start) = prev_row_start { table_start = Some(header_start); @@ -467,7 +403,7 @@ fn protected_regions(body: &str) -> Vec<(usize, usize)> { prev_row_start = None; } } - // A table running to EOF (an unterminated fence is intentionally left unprotected). + // A table running to EOF. if let Some(ts) = table_start.take() { regions.push((ts, table_end)); } @@ -476,24 +412,21 @@ fn protected_regions(body: &str) -> Vec<(usize, usize)> { regions } -/// Whether `trimmed` (already left-trimmed) could be a Markdown table row: non-empty -/// and carrying a `|` column separator. Both header and body rows qualify. +/// Whether `trimmed` could be a table row (header or body). fn is_table_row(trimmed: &str) -> bool { !trimmed.is_empty() && trimmed.contains('|') } -/// Whether `trimmed` is a GFM table **delimiter** row — the `| --- | :-: |` line under -/// the header. Requires a `|` (so a setext `---` underline or a `---` rule is *not* one) -/// and at least one `-`, with every character drawn from `-:| ` and nothing else. +/// Whether `trimmed` is a GFM delimiter row (`| --- | :-: |`). The `|` requirement keeps a +/// `---` rule or setext underline out. fn is_table_delim(trimmed: &str) -> bool { trimmed.contains('|') && trimmed.contains('-') && trimmed.chars().all(|c| matches!(c, '-' | ':' | '|' | ' ')) } -/// Emit `body[start..end]`, trimmed to its non-blank span (so `char_start..char_end` -/// stays exact and the text is clean); skip an all-whitespace span. `seq` is a -/// placeholder — [`chunk_body`] re-numbers at the end. +/// Emit `body[start..end]` trimmed to its non-blank span; skip a blank one. `seq` is set +/// later by [`chunk_body`]. fn push_chunk( chunks: &mut Vec, body: &str, @@ -526,8 +459,7 @@ fn push_chunk( }); } -/// The heading breadcrumb in effect at byte offset `cs` — the path of the last line -/// starting at or before it. +/// The breadcrumb in effect at byte offset `cs`. fn heading_path_at(line_paths: &[(usize, Option)], cs: usize) -> Option { let idx = line_paths.partition_point(|(ls, _)| *ls <= cs); idx.checked_sub(1).and_then(|i| line_paths[i].1.clone()) diff --git a/crates/b2-core/src/db.rs b/crates/b2-core/src/db.rs index 3296ae8..b0919a2 100644 --- a/crates/b2-core/src/db.rs +++ b/crates/b2-core/src/db.rs @@ -1,19 +1,13 @@ -//! Opening the index, the schema migration, and the projection helpers for the -//! Markdown-derived tiers: `notes`, `chunks` (+FTS5), the -//! `embeddings`/`note_centroids` vector tables, and the typed `edges` graph. Every -//! table here is a derived projection of Markdown — nothing is a source of truth -//! (ADR-0002). +//! Opening the index, the schema migration, and the projection helpers for `notes`, +//! `chunks` (+FTS5), the vector tables and the typed `edges` graph. Every table is a +//! derived projection of Markdown (ADR-0002). //! -//! **A note is keyed by its vault-relative path** (ADR-0003): `notes.path` is the -//! primary key and every child references it `ON DELETE CASCADE ON UPDATE CASCADE`. -//! The update half is what makes a B2-performed move a **re-key** rather than a -//! rebuild — one `UPDATE notes SET path` carries every derived row with it. It needs -//! `PRAGMA foreign_keys = ON`, set on every connection alongside `WAL`. +//! A note is keyed by its vault-relative path (ADR-0003); children reference it +//! `ON DELETE CASCADE ON UPDATE CASCADE`, so a B2-performed move is one `UPDATE notes +//! SET path` (needs `PRAGMA foreign_keys = ON`). //! -//! Vectors live in **plain tables**, scored in-process, content-addressed by the -//! blake3 of the chunk text (ADR-0006). The one bookkeeping cost is that a vector no -//! longer dies with its chunk — [`prune_orphan_vectors`] collects what nothing -//! references, on the same derived-data lifecycle as centroids. +//! Vectors live in plain tables, content-addressed by the blake3 of the chunk text +//! (ADR-0006), so they outlive their chunk; [`prune_orphan_vectors`] collects them. use crate::chunk::Chunk; use crate::embed::pack_f32; @@ -27,30 +21,21 @@ use std::path::Path; use std::sync::OnceLock; use std::time::Duration; -/// The B2 index schema version stamped into `meta.schema_version`. Bumping it is the -/// migration gate: on a **stale or unknown** stamp `migrate()` drops the derived tables -/// and lets the next `reindex` rebuild them — there are no migrations, by design -/// (ADR-0002). A stamp *above* this is the one mismatch that is not rebuilt: it was -/// written by a newer `b2`, so the index is refused untouched ([`refuse_if_newer`]). +/// The index schema version stamped into `meta.schema_version`. A stale or unknown +/// stamp drops the derived tables for the next `reindex` to rebuild; there are no +/// migrations (ADR-0002). A stamp above this is refused untouched ([`refuse_if_newer`]). /// -/// **2** dropped the suggestion machinery with the 2026-07-04 relator cut. **3** -/// replaced the `chunks_vec` vec0 virtual table with plain vector tables (ADR-0006); -/// a pre-3 index's orphaned `chunks_vec` entry stays inert in `sqlite_master`, -/// because its module is no longer linked to drop it. **4** added the `resources` -/// inventory and widened `edges` with resource targets. **5** switched `chunks_fts` -/// to `porter unicode61` (the GH #157 A/B's verdict). **6** re-keyed the whole index -/// on the vault-relative path and made `embeddings` content-addressed (GH #170). **7** -/// dropped the columns nothing read (`notes.type`, `description`, `updated`) and the -/// `note_aliases` table. +/// 3: plain vector tables (ADR-0006). 4: `resources`. 5: `porter unicode61` FTS (GH #157). +/// 6: path-keyed, content-addressed `embeddings` (GH #170). 7: unread columns and +/// `note_aliases` dropped. pub const SCHEMA_VERSION: i64 = 7; /// Statements at or over this take the slow-query WARN path (`B2_SLOW_QUERY_MS` /// overrides; see [`slow_query_threshold`]). const SLOW_QUERY_MS_DEFAULT: u64 = 100; -/// The duration at or above which a statement logs as a **slow query** (WARN instead -/// of DEBUG), read once from `B2_SLOW_QUERY_MS`. Observability config only — it never -/// changes what an operation computes. +/// The duration at which a statement logs as a slow query (WARN instead of DEBUG), read +/// once from `B2_SLOW_QUERY_MS`. fn slow_query_threshold() -> Duration { static THRESHOLD: OnceLock = OnceLock::new(); *THRESHOLD.get_or_init(|| { @@ -62,29 +47,18 @@ fn slow_query_threshold() -> Duration { }) } -/// Whether a finished statement is worth the string work in [`on_sqlite_profile`]: -/// true only when something would receive the event. -/// -/// Split out because the `slow && warn` term is load-bearing: drop it (the tempting -/// "just check DEBUG" simplification) and slow-query WARNs silently disappear for a -/// WARN-only subscriber. The unit test below is the truth table. +/// Whether anything would receive a finished statement's event. The `slow && warn` +/// term keeps slow-query WARNs for a WARN-only subscriber; don't reduce it to DEBUG. fn should_emit(slow: bool, warn_enabled: bool, debug_enabled: bool) -> bool { (slow && warn_enabled) || debug_enabled } -/// SQLite's own per-statement profiler (`sqlite3_trace_v2`), surfaced as structured -/// `tracing` events on target `b2::sqlite`: the SQL **template** (`?N` placeholders, -/// never bound values, so no note content lands in the log and events group by -/// statement), `duration_us`, and the `vm_steps`/`fullscan_steps` counters — the "why -/// was it slow" signal, since a high fullscan count means a missing index. At or over -/// [`slow_query_threshold`] it logs at WARN, otherwise DEBUG. -/// -/// **`duration_us` precision is platform-bound** — some platforms (macOS observed) -/// quantize SQLite's profiler clock to ~1ms, so sub-millisecond statements read as -/// `0`. For fine-grained cost use `vm_steps`: VDBE opcodes, deterministic and -/// clock-independent. +/// SQLite's per-statement profiler as `tracing` events on `b2::sqlite`: the SQL template +/// (never bound values, so no note content is logged), `duration_us`, and +/// `vm_steps`/`fullscan_steps` (a high fullscan count means a missing index). /// -/// A plain `fn` because `trace_v2` registers a function pointer. +/// Some platforms (macOS) quantize `duration_us` to ~1ms; `vm_steps` is the +/// deterministic cost measure. fn on_sqlite_profile(event: TraceEvent<'_>) { let TraceEvent::Profile(stmt, elapsed) = event else { return; // only SQLITE_TRACE_PROFILE is masked in, but TraceEvent is non-exhaustive @@ -98,8 +72,7 @@ fn on_sqlite_profile(event: TraceEvent<'_>) { ) { return; } - // Collapse the multi-line SQL literals used in this file to one line, so each - // event stays a single clean record with a stable, groupable `sql` key. + // One line per event, so `sql` is a stable, groupable key. let sql = stmt.sql().split_whitespace().collect::>().join(" "); let duration_us = u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX); let vm_steps = stmt.get_status(StatementStatus::VmStep); @@ -123,20 +96,16 @@ fn on_sqlite_profile(event: TraceEvent<'_>) { /// idempotent migration. Safe to call on a fresh or an already-built index. pub fn open(path: &Path) -> Result { let mut conn = Connection::open(path)?; - // Profile every statement through SQLite's trace_v2 hook (`on_sqlite_profile`). conn.trace_v2( TraceEventCodes::SQLITE_TRACE_PROFILE, Some(on_sqlite_profile), ); // execute_batch tolerates the row PRAGMA mmap_size returns. - // busy_timeout: WAL allows one writer at a time, and two short-statement writers - // can legitimately race (a save during the background embed); a modest wait turns - // that into a few-ms stall instead of SQLITE_BUSY. Set explicitly rather than - // leaned on — rusqlite arms the same 5 s by default, but that is its contract. - // mmap_size + cache_size: whole-space vector scans stream ~100+ MB of blob rows - // per call on a real vault, which under the 2 MB default cache was syscall-bound - // (the bulk of `b2 similar`'s ~4.4 s, #38). mmap_size is a *cap*, not an - // allocation; cache_size is KiB when negative (32 MiB). + // busy_timeout: two writers can race (a save during the background embed); set + // explicitly rather than relying on rusqlite's default. + // mmap_size + cache_size: whole-space vector scans stream 100+ MB per call, which + // the 2 MB default cache made syscall-bound (#38). mmap_size is a cap, not an + // allocation; negative cache_size is KiB (32 MiB). conn.execute_batch( "PRAGMA busy_timeout = 5000; PRAGMA foreign_keys = ON; @@ -149,28 +118,21 @@ pub fn open(path: &Path) -> Result { Ok(conn) } -/// How many times [`enter_wal_mode`] attempts the flip before surfacing the busy error. -/// Large, because these retries *stand in for* `busy_timeout` — the one statement it does -/// not cover (ADR-0021) — so the whole ≈4 s wait budget is here. +/// Attempts at the WAL flip. Large, because these retries stand in for `busy_timeout`, +/// which does not cover it: the whole ≈4 s budget is here (ADR-0021). const WAL_FLIP_ATTEMPTS: u32 = 16; -/// How many times the DDL rebuilds re-attempt their `BEGIN IMMEDIATE`. Small where the -/// flip's is large, and for the opposite reason: these sit *on top of* `busy_timeout`, so -/// each attempt already waits its full 5 s. Past ≈15 s it is a stuck writer (ADR-0021). +/// Attempts at a DDL rebuild's `BEGIN IMMEDIATE`. Small, because each already waits the +/// full 5 s `busy_timeout`; past ≈15 s it is a stuck writer (ADR-0021). const REBUILD_ATTEMPTS: u32 = 3; /// The pause schedule [`retry_while_locked`] uses between attempts. const LOCK_RETRY_BACKOFF_START: Duration = Duration::from_millis(2); const LOCK_RETRY_BACKOFF_MAX: Duration = Duration::from_millis(500); -/// Run `op`, retrying while SQLite reports the lock it wants is held by someone else -/// — the one failure in this module that is a *race* rather than a fault. `what` names -/// the contended step for the log line only. -/// -/// The backoff sleeps but reads no clock and decides nothing from one, so the core's -/// determinism is untouched. Both callers hand it an operation that is idempotent and -/// self-checking: a retry re-reads the state it is about to change, so the attempt -/// after a lost race finds the work already done. +/// Run `op`, retrying while SQLite reports lock contention. `what` names the step for +/// the log. The backoff sleeps but reads no clock. `op` must be idempotent and +/// self-checking, so a retry after a lost race finds the work already done. fn retry_while_locked( what: &str, attempts: u32, @@ -197,21 +159,12 @@ fn retry_while_locked( } } -/// Put the connection in WAL mode, waiting out a concurrent opener (ADR-0021). +/// Put the connection in WAL mode, waiting out a concurrent opener (ADR-0021). The flip +/// takes a write lock that `busy_timeout` does not cover, so the retry is ours. /// -/// `journal_mode = WAL` is the one statement in [`open`] that takes a write lock, and only -/// when it actually *changes* the mode — so once per vault, ever — and it is the one -/// `busy_timeout` cannot cover, so the retry is ours ([`retry_while_locked`]). It converges -/// fast: the next attempt either takes the lock or finds the database already in WAL. -/// -/// **The mode is read back, not assumed.** A filesystem with no shared-memory support -/// declines the flip with `SQLITE_OK` and the *old* mode, which a row-discarding -/// `execute_batch` would report as success. A decline is not an error — B2 is correct in -/// rollback-journal mode — but it is said out loud, and not retried: nothing holds a lock, -/// so waiting changes nothing. +/// The mode is read back: a filesystem without shared memory declines with `SQLITE_OK` +/// and the old mode. That is logged, not retried; B2 works in rollback-journal mode. fn enter_wal_mode(conn: &Connection) -> Result<()> { - // A *declined* flip comes back as `Ok` carrying the old mode, so it leaves the - // retry immediately. let mode = retry_while_locked("journal_mode=WAL", WAL_FLIP_ATTEMPTS, || { Ok(conn.query_row("PRAGMA journal_mode = WAL", [], |r| r.get::<_, String>(0))?) })?; @@ -225,9 +178,8 @@ fn enter_wal_mode(conn: &Connection) -> Result<()> { Ok(()) } -/// Whether an error is SQLite lock contention — the retryable kind. `SQLITE_LOCKED` -/// is matched alongside `SQLITE_BUSY` because the desktop host opens the index from -/// more than one thread. +/// Whether an error is lock contention. `SQLITE_LOCKED` too, because the desktop opens +/// the index from more than one thread. fn is_locked(err: &Error) -> bool { matches!(err, Error::Sqlite(e) if matches!( e.sqlite_error_code(), @@ -235,12 +187,9 @@ fn is_locked(err: &Error) -> bool { )) } -/// The tables [`apply_schema`] creates — the structural half of [`schema_is_current`]. -/// -/// Tables only, and that is sufficient rather than lazy: this guards against a -/// concurrent `DROP TABLE` (#114), and dropping a table takes its indexes and triggers -/// with it, so a missing table is the visible edge of every partial rebuild. The unit -/// test at the foot of this file pins the list to what the DDL actually creates. +/// The tables [`apply_schema`] creates, checked by [`schema_is_current`]. Tables suffice: +/// a dropped table takes its indexes and triggers with it (#114). A unit test pins the +/// list to the DDL. const SCHEMA_TABLES: [&str; 6] = [ "meta", "notes", @@ -250,34 +199,20 @@ const SCHEMA_TABLES: [&str; 6] = [ "edges", ]; -/// Bring the index to [`SCHEMA_VERSION`] — **atomically, and serialized against every other -/// opener** on SQLite's own write lock (ADR-0021, which carries the measured races and why -/// an advisory lock file was rejected). -/// -/// Three properties, each load-bearing. **Serialized:** `migrate` reads, decides, then -/// writes, so two openers that both read a stale version both rebuild and their ~30 DDL -/// statements interleave. **Atomic:** the drop-and-rebuild runs in one transaction, so the -/// stamp and the tables it vouches for commit together, which is what lets -/// [`schema_is_current`] trust it. **The fast path takes no write lock at all**, which is -/// what keeps C1's "a reader is never refused" true: an already-current index costs two -/// reads instead of the ~30 `IF NOT EXISTS` statements it used to re-run on every open. +/// Bring the index to [`SCHEMA_VERSION`], atomically and serialized against other openers +/// on SQLite's write lock (ADR-0021). The stamp commits with the tables it vouches for, and +/// a current index takes no write lock at all, so a reader is never refused (C1). fn migrate(conn: &mut Connection) -> Result<()> { if schema_is_current(conn)? { return Ok(()); } - // Direction, before the rebuild branch can act on "not mine". Checked here as well as - // inside the lock so the common case — a newer index, no contention — is refused - // without ever taking a write lock on a database this build has no business writing. + // Also checked here so a newer index is refused without taking a write lock. refuse_if_newer(conn)?; retry_while_locked("schema migration", REBUILD_ATTEMPTS, || { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - // Re-checked *inside* the write lock, which is the whole point of taking it: - // if we lost the race we waited on the winner's transaction, and this is where - // we find its work already committed. + // Re-checked inside the lock: a lost race finds the winner's work committed. if !schema_is_current(&tx)? { - // The race the outer check cannot cover: the opener we waited on was a - // *newer* b2 that rebuilt this index while we queued. Refusing rolls the - // transaction back untouched. + // The opener we waited on may have been a newer b2. refuse_if_newer(&tx)?; apply_schema(&tx)?; } @@ -286,26 +221,13 @@ fn migrate(conn: &mut Connection) -> Result<()> { }) } -/// Refuse an index stamped **above** [`SCHEMA_VERSION`] — written by a newer `b2` than -/// this binary — instead of letting [`apply_schema`] drop it ([`Error::IndexTooNew`]). -/// -/// [`schema_is_current`] asks only whether the stamp *equals* ours, and every way it can -/// say no used to reach the same unconditional rebuild. That is right for an index of no -/// known version and wrong for one of a *later* known version: the index is disposable -/// (ADR-0002), but this build is not the one entitled to dispose of it, and dropping does -/// not make a shape it predates readable. In the field the cost fell on the user — an -/// older `b2` on `PATH` (a stale `cargo install`, an unupgraded shell) wiped a complete -/// index on a **read-only** command, and the only symptom was a whole-vault re-embed. +/// Refuse an index stamped above [`SCHEMA_VERSION`] ([`Error::IndexTooNew`]) rather than +/// let [`apply_schema`] drop it: an older `b2` on `PATH` must not wipe a newer index. /// -/// **The stamp alone decides, deliberately.** A stamp is written only inside -/// [`apply_schema`]'s transaction, alongside the tables it vouches for, so a stamp above -/// ours means some newer `b2` committed a complete index here. Gating on our own -/// [`SCHEMA_TABLES`] as well would be worse than redundant: a future schema may rename or -/// drop any table on that list, so their absence is evidence of the version gap, not of -/// damage — and it is not our call to make about a shape we cannot read. +/// The stamp alone decides. It commits with its tables, and a future schema may rename +/// any of [`SCHEMA_TABLES`], so their absence is not damage. fn refuse_if_newer(conn: &Connection) -> Result<()> { - // `meta` carries the stamp; on an index that has none there is no claim to compare, - // and `meta_value` would fault on the missing table. + // No `meta`, no stamp to compare. if !table_exists(conn, "meta")? { return Ok(()); } @@ -318,8 +240,7 @@ fn refuse_if_newer(conn: &Connection) -> Result<()> { } } -/// Whether `name` is a table in this database — the one-table form of the presence check -/// [`schema_is_current`] makes over the whole list. +/// Whether `name` is a table in this database. fn table_exists(conn: &Connection, name: &str) -> Result { let n: i64 = conn.query_row( "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", @@ -329,11 +250,8 @@ fn table_exists(conn: &Connection, name: &str) -> Result { Ok(n > 0) } -/// Whether this connection sees a **complete** schema at the current -/// [`SCHEMA_VERSION`]: every table in [`SCHEMA_TABLES`] present *and* the stamp -/// current. Both halves are load-bearing — a current stamp over an incomplete schema -/// is precisely what #114 could leave behind, so an index damaged by an older `b2` is -/// detected here rather than far from its cause on the next `search`. +/// Whether every table in [`SCHEMA_TABLES`] is present and the stamp is current. Both +/// halves matter: #114 could leave a current stamp over an incomplete schema. fn schema_is_current(conn: &Connection) -> Result { let mut stmt = conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")?; let present: HashSet = stmt @@ -345,47 +263,33 @@ fn schema_is_current(conn: &Connection) -> Result { Ok(stamped_version(conn)? == Some(SCHEMA_VERSION)) } -/// The value stored in `meta` under `key`, or `None` when unset. Callers must -/// know `meta` exists — every caller reads it past a check that implies it -/// (a table-presence check, or the embed pass having ensured the space). +/// The value stored in `meta` under `key`, or `None` when unset. Callers must know +/// `meta` exists. fn meta_value(conn: &Connection, key: &str) -> Result> { Ok(conn .query_row("SELECT value FROM meta WHERE key = ?1", [key], |r| r.get(0)) .optional()?) } -/// The `schema_version` recorded in `meta`, or `None` on an index that has never been -/// stamped — or whose stamp was lost, which [`apply_schema`] treats the same way. +/// The `schema_version` recorded in `meta`, or `None` when unstamped. fn stamped_version(conn: &Connection) -> Result> { Ok(meta_value(conn, "schema_version")?.and_then(|s| s.parse().ok())) } -/// Create the schema and stamp `schema_version`, dropping whatever was there first. -/// The DDL mirrors `index-engine.md`; the vector tables are created at embed time -/// instead (ADR-0006, [`ensure_embedding_space`]). +/// Drop whatever is there, create the schema and stamp `schema_version`. The DDL mirrors +/// `index-engine.md`; vector tables are created at embed time ([`ensure_embedding_space`]). /// -/// **One outcome, whatever it finds:** an empty schema at the current version. -/// Reaching here means [`schema_is_current`] said no, and every way it can say no — -/// wrong shape, structurally incomplete (#114), or present but unstamped — is an index -/// of no known version. So the drop is unconditional: guarding it on "was there a prior -/// stamp?" reads like an optimization, but `DROP TABLE IF EXISTS` over an empty catalog -/// is already a no-op, and all the guard would do is wave through the -/// unstamped-but-populated index. Dropping is safe because the index is disposable -/// (ADR-0002), and rows surviving in tables of unknown shape are worse than none: an -/// incremental reindex skips notes whose `body_hash` still matches, so it would leave -/// the recreated tables empty forever. -/// -/// **Called only from inside [`migrate`]'s transaction**, which is what makes -/// drop-then-create safe; it is not a standalone entry point. +/// The drop is unconditional: anything reaching here is of no known version, and +/// surviving rows would be skipped by an incremental reindex (matching `body_hash`), +/// leaving the tables empty forever. Call only inside [`migrate`]'s transaction. fn apply_schema(conn: &Connection) -> Result<()> { // `meta` must exist before the batch below can clear it. conn.execute_batch( "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);", )?; - // Children first (FKs); dropping `chunks` takes its FTS triggers with it. The - // legacy vec0 `chunks_vec` (schema <= 2) is deliberately absent: its module is no - // longer linked, so SQLite cannot DROP it. `DELETE FROM meta` clears the recorded - // embedder, so the next embed pass recreates the vector tables from nothing. + // Children first (FKs). Legacy vec0 `chunks_vec` is absent: its module is no longer + // linked, so SQLite cannot drop it. Clearing `meta` makes the next embed pass + // recreate the vector tables. conn.execute_batch( "DROP TABLE IF EXISTS edge_provenance; DROP TABLE IF EXISTS edges; @@ -499,12 +403,8 @@ pub struct NoteRow<'a> { pub mtime: Option, } -/// Upsert a note keyed by its vault-relative `path`. -/// `indexed_at` is set by SQLite so the projection needs no wall-clock from Rust. -/// -/// `ON CONFLICT(path)` is the *whole* of path reconciliation, which is the point of -/// keying on the path (ADR-0003): the filesystem already guarantees one file per path, -/// so a note deleted and recreated there is simply that path's note now. +/// Upsert a note keyed by its vault-relative `path` (ADR-0003). SQLite sets `indexed_at`, +/// so no wall clock is needed from Rust. pub fn upsert_note(conn: &Connection, row: &NoteRow) -> Result<()> { conn.execute( "INSERT INTO notes (path, title, created, body_hash, mtime, indexed_at) @@ -520,32 +420,24 @@ pub fn upsert_note(conn: &Connection, row: &NoteRow) -> Result<()> { Ok(()) } -/// Delete every `notes` row whose path is not in `seen` — the paths the whole-vault -/// walk actually met, including ones skipped as unreadable (the walk *saw* that file, -/// so evicting it would lie). Returns how many were pruned: the note half of #31, -/// without which a file deleted outside `b2` leaves a ghost row that listings, search, -/// `similar` and the graph keep serving, so an incremental reindex diverges from a -/// from-scratch rebuild (S3). +/// Delete every `notes` row whose path is not in `seen` (every path the walk met, +/// unreadable ones included) and return how many went, so a file deleted outside `b2` +/// leaves no ghost row (#31, S3). /// -/// Chunks (FTS in lockstep via the `chunks_ad` trigger), centroid and -/// **outgoing** edges cascade with the row. Vectors no longer do — they are -/// content-addressed and may be shared, so [`prune_orphan_vectors`] collects them. -/// **Inbound** edges are the caller's concern: `edges.dst_path` carries no FK (it must -/// be free to be NULL — the dangling case), so this must run *before* edge derivation, -/// which then re-dangles the links that pointed here. +/// Chunks, centroid and outgoing edges cascade; vectors are shared, so +/// [`prune_orphan_vectors`] collects them. `edges.dst_path` has no FK, so run this before +/// edge derivation, which re-dangles inbound links. pub fn prune_notes_except(conn: &Connection, seen: &HashSet<&str>) -> Result { prune_members_except(conn, Members::Notes, |path| seen.contains(path)) } -/// Drop the `notes` row at `path` — the index half of deleting a note — and return how -/// many rows went (0 or 1). Everything keyed to the note cascades with the row, as for +/// Drop the `notes` row at `path` and return how many rows went (0 or 1). Cascades as /// [`prune_notes_except`]; inbound edges are the caller's to re-project. pub fn delete_note_row(conn: &Connection, path: &str) -> Result { delete_member(conn, Members::Notes, path) } -/// The two path-keyed member tables: a vault path names a note row or a resource row -/// (L1, L3), and the row-level housekeeping below is the same for both. +/// The two path-keyed member tables: a vault path names a note or a resource (L1, L3). #[derive(Debug, Clone, Copy)] enum Members { Notes, @@ -584,9 +476,8 @@ fn prune_members_except( Ok(pruned) } -/// Every member path under the folder `dir` (vault-relative, no trailing slash), -/// path-ordered. Prefix-matched with `substr` (not `LIKE`) so a folder name containing -/// `%`/`_` never wildcards. +/// Every member path under the folder `dir` (no trailing slash), path-ordered. Uses +/// `substr`, not `LIKE`, so `%`/`_` in a folder name never wildcard. fn members_under_dir(conn: &Connection, members: Members, dir: &str) -> Result> { let prefix = format!("{dir}/"); let mut stmt = conn.prepare(&format!( @@ -603,8 +494,7 @@ fn members_under_dir(conn: &Connection, members: Members, dir: &str) -> Result { pub path: &'a str, @@ -614,8 +504,7 @@ pub struct ResourceRow<'a> { pub content_hash: &'a str, } -/// Upsert a resource keyed by its vault-relative path. `indexed_at` is set by -/// SQLite, like [`upsert_note`]'s — the projection needs no wall-clock from Rust. +/// Upsert a resource keyed by its vault-relative path; SQLite sets `indexed_at`. pub fn upsert_resource(conn: &Connection, row: &ResourceRow) -> Result<()> { conn.execute( "INSERT INTO resources (path, class, size, mtime, content_hash, indexed_at) @@ -631,9 +520,7 @@ pub fn upsert_resource(conn: &Connection, row: &ResourceRow) -> Result<()> { Ok(()) } -/// The stored `(size, mtime)` for an inventoried resource — the change-detection -/// short-circuit: a matching stat means the bytes are not re-read or re-hashed -/// (hashing is the only byte-read the inventory pass performs). +/// The stored `(size, mtime)` for a resource. A matching stat skips re-hashing. pub fn resource_stat(conn: &Connection, path: &str) -> Result)>> { Ok(conn .query_row( @@ -644,7 +531,7 @@ pub fn resource_stat(conn: &Connection, path: &str) -> Result, } -/// One resource's full inventory row (`resource_detail`) — the fallback card's -/// metadata. +/// One resource's full inventory row: the fallback card's metadata. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResourceDetail { pub class: String, @@ -663,8 +549,7 @@ pub struct ResourceDetail { pub content_hash: String, } -/// Every inventoried resource — [`ResourceListing`] rows, path-ordered — the -/// file tree's resource half (`Vault::list_resources`). +/// Every inventoried resource, path-ordered. pub fn list_resources(conn: &Connection) -> Result> { let mut stmt = conn.prepare("SELECT path, class, size, mtime FROM resources ORDER BY path")?; let rows = stmt.query_map([], |r| { @@ -678,8 +563,7 @@ pub fn list_resources(conn: &Connection) -> Result> { Ok(rows.collect::>>()?) } -/// One resource's full inventory row — [`ResourceDetail`] — the fallback card's -/// metadata. `None` when the path is not inventoried. +/// One resource's [`ResourceDetail`], or `None` when not inventoried. pub fn resource_detail(conn: &Connection, path: &str) -> Result> { Ok(conn .query_row( @@ -697,8 +581,7 @@ pub fn resource_detail(conn: &Connection, path: &str) -> Result Result> { let mut stmt = conn.prepare( "SELECT n.path, n.title, e.type, e.caption, e.embed @@ -730,8 +611,7 @@ pub fn inbound_resource_edges(conn: &Connection, path: &str) -> Result>>()?) } -/// One edge a note points *at a resource*, joined with the inventory's `class` -/// (the display glyph) — a row of `explain`'s file-links panel. +/// One edge from a note to a resource, with the resource's `class` (the display glyph). #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResourceEdgeRow { pub path: String, @@ -743,10 +623,8 @@ pub struct ResourceEdgeRow { pub explanation: Option, } -/// Every active edge a note points *at a resource* — the outbound complement of -/// [`inbound_resource_edges`], so `explain` can present all three target kinds a -/// note authors (note / resource / dangling — GH #22) instead of silently hiding -/// its file links. Ordered for deterministic display. +/// Every edge from a note to a resource, so `explain` shows file links alongside note +/// and dangling targets (GH #22). Ordered. pub fn outbound_resource_edges(conn: &Connection, note_path: &str) -> Result> { let mut stmt = conn.prepare( "SELECT e.dst_resource_path, r.class, e.type, e.origin, e.caption, e.embed, e.explanation @@ -768,37 +646,29 @@ pub fn outbound_resource_edges(conn: &Connection, note_path: &str) -> Result>>()?) } -/// The bounded inbound set a **resource move** must rewrite — [`InboundEdge`] -/// rows. The resource sibling of [`inbound_edge_targets`]; ordered for -/// deterministic rewriting. +/// The inbound edges a resource move must rewrite; the resource sibling of +/// [`inbound_edge_targets`]. pub fn inbound_resource_edge_targets(conn: &Connection, path: &str) -> Result> { inbound_edges_on(conn, "dst_resource_path", path) } -/// Delete every `resources` row whose path is not in `seen` (the walk's survivors) -/// and return how many were pruned. Inbound edges **re-dangle** automatically — -/// `edges.dst_resource_path` is `ON DELETE SET NULL`, `dst_path_raw` retained — -/// so a stale inventory row never outlives its file (the resource half of #31). +/// Delete every `resources` row whose path is not in `seen` and return how many went +/// (#31). Inbound edges re-dangle via `ON DELETE SET NULL`. pub fn prune_resources_except(conn: &Connection, seen: &HashSet) -> Result { prune_members_except(conn, Members::Resources, |path| seen.contains(path)) } -/// Drop the inventory row at `path` — the index half of deleting a resource — and -/// return how many rows went (0 or 1). Inbound edges re-dangle as for -/// [`prune_resources_except`]. +/// Drop the inventory row at `path` and return how many rows went (0 or 1). Inbound +/// edges re-dangle. pub fn delete_resource_row(conn: &Connection, path: &str) -> Result { delete_member(conn, Members::Resources, path) } -/// Re-key the inventory row at `old_path` to `new_path` — the resource sibling of -/// [`repoint_note_path`], and the index half of moving one. The bytes are the same, so -/// `size` and `content_hash` carry over; `class` (from the new extension) and `mtime` -/// (the moved file's) are the caller's. Returns whether `old_path` was inventoried — -/// `false` changes nothing. +/// Re-key the inventory row at `old_path` to `new_path`, keeping `size` and +/// `content_hash`. Returns whether `old_path` was inventoried. /// -/// An upsert of the new row then a delete of the old, not an `UPDATE`: -/// `edges.dst_resource_path` has no `ON UPDATE CASCADE`, so the old row's inbound edges -/// re-dangle (`ON DELETE SET NULL`) until the caller re-projects their sources. +/// Upsert then delete, not `UPDATE`: `edges.dst_resource_path` has no `ON UPDATE +/// CASCADE`, so inbound edges re-dangle until the caller re-projects their sources. pub fn repoint_resource( conn: &Connection, old_path: &str, @@ -827,31 +697,19 @@ pub fn repoint_resource( // chunks (FTS kept in lockstep by the triggers in apply_schema()) // --------------------------------------------------------------------------- -/// The content address of one chunk's embed input: blake3 of the chunk text. The text -/// stored on the row *is* what the embedder is handed (`chunk.rs` folds the heading -/// breadcrumb in before it lands here), so this hash keys the vector store exactly -/// (ADR-0006) — two chunks with byte-identical text must have the same vector, which is -/// a correctness statement before it is a saving. -/// -/// The model identity is deliberately *not* mixed in: a model swap drops the whole -/// `embeddings` table (ADR-0007), so the space is per-model by construction and a wider -/// key would only make that drop look optional. +/// The content address of a chunk's embed input: blake3 of the stored text, which is +/// exactly what the embedder sees (ADR-0006). No model id: a model swap drops the whole +/// table (ADR-0007). pub fn text_hash(text: &str) -> String { blake3::hash(text.as_bytes()).to_hex().to_string() } -/// Replace a note's chunks (delete + reinsert) and return the new chunk ids in `seq` -/// order; the FTS triggers emit the `'delete'` sentinel for the removed rows. +/// Replace a note's chunks and return the new chunk ids in `seq` order. /// -/// Stored vectors do **not** cascade — they are content-addressed and may be shared, -/// so they outlive any one chunk row and the whole-vault pass collects what nothing -/// references (ADR-0006, [`prune_orphan_vectors`]). That is exactly what makes a -/// re-chunk (or a move) re-embed nothing when the text is unchanged. The note's -/// centroid summarizes the *old* chunk set, so it is dropped here and the next embed -/// pass recomputes it. The caller embeds the returned ids. +/// Vectors are shared, so they survive (ADR-0006, [`prune_orphan_vectors`]) and a re-chunk +/// of unchanged text re-embeds nothing. The centroid is stale, so it is dropped. pub fn replace_chunks(conn: &Connection, note_path: &str, chunks: &[Chunk]) -> Result> { - // Guarded on existence so the model-free projection pass still never *creates* - // the embedding space (index-engine.md). + // The model-free projection must never create the embedding space (index-engine.md). if embedding_space_exists(conn)? { conn.execute( "DELETE FROM note_centroids WHERE note_path = ?1", @@ -882,10 +740,9 @@ pub fn replace_chunks(conn: &Connection, note_path: &str, chunks: &[Chunk]) -> R Ok(new_ids) } -/// The closed set of tokenizers `chunks_fts` can be rebuilt with — an enum, so the -/// string spliced into [`rebuild_fts`]'s DDL is never caller-supplied text. -/// `PorterUnicode61` is the shipped default (the GH #157 verdict); `Unicode61` is the -/// unstemmed ablation arm the eval harness keeps measurable. +/// The tokenizers `chunks_fts` can be rebuilt with; an enum so [`rebuild_fts`]'s DDL never +/// splices caller text. `PorterUnicode61` ships (GH #157); `Unicode61` is the eval's +/// unstemmed arm. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FtsTokenizer { Unicode61, @@ -893,8 +750,7 @@ pub enum FtsTokenizer { } impl FtsTokenizer { - /// The FTS5 `tokenize =` value this variant names — public so the eval's - /// recorded rows spell the tokenizer exactly as the schema does. + /// The FTS5 `tokenize =` value, spelled as the schema does. pub fn sql(self) -> &'static str { match self { FtsTokenizer::Unicode61 => "unicode61", @@ -903,14 +759,9 @@ impl FtsTokenizer { } } -/// The tokenizer `chunks_fts` was actually created with, read back from the schema -/// rather than assumed — because it *moves*: [`rebuild_fts`] swaps it under the GH #157 -/// ablation, so a caller that must tokenize a query the way the index does (see -/// [`search::lexical_evidence`](crate::search::lexical_evidence)) has to ask. -/// -/// The recorded value is matched against the closed [`FtsTokenizer`] set, never spliced -/// onward as text. An unreadable or unrecognised schema degrades to the shipped -/// default, which is the one `migrate` creates. +/// The tokenizer `chunks_fts` was created with, read back because [`rebuild_fts`] can +/// swap it (see [`search::lexical_evidence`](crate::search::lexical_evidence)). An +/// unrecognised schema degrades to the shipped default. pub fn index_tokenizer(conn: &Connection) -> Result { const DEFAULT: FtsTokenizer = FtsTokenizer::PorterUnicode61; let sql: Option = conn @@ -933,20 +784,12 @@ pub fn index_tokenizer(conn: &Connection) -> Result { .unwrap_or(DEFAULT)) } -/// Drop and recreate `chunks_fts` with `tokenizer`, repopulated from the untouched -/// `chunks` content table (FTS5's external-content `'rebuild'`). Chunk rows, vectors -/// and centroids are untouched — the tokenizer only changes how the lexical half -/// indexes the same text, which is what makes the GH #157 stemmer A/B runnable without -/// re-chunking or re-embedding. -/// -/// A drop-and-rebuild like the migration, so it takes the same write-lock discipline; -/// the op is idempotent, so an attempt after a lost race lands on the same end state. -/// The `chunks_*` triggers live on `chunks` and reference this table by name, so they -/// survive the swap. +/// Drop and recreate `chunks_fts` with `tokenizer`, repopulated from `chunks`, without +/// re-chunking or re-embedding (GH #157). Same write-lock discipline as the migration; +/// the `chunks_*` triggers reference this table by name, so they survive. pub fn rebuild_fts(conn: &Connection, tokenizer: FtsTokenizer) -> Result<()> { retry_while_locked("chunks_fts rebuild", REBUILD_ATTEMPTS, || { - // `new_unchecked` for the same reason as `ensure_embedding_space`: this takes - // `&Connection`, and nothing in this crate can already be inside a transaction. + // `new_unchecked`: see `ensure_embedding_space`. let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; tx.execute_batch(&format!( "DROP TABLE IF EXISTS chunks_fts; @@ -965,9 +808,8 @@ pub fn rebuild_fts(conn: &Connection, tokenizer: FtsTokenizer) -> Result<()> { } // --------------------------------------------------------------------------- -// embeddings — the vector tables are created at embed time, not in apply_schema(): -// their *existence* is the "this vault has an embedding space" signal the -// projected-but-unembedded fallbacks key on (ADR-0006). +// embeddings: created at embed time, not in apply_schema(); their existence is the +// "this vault has an embedding space" signal (ADR-0006). // --------------------------------------------------------------------------- /// Whether the embedding space (the `embeddings` table) currently exists. @@ -975,24 +817,18 @@ pub fn embedding_space_exists(conn: &Connection) -> Result { table_exists(conn, "embeddings") } -/// Ensure the vector tables exist, recording `(embed_model_id, embed_dim)` in `meta`. If -/// either differs from what is recorded — a model swap — the tables are dropped and -/// recreated empty, so a full re-embed follows: `meta` is the only place a swap is -/// detectable, so vectors never go silently stale (ADR-0007). (`dim` is bookkeeping only now -/// that vectors are plain BLOBs, but it still gates the swap and the read-time fail-fast.) +/// Ensure the vector tables exist for `(model_id, dim)`, recorded in `meta`. A model swap +/// drops and recreates them empty, so vectors never go silently stale (ADR-0007). /// -/// **Serialized and atomic on the same terms as [`migrate`]** (ADR-0021): two embed passes -/// genuinely overlap — the desktop's reindex task and a `b2 reindex`, which the CLI's own -/// advisory lock does not cover — and B's `DROP` after A's `CREATE` leaves A inserting into -/// a table that no longer exists. +/// Serialized and atomic like [`migrate`] (ADR-0021): the desktop's reindex and a +/// `b2 reindex` can overlap, and one's `DROP` after the other's `CREATE` breaks it. pub fn ensure_embedding_space(conn: &Connection, model_id: &str, dim: usize) -> Result<()> { if embedding_space_matches(conn, model_id, dim)? { return Ok(()); } retry_while_locked("embedding-space rebuild", REBUILD_ATTEMPTS, || { - // `new_unchecked` because this takes `&Connection`, not `&mut` — the embed - // pass threads a shared connection through. Sound because `b2-core` opens no - // other transaction on it. + // `new_unchecked` because the embed pass shares a `&Connection`. Sound because + // `b2-core` opens no other transaction on it. let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; if !embedding_space_matches(&tx, model_id, dim)? { tx.execute_batch( @@ -1016,10 +852,7 @@ pub fn ensure_embedding_space(conn: &Connection, model_id: &str, dim: usize) -> }) } -/// Whether the vector tables exist *and* were built by this exact embedder identity — -/// the "nothing to do" test [`ensure_embedding_space`] runs twice, once cheaply and -/// once under the write lock. Identity first: a differing model settles it without the -/// `sqlite_master` lookup. +/// Whether the vector tables exist and were built by this exact embedder. fn embedding_space_matches(conn: &Connection, model_id: &str, dim: usize) -> Result { let unchanged = matches!( recorded_embedder(conn)?, @@ -1028,9 +861,8 @@ fn embedding_space_matches(conn: &Connection, model_id: &str, dim: usize) -> Res Ok(unchanged && embedding_space_exists(conn)?) } -/// The `(embed_model_id, embed_dim)` a prior ingest recorded, if any; `None` means the -/// vault has never been embedded. The only place a model swap is detectable, so a read -/// compares it to the active embedder and fails fast on a mismatch (ADR-0007). +/// The `(embed_model_id, embed_dim)` a prior ingest recorded; `None` if never embedded. +/// Reads compare it to the active embedder and fail fast on a mismatch (ADR-0007). pub fn recorded_embedder(conn: &Connection) -> Result> { let model = meta_value(conn, "embed_model_id")?; let dim = meta_value(conn, "embed_dim")?; @@ -1049,10 +881,8 @@ fn upsert_meta(conn: &Connection, key: &str, value: &str) -> Result<()> { Ok(()) } -/// Store one embedding under the content address of the text it came from. `OR IGNORE` -/// because the store is shared: two notes with byte-identical chunk text address the -/// same row, and the second writer is agreeing with the first. That also makes a -/// resumed or overlapping embed pass idempotent. +/// Store one embedding under its text's content address. `OR IGNORE`: identical text +/// addresses the same row, which also makes overlapping embed passes idempotent. pub fn set_vector(conn: &Connection, text_hash: &str, embedding: &[f32]) -> Result<()> { conn.execute( "INSERT OR IGNORE INTO embeddings(text_hash, vector) VALUES (?1, ?2)", @@ -1061,15 +891,9 @@ pub fn set_vector(conn: &Connection, text_hash: &str, embedding: &[f32]) -> Resu Ok(()) } -/// Delete every stored vector no chunk references — the collection pass -/// content-addressing costs us, run by the whole-vault projection once this run's chunk -/// set is final. -/// -/// A vector cannot be dropped with its chunk precisely because it may be shared, and -/// that sharing is the point: a moved note's chunks are deleted and re-inserted under a -/// new path, and their vectors have to survive the gap (ADR-0006). So the lifecycle is -/// the centroids' — derived data reconciled by the pass that knows the whole picture, -/// never by a per-row rule that cannot. Requires the embedding space to exist. +/// Delete every stored vector no chunk references, once the whole-vault pass's chunk set +/// is final. Not per-row: a moved note's vectors must survive re-insertion (ADR-0006). +/// Requires the embedding space to exist. pub fn prune_orphan_vectors(conn: &Connection) -> Result { Ok(conn.execute( "DELETE FROM embeddings @@ -1089,9 +913,8 @@ pub fn note_for_chunk(conn: &Connection, chunk_id: i64) -> Result .optional()?) } -/// A ranked chunk resolved for display in one read: its note, that note's title, and the -/// chunk's heading breadcrumb and text. One statement, so the note and the chunk come -/// from the same snapshot: a hit is either whole or `None` (GH #137). +/// A ranked chunk resolved for display. One statement, so the note and chunk come from +/// one snapshot: a hit is whole or `None` (GH #137). #[derive(Debug, Clone, PartialEq, Eq)] pub struct ChunkHit { pub note_path: String, @@ -1129,8 +952,7 @@ pub fn chunk_text(conn: &Connection, chunk_id: i64) -> Result> { .optional()?) } -/// A chunk's heading breadcrumb + text in one read (None if the chunk id is -/// unknown) — the chunk-level hit resolution (`Vault::search_chunks`). +/// A chunk's heading breadcrumb and text (None if the chunk id is unknown). pub fn chunk_detail(conn: &Connection, chunk_id: i64) -> Result, String)>> { Ok(conn .query_row( @@ -1141,9 +963,8 @@ pub fn chunk_detail(conn: &Connection, chunk_id: i64) -> Result Result> { Ok(conn .query_row( @@ -1154,14 +975,8 @@ pub fn note_body_hash(conn: &Connection, note_path: &str) -> Result Result { let n_missing: i64 = conn.query_row( "SELECT COUNT(*) @@ -1173,26 +988,14 @@ pub fn note_fully_embedded(conn: &Connection, note_path: &str) -> Result { Ok(n_missing == 0) } -/// The vault's embedding coverage as `(notes_embedded, notes_total)` — the honest -/// "N/M embedded" signal (#26). `notes_embedded` counts notes with **no chunk awaiting a -/// vector**; it is `0` before any embed on a vault of ordinary notes, so this reads -/// cleanly on a projected-but-unembedded one. **Model-free:** a pure count over the -/// projection. +/// Embedding coverage as `(notes_embedded, notes_total)`: notes with no chunk awaiting a +/// vector (#26). Model-free. /// -/// **A note with no chunks counts as embedded** — an empty body: a frontmatter-only stub, -/// a `Untitled.md` the tree just created, a file with nothing in it. It has no chunk, so -/// there is no vector it is waiting for, and the older `>= 1 chunk` requirement left it -/// permanently outside the numerator. That is not a rounding error: this fraction is the -/// vault's "am I done?" and every consumer treats `embedded < total` as outstanding work -/// — the chat pane's grounding note, the search caveat, the graph pane's ghost hint, and -/// the desktop's auto-index-on-open and fs-watch heal, which each scheduled a no-op embed -/// pass forever. A vault holding one empty note could never read as finished, however -/// often it was reindexed. Vacuous truth is the honest answer, not a lenient one. +/// A chunkless note counts as embedded. Consumers treat `embedded < total` as pending +/// work, so otherwise one empty note would schedule no-op embed passes forever. pub fn embed_progress(conn: &Connection) -> Result<(usize, usize)> { let total: i64 = conn.query_row("SELECT COUNT(*) FROM notes", [], |r| r.get(0))?; - // No embeddings table yet: the join below would reference a missing table, so the - // same predicate is asked of the projection alone. Only the chunkless notes can - // satisfy it here — every chunk in the vault is awaiting a vector by definition. + // No embeddings table yet: only chunkless notes are embedded. if !embedding_space_exists(conn)? { let chunkless: i64 = conn.query_row( "SELECT COUNT(*) FROM notes n @@ -1202,8 +1005,7 @@ pub fn embed_progress(conn: &Connection) -> Result<(usize, usize)> { )?; return Ok((chunkless as usize, total as usize)); } - // A note counts as embedded iff no chunk of it lacks a vector — `note_fully_embedded`, - // aggregated. A chunkless note satisfies the NOT EXISTS vacuously, as it should. + // `note_fully_embedded`, aggregated. let embedded: i64 = conn.query_row( "SELECT COUNT(*) FROM notes n WHERE NOT EXISTS ( @@ -1217,14 +1019,12 @@ pub fn embed_progress(conn: &Connection) -> Result<(usize, usize)> { Ok((embedded as usize, total as usize)) } -/// One chunk still lacking a stored vector — a row of the DB-derived pending set -/// ([`chunks_missing_vectors`]) the embed pass fills. +/// One chunk still lacking a stored vector. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PendingChunk { pub note_path: String, pub text: String, - /// The content address the embed pass stores this vector under — carried on the - /// row so the pass never re-hashes text SQLite already hashed at chunk time. + /// The content address to store the vector under, so the pass never re-hashes. pub text_hash: String, } @@ -1237,17 +1037,11 @@ fn pending_chunk(r: &rusqlite::Row) -> rusqlite::Result { }) } -/// Every chunk still lacking a stored vector, in `(path, seq)` order — the -/// **DB-derived pending set** the embed pass fills. Deriving it here is what decouples -/// projection from embedding: nothing is handed between the passes in memory, so any -/// stop point (a cancelled embed, a crash between passes) heals on the next embed. The -/// ordering reproduces the fused reindex's per-note batching and progress. Requires the -/// embedding space to exist. +/// Every chunk still lacking a stored vector, in `(path, seq)` order: the pending set the +/// embed pass fills. Derived from the DB, so any interrupted pass heals on the next. +/// Requires the embedding space. /// -/// A row is a *chunk*, not a distinct vector: two chunks sharing text appear twice, -/// because the caller needs to know which notes are waiting on work. Deduplicating the -/// embedder calls is [`crate::ingest::embed_vault`]'s job — it is the only layer that -/// knows the order the notes will be worked in. +/// Chunks sharing text appear twice; [`crate::ingest::embed_vault`] dedups embedder calls. pub fn chunks_missing_vectors(conn: &Connection) -> Result> { let mut stmt = conn.prepare( "SELECT c.note_path, c.text, c.text_hash @@ -1260,10 +1054,8 @@ pub fn chunks_missing_vectors(conn: &Connection) -> Result> { Ok(rows.collect::>>()?) } -/// [`chunks_missing_vectors`] for **one** note, in `seq` order — what the inline -/// ingest path (`add`/`link`/`mv`) embeds after re-projecting a single file. Its own -/// indexed query rather than a filter over the whole-vault set: a directory move -/// re-projects every note it moved, and filtering there would be O(vault × moved). +/// [`chunks_missing_vectors`] for one note, for the inline ingest path. Its own indexed +/// query, because a directory move would make filtering O(vault × moved). pub fn note_chunks_missing_vectors( conn: &Connection, note_path: &str, @@ -1279,8 +1071,7 @@ pub fn note_chunks_missing_vectors( Ok(rows.collect::>>()?) } -/// A note's chunk ids in reading order (`seq`), embedded or not — the unit a reader -/// walks a note by. Empty for an unknown or empty note. +/// A note's chunk ids in `seq` order, embedded or not. Empty for an unknown or empty note. pub fn note_chunk_ids(conn: &Connection, note_path: &str) -> Result> { let mut stmt = conn.prepare_cached("SELECT id FROM chunks WHERE note_path = ?1 ORDER BY seq")?; @@ -1300,9 +1091,8 @@ pub fn note_title(conn: &Connection, note_path: &str) -> Result> .flatten()) } -/// A note's `(title, created)` in one read (both `None` if the note is absent), resolved -/// from the projection (GH #22): a neighbor is titled and dated for display without an -/// adapter ever re-reading the file. +/// A note's `(title, created)` from the projection (both `None` if absent), so adapters +/// never re-read the file (GH #22). pub fn note_header(conn: &Connection, note_path: &str) -> Result<(Option, Option)> { Ok(conn .query_row( @@ -1314,9 +1104,7 @@ pub fn note_header(conn: &Connection, note_path: &str) -> Result<(Option .unwrap_or_default()) } -/// Every indexed note's `(path, title)`, ordered by `path` — the flat listing -/// the desktop UI's file tree is built from (`Vault::list_notes`). Path order means -/// the adapter can assemble the folder tree in one pass without re-sorting. +/// Every indexed note's `(path, title)`, path-ordered so the file tree builds in one pass. pub fn all_notes(conn: &Connection) -> Result)>> { let mut stmt = conn.prepare("SELECT path, title FROM notes ORDER BY path")?; let rows = stmt.query_map([], |r| { @@ -1325,12 +1113,9 @@ pub fn all_notes(conn: &Connection) -> Result)>> { Ok(rows.collect::, _>>()?) } -/// A note's stored chunk vectors as `(chunk_id, vector)` in `seq` order — one indexed -/// join, not a per-chunk round-trip. Reading a note's own vectors back is what lets -/// discovery search from them without re-embedding (passage-to-passage, no -/// `embed_query`); it is also discovery's rescore unit and the input to a centroid -/// refresh. Call only when the embedding space exists. `prepare_cached` because -/// discovery calls this once per shortlisted note. +/// A note's stored chunk vectors as `(chunk_id, vector)` in `seq` order, so discovery +/// searches from them without re-embedding. Requires the embedding space. Cached: +/// discovery calls it once per shortlisted note. pub fn note_chunk_vectors(conn: &Connection, note_path: &str) -> Result)>> { let mut stmt = conn.prepare_cached( "SELECT c.id, e.vector FROM chunks c @@ -1346,10 +1131,8 @@ pub fn note_chunk_vectors(conn: &Connection, note_path: &str) -> Result>>()?) } -/// Recompute and store `note_path`'s centroid from its currently stored chunk vectors -/// (the row is deleted when it has none). The embed pass calls this after finishing a -/// note, so a centroid row exists exactly for embedded notes and always summarizes -/// their *current* vectors — derived data with no separate invalidation (ADR-0006). +/// Recompute `note_path`'s centroid from its stored chunk vectors, deleting it when there +/// are none (ADR-0006). pub fn refresh_note_centroid(conn: &Connection, note_path: &str) -> Result<()> { let vectors: Vec> = note_chunk_vectors(conn, note_path)? .into_iter() @@ -1373,20 +1156,10 @@ pub fn refresh_note_centroid(conn: &Connection, note_path: &str) -> Result<()> { Ok(()) } -/// Every fully-embedded note with **no** centroid row, path-ordered — the second half -/// of the embed pass's work list (GH #170). -/// -/// It exists because content-addressing broke a coupling the old design leaned on. -/// `replace_chunks` drops a note's centroid and used to drop its vectors too, so a -/// re-chunked note always had pending chunks and refreshing the centroid inside the -/// embed loop sufficed. Vectors now survive a re-chunk — that is the whole point — so a -/// note can reach the embed pass with a complete vector set, no centroid, and nothing -/// pending to bring it back. Silently: discovery's coarse stage scans centroids only, -/// so such a note would stop being discoverable while looking perfectly indexed -/// (S3, which is how the property suite caught it). -/// -/// "Fully embedded" is the gate, not "has any vector": a centroid over a partial set -/// would be wrong rather than merely stale. +/// Every fully-embedded note with no centroid row, path-ordered (GH #170). A re-chunk +/// keeps vectors but drops the centroid, leaving nothing pending; without this the note +/// silently leaves discovery's coarse stage (S3). Fully embedded only: a partial +/// centroid would be wrong. pub fn notes_missing_centroids(conn: &Connection) -> Result> { let mut stmt = conn.prepare( "SELECT DISTINCT c.note_path FROM chunks c @@ -1402,17 +1175,14 @@ pub fn notes_missing_centroids(conn: &Connection) -> Result> { Ok(rows.collect::>>()?) } -/// Stream every stored `(note_path, centroid_blob)` through `f`, one row at a time — -/// discovery's first-stage coarse scan. O(notes), the whole point of the two-stage -/// shape (#38): the O(chunks) work happens only for the shortlisted notes. The blob -/// is *borrowed* for the callback (`get_ref`), so scoring adds no per-row allocation. +/// Stream every `(note_path, centroid_blob)` through `f`: discovery's O(notes) coarse +/// scan (#38). The blob is borrowed, so scoring allocates nothing per row. pub fn for_each_note_centroid(conn: &Connection, mut f: impl FnMut(&str, &[u8])) -> Result<()> { let mut stmt = conn.prepare("SELECT note_path, centroid FROM note_centroids")?; let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { - // Match the ValueRefs rather than `.as_str()?` — their `FromSqlError` isn't in - // our error enum, and the column types are fixed by our own DDL, so a - // mismatched row is skipped rather than an error. + // Match ValueRefs: `FromSqlError` isn't in our error enum, and our DDL fixes the + // types, so a mismatched row is skipped. let rusqlite::types::ValueRef::Text(text) = row.get_ref(0)? else { continue; }; @@ -1426,15 +1196,9 @@ pub fn for_each_note_centroid(conn: &Connection, mut f: impl FnMut(&str, &[u8])) Ok(()) } -/// Stream every embedded chunk's `(chunk_id, vector_blob)` through `f`, one row at a -/// time — the scan behind every vector read, which never materializes the whole space -/// at once (ADR-0006). The blob is *borrowed* for the callback, so scoring adds no -/// per-row allocation. -/// -/// Ranking is per **chunk**, so content-addressing has to be undone here: the join -/// hands each vector to every chunk that addresses it, which is why two notes with -/// identical text still get one rank each. A shared vector is read once and scored -/// twice rather than stored twice. +/// Stream every embedded chunk's `(chunk_id, vector_blob)` through `f` without +/// materializing the space (ADR-0006). The join hands a shared vector to each chunk that +/// addresses it, since ranking is per chunk. pub fn for_each_stored_vector(conn: &Connection, mut f: impl FnMut(i64, &[u8])) -> Result<()> { let mut stmt = conn.prepare( "SELECT c.id, e.vector FROM embeddings e JOIN chunks c ON c.text_hash = e.text_hash", @@ -1442,8 +1206,7 @@ pub fn for_each_stored_vector(conn: &Connection, mut f: impl FnMut(i64, &[u8])) let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { let chunk_id: i64 = row.get(0)?; - // Match the ValueRef rather than `as_blob()?` — its `FromSqlError` isn't in our - // error enum, and a stored vector is always a Blob. + // Match the ValueRef: `FromSqlError` isn't in our error enum. if let rusqlite::types::ValueRef::Blob(blob) = row.get_ref(1)? { f(chunk_id, blob); } @@ -1451,9 +1214,8 @@ pub fn for_each_stored_vector(conn: &Connection, mut f: impl FnMut(i64, &[u8])) Ok(()) } -/// Every chunk's squared-L2 distance to `query`, sorted nearest first (ties broken by -/// `chunk_id` for determinism) — the scan behind [`vector_search`], computed in-process over the [`for_each_stored_vector`] -/// stream: one sequential statement, one reused decode buffer (ADR-0006). +/// Every chunk's squared-L2 distance to `query`, nearest first, ties by `chunk_id` +/// (ADR-0006). fn scan_vector_distances(conn: &Connection, query: &[f32]) -> Result> { let mut out: Vec<(i64, f32)> = Vec::new(); let mut scratch: Vec = Vec::new(); @@ -1465,11 +1227,8 @@ fn scan_vector_distances(conn: &Connection, query: &[f32]) -> Result Result> { let mut hits = scan_vector_distances(conn, query)?; hits.truncate(k); @@ -1480,34 +1239,28 @@ pub fn vector_search(conn: &Connection, query: &[f32], k: usize) -> Result, - /// The resolved **resource** target (vault-relative path into `resources`), - /// when the link names a non-`.md` file — mutually exclusive with `dst_path` - /// in practice (a target resolves as a note or a resource, never both). + /// The resolved resource target (a non-`.md` file); exclusive with `dst_path`. pub dst_resource_path: Option, pub dst_path_raw: String, pub r#type: String, pub origin: String, pub explanation: Option, - /// An embed form (`![alt](…)` / `![[…]]`) — display nicety, not a verb. + /// An embed form (`![alt](…)` / `![[…]]`): display only, not a verb. pub embed: bool, /// The authored alt/link/alias text. pub caption: Option, pub occurrence_index: i64, } -/// Replace a note's edges. Every edge is authored (body links ∪ frontmatter -/// `b2_relations:`), so this deletes the note's edges and re-inserts them from the -/// current Markdown (Flow ①) — the whole graph is a projection of Markdown (G1). +/// Replace a note's edges from its current Markdown (Flow ①, G1). pub fn replace_authored_edges(conn: &Connection, src_path: &str, edges: &[EdgeRow]) -> Result<()> { conn.execute("DELETE FROM edges WHERE src_path = ?1", [src_path])?; for e in edges { @@ -1538,9 +1291,7 @@ pub fn replace_authored_edges(conn: &Connection, src_path: &str, edges: &[EdgeRo // resolver: the authored `[[path]]` → the note it names // --------------------------------------------------------------------------- -/// Whether `path` names an indexed note — the existence check that replaced the -/// two-way `b2id ⇄ path` resolver (GH #170): with the path *being* the identity, -/// resolution is one membership test rather than a translation. +/// Whether `path` names an indexed note; the path is the identity (GH #170). pub fn note_exists(conn: &Connection, path: &str) -> Result { let found: Option = conn .query_row("SELECT 1 FROM notes WHERE path = ?1", [path], |r| r.get(0)) @@ -1548,36 +1299,29 @@ pub fn note_exists(conn: &Connection, path: &str) -> Result { Ok(found.is_some()) } -/// One inbound edge a move/delete must act on: the source note's vault-relative -/// path and the exact authored link text (`dst_path_raw`) written there. +/// One inbound edge a move or delete must act on: the source note and the authored link +/// text. #[derive(Debug, Clone, PartialEq, Eq)] pub struct InboundEdge { pub src_path: String, pub dst_raw: String, } -/// Every active authored edge pointing *at* the note `dst_path`, as [`InboundEdge`] -/// rows. This is the bounded set a move must rewrite — the -/// materialized graph names the files to touch, so a move never scans the vault -/// (index-engine.md §8). Ordered for deterministic rewriting. +/// Every edge pointing at the note `dst_path`: the set a move rewrites without scanning +/// the vault (index-engine.md §8). Ordered. pub fn inbound_edge_targets(conn: &Connection, dst_path: &str) -> Result> { inbound_edges_on(conn, "dst_path", dst_path) } -/// Which member of a set an [`inbound_edges_of`] edge points at: an index into the -/// note paths or the resource paths the set was read from. +/// Which member an [`inbound_edges_of`] edge points at, as an index into its input. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum InboundTarget { Note(usize), Resource(usize), } -/// Every edge pointing at one of a set of members — the notes at `notes`, the -/// resources at `resources` — tagged with the member it targets: the notes' edges -/// first, then the resources', each member's in [`inbound_edge_targets`] order. The one -/// graph read a move (which rewrites these links) and a delete (which dangles them) -/// both start from, bounded by the inbound count rather than a vault scan -/// (index-engine.md §8). +/// Every edge pointing at one of `notes` or `resources`, tagged with its target; notes +/// first. The graph read a move or delete starts from (index-engine.md §8). pub fn inbound_edges_of( conn: &Connection, notes: &[&str], @@ -1595,8 +1339,7 @@ pub fn inbound_edges_of( Ok(out) } -/// The notes linking into a set of members ([`inbound_edges_of`]), sorted and deduped — -/// the files whose edges must re-project once the set moves or goes. +/// The notes linking into a set of members, sorted and deduped. pub fn inbound_sources( conn: &Connection, notes: &[&str], @@ -1630,31 +1373,22 @@ fn inbound_edges_on( Ok(rows.collect::>>()?) } -/// Every indexed note path under the directory `dir` (vault-relative, no trailing -/// slash), path-ordered — the moved (or deleted) set of a **directory** op. +/// Every indexed note path under `dir` (no trailing slash), path-ordered. pub fn notes_under_dir(conn: &Connection, dir: &str) -> Result> { members_under_dir(conn, Members::Notes, dir) } -/// Every inventoried resource path under the directory `dir` — the resource half -/// of [`notes_under_dir`], same prefix semantics, path-ordered. +/// Every inventoried resource path under `dir`, as [`notes_under_dir`]. pub fn resources_under_dir(conn: &Connection, dir: &str) -> Result> { members_under_dir(conn, Members::Resources, dir) } -/// Re-key a note from `old_path` to `new_path` — **the** index-side move (ADR-0003). -/// -/// One statement, and the FK graph does the rest: `chunks`, -/// `note_centroids` and `edges.src_path` all declare `ON UPDATE CASCADE`, so every -/// derived row travels with the note atomically and its chunk *vectors* are never -/// touched at all — they are content-addressed, so they belong to the text, not the -/// path (ADR-0006). What does **not** cascade is `edges.dst_path`: it carries no FK, -/// because it must be free to be NULL for a dangling link, so callers re-project every -/// inbound source afterwards. +/// Re-key a note from `old_path` to `new_path`: the index-side move (ADR-0003). Child rows +/// cascade; vectors belong to the text (ADR-0006). `edges.dst_path` has no FK, so callers +/// re-project inbound sources afterwards. /// -/// A **directory move** runs this for every moved note *before* re-projecting any file, -/// so link resolution stays independent of re-projection order. Requires -/// `PRAGMA foreign_keys = ON`; without it the cascades silently do not fire. +/// A directory move re-keys every note before re-projecting any file, so resolution is +/// order-independent. Requires `PRAGMA foreign_keys = ON`. pub fn repoint_note_path(conn: &Connection, old_path: &str, new_path: &str) -> Result<()> { conn.execute( "UPDATE notes SET path = ?1 WHERE path = ?2", @@ -1674,10 +1408,8 @@ pub fn resolve_link_target(conn: &Connection, link_path: &str) -> Result Result> { Ok(conn .query_row("SELECT path FROM resources WHERE path = ?1", [path], |r| { @@ -1687,12 +1419,11 @@ pub fn resolve_resource_target(conn: &Connection, path: &str) -> Result(0)) .unwrap() .map(|n| n.unwrap()) - // FTS5 keeps its own shadow tables (`chunks_fts_data`, `_idx`, …) alongside - // the virtual table; they are SQLite's bookkeeping, created and dropped with - // it, never ours to list. `sqlite_%` is reserved for the same reason. + // FTS5 shadow tables and `sqlite_%` are SQLite's own. .filter(|n| !n.starts_with("chunks_fts_") && !n.starts_with("sqlite_")) .collect(); @@ -1743,12 +1470,9 @@ mod tests { ); } - /// The full truth table for the profiler's emit guard (all 8 combinations). - /// Both listening paths have to survive independently: a slow statement logs at - /// WARN, so it must emit with DEBUG **off**, and DEBUG on emits regardless of speed. + /// All 8 combinations: slow emits with only WARN on; DEBUG emits regardless. #[test] fn emits_only_when_some_level_would_receive_the_event() { - // Something is listening at the level this event would use. assert!( should_emit(true, true, false), "slow + WARN on: the slow-query log, and the case a naive simplification drops" @@ -1764,7 +1488,6 @@ mod tests { "DEBUG on: every statement logs" ); - // Nothing would receive it — skip the string work. assert!( !should_emit(false, true, false), "fast statement, only WARN on: nothing to say" diff --git a/crates/b2-core/src/dirs.rs b/crates/b2-core/src/dirs.rs index 8268b3c..ea30f73 100644 --- a/crates/b2-core/src/dirs.rs +++ b/crates/b2-core/src/dirs.rs @@ -1,9 +1,5 @@ -//! Folder-structure ops — the structure half of "the vault directory is the source -//! of truth": Markdown files carry content, the directory tree carries **structure** -//! (data-model.md §1), and a folder — empty or not — is user-authored vault material -//! exactly like a note. Folders are never projected into the index (nothing to -//! chunk, embed, or link), so both ops here read/write the filesystem directly: -//! the walk *is* the projection, and a listing can never go stale against disk. +//! Folder ops. A folder, empty or not, is vault material like a note (data-model.md §1), +//! but is never indexed, so these read and write the filesystem directly. use std::fs; use std::path::Path; @@ -18,12 +14,8 @@ pub struct DirCreateReport { pub dir: String, } -/// Every folder under `vault_root` (empty ones included), vault-relative with `/` -/// separators and no trailing slash, sorted. Dot-prefixed directories are skipped — the -/// same hidden rule the ingest walk applies (GH #136), so the tree and the index agree on -/// what a vault member is. That mirroring includes symlink behaviour: `is_dir()` follows -/// directory symlinks exactly as the ingest walk does, so a vault-wide symlink policy, if -/// one ever lands, must change both walks together. +/// Every folder under `vault_root`, sorted, skipping dot-prefixed ones as the ingest walk +/// does (GH #136). `is_dir()` follows symlinks as that walk does; change both together. pub fn list_dirs(vault_root: &Path) -> Result> { let mut out = Vec::new(); collect_dirs(vault_root, vault_root, &mut out)?; @@ -41,8 +33,7 @@ fn collect_dirs(root: &Path, dir: &Path, out: &mut Vec) -> Result<()> { if crate::pathspec::is_hidden(&path) { continue; } - // `path` was produced by walking `root`, so `strip_prefix` cannot fail; - // handle it gracefully anyway rather than panic on the invariant. + // Cannot fail under `root`; skip rather than panic. let Ok(rel) = path.strip_prefix(root) else { continue; }; @@ -52,12 +43,8 @@ fn collect_dirs(root: &Path, dir: &Path, out: &mut Vec) -> Result<()> { Ok(()) } -/// Create the folder `dir_input` (vault-relative; a trailing `/` is tolerated), -/// missing parents included — matching `create_note`'s parent creation and the -/// UI's nested-name input — but, unlike `mkdir -p`, **refusing an occupied -/// target**: the user asked to *create*, and it's already there. Errors with -/// [`Error::DirDestination`] for an invalid path and [`Error::DirTargetExists`] -/// when anything (file or folder) already sits there. +/// Create the folder `dir_input` with missing parents. Unlike `mkdir -p`, an occupied +/// target is refused. pub fn create_dir(vault_root: &Path, dir_input: &str) -> Result { let dir = crate::pathspec::normalize_rel_dir(dir_input).map_err(Error::DirDestination)?; let abs = vault_root.join(&dir); diff --git a/crates/b2-core/src/discover.rs b/crates/b2-core/src/discover.rs index c0bb293..35ef4a4 100644 --- a/crates/b2-core/src/discover.rs +++ b/crates/b2-core/src/discover.rs @@ -1,32 +1,11 @@ -//! Connection-discovery candidate generation — the engine behind **`b2 similar`**. It -//! surfaces the notes to *consider* linking; the human is the precision gate, and -//! `b2 link` commits one (ADR-0009). It is the only discovery stage, and the only one -//! that reads the graph. +//! Connection discovery, the engine behind `b2 similar`: notes semantically near the anchor +//! but not already linked, for the human to judge (ADR-0009). Recall-oriented: the ranked +//! list is always served and no statistic gates membership (ADR-0014); z only grades. //! -//! A candidate is a note **semantically near the anchor but not already connected** — -//! the *complement* of the graph, not the intersection. Generation is deliberately **recall-oriented**, and the **ranked list is -//! always served**: no statistic gates membership, and `limit` under-fills only for want -//! of scorable notes. ADR-0014 carries the retired existence gate's whole evidence trail -//! — including why an anchor-local statistic cannot tell *nothing is related* from -//! *everything is related*, and the terms any replacement signal must win on. The z -//! survives ungated as the strength band's input. -//! -//! Mechanics are **two-stage**: -//! -//! 1. **Coarse, O(notes):** rank every note by its stored *centroid*'s distance to the -//! anchor's, minus the anchor and its 1-hop neighbours, and keep a shortlist many -//! times larger than `limit`. -//! 2. **Exact, O(shortlist):** for each shortlisted note, load its chunk vectors and -//! score the exact max-sim across the anchor's chunks, keeping the chunk that -//! achieved it as evidence. -//! -//! Stage 2's scoring is the same exact max-sim a whole-space scan computed, so a -//! shortlist that covers the vault reproduces it exactly; what the shape buys is reading -//! N_notes centroid rows instead of N_chunks vector rows (#38). Discovery is vector-only -//! and **re-embeds nothing** — the anchor is represented by the vectors already stored, -//! never by an `embed_query` of its text (bge's asymmetric query prefix is the wrong -//! side). Graph distance beyond the 1-hop exclusion is **not** a ranking signal; -//! weighting it is a deferred, eval-gated experiment. +//! Two stages (#38): a coarse O(notes) centroid scan, minus the anchor and its 1-hop +//! neighbours, keeps a wide shortlist; then exact chunk max-sim scores it, keeping the +//! winning chunk as evidence. Re-embeds nothing: the anchor is its stored vectors, never an +//! `embed_query` (bge's query prefix is the wrong side). use crate::db; use crate::embed::{centroid_of, l2_sq, unpack_f32_into}; @@ -34,57 +13,38 @@ use crate::error::Result; use crate::graph; use rusqlite::Connection; -/// The exclusion radius: a candidate must not already be *directly* linked to the -/// anchor. Fixed at 1 so triadic-closure candidates — two hops away, transitively -/// related but with no direct edge — stay in the pool. +/// The exclusion radius. 1, so two-hop (triadic-closure) candidates stay in the pool. const EXCLUDE_HOPS: usize = 1; -/// Floor on the stage-1 shortlist. Generous relative to any `limit` a human-facing -/// surface asks for: discovery is recall-oriented (the human is the precision gate), -/// so the coarse stage must never be the reason a nearby note goes missing. On any -/// vault at or below this many candidate notes the two-stage result is *exactly* -/// the old whole-space scan's. +/// Floor on the stage-1 shortlist, generous so the coarse stage never loses a nearby note. +/// At or below this many notes the result equals a whole-space scan. const SHORTLIST_MIN: usize = 200; -/// Stage-1 shortlist size per requested result: `limit × this`, floored at -/// [`SHORTLIST_MIN`]. A wide margin over `limit` because a note's centroid can rank -/// a few places below where its single best chunk deserves (the centroid smooths -/// over the note's chunks); the exact stage re-ranks whatever survives. +/// Stage-1 shortlist per requested result, floored at [`SHORTLIST_MIN`]. Wide, because a +/// centroid can rank below its note's best chunk. const SHORTLIST_PER_RESULT: usize = 20; -/// Scored pools smaller than this carry no statistic: a z over a handful of distances -/// is noise, so under it every candidate is served **ungraded** and an adapter says so -/// rather than banding. It moves *banding only*, never membership — serving is -/// continuous in population size (ADR-0014; the retired gate shared this guard, which -/// made it a serve-everything/serve-nothing cliff at n = 12). +/// Below this pool size candidates are served ungraded: a z over a handful is noise. +/// Affects banding only, never membership (ADR-0014). const STATS_MIN_POPULATION: usize = 12; -/// One discovery candidate: a note near the anchor and not already connected, ranked by -/// best-passage `score`. Owned, so the façade can resolve it to a -/// [`SimilarView`](crate::vault::SimilarView) without threading a lifetime through. +/// One discovery candidate, ranked by best-passage `score`. #[derive(Debug, Clone, PartialEq)] pub struct CandidateNote { - /// The candidate note's vault-relative path — its identity (L1). + /// The candidate's identity (L1). pub note_path: String, - /// Best chunk-pair similarity across the anchor's chunks × this note's chunks — - /// higher is nearer (negated L2 distance, matching [`Hit`](crate::search::Hit)). + /// Best chunk-pair similarity, as negated L2 distance (like [`Hit`](crate::search::Hit)). pub score: f64, - /// The candidate's chunk that achieved `score` — the passage that made this note - /// similar, surfaced by `b2 similar` as the evidence for *why* it appeared. + /// The candidate's chunk that achieved `score`, shown as evidence. pub evidence_chunk_id: i64, - /// The candidate's stage-2 best-passage z against the anchor's scored shortlist - /// population. It **gates nothing** (ADR-0014): it is the strength band's input and - /// nothing else. `None` when no statistic was computed (an ungraded space, a pool - /// under [`STATS_MIN_POPULATION`], or zero variance). It is strictly monotonic in - /// `score` within one query, so the band and the row order are one number by - /// construction. + /// Best-passage z against the scored shortlist: the strength band's input, gating + /// nothing (ADR-0014). Monotonic in `score`, so band and row order agree. `None` when + /// ungraded. pub z: Option, } -/// The statistic behind the strength band: the mean and spread of the scored -/// population's best-passage **squared** distances. It grades; it never gates -/// (ADR-0014). Any distance in the same space can be read against it, which is how a -/// passage pair in the explain view is graded on the same yardstick as the cards. +/// Mean and spread of the scored population's best-passage squared distances. Grades, +/// never gates (ADR-0014); explain grades passage pairs on the same yardstick. #[derive(Debug, Clone, Copy, PartialEq)] struct Population { mean: f64, @@ -98,53 +58,44 @@ impl Population { } } -/// One anchor's whole discovery field: the stage-1 order, the stage-2 scores, and the -/// statistic. [`candidates`] serves a prefix of it and [`explain`] reads one note's place -/// in it, so a card and its explanation are two reads of one computation and can never -/// disagree. +/// One anchor's whole discovery field. [`candidates`] and [`explain`] both read it, so a +/// card and its explanation can't disagree. struct Field { - /// Every unlinked note with a centroid, nearest centroid first (ties by path), - /// before the shortlist cut. + /// Every unlinked note with a centroid, nearest first, before the shortlist cut. coarse: Vec, /// How many of `coarse` stage 2 was allowed to score. shortlist: usize, - /// Stage 2: `(squared distance, note, evidence chunk)` for every shortlisted note - /// with stored vectors, nearest first (ties by path). + /// Stage 2: `(squared distance, note, evidence chunk)`, nearest first. scored: Vec<(f32, String, i64)>, - /// `None` when ungraded: not asked for, a pool under [`STATS_MIN_POPULATION`], or - /// zero variance. + /// `None` when ungraded. population: Option, - /// The anchor and its 1-hop neighbours: excluded from discovery. + /// The anchor and its 1-hop neighbours. excluded: std::collections::HashSet, } -/// Compute `anchor`'s field at the shortlist a `limit`-sized list uses. `None` when -/// there is nothing to search from: no embedding space, or an anchor with no stored -/// vectors. +/// Compute `anchor`'s field for a `limit`-sized list. `None` with no embedding space or +/// no anchor vectors. fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result> { if !db::embedding_space_exists(conn)? { return Ok(None); } - // The anchor's own stored vectors, loaded once (re-embeds nothing — index-engine.md §3); - // none ⇒ nothing to search from. Its centroid is computed in-process from them - // rather than read back, so an anchor mid-embed still discovers from what it has. + // Stored vectors only (index-engine.md §3). The centroid is computed here, not read + // back, so an anchor mid-embed still discovers from what it has. let (anchor_ids, anchor_vecs): (Vec, Vec>) = db::note_chunk_vectors(conn, anchor)?.into_iter().unzip(); let Some(anchor_centroid) = centroid_of(&anchor_vecs) else { return Ok(None); }; - // The only use of the graph in generation: subtract what's already linked — the - // anchor and everything within 1 hop (self + direct neighbors). + // The graph's only use here: subtract what's already linked. let excluded = graph::reachable_within(conn, anchor, EXCLUDE_HOPS)?; - // Stage 1 — coarse shortlist over note centroids: one O(notes) scan, excluded - // notes skipped up front so they never occupy a shortlist slot. + // Stage 1: coarse scan over centroids, skipping excluded notes so they take no slot. let mut coarse: Vec<(f32, String)> = Vec::new(); let mut scratch: Vec = Vec::new(); db::for_each_note_centroid(conn, |note, blob| { if excluded.contains(note) { - return; // the anchor or a direct neighbor — already connected + return; } unpack_f32_into(blob, &mut scratch); coarse.push((l2_sq(&anchor_centroid, &scratch), note.to_string())); @@ -152,19 +103,13 @@ fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result = coarse.into_iter().map(|(_, note)| note).collect(); - // Stage 1 ends here, and nothing judges it: the shortlist is a recall device, never - // a quality gate (GH #192). It used to be both — cutting the coarse list on centroid - // z meant max-sim could never rescue a candidate whose best passage is far nearer - // than its centroid suggests, which is exactly a multi-topic note's shape (ADR-0014). + // The shortlist is a recall device, never a quality gate (GH #192, ADR-0014). let shortlist = limit .saturating_mul(SHORTLIST_PER_RESULT) .max(SHORTLIST_MIN); - // Stage 2 — exact max-sim over the whole shortlist: per note, the best (smallest - // squared-L2) of its [`nearest_pairs`] — the same kernel the explanation reads, so - // the card's evidence is by construction the explanation's first pair. The earliest - // chunk wins a tie. A shortlisted note with no stored chunk vectors (possible - // mid-embed) scores nothing and drops out. + // Stage 2: exact max-sim via [`nearest_pairs`], the kernel explain reads too. The + // earliest chunk wins a tie; a note with no vectors yet drops out. let mut scored: Vec<(f32, String, i64)> = Vec::new(); for note_path in coarse.iter().take(shortlist) { let pairs = nearest_pairs( @@ -179,16 +124,13 @@ fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result= STATS_MIN_POPULATION { let n = scored.len() as f64; @@ -213,21 +155,9 @@ fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result, } @@ -287,25 +213,19 @@ pub struct GradedPair { #[derive(Debug, Clone, PartialEq)] pub struct Explanation { pub standing: Standing, - /// The candidate's rank by whole-note centroid in stage 1 (1-based), when it entered - /// stage 1 at all. Beside a best-passage rank it shows a buried gem: a note whose - /// one section matches far better than the note as a whole. + /// The candidate's 1-based centroid rank in stage 1. Beside its passage rank it shows + /// a note whose one section matches far better than the whole. pub centroid_rank: Option, /// The candidate's z, when it was scored in a graded field. pub z: Option, - /// Every scored note's z in served order (so descending): the field the band is - /// relative to. Empty when ungraded. + /// Every scored note's z, descending. Empty when ungraded. pub population: Vec, /// One pair per candidate passage, nearest first ([`passage_pairs`]), each graded. pub pairs: Vec, } -/// Explain one note's place in `anchor`'s discovery field, at the shortlist a -/// `limit`-sized list uses. The same computation [`candidates`] serves from, so a -/// served row's rank, z and winning pair are exactly the card's. Answers for any note, -/// not only a served one: a linked, unembedded or unshortlisted note says which, and its -/// passages are still compared when both sides have vectors. A pure read over stored -/// vectors, re-embedding nothing. +/// Explain one note's place in `anchor`'s discovery field, from the same computation +/// [`candidates`] serves. Answers for any note, served or not. Re-embeds nothing. pub fn explain( conn: &Connection, anchor: &str, @@ -357,38 +277,24 @@ pub fn explain( shortlist: field.shortlist, } } else { - // Not scored and not past the shortlist: it has no centroid or no chunk - // vectors yet. + // Not scored and not past the shortlist: no centroid or vectors yet. Standing::Unembedded }; Ok(out) } -/// One matched passage pair between an anchor and a candidate: the candidate's chunk, -/// the anchor's chunk nearest to it, and how near they are. +/// A candidate's chunk, the anchor chunk nearest to it, and how near. #[derive(Debug, Clone, PartialEq)] pub struct PassagePair { - /// The anchor's chunk nearest to `candidate_chunk_id`. pub anchor_chunk_id: i64, - /// The candidate's chunk this pair is about. pub candidate_chunk_id: i64, - /// Negated L2 distance between the two — higher is nearer, the same unit as - /// [`CandidateNote::score`]. + /// Negated L2 distance, as in [`CandidateNote::score`]. pub score: f64, } -/// The evidence behind one discovery row: up to `limit` passage pairs between `anchor` -/// and `candidate`, nearest first — **one pair per candidate chunk**, each matched to the -/// anchor chunk nearest to it. The first pair is exactly the pair [`candidates`] scored -/// the note on (same distances, same strictly-less tie rule, so the same -/// `evidence_chunk_id` at the same `score`): an explanation built from these is about -/// the passage the card showed, never a second ranking that could disagree with it. -/// -/// [`candidates`] keeps only the winning candidate chunk because a list needs no more; -/// this is the read for the moment a human asks *why* one row is there, where the -/// anchor's half of the pair is the other half of the answer. A pure read over stored -/// vectors, re-embedding nothing. Empty when the vault has no embedding space, when -/// either note has no stored vectors, or when `limit` is 0. +/// Up to `limit` passage pairs between `anchor` and `candidate`, nearest first, one per +/// candidate chunk. The first is exactly the pair [`candidates`] scored the note on. +/// Re-embeds nothing. pub fn passage_pairs( conn: &Connection, anchor: &str, @@ -401,8 +307,7 @@ pub fn passage_pairs( .collect()) } -/// [`passage_pairs`] with each pair's squared distance kept beside it, the unit the -/// population statistic reads. +/// [`passage_pairs`] with each squared distance kept, for grading. fn passage_pairs_sq( conn: &Connection, anchor: &str, @@ -419,8 +324,7 @@ fn passage_pairs_sq( &anchor_vecs, db::note_chunk_vectors(conn, candidate)?, ); - // Stable, so equal distances keep the candidate's chunk order — the earliest chunk - // wins a tie, as it does in `candidates`. + // Stable, so the earliest chunk wins a tie, as in `candidates`. pairs.sort_by(|a, b| a.0.total_cmp(&b.0)); Ok(pairs .into_iter() @@ -438,12 +342,9 @@ fn passage_pairs_sq( .collect()) } -/// Discovery's max-sim kernel: for each candidate chunk, in order, the anchor chunk -/// nearest to it, as `(squared distance, anchor chunk, candidate chunk)`. Strictly-less -/// keeps the earliest anchor chunk on a tie. Stage 2 takes the minimum of this and the -/// explanation sorts it, so a card and its explanation read one computation. Squared L2 -/// is the ranking key without the per-comparison `sqrt`; the readers take it once per -/// surfaced pair. Empty when either side has no vectors. +/// Discovery's max-sim kernel: per candidate chunk, the nearest anchor chunk, as +/// `(squared distance, anchor chunk, candidate chunk)`. Strictly-less keeps the earliest +/// anchor chunk on a tie. fn nearest_pairs( anchor_ids: &[i64], anchor_vecs: &[Vec], diff --git a/crates/b2-core/src/embed.rs b/crates/b2-core/src/embed.rs index 6f1cfe1..6a4a485 100644 --- a/crates/b2-core/src/embed.rs +++ b/crates/b2-core/src/embed.rs @@ -1,57 +1,41 @@ -//! The embedder seam (index-engine.md §6; the "build for tomorrow's -//! model" tenet in invariants.md). Producing embeddings inside a single binary -//! is the one genuinely hard part, and it is *orthogonal* to the store — so it -//! sits behind this trait. The engine is built and tested against a deterministic -//! fake; the real local model (`b2-embed`'s candle-backed `LocalEmbedder`, -//! index-engine.md §6) drops in through this same seam with no schema or flow change. +//! The embedder seam (index-engine.md §6). The engine is tested against a deterministic +//! fake; `b2-embed`'s real model drops in here with no schema or flow change. use crate::error::Result; /// Turns note text into a vector. The dimension is fixed per model and recorded as /// `meta.embed_dim` (ADR-0007). /// -/// `embed` is **fallible**: the fake never fails, but a real model runs tensor math that -/// can, and the index path must surface that rather than panic. Retrieval is -/// **asymmetric-ready** — [`embed`](Self::embed) embeds a passage, -/// [`embed_query`](Self::embed_query) a query; models that prefix the two differently -/// override the latter, whose default is symmetric. +/// Fallible because a real model can fail. [`embed`](Self::embed) embeds a passage and +/// [`embed_query`](Self::embed_query) a query, for asymmetric models. pub trait Embedder { - /// Stable identifier recorded in `meta.embed_model_id`. A change to it (or to - /// `dim`) is a model swap → drop the stored vectors + re-embed (index-engine.md §8). + /// Stable identifier recorded in `meta.embed_model_id`. A change is a model swap + /// (index-engine.md §8). fn model_id(&self) -> &str; /// Vector dimension; must equal the recorded `meta.embed_dim`. fn dim(&self) -> usize; /// Embed one document/passage for indexing. fn embed(&self, text: &str) -> Result>; - /// Embed a search query. Default is symmetric (query == document); asymmetric - /// models override this to apply their query-side prompt prefix. + /// Embed a search query. Symmetric by default; asymmetric models add their prefix. fn embed_query(&self, text: &str) -> Result> { self.embed(text) } - /// Embed a batch of documents/passages for indexing, one vector per input in - /// order. The default maps [`embed`](Self::embed) over the slice — always - /// correct; a real model overrides this to run **one batched forward pass**, - /// which is dramatically faster on CPU than N single passes (the reindex hot - /// path). Chunk vectors are independent, so batch boundaries never change a - /// result. + /// Embed a batch of passages, one vector per input in order. A real model overrides + /// this with one batched forward pass. Batch boundaries never change a result. fn embed_batch(&self, texts: &[&str]) -> Result>> { texts.iter().map(|t| self.embed(t)).collect() } } -/// Deterministic, content-addressed embedder for tests/dev: identical text → -/// identical vector, so KNN is reproducible and drop-&-rebuild yields the same -/// vectors. It is **not** semantic — it stands in for `b2-embed`'s real local model -/// behind the same seam, so the engine suite runs offline (`B2_EMBEDDER=fake`, -/// docs/architecture.md). +/// Deterministic embedder for tests and dev (`B2_EMBEDDER=fake`): identical text gives +/// an identical vector. Not semantic. #[derive(Debug, Clone, Copy)] pub struct FakeEmbedder { dim: usize, } impl FakeEmbedder { - /// A fake of dimension `dim`. A vector has at least one component, so a `dim` of 0 - /// is clamped to 1 rather than refused. + /// A fake of dimension `dim`, clamped to at least 1. pub fn new(dim: usize) -> Self { Self { dim: dim.max(1) } } @@ -64,11 +48,8 @@ impl Default for FakeEmbedder { } } -/// The fake embedder's recorded model id — `pub` because the identity is load-bearing -/// beyond tests: a vault whose `meta.embed_model_id` equals this holds hash vectors -/// with no semantic geometry, so `Vault::similar` never grades it — the ranked list -/// is served with no z, since a statistic over hash noise would be a band worth of -/// noise (GH #150/#197). +/// The fake embedder's model id. A vault recorded with it holds hash vectors, so +/// `Vault::similar` never grades it (GH #150/#197). pub const FAKE_MODEL_ID: &str = "fake-deterministic-v1"; impl Embedder for FakeEmbedder { @@ -81,8 +62,7 @@ impl Embedder for FakeEmbedder { } fn embed(&self, text: &str) -> Result> { - // blake3 XOF → dim little-endian u32 words → floats in [0,1). Deterministic - // in text, so the same chunk always embeds to the same vector. + // blake3 XOF → dim little-endian u32 words → floats in [0,1). let mut hasher = blake3::Hasher::new(); hasher.update(text.as_bytes()); let mut reader = hasher.finalize_xof(); @@ -98,10 +78,8 @@ impl Embedder for FakeEmbedder { } } -/// Pack a vector as a compact little-endian float32 BLOB — the stored form of every -/// vector in the index (`embeddings.vector`, `note_centroids.centroid`; -/// index-engine.md §3). The query side packs the same way so an exact match has -/// distance 0. +/// Pack a vector as a little-endian float32 BLOB, the stored form of every vector +/// (index-engine.md §3). pub fn pack_f32(v: &[f32]) -> Vec { let mut out = Vec::with_capacity(v.len() * 4); for x in v { @@ -110,21 +88,15 @@ pub fn pack_f32(v: &[f32]) -> Vec { out } -/// Inverse of [`pack_f32`]: read a stored BLOB (little-endian float32, no header) -/// back into a vector. Used to reuse a note's *stored* chunk vectors as discovery -/// queries without re-embedding (index-engine.md §3). A trailing partial group can't occur -/// for a vector written by [`pack_f32`], so a non-multiple-of-4 length is simply -/// truncated rather than treated as an error. +/// Inverse of [`pack_f32`]. A trailing partial group is truncated, not an error. pub fn unpack_f32(bytes: &[u8]) -> Vec { let mut out = Vec::with_capacity(bytes.len() / 4); unpack_f32_into(bytes, &mut out); out } -/// [`unpack_f32`] into a caller-owned scratch buffer, reusing its capacity. The -/// whole-space scans decode one stored vector per visited row; a fresh `Vec` per row -/// was a measurable slice of the `b2 similar` stall (#38), where this costs nothing -/// after the first row. +/// [`unpack_f32`] into a reused scratch buffer: a fresh `Vec` per scanned row was +/// measurable (#38). pub fn unpack_f32_into(bytes: &[u8], out: &mut Vec) { out.clear(); out.extend( @@ -134,15 +106,11 @@ pub fn unpack_f32_into(bytes: &[u8], out: &mut Vec) { ); } -/// Squared Euclidean distance between two equal-length vectors — the index's one ranking -/// key, minus the final `sqrt` (monotonic, so dropping it never changes an ordering; it is -/// applied once per *surfaced* result). A length mismatch scores the shared prefix rather -/// than panicking. +/// Squared Euclidean distance, the index's ranking key. A length mismatch scores the +/// shared prefix rather than panicking. /// -/// Eight independent accumulators, summed at the end: float addition is non-associative, -/// so a single running sum forms one serial dependency chain the compiler must execute as -/// written — splitting it lets LLVM autovectorize. Measured at the #38 scale, the naive -/// iterator shape cost ~530 ms against this shape's ~75 ms. +/// Eight accumulators: float addition is non-associative, so one running sum can't +/// autovectorize (~530 ms vs ~75 ms at the #38 scale). pub fn l2_sq(a: &[f32], b: &[f32]) -> f32 { let n = a.len().min(b.len()); let (a, b) = (&a[..n], &b[..n]); @@ -164,18 +132,13 @@ pub fn l2_sq(a: &[f32], b: &[f32]) -> f32 { sum } -/// The **centroid** of a note's chunk vectors: their arithmetic mean, L2-normalized -/// (the spherical mean — the standard coarse representative when the underlying -/// vectors are cosine-normalized, as b2-embed's are). `None` for an empty set — a -/// note with no stored vectors has no centroid row. Deterministic: summation runs in -/// the given (chunk `seq`) order. Discovery's first stage ranks whole *notes* by -/// centroid distance, so its heavy scan is O(notes), not O(chunks) (#38). +/// A note's centroid: the L2-normalized mean of its chunk vectors, summed in the given +/// order; `None` for an empty set. Discovery's first stage scans these (#38). pub fn centroid_of(vectors: &[Vec]) -> Option> { let first = vectors.first()?; let mut mean = vec![0.0f32; first.len()]; for v in vectors { - // A length mismatch can't occur within one embedding space; fold the shared - // prefix rather than panic, mirroring `l2_sq`. + // Fold the shared prefix on a length mismatch, as `l2_sq` does. for (m, x) in mean.iter_mut().zip(v) { *m += x; } diff --git a/crates/b2-core/src/error.rs b/crates/b2-core/src/error.rs index 9797107..14b0457 100644 --- a/crates/b2-core/src/error.rs +++ b/crates/b2-core/src/error.rs @@ -1,8 +1,7 @@ -//! The crate error type — a `thiserror` enum, because the adapters match on its -//! variants to choose a user-facing message. +//! The crate error type. Adapters match its variants to choose a user-facing message. -/// Errors surfaced by the index engine. Kept internal/structured — user-facing -/// surfaces (the CLI, the desktop app) translate these into generic, actionable messages. +/// Errors surfaced by the engine. Adapters translate them into generic, actionable +/// messages. #[derive(thiserror::Error, Debug)] pub enum Error { #[error("sqlite error: {0}")] @@ -14,143 +13,99 @@ pub enum Error { #[error("frontmatter edit unsupported: {0}")] Frontmatter(String), - /// A note reference (a vault-relative path) did not resolve to any indexed - /// note — the one domain error the façade distinguishes from "found, no - /// results". + /// A note reference did not resolve to an indexed note. #[error("note not found: {0}")] NoteNotFound(String), - /// The embedder failed to produce a vector (real-model tensor/runtime error). - /// Kept as a message so `b2-core` stays free of the embedding runtime's types. + /// The embedder failed. A message, so `b2-core` stays free of the runtime's types. #[error("embedding failed: {0}")] Embed(String), - /// The chat provider failed outright — connection refused, an HTTP error, a - /// malformed stream — on the *answer* call of flow ④ (a condensation - /// failure never surfaces: it degrades to the raw question, GH #153). - /// The [`Error::Embed`] posture applied to the second seam: kept as a - /// message so `b2-core` stays free of the LLM runtime's types. + /// The chat provider failed on flow ④'s answer call (a failed condense degrades + /// instead, GH #153). A message, as for [`Error::Embed`]. #[error("llm call failed: {0}")] Llm(String), - /// One model reply asked for more tool calls than the provider's configured cap. A - /// typed sibling of [`Error::Llm`] rather than a message, because callers treat it - /// differently: a tool-using turn degrades on an ordinary failed lookup round (the - /// model may simply not support tools), but must **not** degrade on this — a reply - /// this far outside the protocol is a broken or hostile server, and answering anyway - /// would hide it. + /// One model reply asked for more tool calls than the configured cap. Separate from + /// [`Error::Llm`] because a tool-using turn must not degrade on it: it signals a + /// broken or hostile server. #[error("the model asked for more than {limit} tool calls in one reply")] ToolCallLimit { limit: usize }, - /// The index's recorded embedding model/dim differs from the active embedder, - /// so its vectors are incomparable with new query vectors. A read (search) - /// fails fast with this rather than returning silently wrong results; the fix - /// is a `reindex` (which re-embeds). See index-engine.md §8 and GitHub Issues. + /// The index's recorded embedder differs from the active one, so a search would be + /// silently wrong (index-engine.md §8). #[error("index built with embedding model {indexed}, but the active model is {active}; run `b2 reindex`")] ModelMismatch { indexed: String, active: String }, - /// `b2 mv` was given a destination that isn't a valid vault-relative Markdown - /// path — empty, absolute, escaping the vault via `..`, dot-prefixed (b2 - /// indexes no hidden member, so it would move the file out of the vault's - /// managed subtree — GH #136), or the source itself. + /// `b2 mv` was given an invalid destination: empty, absolute, escaping via `..`, + /// dot-prefixed (GH #136), or the source itself. #[error("invalid move destination: {0}")] MoveDestination(String), - /// `b2 mv` would overwrite an existing file — refused (the vault never clobbers, - /// data-model.md §1). The path is echoed for the user-facing message. + /// `b2 mv` would overwrite an existing file (data-model.md §1). #[error("move target already exists: {0}")] MoveTargetExists(String), - /// A move failed part-way and B2 could not put back every file it had already - /// changed (GH #230). Rare: an ordinary failure is rolled back and surfaces as - /// its own error. Carries the vault-relative paths still holding the move's - /// rewrite, so the user can check those links by hand, and the failure that - /// started the undo, so the adapters' internal log keeps the root cause. + /// A move failed part-way and could not be fully undone (GH #230). Carries the paths + /// still holding the rewrite, for the user to check, and the root cause. #[error("move failed and could not be fully undone: {} (cause: {source})", paths.join(", "))] MoveIncomplete { paths: Vec, source: Box, }, - /// `b2 mv` was given a source folder that doesn't exist in the vault — the - /// directory sibling of [`Error::NoteNotFound`] (a folder is a path prefix, - /// never an indexed row, so it resolves against the filesystem). + /// A source folder doesn't exist in the vault. #[error("directory not found: {0}")] DirNotFound(String), - /// `b2 add` was given a destination that isn't a valid vault-relative Markdown - /// path — empty, absolute, escaping the vault via `..`, or dot-prefixed (a - /// `.scratch.md` would never be indexed, so b2 refuses to author one — GH - /// #136). The `mv` parallel of [`Error::MoveDestination`], distinct so the CLI - /// can phrase it for note creation rather than a move. + /// `b2 add` was given an invalid path: empty, absolute, escaping via `..`, or + /// dot-prefixed (GH #136). #[error("invalid new-note path: {0}")] AddDestination(String), - /// `b2 add` would overwrite an existing file — refused (the vault never clobbers, - /// data-model.md §1). The path is echoed for the user-facing message. + /// `b2 add` would overwrite an existing file (data-model.md §1). #[error("note already exists: {0}")] AddTargetExists(String), - /// An import ([`crate::import`]) was handed a destination it can't place a file - /// at — a name carrying path separators, or a folder/name pair that normalizes - /// to an empty, absolute, escaping, or dot-prefixed path. The import sibling of - /// [`Error::AddDestination`], distinct so the adapters can phrase it for a file - /// arriving from outside rather than a note being authored. + /// An import was given an invalid destination: a name with separators, or a path + /// that is empty, absolute, escaping or dot-prefixed. #[error("invalid import destination: {0}")] ImportDestination(String), - /// An import would overwrite an existing vault file — refused, exactly as - /// [`Error::AddTargetExists`] is, and separate for the same reason: the thing - /// arriving may be a PDF, so the message must not call it a note. + /// An import would overwrite an existing file. Separate from + /// [`Error::AddTargetExists`] because the file may not be a note. #[error("file already exists: {0}")] ImportTargetExists(String), - /// `create_dir` was given a path that isn't a valid vault-relative folder — - /// empty, absolute, escaping via `..`, or dot-prefixed (b2 never manages - /// hidden paths). The folder sibling of [`Error::AddDestination`]. + /// `create_dir` was given an invalid folder path. #[error("invalid folder path: {0}")] DirDestination(String), - /// `create_dir` would land on an existing file or folder — refused rather than - /// silently merged (the user asked to *create*; it's already there). The folder - /// sibling of [`Error::AddTargetExists`]. + /// `create_dir` would land on an existing file or folder. #[error("folder already exists: {0}")] DirTargetExists(String), - /// `Vault::write` was handed a `base_revision` that no longer matches the file - /// on disk — an external editor changed the note since it was read. Refused - /// rather than clobbered: the caller re-reads (getting - /// the current revision) and either reloads or knowingly re-writes. The path is - /// carried for the debug detail, never for the user-facing message. + /// A save's `base_revision` no longer matches the file: it changed on disk since it + /// was read. The path is for debug detail only. #[error("write conflict: {0} changed on disk since it was read")] WriteConflict(String), - /// `b2 link` was given a `--type` that is not a core relation verb - /// (data-model.md §2). The core is the palette `b2 link` offers; a tail verb can - /// still be hand-authored in the Markdown, but the command validates to the core - /// so a typo (`support` for `supports`) is caught rather than silently stored. + /// `b2 link` was given a `--type` that is not a core verb (data-model.md §2), so a + /// typo is caught rather than stored. #[error("not a core relation verb: {0}")] InvalidRelation(String), - /// The index on disk was written by a **newer** `b2` than this binary — its - /// `meta.schema_version` is above [`crate::SCHEMA_VERSION`]. Refused rather than - /// rebuilt: the migration has no migrations by design (ADR-0002), so its answer to - /// an unreadable index is to drop it, and a *newer* index is unreadable for a - /// reason that dropping does not fix. An older `b2` left on `PATH` would otherwise - /// destroy a complete index on a read-only command. The fix is to run the newer - /// `b2`, not to reindex. + /// The index was written by a newer `b2` (schema above [`crate::SCHEMA_VERSION`]). + /// Refused rather than dropped (ADR-0002); the fix is the newer `b2`. #[error("index schema {found} was written by a newer b2 (this build reads {supported}); run the newer b2")] IndexTooNew { found: i64, supported: i64 }, - /// A resource reference (vault-relative path) did not resolve to any - /// inventoried resource — the resource sibling of [`Error::NoteNotFound`]. + /// A resource path did not resolve to an inventoried resource. #[error("resource not found: {0}")] ResourceNotFound(String), - /// The operation exists for notes but not (yet) for resources — e.g. - /// `b2 similar `, until resources have chunks and centroids of their - /// own. Distinct from [`Error::ResourceNotFound`] so the adapters can - /// say "not yet" rather than "no such file". + /// The operation works on notes but not yet on resources (e.g. `b2 similar`), so + /// adapters say "not yet" rather than "no such file". #[error("not supported for resources yet: {0}")] ResourceUnsupported(String), } diff --git a/crates/b2-core/src/graph.rs b/crates/b2-core/src/graph.rs index 4e99af2..a370482 100644 --- a/crates/b2-core/src/graph.rs +++ b/crates/b2-core/src/graph.rs @@ -1,7 +1,5 @@ -//! Graph queries over the typed `edges` table. Inversion (backlinks) is the -//! reason the graph is materialized rather than parsed at read time -//! (index-engine.md §3): a note's inbound edges live in every *other* -//! note, so `neighbors` is one indexed lookup, not a full-vault scan. +//! Graph queries over the typed `edges` table. Materialized so backlinks are one indexed +//! lookup rather than a full-vault scan (index-engine.md §3). use crate::error::Result; use crate::relation; @@ -15,31 +13,21 @@ pub enum Direction { Inbound, } -/// One neighbor of a note: the note at the other end of an active edge, plus the -/// display label (the verb for outbound, the inverse for inbound). +/// One neighbor of a note: the other end of an edge, plus its display label. #[derive(Debug, Clone)] pub struct Neighbor { - /// The vault-relative path at the other end of the edge — the note's identity - /// (L1). + /// The note at the other end (L1). pub other: String, - /// The stored relation verb. pub edge_type: String, pub direction: Direction, - /// Display label: the verb itself outbound, the inverse label inbound - /// (data-model.md §2). Symmetric verbs read the same both ways. + /// The verb outbound, its inverse label inbound (data-model.md §2). pub label: String, pub explanation: Option, - /// Edge provenance — which of the two homes authored it (data-model.md §0): - /// `inline` (a body link) or `frontmatter` (a `b2_relations:` entry, whether - /// `b2 link` wrote it on the human's command or they wrote it by hand). `b2 explain` - /// surfaces this so a plain body link reads distinctly from a typed relation. + /// `inline` (a body link) or `frontmatter` (a `b2_relations:` entry) (data-model.md §0). pub origin: String, } -/// All neighbors of the note at `note_path` — outbound edges (this note → others) -/// then inbound edges (others → this note), each labeled for display. Every edge is -/// authored and active (there is no suggestion lifecycle), so this is the note's full -/// typed graph. +/// All neighbors of `note_path`: outbound, then inbound. pub fn neighbors(conn: &Connection, note_path: &str) -> Result> { let mut out = collect_neighbors( conn, @@ -60,9 +48,7 @@ pub fn neighbors(conn: &Connection, note_path: &str) -> Result> { Ok(out) } -/// One direction's half of [`neighbors`]: run a query yielding -/// `(other, type, explanation, origin)` rows and label each for `direction` — -/// the verb itself outbound, its inverse inbound (data-model.md §2). +/// One direction's half of [`neighbors`], from `(other, type, explanation, origin)` rows. fn collect_neighbors( conn: &Connection, sql: &str, @@ -88,28 +74,21 @@ fn collect_neighbors( Ok(rows.collect::>>()?) } -/// One outbound link that resolved to **nothing** — neither a note nor a resource -/// exists at its target, so it is a *dangling* edge (`dst_path IS NULL AND -/// dst_resource_path IS NULL`). A note is one `.md` file (data-model.md §1), so a -/// `[[Hermes]]` that names a *folder* — or a plain typo — never resolves. These are -/// surfaced rather than silently dropped so a broken link is visible (GH #12). +/// A dangling outbound link: no note or resource at its target (a typo, or a folder; +/// data-model.md §1). Surfaced, not dropped (GH #12). #[derive(Debug, Clone, PartialEq, Eq)] pub struct Unresolved { - /// The target exactly as authored (`dst_path_raw`) — e.g. `Hermes`. + /// The target as authored (`dst_path_raw`). pub target: String, - /// The relation verb (`references` for a bare link). pub edge_type: String, - /// Edge origin — `inline` (a body link) or `frontmatter` (a `b2_relations:` entry). + /// `inline` or `frontmatter`. pub origin: String, pub explanation: Option, } -/// A note's **dangling** outbound links: the edges it authored whose target resolves -/// to no note and no resource, in a deterministic order. The complement of -/// [`neighbors`]'s outbound half (which keeps only `dst_path IS NOT NULL`) — together -/// they cover every outbound edge, so a link is either a resolved neighbor or a -/// surfaced unresolved link, never silently gone (GH #12). Backed by the -/// `edges_dangling_idx` partial index, whose predicate this query mirrors. +/// A note's dangling outbound links, in a deterministic order: the complement of +/// [`neighbors`]'s outbound half, so a link is never silently gone (GH #12). The query +/// mirrors `edges_dangling_idx`'s predicate. pub fn unresolved_outbound(conn: &Connection, note_path: &str) -> Result> { let mut stmt = conn.prepare( "SELECT dst_path_raw, type, origin, explanation FROM edges @@ -127,10 +106,7 @@ pub fn unresolved_outbound(conn: &Connection, note_path: &str) -> Result>>()?) } -/// The set of notes within `hops` typed hops of `anchor` (inclusive of `anchor`), -/// traversing `active` edges **undirected** — a note related to the anchor either -/// way is reachable. Discovery subtracts the anchor's 1-hop set (what is already linked) -/// from its candidates. `hops = 0` is just the anchor. +/// Notes within `hops` undirected hops of `anchor`, the anchor included. pub fn reachable_within(conn: &Connection, anchor: &str, hops: usize) -> Result> { let mut seen = HashSet::from([anchor.to_string()]); let mut frontier = vec![anchor.to_string()]; diff --git a/crates/b2-core/src/import.rs b/crates/b2-core/src/import.rs index 4e64af9..a5471a3 100644 --- a/crates/b2-core/src/import.rs +++ b/crates/b2-core/src/import.rs @@ -1,19 +1,9 @@ -//! Bring an *outside* file into the vault — CRUD's **import** arm, and the kernel -//! behind the desktop's drag-from-Finder-onto-the-tree gesture. +//! Import an outside file into the vault: a byte-honest copy, authoring nothing, which is +//! what makes it a permitted write (ADR-0004). Unlike [`crate::add`], which authors a note. //! -//! What separates this from [`crate::add`]: `add` **authors** a new document, while an -//! import is a **byte-honest copy** of a file the human already has. B2 writes the bytes -//! it was handed and authors nothing, so a dropped `.md` keeps its own frontmatter -//! verbatim and a dropped PDF keeps its bytes. That is what makes it a permitted write -//! (ADR-0004): placing a file is the same category of act as moving or deleting one. -//! -//! **Vault first**, the order `add`/`mv`/`link` also write in: place the file, then -//! project *from disk*, so the index is derived from what actually landed. **Model-free** -//! (a [`ProjectionCtx`], like `create_note`): an imported note's chunks join the pending -//! set for the next embed pass, so importing works with no model provisioned. -//! -//! Two entry points because the two gestures arrive differently: a file dropped on the -//! webview is **bytes**, a file chosen in an OS picker is a **path**. +//! Vault first: place the file, then project from disk. Model-free: imported chunks wait +//! for the next embed pass. [`import_bytes`] serves a drag-and-drop, [`import_path`] an OS +//! picker. use crate::error::{Error, Result}; use crate::ingest::{self, ProjectionCtx}; @@ -23,21 +13,16 @@ use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; -/// What an import did: where the file landed — which, for a note and a resource -/// alike, is the whole of what arrived (invariants L1/L3: both are path-keyed). +/// Where the imported file landed (L1/L3). #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ImportReport { pub path: String, - /// Whether the arriving file was routed as a **note** (a `.md`, projected with - /// chunks/FTS/edges) rather than a resource (one inventory row). The adapters - /// use it to decide whether the drop is openable in the editor. + /// Routed as a note rather than a resource, so the editor can open it. pub note: bool, } -/// Import `bytes` into the vault folder `dir` (`""` for the root) under `file_name` — -/// the drag-and-drop arm, where the OS hands the page content rather than a path. -/// Refuses [`Error::ImportDestination`] for a name/folder pair that isn't a valid -/// vault-relative path and [`Error::ImportTargetExists`] rather than clobber. +/// Import `bytes` into folder `dir` (`""` for the root) as `file_name`. Refuses an invalid +/// destination and an occupied one. pub fn import_bytes( ctx: ProjectionCtx, dir: &str, @@ -49,12 +34,8 @@ pub fn import_bytes( project_placed(ctx, rel, &abs) } -/// Import the file at `source` into the vault folder `dir`, keeping its name — the -/// OS-picker arm, and the reason it exists rather than `import_bytes(fs::read(source)?)` -/// at the call site: the adapter then holds no logic and the bytes never round-trip -/// through it. Same refusals as [`import_bytes`], plus [`Error::ImportDestination`] for a -/// source that is a folder or has no file name. A source *inside* the vault is a -/// duplicate, not an error — a copy landing at a free path is simply a second note. +/// Import the file at `source` into folder `dir`, keeping its name, so the adapter holds +/// no logic. A source inside the vault just makes a copy. pub fn import_path(ctx: ProjectionCtx, dir: &str, source: &Path) -> Result { if source.is_dir() { return Err(Error::ImportDestination(format!( @@ -69,20 +50,15 @@ pub fn import_path(ctx: ProjectionCtx, dir: &str, source: &Path) -> Result Result { let name = file_name.trim(); if name.is_empty() || name.contains('/') || name.contains('\\') { @@ -99,37 +75,22 @@ fn destination(dir: &str, file_name: &str) -> Result { crate::pathspec::normalize_rel(&joined).map_err(Error::ImportDestination) } -/// The validated destination as both halves the rest of the op needs: the -/// vault-relative path (what the index and the report speak in) and its absolute twin -/// (what the filesystem does). +/// The validated destination, vault-relative and absolute. fn destination_paths(vault_root: &Path, dir: &str, file_name: &str) -> Result<(String, PathBuf)> { let rel = destination(dir, file_name)?; let abs = vault_root.join(&rel); Ok((rel, abs)) } -/// Reserve the import's destination and fill it — [`place_new`], refusing an occupied -/// destination as [`Error::ImportTargetExists`]. That is also why `import_path` streams -/// instead of calling `fs::copy`, which would truncate an occupied destination. -/// -/// The destination is a file B2 created, so it carries ordinary new-file permissions -/// rather than the source's mode — byte-honesty is about content. +/// [`place_new`] with [`Error::ImportTargetExists`]. The file gets new-file permissions, +/// not the source's mode. fn place(rel: &str, abs: &Path, fill: impl FnOnce(&mut fs::File) -> io::Result<()>) -> Result<()> { place_new(abs, || Error::ImportTargetExists(rel.to_string()), fill) } -/// Create a new file at `abs` (missing parent folders included) and fill it — or leave -/// nothing behind. Shared by every op that writes a file B2 did not have before: an -/// import here, and `add`'s new note. -/// -/// **`create_new` is the refusal**, not a check before one: "does it exist" and "claim -/// it" are a single syscall, so a file appearing in between — another window, a sync -/// client, the CLI — cannot be overwritten. An [`io::ErrorKind::AlreadyExists`] *is* the -/// op's own target-exists error (`taken`), so the race and the ordinary "that name is -/// taken" reach the user as one message. -/// -/// A `fill` that fails partway takes the reserved file with it: half a file is not a -/// new file. +/// Create a new file at `abs` (with parent folders) and fill it, or leave nothing behind. +/// Shared with `add`. `create_new` is the refusal itself, one syscall, so a file appearing +/// concurrently can't be overwritten; `AlreadyExists` maps to `taken`. pub(crate) fn place_new( abs: &Path, taken: impl FnOnce() -> Error, @@ -144,22 +105,15 @@ pub(crate) fn place_new( Err(e) => return Err(e.into()), }; if let Err(e) = fill(&mut file) { - drop(file); // close before unlinking, so every platform agrees what happens + drop(file); // close before unlinking, for every platform let _ = fs::remove_file(abs); return Err(e.into()); } Ok(()) } -/// Project the just-placed file from disk, routing on its extension exactly as the vault -/// walk does: `.md` is a note (chunks, FTS, edges), everything else a resource (one -/// inventory row). B2 adds nothing to either. -/// -/// **On a refusal the placed file is removed again** — B2 undoing its own half-finished -/// write, not deleting vault material: the file becomes vault material only if this -/// returns `Ok`. The removal is best-effort and never masks the projection error, which -/// is the actionable one; a leftover file is then just an unindexed file, the state a -/// Finder copy produces and the next whole-vault pass picks up. +/// Project the placed file, routed as the walk routes it. On failure, B2 removes its own +/// half-finished write (best-effort, never masking the projection error). fn project_placed(ctx: ProjectionCtx, rel: String, abs: &Path) -> Result { match project_from_disk(ctx, &rel) { Ok(note) => Ok(ImportReport { path: rel, note }), @@ -170,18 +124,15 @@ fn project_placed(ctx: ProjectionCtx, rel: String, abs: &Path) -> Result Result { match ResourceClass::of_path(rel) { None => { ingest::project_file(ctx, rel)?; Ok(true) } - // `force`: the walk may skip a file whose `(size, mtime)` is unchanged, but an - // import never may. The row it would be trusting can describe a file that was - // deleted out of band and not yet pruned, and this one was written moments ago — - // so a same-size replacement inside the same second would keep a `content_hash` - // for bytes that no longer exist. Hash what was actually placed. + // `force`: an existing row may describe a file deleted out of band; hash what + // was actually placed. Some(class) => { ingest::project_resource_file(ctx.conn, ctx.root, rel, class, true)?; Ok(false) @@ -202,8 +153,6 @@ mod tests { #[test] fn a_file_name_is_a_name_never_a_path() { - // The whole point of the separator refusal: neither of these may relocate the - // import out of the folder the human dropped on. assert!(destination("papers", "../../etc/passwd").is_err()); assert!(destination("papers", "sub/a.pdf").is_err()); assert!(destination("papers", "sub\\a.pdf").is_err()); @@ -214,7 +163,7 @@ mod tests { fn the_shared_path_rules_still_apply_to_the_pair() { assert!(destination("..", "a.pdf").is_err()); // escaping folder assert!(destination("/abs", "a.pdf").is_err()); // absolute folder - assert!(destination("papers", ".hidden.pdf").is_err()); // never indexed, so never written + assert!(destination("papers", ".hidden.pdf").is_err()); // never indexed assert!(destination(".b2", "a.pdf").is_err()); } } diff --git a/crates/b2-core/src/ingest.rs b/crates/b2-core/src/ingest.rs index 0990e5c..0857844 100644 --- a/crates/b2-core/src/ingest.rs +++ b/crates/b2-core/src/ingest.rs @@ -1,14 +1,9 @@ -//! Ingest (flow ①): parse -> project into `notes`, `chunks` (+FTS) and -//! the typed `edges` graph, all keyed by the note's vault-relative path (ADR-0003). -//! **Ingest writes nothing to the vault** (ADR-0004) — it is a pure read of it. +//! Ingest (flow ①): parse the vault and project it into `notes`, `chunks` (+FTS) and +//! `edges`, keyed by vault-relative path (ADR-0003). Writes nothing to the vault (ADR-0004). //! -//! A full ingest is **two separately-invokable passes**: [`project_vault`], the -//! model-free one, which runs in two phases so link resolution never depends on file -//! order (phase 1 projects every note + its chunks, phase 2 derives edges against the -//! now-complete resolver); and [`embed_vault`], the model-bound one, which fills -//! whatever chunks still lack a vector — a pending set **derived from the DB**, never -//! handed over in memory. `Vault::reindex` is their composition; [`ingest_file`] -//! re-projects a single note inline against an already-built index. +//! Two passes: [`project_vault`] (model-free; notes and chunks first, then edges, so link +//! resolution never depends on file order) and [`embed_vault`] (fills chunks lacking a +//! vector, a pending set derived from the DB). [`ingest_file`] re-projects one note. use crate::chunk::{chunk_body, ChunkConfig}; use crate::db::{self, EdgeRow, NoteRow}; @@ -23,23 +18,13 @@ use std::fs; use std::ops::ControlFlow; use std::path::Path; -/// How many chunks to embed per forward pass. Batching amortizes one matmul over many -/// texts, traded against the **padding waste** of batching short chunks with long ones -/// — the tokenizer pads every chunk to the batch's longest. Measured on a real vault, -/// 16 beat 32 (~40% faster) and 8. It is also the reindex **cancel granularity**: the -/// flag is checked once per batch. +/// Chunks per forward pass. Larger batches pad short chunks to the longest; measured, 16 +/// beat 8 and 32. Also the reindex cancel granularity. const EMBED_BATCH: usize = 16; -/// Everything a **projection** needs: the index connection, the vault root, and the -/// vault's chunking policy. The three travel together through every write-side op, so -/// they are one parameter rather than three in two orders (GH #134). -/// -/// It carries a **posture**, and that is the point: an op that takes a `ProjectionCtx` -/// holds no embedder and therefore *cannot* embed. The model-free rule for -/// `rm`/`create_note`/`write` is the type system's to keep. -/// -/// A short-lived, `Copy`, borrow-only view struct never stored — the sanctioned -/// exception to "prefer owned fields" (CLAUDE.md), as `NoteRow` is. +/// What a projection needs: connection, vault root and chunking policy (GH #134). It holds +/// no embedder, so an op taking it cannot embed: that keeps `rm`/`create_note`/`write` +/// model-free. A short-lived `Copy` view, like `NoteRow`. #[derive(Clone, Copy)] pub struct ProjectionCtx<'a> { pub(crate) conn: &'a Connection, @@ -48,18 +33,14 @@ pub struct ProjectionCtx<'a> { } impl<'a> ProjectionCtx<'a> { - /// Bundle the three projection inputs. `cfg` is the vault's *one* chunking policy - /// (never re-defaulted per call), so every path that chunks a given vault cuts - /// identically and `incremental ≡ full rebuild` holds by construction. + /// `cfg` is the vault's one chunking policy, never re-defaulted per call, so + /// `incremental ≡ full rebuild` holds. pub fn new(conn: &'a Connection, root: &'a Path, cfg: &'a ChunkConfig) -> Self { Self { conn, root, cfg } } } -/// A [`ProjectionCtx`] plus the embedder — the **embedding** posture, and the other -/// half of GH #134's split. The ops that re-embed what they touch take this; the -/// model-free ones take the projection context alone, so the two are unmixable by the -/// compiler. Same view-struct rules as [`ProjectionCtx`]. +/// A [`ProjectionCtx`] plus the embedder, for ops that re-embed what they touch (GH #134). #[derive(Clone, Copy)] pub struct EmbedCtx<'a> { pub(crate) proj: ProjectionCtx<'a>, @@ -73,48 +54,40 @@ impl<'a> EmbedCtx<'a> { } } -/// A file's mtime as Unix seconds — the projection's shared stat reading -/// (notes, resources, and a moved resource's repoint all record the same -/// shape). `None` when the platform clock can't supply one. +/// A file's mtime as Unix seconds, or `None` when the platform can't supply one. pub(crate) fn unix_mtime(meta: &fs::Metadata) -> Option { let modified = meta.modified().ok()?; let since_epoch = modified.duration_since(std::time::UNIX_EPOCH).ok()?; Some(since_epoch.as_secs() as i64) } -/// What [`project_note_and_chunks`] returns for one note: the material later phases -/// need — the body for edge derivation, and the `(text_hash, text)` pairs still -/// needing a vector for the embed step. +/// One projected note: the body for edge derivation, and `(text_hash, text)` pairs still +/// needing a vector. struct ProjectedNote { body: String, relations: Vec, pending: Vec<(String, String)>, } -/// A [`plan_reindex`] preview (`reindex --dry-run`): what a real reindex *would* do, -/// decided read-only. Only one question is left to preview — a run that writes nothing -/// (ADR-0004) has nothing to warn about beyond the work it will do. +/// A [`plan_reindex`] preview (`reindex --dry-run`), decided read-only. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Plan { - /// Notes a real reindex would project (every readable `.md` file the walk collects). + /// Notes a real reindex would project. pub notes: usize, /// …of which this many would be (re)embedded (changed, fresh, or forced). pub would_embed: usize, } -/// When [`project_note_and_chunks`] re-cuts a note's chunks. Three callers, three -/// questions — which is why this is not a pair of booleans. +/// When [`project_note_and_chunks`] re-cuts a note's chunks. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Rechunk { - /// The projection pass: re-chunk a new or body-changed note, reading only `notes`, - /// never vector state ("unchanged body but missing vectors" is [`embed_vault`]'s - /// job, which keeps [`project_vault`] free of the vector tables). + /// The projection pass: re-chunk a new or body-changed note. Reads only `notes`, never + /// vector state (missing vectors are [`embed_vault`]'s job). IfBodyChanged, /// A forced projection (`reindex --force`): re-chunk every note. Always, - /// The inline path ([`ingest_file`]), which embeds what it re-cuts: also re-chunk a - /// note left mid-embed ([`would_reembed`]'s vector-state check), and hand back the - /// chunks still missing a vector. + /// The inline path ([`ingest_file`]): also re-chunk a note left mid-embed, and hand + /// back the chunks still missing a vector. IfBodyChangedOrUnembedded, } @@ -128,45 +101,31 @@ impl Rechunk { } } -/// A note body's content hash — the incremental pass's "did the body change?" key, -/// shared by the real projection and the dry-run so the forecast cannot drift from it. +/// The incremental "did the body change?" key, shared with the dry-run so they can't drift. fn body_hash(body: &str) -> String { blake3::hash(body.as_bytes()).to_hex().to_string() } -/// Progress during the embed phase, reported **per batch** so a large vault never -/// looks frozen while it embeds. Purely observational. -/// -/// The counts describe the notes that actually (re)embed this run, not every note: an -/// incremental reindex reuses most notes' vectors, so `notes_to_embed` is the real unit -/// of work. Reporting position in the full note list would jump to "note 14/18" while -/// only a handful are doing anything. `Serialize` so the desktop can stream it to the -/// webview; the field names are the JSON keys the frontend reads. +/// Embed-phase progress, reported per batch. Counts cover only the notes that (re)embed +/// this run, not the whole vault. Field names are the JSON keys the desktop frontend reads. #[derive(Debug, Clone, Serialize)] pub struct ReindexProgress { /// Vault-relative path of the note currently embedding. pub note_path: String, - /// Number of chunks in the current note (this file's own chunk count). + /// Chunks in the current note. pub note_chunks: usize, /// How many notes have begun embedding so far (1-based)… pub notes_embedded: usize, - /// …out of this many notes that need (re)embedding this run — the changed/fresh - /// notes (or every note under `force`), not the whole vault. + /// …out of this many notes that need (re)embedding this run. pub notes_to_embed: usize, /// Chunks embedded so far, cumulative across every note this run. pub chunks_done: usize, } -/// Project one note's frontmatter + chunks — everything derivable without resolving -/// links. Returns its body (kept so phase 2 derives edges without re-reading), its -/// frontmatter relations, and the `(text_hash, text)` pairs still needing a vector; -/// embedding is deferred. No embedder here, and **no write to the vault**. -/// -/// **Incremental:** unless [`Rechunk::Always`], a note whose body hash is unchanged is -/// left untouched and `pending` comes back empty; frontmatter-only edits still -/// re-project the note row and its edges. The invariant `incremental ≡ full rebuild` -/// holds because the re-used rows are byte-for-byte what a fresh projection would -/// produce. `pending` is only ever filled for [`Rechunk::IfBodyChangedOrUnembedded`]. +/// Project one note's row and chunks: everything derivable without resolving links. +/// Unless [`Rechunk::Always`], an unchanged body keeps its chunks (`incremental ≡ full +/// rebuild`: they are what a fresh projection would produce). `pending` is filled only for +/// [`Rechunk::IfBodyChangedOrUnembedded`]. fn project_note_and_chunks( ctx: ProjectionCtx, rel_path: &str, @@ -181,9 +140,8 @@ fn project_note_and_chunks( let body_hash = body_hash(&body); let mtime = fs::metadata(&abs).ok().as_ref().and_then(unix_mtime); - // Decide the re-chunk BEFORE the upsert overwrites `body_hash`. The inline path - // also reads vector state (its caller ensured the space, hence - // `space_exists = true`); the projection pass reads only `notes`. + // Decide before the upsert overwrites `body_hash`. The inline path's caller ensured + // the embedding space, hence `space_exists = true`. let rechunk = match when { Rechunk::Always => true, Rechunk::IfBodyChanged => { @@ -195,8 +153,7 @@ fn project_note_and_chunks( }; let fields = parsed.fields(); - // A note's display title is its **filename**; a frontmatter `title:` is inert. - // Projected here so every read path shows it with no per-call derivation. + // The display title is the filename; a frontmatter `title:` is inert. let title = note::display_title(rel_path); db::upsert_note( conn, @@ -211,18 +168,15 @@ fn project_note_and_chunks( let relations = fields.relations.clone(); - // Incremental fast path: an unchanged body means identical chunks — reuse them and - // return no pending work. A re-chunk hands back pairs only for chunks with **no - // stored vector**: the store is content-addressed (ADR-0006), so re-chunking a note - // whose text is unchanged — what a move produces — yields nothing pending at all. + // Vectors are content-addressed (ADR-0006), so re-chunking unchanged text (a move) + // yields nothing pending. let pending = if rechunk { let chunks = chunk_body(&body, cfg); db::replace_chunks(conn, rel_path, &chunks)?; if when == Rechunk::IfBodyChangedOrUnembedded { pending_for_note(conn, rel_path)? } else { - // The projection pass never reads vector state; the embed pass derives - // its own pending set from the DB. + // The embed pass derives its own pending set from the DB. Vec::new() } } else { @@ -236,10 +190,8 @@ fn project_note_and_chunks( }) } -/// The `(text_hash, text)` pairs of one note's chunks that still lack a vector — the -/// single-note form of [`db::chunks_missing_vectors`], for the inline path. Its own -/// query rather than a filter over the whole-vault set, because `move_dir` calls this -/// once per moved note. Requires the embedding space to exist. +/// One note's chunks still lacking a vector. Its own query, not a filter over the whole +/// vault, because `move_dir` calls it per note. Requires the embedding space. fn pending_for_note(conn: &Connection, note_path: &str) -> Result> { Ok(db::note_chunks_missing_vectors(conn, note_path)? .into_iter() @@ -247,13 +199,9 @@ fn pending_for_note(conn: &Connection, note_path: &str) -> Result Result<()> { - // Gather authored links: body first (inline), then frontmatter (frontmatter). let mut staged: Vec<(crate::link::ParsedLink, &'static str)> = Vec::new(); for link in crate::link::parse_links(body) { staged.push((link, "inline")); @@ -352,11 +281,9 @@ fn project_edges( } } - // The source note's directory — the base for a Markdown-form relative target, read - // straight off the path now that the path *is* the identity (ADR-0003). + // The base for a Markdown-form relative target. let src_dir = crate::pathspec::parent_dir(src_path); - // Resolve targets; record which (target, type) the frontmatter authors. let mut fm_keys: HashSet<(String, String)> = HashSet::new(); let mut resolved = Vec::with_capacity(staged.len()); for (link, origin) in staged { @@ -399,9 +326,8 @@ fn project_edges( db::replace_authored_edges(conn, src_path, &rows) } -/// Resolve one parsed link to `(dst_path, dst_resource_path)` — at most one is -/// `Some`; both `None` means dangling. The lookup path is the authored target -/// minus any `#fragment`; kind dispatch is extension-only (see [`project_edges`]). +/// Resolve a link to `(dst_path, dst_resource_path)`: at most one is `Some`, both `None` +/// means dangling. A `#fragment` is stripped for the lookup only. fn resolve_target( conn: &Connection, src_dir: &str, @@ -417,9 +343,8 @@ fn resolve_target( return Ok((None, None)); // fragment-only wikilink — dangling } - // Candidate paths, most specific first: a Markdown-form target is - // note-relative per standard Markdown, falling back to vault-root (the - // wikilink habit); wikilinks are vault-root only, as today. + // A Markdown-form target tries note-relative first, then vault-root; wikilinks are + // vault-root only. let mut candidates: Vec = Vec::with_capacity(2); if link.md_form { if let Some(joined) = crate::pathspec::join_relative(src_dir, lookup) { @@ -430,8 +355,7 @@ fn resolve_target( candidates.push(lookup.to_string()); } - // Extension-only kind dispatch, the one rule, shared with the adapters' argument - // dispatch: an extension other than `md` means resource; `.md` or none means note. + // Extension-only kind dispatch, shared with the adapters: non-`md` means resource. let is_resource = crate::resource::doc_kind(lookup) == crate::resource::DocKind::Resource; for candidate in &candidates { if is_resource { @@ -445,10 +369,8 @@ fn resolve_target( Ok((None, None)) } -/// Deterministic id for an authored edge from its identity tuple (ADR-0010): stable -/// across re-index, so the same body at the same path always yields the same id — and -/// deliberately *not* stable across a move, since both ends are paths. Edge ids live -/// only in the disposable index and are re-derived by a move's re-projection. +/// Deterministic edge id from its identity tuple (ADR-0010): stable across re-index, not +/// across a move, since both ends are paths. fn derive_edge_id(src_path: &str, target_key: &str, edge_type: &str, occurrence: i64) -> String { let mut h = blake3::Hasher::new(); for part in [src_path, target_key, edge_type] { @@ -459,19 +381,14 @@ fn derive_edge_id(src_path: &str, target_key: &str, edge_type: &str, occurrence: h.finalize().to_hex()[..32].to_string() } -/// Ingest a single note at `ctx.root/rel_path` against an already-built index -/// (the incremental path). Projects note + chunks + edges. The context's chunking -/// policy is the same one every other path uses, so a single-note re-projection cuts -/// identically to a full rebuild. +/// Ingest a single note against an already-built index: note, chunks, vectors, edges. pub fn ingest_file(ctx: EmbedCtx, rel_path: &str) -> Result<()> { let EmbedCtx { proj, embedder } = ctx; let conn = proj.conn; db::ensure_embedding_space(conn, embedder.model_id(), embedder.dim())?; - // Incremental (force=false): a frontmatter-only edit leaves the body unchanged, so - // this re-projects note + edges without re-embedding. Vector state IS consulted — - // this path embeds inline, so a note left mid-embed re-chunks and re-embeds here. + // A frontmatter-only edit re-projects without re-embedding; a note left mid-embed + // re-chunks and re-embeds here. let p = project_note_and_chunks(proj, rel_path, Rechunk::IfBodyChangedOrUnembedded)?; - // A single-note re-projection is never cancelled — always run to completion. embed_pending(conn, embedder, rel_path, &p.pending, |_| { ControlFlow::Continue(()) })?; @@ -479,10 +396,8 @@ pub fn ingest_file(ctx: EmbedCtx, rel_path: &str) -> Result<()> { Ok(()) } -/// Reindex every `.md` file under `vault_root` — [`project_vault`] then [`embed_vault`], -/// incrementally and with no progress reporting; dot-folders are skipped. A convenience -/// wrapper the test suite drives: it builds the [`EmbedCtx`] itself around the default -/// [`ChunkConfig`], which is why it takes loose arguments. +/// [`project_vault`] then [`embed_vault`] with the default [`ChunkConfig`] and no +/// progress. A convenience for the test suite. pub fn ingest_vault(conn: &Connection, vault_root: &Path, embedder: &dyn Embedder) -> Result<()> { let cfg = ChunkConfig::default(); project_vault(ProjectionCtx::new(conn, vault_root, &cfg), false)?; @@ -490,28 +405,21 @@ pub fn ingest_vault(conn: &Connection, vault_root: &Path, embedder: &dyn Embedde Ok(()) } -/// Re-project a single note **model-free** — the single-note sibling of -/// [`project_vault`], and the pass `Vault::write` runs after its body splice. A changed -/// body re-chunks, and the chunks join the DB-derived pending set for any later embed -/// pass, so the save path needs no embedder. Contrast [`ingest_file`], which embeds -/// inline for `add`/`link`/`mv` — ops that already require the model. +/// Re-project a single note model-free (what `Vault::write` runs). New chunks join the +/// DB-derived pending set for a later embed pass, so saving needs no embedder. pub fn project_file(ctx: ProjectionCtx, rel_path: &str) -> Result<()> { let p = project_note_and_chunks(ctx, rel_path, Rechunk::IfBodyChanged)?; project_edges(ctx.conn, rel_path, &p.body, &p.relations) } -/// A vault file (note **or** resource) the projection pass could not read, and -/// therefore skipped, so one unreadable file never aborts a whole-vault reindex. -/// `reason` is about the *file itself* ("not valid UTF-8 text", "permission denied"), -/// never a B2 internal, so it is safe to show and to log. Only a *filesystem* failure -/// on one file is recoverable this way; a systemic error still aborts the pass. +/// A vault file the projection pass could not read and skipped. `reason` describes the +/// file, never a B2 internal, so it is safe to show. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct SkippedNote { pub path: String, pub reason: String, } -/// A file the walk met but could not read, as the pass reports it. fn skipped(path: &str, err: &std::io::Error) -> SkippedNote { SkippedNote { path: path.to_string(), @@ -519,10 +427,7 @@ fn skipped(path: &str, err: &std::io::Error) -> SkippedNote { } } -/// Classify an I/O error hit while reading one file into a short, clean, -/// user-appropriate reason — the file's problem stated plainly, with no raw OS jargon -/// (e.g. "stream did not contain valid UTF-8") and no B2 internal. Anything unusual -/// falls back to a generic "could not be read". +/// A short, user-facing reason for an I/O error on one file, free of OS jargon. fn skip_reason(err: &std::io::Error) -> String { use std::io::ErrorKind; match err.kind() { @@ -533,41 +438,32 @@ fn skip_reason(err: &std::io::Error) -> String { } } -/// The result of the model-free **projection pass** over the whole vault -/// ([`project_vault`]): every projected note's path, in the deterministic (sorted-path) -/// walk order, plus any files skipped as unreadable (empty on a clean vault). +/// The result of [`project_vault`]: projected note paths in sorted walk order, plus +/// files skipped as unreadable. #[derive(Debug, Clone)] pub struct ProjectOutcome { pub notes: Vec, pub skipped: Vec, - /// Ghost rows pruned this pass (#31) — notes whose files were deleted outside - /// b2 with no replacement. Zero on a vault with no out-of-band deletions. + /// Rows pruned for notes deleted outside b2 (#31). pub notes_pruned: usize, - /// Resources inventoried this pass (unchanged ones included), and stale - /// inventory rows pruned. + /// Resources inventoried (unchanged included), and stale rows pruned. pub resources_indexed: usize, pub resources_pruned: usize, } -/// The result of a (possibly cancelled) **embed pass** ([`embed_vault`]): which -/// notes fully embedded this run, and whether a cooperative cancel cut it short. +/// The result of a (possibly cancelled) [`embed_vault`]. #[derive(Debug, Clone)] pub struct EmbedOutcome { - /// Paths of the notes that fully embedded this run, in the order they were - /// worked (path order). Notes whose vectors were already complete do no work and - /// are not listed. + /// Notes that fully embedded this run, in path order. Already-complete notes are + /// not listed. pub embedded: Vec, /// The pass stopped early because `on_progress` returned [`ControlFlow::Break`]. pub cancelled: bool, } -/// The **projection pass**: project every `.md` file under `ctx.root` — phase 1 -/// (note, chunks, FTS) then phase 2 (the typed edges) — with **no embedder and no embedding -/// space**, so a projected-but-unembedded index is already complete for keyword search -/// and the graph. Incremental: unless `force`, a note is re-chunked only when its body -/// changed or it is new, read purely from `notes` and never from vector state (missing -/// vectors are [`embed_vault`]'s job), so `project(force)` then `embed()` is the full -/// rebuild. It writes nothing to the vault (ADR-0004). +/// The projection pass: notes, chunks and FTS, then edges, with no embedder, so an +/// unembedded index already serves keyword search and the graph. Unless `force`, only new +/// or body-changed notes re-chunk. Writes nothing to the vault (ADR-0004). pub fn project_vault(ctx: ProjectionCtx, force: bool) -> Result { let ProjectionCtx { conn, root, .. } = ctx; let VaultWalk { @@ -575,29 +471,21 @@ pub fn project_vault(ctx: ProjectionCtx, force: bool) -> Result resources: resource_files, } = walk_vault(root)?; - // Phase 1: project every note + its chunks, which fills the resolver so phase 2 - // never depends on file order. No pending pairs come back — the embed pass derives - // its work from the DB, so nothing is handed over in memory. + // Phase 1: notes and chunks, filling the resolver so phase 2 is order-independent. let mut staged = Vec::with_capacity(rel_paths.len()); let mut skipped = Vec::new(); for rel in &rel_paths { match project_note_and_chunks(ctx, rel, Rechunk::forced(force)) { Ok(p) => staged.push((rel.clone(), p.body, p.relations)), - // A note we cannot read is *skipped*, not fatal: one bad file must never - // abort a whole-vault reindex. Only filesystem failures reading THIS note - // land here — the DB layer's `Error::Sqlite` is systemic and still aborts. - // The read fails before any upsert, so no partial row is written. + // An unreadable note is skipped, not fatal; a DB error still aborts. The read + // fails before any upsert, so no partial row is written. Err(Error::Io(e)) => skipped.push(self::skipped(rel, &e)), Err(other) => return Err(other), } } - // Deletion reconciliation (#31): prune the rows of notes whose files are gone, so - // an incremental reindex converges on what a from-scratch rebuild would hold instead - // of serving ghosts. "Gone" means "the walk did not meet this path" — a file it saw - // but could not read is kept, since evicting it would lie. Runs before phase 2 so - // links at a deleted note re-dangle, exactly as a full rebuild resolves them. The - // single-note paths touch one note and never prune. + // Prune notes the walk did not meet (#31); a seen-but-unreadable file is kept. Before + // phase 2 so links at a deleted note re-dangle, as in a full rebuild. let seen: HashSet<&str> = staged .iter() .map(|(path, ..)| path.as_str()) @@ -605,22 +493,18 @@ pub fn project_vault(ctx: ProjectionCtx, force: bool) -> Result .collect(); let notes_pruned = db::prune_notes_except(conn, &seen)?; - // Vectors are content-addressed (ADR-0006), so they do NOT die with the chunk rows - // just removed — that survival is what lets a moved note re-use them. Collecting - // what is now unreferenced belongs here, where the run's chunk set is final, and is - // guarded so the model-free pass never touches a never-embedded vault. + // Content-addressed vectors (ADR-0006) outlive their chunk rows; collect the + // unreferenced ones now the chunk set is final. if db::embedding_space_exists(conn)? { db::prune_orphan_vectors(conn)?; } - // Resource inventory — between the phases so the rows exist before phase 2 - // resolves links (a `![[img.png]]` edge resolves against `resources`, spec §3). + // Resources before phase 2, which resolves `![[img.png]]` against them (spec §3). let (resources_indexed, resources_pruned, mut resource_skips) = project_resources(conn, root, &resource_files)?; skipped.append(&mut resource_skips); - // Phase 2: edges, resolved against the now-complete resolver. A skipped note has no - // rows and no edges; a link pointing at it stays unresolved, as for any absent target. + // Phase 2: edges. A link at a skipped note stays unresolved. let mut notes = Vec::with_capacity(staged.len()); for (path, body, relations) in staged { project_edges(conn, &path, &body, &relations)?; @@ -645,14 +529,9 @@ pub fn project_vault(ctx: ProjectionCtx, force: bool) -> Result }) } -/// The **embed pass**: fill a vector for every chunk that lacks one. Ensures the -/// embedding space first (a model swap drops and resets it, so *all* chunks then count -/// as missing), then works the DB-derived pending set note by note through the batched -/// [`embed_pending`] loop, firing `on_progress` per batch and honoring its -/// [`ControlFlow::Break`] as the cancel checkpoint. Takes **no `force`**: re-chunking is -/// a projection concern, so this pass is purely "fill what's missing" — which is also -/// why any interruption heals on the next call. Pending notes are counted before any -/// work starts, so progress is determinate from the first batch. +/// The embed pass: fill a vector for every chunk lacking one (after a model swap, all of +/// them). `on_progress` fires per batch; [`ControlFlow::Break`] cancels. No `force`: it only +/// fills what's missing, so an interruption heals on the next call. pub fn embed_vault( conn: &Connection, embedder: &dyn Embedder, @@ -660,13 +539,9 @@ pub fn embed_vault( ) -> Result { db::ensure_embedding_space(conn, embedder.model_id(), embedder.dim())?; - // Group the (path, seq)-ordered pending chunks by note; consecutive rows share a - // note, so per-note batching and progress reproduce the fused reindex's shape. - // - // A hash is embedded **once per run** however many notes hold that text: the store - // is content-addressed (ADR-0006). Notes are worked in order and each batch is - // written before the next note starts, so a de-duplicated note finds its vectors - // already in the table, completes immediately, and still refreshes its centroid. + // Group pending chunks by note. Each hash is embedded once per run (ADR-0006); a note + // whose chunks were all claimed earlier finds its vectors stored and still refreshes + // its centroid. type PendingNote = (String, Vec<(String, String)>); let mut by_note: Vec = Vec::new(); let mut claimed: HashSet = HashSet::new(); @@ -690,14 +565,13 @@ pub fn embed_vault( let mut chunks_done = 0usize; let mut cancelled = false; for (i, (path, pending)) in by_note.iter().enumerate() { - // Per-note span: under a span-close subscriber each note reports how long - // its embed took — the kernel's slowest step, hence the one worth plotting. + // Per-note span, for timing the slowest step. let _note_span = tracing::debug_span!( target: "b2::ingest", "embed_note", path = path.as_str(), chunks = pending.len() ) .entered(); - let notes_embedded = i + 1; // 1-based position for the progress line + let notes_embedded = i + 1; // 1-based let note_chunks = pending.len(); let outcome = embed_pending(conn, embedder, path, pending, |n| { chunks_done += n; @@ -714,14 +588,12 @@ pub fn embed_vault( } if outcome.cancelled { cancelled = true; - break; // cooperative cancel: stop starting new notes + break; } } - // The centroid half of the pass. A note re-cut but textually unchanged has every - // vector already stored, so it never enters the loop above — yet `replace_chunks` - // dropped the centroid summarizing its old chunks. Left unrefreshed it would vanish - // from discovery's coarse scan while looking fully indexed (S3). Runs after a cancel - // too: the query offers only *fully* embedded notes. + // A re-cut but unchanged note skips the loop above, yet `replace_chunks` dropped its + // centroid; without one it vanishes from discovery (S3). Safe after a cancel: only + // fully embedded notes are offered. for note_path in db::notes_missing_centroids(conn)? { db::refresh_note_centroid(conn, ¬e_path)?; } @@ -738,34 +610,22 @@ pub fn embed_vault( }) } -/// A **read-only** preview of a reindex (`reindex --dry-run`). Walks every `.md` file -/// in the same order as [`project_vault`] and decides, per note, whether a real run would -/// (re)embed its body. That is the *whole* preview, and the shrinkage is the feature: -/// the dry-run existed largely because a real run wrote to the vault, and one that -/// writes nothing (ADR-0004) has only work to forecast. -/// -/// The embed decision reads the *currently stored* vectors, so it previews an -/// incremental run under the embedder the index was built with; it does **not** detect a -/// pending model swap, which would need the real model a dry-run avoids loading. +/// Read-only preview of a reindex (`reindex --dry-run`): per note, would a real run +/// (re)embed it? Reads the stored vectors, so it does not detect a pending model swap. pub fn plan_reindex(conn: &Connection, vault_root: &Path, force: bool) -> Result { let space_exists = db::embedding_space_exists(conn)?; - // The dry-run previews *notes* (the embed decision); the resource inventory has - // no per-file decisions to preview, so its half of the walk is unused here. let rel_paths = walk_vault(vault_root)?.notes; let mut plan = Plan { notes: 0, would_embed: 0, }; for rel in rel_paths { - // Skip an unreadable file rather than abort — a real reindex would skip it too, - // so the dry-run must not be the one place it still crashes the run. + // A real reindex skips an unreadable file too. let raw = match fs::read_to_string(vault_root.join(&rel)) { Ok(raw) => raw, Err(_) => continue, }; let body_hash = body_hash(note::parse(&raw).body()); - // An unindexed path has no stored body hash, so this reads `true` for a note - // new to the index — exactly as the real run decides it. plan.notes += 1; if would_reembed(conn, &rel, &body_hash, force, space_exists)? { plan.would_embed += 1; @@ -774,8 +634,7 @@ pub fn plan_reindex(conn: &Connection, vault_root: &Path, force: bool) -> Result Ok(plan) } -/// What one vault walk found: every note path and every resource with its class, each -/// list sorted so the passes that share the walk run in one deterministic order. +/// What one vault walk found, each list sorted for a deterministic order. struct VaultWalk { notes: Vec, resources: Vec<(String, ResourceClass)>, @@ -791,14 +650,9 @@ fn walk_vault(root: &Path) -> Result { Ok(VaultWalk { notes, resources }) } -/// Walk the vault once, routing every file: `.md` (case-insensitive) to `notes`, -/// everything else to `resources` with its class — the `index = projection of the vault -/// directory` walk (ADR-0002). -/// -/// **Hidden means hidden** (GH #136): a dot-prefixed entry is not vault material, so -/// [`is_hidden`](crate::pathspec::is_hidden) is applied *above* the note/resource -/// dispatch and the recursion alike. A `.DS_Store` and a `.scratch.md` are equally -/// invisible; the files stay on disk untouched, simply outside the projection. +/// Walk the vault, routing `.md` (case-insensitive) to `notes` and everything else to +/// `resources` (ADR-0002). Dot-prefixed entries, files and folders alike, are skipped +/// (GH #136). fn collect_vault_files( root: &Path, dir: &Path, @@ -815,8 +669,7 @@ fn collect_vault_files( collect_vault_files(root, &path, notes, resources)?; continue; } - // `path` was produced by walking `root`, so `strip_prefix` cannot fail; - // handle it gracefully anyway rather than panic on the invariant. + // Cannot fail under `root`; skip rather than panic. let Ok(rel) = path.strip_prefix(root) else { continue; }; @@ -829,13 +682,9 @@ fn collect_vault_files( Ok(()) } -/// The **resource inventory pass**: stat every walked resource, short-circuit on an -/// unchanged `(size, mtime)`, otherwise read the bytes once to blake3 them and upsert; -/// then prune the rows the walk no longer saw (inbound edges re-dangle via the schema's -/// `ON DELETE SET NULL`). Model-free and chunk-free — hashing is the only byte-read. An -/// unreadable file is skipped and any prior row survives: the file was seen on disk, so -/// pruning it would lie. Returns `(indexed, pruned, skipped)`, `indexed` counting -/// unchanged resources too. +/// The resource inventory pass: inventory every walked resource, then prune rows the walk +/// no longer saw (inbound edges re-dangle via `ON DELETE SET NULL`). An unreadable file is +/// skipped and its prior row kept. Returns `(indexed, pruned, skipped)`. fn project_resources( conn: &Connection, vault_root: &Path, @@ -845,13 +694,11 @@ fn project_resources( let mut seen: HashSet = HashSet::with_capacity(resources.len()); let mut indexed = 0; for (rel, class) in resources { - // The walk saw the file, so it exists: it is never pruned this pass, even - // if reading it fails below. + // Seen, so never pruned, even if reading it fails. seen.insert(rel.clone()); match project_resource_file(conn, vault_root, rel, *class, false) { Ok(()) => indexed += 1, - // Only a *filesystem* failure on this one file is recoverable — anything - // else (SQLite, …) is systemic and aborts the pass, as everywhere else. + // Only a filesystem failure is recoverable; anything else aborts. Err(Error::Io(e)) => skipped.push(self::skipped(rel, &e)), Err(e) => return Err(e), } @@ -860,19 +707,9 @@ fn project_resources( Ok((indexed, pruned, skipped)) } -/// Inventory **one** resource: short-circuit on an unchanged `(size, mtime)`, otherwise -/// read the bytes once to blake3 them and upsert. The per-file kernel -/// [`project_resources`] loops over, and the resource arm of an import. -/// -/// `force` skips that short-circuit, and the two callers differ on it because they know -/// different things. The **walk** meets files it has seen before, so an unchanged stat -/// means "the row already describes this" — the optimization that keeps a reindex from -/// re-reading every PDF. An **import** just created the file, so any row at that path is -/// about a *different* file, and trusting a matching stat would keep a `content_hash` -/// for bytes that are gone. -/// -/// I/O failures travel as [`Error::Io`] and each caller decides what they mean: the walk -/// classifies one into a skip, the import treats it as the failure it is. +/// Inventory one resource: skip on an unchanged `(size, mtime)`, else blake3 and upsert. +/// `force` bypasses the stat check: an import just created the file, so an existing row +/// describes different bytes. I/O errors return as [`Error::Io`] for the caller to judge. pub(crate) fn project_resource_file( conn: &Connection, vault_root: &Path, @@ -885,7 +722,7 @@ pub(crate) fn project_resource_file( let size = meta.len() as i64; let mtime = unix_mtime(&meta); if !force && db::resource_stat(conn, rel)? == Some((size, mtime)) { - return Ok(()); // unchanged — inventoried without touching the bytes + return Ok(()); // unchanged } let bytes = fs::read(&abs)?; let content_hash = blake3::hash(&bytes).to_hex().to_string(); @@ -906,13 +743,8 @@ pub(crate) fn project_resource_file( mod tests { use super::*; - /// The `(size, mtime)` shortcut is the **walk's**, and an import must not inherit - /// it: a row can outlive the file it describes, so a same-size replacement landing - /// inside the same second would keep a `content_hash` for bytes that are gone — and - /// that hash is what the out-of-band move repair matches a dangling link against. - /// - /// The stat equality is **constructed**, not raced for: `set_modified` pins the - /// replacement's mtime to the original's, so the case under test runs every time. + /// A same-size replacement within the same second must not keep a stale `content_hash` + /// on import (the move repair matches on it). `set_modified` pins the mtime. #[test] fn the_unchanged_stat_shortcut_is_the_walks_alone() { let tmp = tempfile::TempDir::new().unwrap(); @@ -934,8 +766,7 @@ mod tests { let first = hash(&conn); let stamped = fs::metadata(&abs).unwrap().modified().unwrap(); - // Different bytes, same length, same mtime — the one state a stat cannot tell - // apart from "nothing happened". + // Different bytes, same length, same mtime. fs::write(&abs, b"BBBB").unwrap(); fs::File::options() .write(true) diff --git a/crates/b2-core/src/link.rs b/crates/b2-core/src/link.rs index 3c23ae4..cb69511 100644 --- a/crates/b2-core/src/link.rs +++ b/crates/b2-core/src/link.rs @@ -1,23 +1,14 @@ -//! Parse a note body into the links that become edges — B2's **one link grammar**. +//! B2's one link grammar: parse a note body into the links that become edges. //! -//! **The body carries no B2 syntax** (ADR-0010): every body construct is ordinary -//! Obsidian Markdown and yields an untyped `references` edge — prose around a link is just -//! prose. Three body constructs: a bare `[[path|alias]]`; Markdown's own `[text](path)` / -//! `![alt](path)` (relative vault targets only, the `!` marking an **embed** and the -//! text/alt captured as the edge's **caption**); and the `![[file.ext|alias]]` embed. A -//! *typed* edge — ` [[path|alias]] — explanation` — exists only as a frontmatter -//! `b2_relations:` entry, parsed by [`parse_relation`] and written by [`render_relation`]. +//! The body carries no B2 syntax (ADR-0010): `[[path|alias]]`, `![[file|alias]]`, +//! `[text](path)` and `![alt](path)` all yield untyped `references` edges. A typed edge +//! (` [[path]] — explanation`) exists only in frontmatter `b2_relations:` +//! ([`parse_relation`], [`render_relation`]). //! -//! Everything that reads link text goes through [`link_spans`]: ingest projects edges from -//! it ([`parse_links`]) and a move rewrites targets in place from it (`mv.rs`), so what a -//! move repairs is exactly what ingest projected. The scan is **per line**: a stray `[[` -//! never pairs with a `]]` on a later line. -//! -//! Hand-rolled and deliberately minimal. Known simplifications, to revisit when queries -//! need them: a typed frontmatter entry yields exactly one edge (extra wikilinks in its -//! trailing text read as explanation); links inside code spans/fences are not excluded; -//! only `—`/`:` introduce an explanation; a Markdown link's text stops at the first `]` -//! and its target at the first `)`. +//! Ingest and `mv.rs` both read links via [`link_spans`], so a move repairs exactly what +//! ingest projected. The scan is per line. Known simplifications: links in code are not +//! excluded; only `—`/`:` introduce an explanation; a Markdown link's text stops at the +//! first `]` and its target at the first `)`. use std::ops::Range; @@ -26,21 +17,15 @@ use std::ops::Range; pub struct ParsedLink { /// `references` for a bare link, otherwise the relation verb. pub edge_type: String, - /// The target as written — `[[path|alias]]`'s path part or `[…](path)`'s - /// parenthesized target (becomes `dst_path_raw`; a `#fragment` suffix is - /// stripped at *resolution*, never here). + /// The target as written (`dst_path_raw`); a `#fragment` is stripped at resolution. pub target_path: String, /// Trailing text after `—`/`:` on a typed frontmatter entry. pub explanation: Option, - /// True for an embed form (`![alt](path)` / `![[file]]`) — recorded on the - /// edge as a display nicety, never a distinct verb (data-model.md §3). + /// An embed form (`![…]`): a display nicety, never a distinct verb (data-model.md §3). pub embed: bool, - /// True when the target came from a Markdown-form link (`[…](target)`). - /// Resolution treats these with standard Markdown semantics — note-relative - /// first, then vault-root — while wikilink targets stay vault-root. + /// A Markdown-form link, resolved note-relative first, then vault-root. pub md_form: bool, - /// The authored display text — `![alt](…)`'s alt, `[text](…)`'s text, or a - /// wikilink's alias. Stored as the edge's `caption` (data-model.md §3). + /// The alt, link text or alias; the edge's `caption` (data-model.md §3). pub caption: Option, } @@ -53,25 +38,20 @@ pub(crate) enum LinkForm { Markdown, } -/// Where one link sits in the scanned text, as byte ranges into it — so the one grammar -/// serves both reading a link ([`parse_links`]) and splicing a new target into it in -/// place (the move's rewrite). +/// Where one link sits in the text, as byte ranges, so a move can splice in place. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct LinkSpan { pub form: LinkForm, /// A `!` directly before the opening bracket. pub embed: bool, - /// The target token, trimmed: `concepts/memory` in `[[ concepts/memory | M ]]`, - /// `img.png` in `![a]( img.png )`. Never empty. + /// The target token, trimmed. Never empty. pub target: Range, - /// The display text, trimmed: a wikilink's alias (present — possibly empty — whenever - /// the link has a `|`), or a Markdown link's text (absent when blank). + /// Trimmed: a wikilink's alias (present, maybe empty, when there is a `|`), or a + /// Markdown link's text (absent when blank). pub caption: Option>, } -/// Every vault link in `text`, in document order, scanned line by line. A Markdown-form -/// link at an external target (a scheme, an absolute path, a fragment-only anchor) is -/// not a vault link and yields no span; nor does a wikilink with an empty target. +/// Every vault link in `text`, in document order, scanned line by line. pub(crate) fn link_spans(text: &str) -> Vec { let mut spans = Vec::new(); let mut line_start = 0; @@ -86,8 +66,7 @@ pub(crate) fn link_spans(text: &str) -> Vec { spans } -/// Parse every link in `body`, in document order — all untyped `references` -/// edges: the body carries no typed syntax (data-model §2). +/// Every link in `body` as an untyped `references` edge (data-model §2). pub fn parse_links(body: &str) -> Vec { link_spans(body) .iter() @@ -107,11 +86,9 @@ fn reference(text: &str, span: &LinkSpan) -> ParsedLink { } } -/// Parse a typed spec ` [[path|alias]] [— explanation]` — the shape of a -/// frontmatter `b2_relations:` entry (data-model §2). `None` if it isn't -/// ` `. +/// Parse a typed spec ` [[path|alias]] [— explanation]` (data-model §2). fn parse_typed_spec(rest: &str) -> Option { - // The verb: a lowercase-kebab token immediately before the wikilink. + // A lowercase-kebab verb. let verb_end = rest.find(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')); let verb_end = match verb_end { Some(0) | None => return None, // no verb (e.g. "[[..]]") or no following token @@ -122,7 +99,6 @@ fn parse_typed_spec(rest: &str) -> Option { return None; } - // The wikilink must follow the verb directly (whitespace allowed). let after_verb = rest[verb_end..].trim_start(); let inner_and_rest = after_verb.strip_prefix("[[")?; let close = inner_and_rest.find("]]")?; @@ -143,27 +119,21 @@ fn parse_typed_spec(rest: &str) -> Option { }) } -/// Parse one frontmatter `b2_relations:` entry (the string value of one YAML -/// list item): a typed spec ` [[path|alias]] — …`, or a bare -/// `[[path|alias]]` ⇒ `references`. `None` if it holds no wikilink. The caller -/// assigns `origin=frontmatter`. This is the **only** parser that yields a verb -/// or explanation — the body never does (data-model §2). +/// Parse one `b2_relations:` entry: a typed spec, or a bare `[[…]]` ⇒ `references`. The +/// only parser that yields a verb or explanation (data-model §2). pub fn parse_relation(spec: &str) -> Option { let spec = spec.trim(); if let Some(link) = parse_typed_spec(spec) { return Some(link); } - // bare link fallback → references (the entry is one YAML value, scanned whole) + // Bare link → references. let mut spans = Vec::new(); scan_line(spec, 0, &mut spans); spans.first().map(|span| reference(spec, span)) } -/// Render a typed frontmatter `b2_relations:` entry — the inverse of [`parse_relation`] -/// for a typed spec: ` [[target_path]]`, then ` — explanation` when one is given -/// (data-model §2). `target_path` is written verbatim as the link text, so the caller -/// chooses the convention (a note's `.md` dropped, as Obsidian writes links). A blank -/// explanation is omitted and a given one trimmed, exactly as parsing reads it back. +/// Render a typed `b2_relations:` entry, the inverse of [`parse_relation`] (data-model +/// §2). `target_path` is written verbatim; a blank explanation is omitted. pub fn render_relation(verb: &str, target_path: &str, explanation: Option<&str>) -> String { match explanation.map(str::trim).filter(|e| !e.is_empty()) { Some(e) => format!("{verb} [[{target_path}]] — {e}"), @@ -171,11 +141,8 @@ pub fn render_relation(verb: &str, target_path: &str, explanation: Option<&str>) } } -/// Collect every link in `line` as a [`LinkSpan`] offset by `base`, in written order: -/// `[[path|alias]]` and `![[path|alias]]` (wikilink + embed), and Markdown's own -/// `[text](target)` / `![alt](target)` — the latter only for **vault** targets -/// (a scheme, an absolute path, or a fragment-only target is not a vault member -/// and yields nothing; data-model.md §10). +/// Collect every link in `line` as a [`LinkSpan`] offset by `base`. Markdown-form links +/// count only for vault targets (data-model.md §10). fn scan_line(line: &str, base: usize, out: &mut Vec) { let mut i = 0; while i < line.len() { @@ -185,7 +152,7 @@ fn scan_line(line: &str, base: usize, out: &mut Vec) { Some(r) if r.starts_with('[') => (true, r), _ if rest.starts_with('[') => (false, rest), _ => { - // Not a link start — skip one char (multi-byte safe). + // Skip one char, multi-byte safe. i += rest.chars().next().map_or(1, char::len_utf8); continue; } @@ -241,8 +208,7 @@ struct MdLink { consumed: usize, } -/// Parse a leading `[text](target)`. Minimal by design (module doc): text stops at the -/// first `]`, the target at the first `)`, and `](` must be adjacent. +/// Parse a leading `[text](target)`. Minimal by design (module doc); `](` must be adjacent. fn parse_md_link(s: &str) -> Option { let inner = s.strip_prefix('[')?; let close = inner.find(']')?; @@ -261,8 +227,7 @@ fn parse_md_link(s: &str) -> Option { }) } -/// A wikilink's inner text (between `[[` and `]]`, starting at byte `base`) split at the -/// first `|` into the trimmed target and, when a `|` is present, the trimmed alias. +/// Split a wikilink's inner text (at byte `base`) at the first `|` into target and alias. fn split_wiki_inner(inner: &str, base: usize) -> (Range, Option>) { match inner.split_once('|') { Some((path, alias)) => ( @@ -279,15 +244,11 @@ fn trimmed(s: &str, base: usize) -> Range { start..start + s.trim().len() } -/// `r` moved `by` bytes to the right. fn shift(r: &Range, by: usize) -> Range { r.start + by..r.end + by } -/// A Markdown-form target that is **not** a vault member: any scheme -/// (`https://…`, `mailto:…`), an absolute path, or a fragment-only anchor. -/// Wikilink targets never come through here — they are vault paths by -/// construction. +/// A Markdown-form target outside the vault: a scheme, an absolute path, or `#anchor`. fn is_external_target(target: &str) -> bool { if target.starts_with('/') || target.starts_with('#') { return true; @@ -298,8 +259,7 @@ fn is_external_target(target: &str) -> bool { } } -/// Read the explanation after a typed link: trailing text introduced by an -/// em-dash or a colon (data-model §2). Anything else means no explanation. +/// The explanation after a typed link, introduced by `—` or `:` (data-model §2). fn extract_explanation(tail: &str) -> Option { let t = tail.trim_start(); let body = t.strip_prefix('—').or_else(|| t.strip_prefix(':'))?; @@ -311,10 +271,9 @@ fn extract_explanation(tail: &str) -> Option { mod tests { use super::*; - /// One expected link, as the case tables spell it: `(type, target, caption, embed)`. + /// `(type, target, caption, embed)`. type Expected<'a> = (&'a str, &'a str, Option<&'a str>, bool); - /// Shorthand: parse one line, return `(type, target, caption, embed)`. fn parsed(line: &str) -> Vec<(String, String, Option, bool)> { parse_links(line) .into_iter() @@ -407,8 +366,7 @@ mod tests { #[test] fn a_verb_shaped_body_list_item_is_just_a_reference() { - // The body carries no typed syntax (data-model §2): a list item opening - // with a verb-looking word is prose, and only its wikilink projects. + // The body carries no typed syntax (data-model §2). let links = parse_links("- supports [[papers/x.pdf|the paper]] — key evidence"); assert_eq!(links.len(), 1); let l = &links[0]; @@ -419,10 +377,7 @@ mod tests { assert!(!l.embed); } - /// The exact hazard that killed the body typed-line syntax (decision - /// 2026-07-21): a *lowercase* verb lookalike opening a list item must stay - /// prose. `- see [[x]]` becoming a typed edge of verb "see" is the failure - /// mode; only the wikilink may project, always untyped. + /// The hazard that retired body typed-line syntax (decision 2026-07-21). #[test] fn lowercase_verb_lookalikes_in_prose_stay_prose() { let links = parse_links("- see [[concepts/memory|Human memory]] for the mechanism\n"); @@ -430,9 +385,6 @@ mod tests { assert_eq!(links[0].edge_type, "references"); } - /// A prose link and a verb-led list item are the same thing to the parser: two - /// `references` edges in document order, neither carrying an explanation — no - /// body shape is ever "special". #[test] fn body_links_never_gain_a_type_from_surrounding_prose() { let body = "Spaced repetition exploits the [[concepts/memory|Human memory]] retrieval curve.\n\n## Relations\n- supports [[concepts/memory|Human memory]] — applies the forgetting curve\n"; @@ -442,8 +394,6 @@ mod tests { assert!(links.iter().all(|l| l.explanation.is_none())); } - /// A wikilink's caption is its own `|`-part: absent means `None`, never an empty - /// string. #[test] fn a_wikilink_without_an_alias_has_no_caption() { let links = parse_links("Refer to [[concepts/memory]].\n"); @@ -459,18 +409,13 @@ mod tests { assert_eq!(l.caption.as_deref(), Some("the paper")); assert_eq!(l.explanation.as_deref(), Some("key evidence")); - // A bare entry (no verb) falls back to an untyped reference. let bare = parse_relation("[[notes/a|A]]").unwrap(); assert_eq!(bare.edge_type, "references"); assert_eq!(bare.target_path, "notes/a"); - // No wikilink at all ⇒ no edge. assert!(parse_relation("just some words").is_none()); } - /// The two accepted explanation separators and the tolerated verb tail - /// (relation.rs): `—` is asserted above, `:` here, and a non-core verb is - /// stored verbatim rather than coerced into the closed core. #[test] fn relation_accepts_a_colon_separator_and_keeps_a_tail_verb_verbatim() { let colon = @@ -483,9 +428,6 @@ mod tests { assert_eq!(tail.explanation, None); } - /// What `b2 link` writes is what ingest reads back: rendering then parsing a - /// relation returns the verb, target and explanation it was rendered from — and a - /// blank explanation renders as none at all. #[test] fn a_rendered_relation_parses_back_to_what_it_was_rendered_from() { let cases: &[(&str, &str, Option<&str>)] = &[ @@ -530,8 +472,7 @@ mod tests { false )] ); - // same-line recovery: the link is still found (its caption may swallow - // the stray bracket — the documented first-`]` minimalism) + // same-line recovery (its caption may swallow the stray bracket) let got = parsed("broken [[x then fine [ok](a.png)"); assert_eq!(got.len(), 1); assert_eq!(got[0].1, "a.png"); @@ -541,8 +482,6 @@ mod tests { assert_eq!(parsed("hey! [x](a.png)").len(), 1); } - /// The spans are what a move splices into, so their ranges must name exactly the - /// trimmed target (and caption) bytes, offset across lines — CRLF endings included. #[test] fn spans_address_the_trimmed_target_bytes_across_lines() { let text = "a [[ x/y | Why ]] b\r\n![alt]( img.png ) and [t](https://e.x)\n[[z]]"; @@ -568,8 +507,6 @@ mod tests { ); } - /// A stray `[[` never pairs with a `]]` on a later line, so the later line's link - /// is found — by ingest and, through the same spans, by a move's rewrite. #[test] fn a_stray_open_bracket_never_swallows_the_next_lines_link() { let text = "broken [[x\nsee [[old]]\n"; diff --git a/crates/b2-core/src/llm.rs b/crates/b2-core/src/llm.rs index dbb391b..c7d52bc 100644 --- a/crates/b2-core/src/llm.rs +++ b/crates/b2-core/src/llm.rs @@ -1,26 +1,16 @@ -//! The chat seam — `LlmProvider`, the second enumerated AI seam (ADR-0005). Sibling of -//! [`crate::embed`]: the engine is built and tested against the deterministic [`FakeLlm`], -//! and `b2-llm`'s real provider drops in through the same trait with no schema or flow -//! change. +//! The chat seam, `LlmProvider` (ADR-0005). The engine is tested against [`FakeLlm`]; +//! `b2-llm`'s real provider drops in through the same trait. //! -//! Two deliberate contrasts with the embedder seam: -//! -//! - **No index identity** (contrast ADR-0007): chat output is never stored, so -//! [`LlmProvider::model_id`] is display only — no `meta` row, no reindex on a model -//! swap, which is what makes "change models at any time" true by construction. -//! - **Streaming is the contract, not a nicety**: tokens flow up through a callback, whose -//! return steers cooperative cancellation at token granularity. In the trait from day -//! one because retrofitting it would touch every implementor and call site. -//! -//! Sync, no runtime (ADR-0011): cancellation is returning early from a blocking read loop. +//! Unlike the embedder, chat has no index identity (contrast ADR-0007): output is never +//! stored, so a model swap never reindexes. Streaming is the contract: the token callback's +//! return value cancels. Sync (ADR-0011): cancelling returns early from a blocking read. use crate::error::Result; use serde::{Deserialize, Serialize}; use std::ops::ControlFlow; -/// One side of a chat turn. `User` turns are the human's; `Assistant` turns are -/// prior model answers the adapter carried forward (session-only history — a -/// persisted transcript would be B2-derived state outside Markdown, S4). +/// One side of a chat turn. History is session-only: a persisted transcript would be +/// B2-derived state outside Markdown (S4). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum Role { @@ -28,12 +18,8 @@ pub enum Role { Assistant, } -/// One turn of the conversation, oldest first in [`ChatRequest::turns`]. -/// -/// Serializable **both ways**, unlike the read-only view types: history is the adapter's -/// and session-only, so a GUI carrying a conversation across the IPC hands it back turn by -/// turn rather than defining a parallel DTO. It crosses a process boundary; it never -/// reaches disk. +/// One turn of the conversation. Deserializable too, so the desktop hands history back +/// over IPC; it never reaches disk. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ChatTurn { pub role: Role, @@ -41,7 +27,6 @@ pub struct ChatTurn { } impl ChatTurn { - /// A turn the human typed — the shape adapters append per question asked. pub fn user(content: &str) -> Self { Self { role: Role::User, @@ -49,7 +34,6 @@ impl ChatTurn { } } - /// A prior model answer the adapter carries forward as context. pub fn assistant(content: &str) -> Self { Self { role: Role::Assistant, @@ -58,25 +42,21 @@ impl ChatTurn { } } -/// Which flow-④ step a request serves — carried **structurally** so a provider -/// (or the fake) never infers it from prompt text, and so a grounded request -/// whose retrieval came back empty is never mistaken for condensation (they -/// both carry no passages, for different reasons). +/// Which flow-④ step a request serves, carried structurally so nothing infers it from +/// prompt text or from an empty passage list. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RequestKind { /// Step 0: rewrite the latest turn into a standalone retrieval query. Condense, /// Steps 2–3: answer the question from the numbered passages. Chat, - /// A tool-using turn ([`ChatRequest::tools`] is what the model may call): the - /// passages reach the model inside tool results, so [`ChatRequest::passages`] is - /// the *citation ledger* for this kind and is not rendered into the system message. + /// A tool-using turn. Passages reach the model in tool results, so + /// [`ChatRequest::passages`] is only the citation ledger here. Agent, } -/// One B2 tool the model may call — a read-only `Vault` op described for the model. -/// `parameters` is a JSON Schema object, passed to the provider as is (the OpenAI -/// `function.parameters` shape). +/// One read-only `Vault` op the model may call. `parameters` is a JSON Schema object +/// (OpenAI's `function.parameters`). #[derive(Debug, Clone, PartialEq, Eq)] pub struct ToolSpec { pub name: String, @@ -84,9 +64,8 @@ pub struct ToolSpec { pub parameters: serde_json::Value, } -/// One call the model asked for. `arguments` is the model's JSON **text**, unparsed: -/// it is untrusted output, so whoever runs the tool parses it and answers a malformed -/// call with an error *result* rather than failing the turn. +/// One call the model asked for. `arguments` is untrusted, unparsed JSON text: a +/// malformed call gets an error result, not a failed turn. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ToolCall { /// The provider's id for the call, echoed back with its result. @@ -95,69 +74,48 @@ pub struct ToolCall { pub arguments: String, } -/// A call and what B2 answered — one step of a tool-using turn, replayed to the model -/// on the next round so it can read what it asked for. +/// A call and B2's answer, replayed to the model on the next round. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ToolExchange { pub call: ToolCall, pub result: String, } -/// What the grounded prompt instructs the model to say when the passages don't -/// support an answer. Declared beside the seam because both sides must agree on -/// it: the flow's system prompt cites it (`chat::GROUNDED_SYSTEM_PROMPT` -/// contains it verbatim — asserted by the suite), and [`FakeLlm`] obeys it for -/// a chat request with no passages. +/// What the model says when the passages don't support an answer. Shared by +/// `chat::GROUNDED_SYSTEM_PROMPT` and [`FakeLlm`]. pub const NO_EVIDENCE_ANSWER: &str = "I don't find that in your notes."; -/// One numbered context passage handed to the model — the retrieval unit of -/// flow ④, carried structured (not pre-rendered) so a fake can read it and a -/// wire client can render it once, via [`ChatRequest::system_message`]. -/// Numbering is positional and 1-based: passage `i` is cited as `[i + 1]`. +/// One context passage for flow ④, kept structured until [`ChatRequest::system_message`] +/// renders it. Passage `i` is cited as `[i + 1]`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ContextPassage { - /// Vault-relative path of the note the passage came from — its identity (L1), - /// and what a citation resolves back to. + /// The source note (L1), what a citation resolves to. pub path: String, - /// The chunk's heading breadcrumb, when the chunker recorded one. pub heading_path: Option, - /// The passage text, verbatim (the chunk's stored text). + /// The chunk's stored text, verbatim. pub text: String, } -/// What one provider call is asked to complete: a system prompt, the -/// conversation so far (the final turn is the current user message), and the -/// numbered context passages grounding the answer. A condensation request -/// (flow ④ step 0) carries **no passages** — nothing has been retrieved yet; -/// retrieval is what condensation feeds. +/// What one provider call is asked to complete. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ChatRequest { - /// Which step this request serves (condense vs. grounded chat). pub kind: RequestKind, - /// The instruction text (prompt assembly is core logic — `chat.rs` authors - /// it, so it is testable against [`FakeLlm`]). + /// The instruction text, authored in `chat.rs`. pub system: String, - /// The conversation, oldest first; the final turn is the current user - /// message. + /// The conversation, oldest first; the last turn is the current message. pub turns: Vec, - /// The numbered passages grounding a chat answer; empty for condensation. For - /// [`RequestKind::Agent`] these are the passages tool results have handed over so - /// far — what `[n]` markers resolve against — and are not rendered again. + /// The passages grounding the answer; empty for condensation. For an agent turn, + /// those tool results handed over so far. pub passages: Vec, - /// The tools the model may call this round; empty for every non-agent request, and - /// for the last round of an agent turn (which must answer). + /// Tools the model may call; empty outside agent turns and on an agent's last round. pub tools: Vec, - /// The calls made so far this turn with their results, oldest first — replayed - /// after [`turns`](Self::turns). + /// Calls made so far this turn, replayed after [`turns`](Self::turns). pub exchanges: Vec, } impl ChatRequest { - /// Render the system prompt plus the numbered passage block into the one - /// system message a wire client sends. Rendering lives here — beside the - /// structured passages — so every provider numbers passages exactly as - /// citation resolution ([`crate::chat::cited_markers`]) counts them: - /// 1-based `[n]`, in passage order. + /// The system prompt plus the numbered passages, as one system message. Numbered here so + /// every provider matches [`crate::chat::cited_markers`]. pub fn system_message(&self) -> String { let mut out = self.system.clone(); // An agent turn's passages already reached the model inside tool results. @@ -174,9 +132,8 @@ impl ChatRequest { } impl ContextPassage { - /// The passage as the model reads it, cited as `[marker]`: the marker, the path and - /// any heading breadcrumb on one line, then the text. The one layout, whichever way - /// a passage reaches the model — the system message's block or a tool result. + /// The passage as the model reads it, cited as `[marker]`, in the system message or a + /// tool result alike. pub fn block(&self, marker: usize) -> String { let heading = match &self.heading_path { Some(h) => format!(" — {h}"), @@ -186,36 +143,25 @@ impl ContextPassage { } } -/// How a completion ended and what it produced: everything delivered to the -/// callback so far, plus the completed-vs-cancelled marker — so an adapter can -/// render a truncated answer honestly rather than passing it off as whole. +/// A completion's text and whether it was cut short, so a truncated answer is shown as one. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Completion { - /// The accumulated text: every token handed to `on_token`, in order, - /// including the one whose callback returned `Break`. + /// Every token handed to `on_token`, including the one that returned `Break`. pub text: String, - /// `true` when the stream was cut short — by the callback breaking, or by - /// the provider's own early stop. `text` is then an honest prefix. + /// Cut short by the callback or the provider; `text` is then a prefix. pub cancelled: bool, - /// The tools the model asked to run instead of (or before) answering. Empty for a - /// plain answer, and always empty when the request offered no tools. + /// Tools the model asked to run instead of answering. pub tool_calls: Vec, } -/// The chat seam (sibling of `Embedder`; invariant M1). Messages in, streamed -/// text out. Unlike `Embedder::model_id`, `model_id` here is for display and -/// logging only — chat carries **no** index identity (contrast M2): nothing is -/// recorded in `meta`, and swapping models never touches the index. +/// The chat seam (invariant M1): messages in, streamed text out. No index identity +/// (contrast M2). pub trait LlmProvider { - /// The provider's display name for logs and UI badges — never an identity - /// anything keys on (no `meta` row; contrast `Embedder::model_id`). + /// Display name for logs and UI; nothing keys on it. fn model_id(&self) -> &str; - /// Stream a completion. `on_token` receives tokens as they arrive and - /// steers the stream: returning `ControlFlow::Break(())` cancels — the - /// implementation must stop promptly and drop the connection (the pane's - /// Esc, a closed pane, the CLI's cancel all land here). Returns the text - /// accumulated so far plus how the stream ended (completed / cancelled). + /// Stream a completion. `on_token` returning `Break` cancels: stop promptly and drop + /// the connection. fn complete( &self, req: &ChatRequest, @@ -223,24 +169,17 @@ pub trait LlmProvider { ) -> Result; } -/// The fake provider's display id — the `FAKE_MODEL_ID` sibling. Nothing keys -/// on it (chat carries no index identity); it exists so logs read honestly. +/// The fake provider's display id. pub const FAKE_LLM_MODEL_ID: &str = "fake-llm-v1"; -/// Deterministic provider for tests/dev — the [`crate::embed::FakeEmbedder`] sibling, -/// keyed on [`ChatRequest::kind`]. A **chat** request streams a fixed grounded answer -/// citing every passage it was handed, one `[n]` marker per token, so the suite can assert -/// the whole flow-④ pipeline — and mid-stream cancellation at an exact token — model-free; -/// handed no passages it answers [`NO_EVIDENCE_ANSWER`]. A **condensation** request echoes -/// the latest user turn verbatim. An **agent** request is a two-step script read off the -/// request's structure: while an offered tool that needs no arguments has not been called -/// yet, call each such tool once with `{}`; after that, answer as a chat request does. +/// Deterministic provider for tests and dev. Chat streams an answer citing every passage, +/// one `[n]` per token (so cancellation can land on an exact token), or +/// [`NO_EVIDENCE_ANSWER`] with none. Condense echoes the last user turn. Agent calls each +/// uncalled no-argument tool once with `{}`, then answers as chat. #[derive(Debug, Clone, Copy, Default)] pub struct FakeLlm; impl LlmProvider for FakeLlm { - /// The fixed display id [`FAKE_LLM_MODEL_ID`] — logging only, like every - /// provider's. fn model_id(&self) -> &str { FAKE_LLM_MODEL_ID } @@ -279,8 +218,7 @@ impl LlmProvider for FakeLlm { } let tokens: Vec = match req.kind { RequestKind::Condense => { - // Echo the question. One token — cancellation scripting - // belongs to the multi-token chat branch. + // Echo the question as one token. let echo = req .turns .iter() @@ -294,13 +232,10 @@ impl LlmProvider for FakeLlm { vec![echo] } } - // Nothing retrieved, nothing to cite: the grounded prompt's - // no-evidence response, as a real model would give it. RequestKind::Chat | RequestKind::Agent if req.passages.is_empty() => { vec![NO_EVIDENCE_ANSWER.to_string()] } RequestKind::Chat | RequestKind::Agent => { - // A grounded answer citing every passage, marker-per-token. let mut t: Vec = vec!["Grounded".into(), " in".into()]; t.extend((1..=req.passages.len()).map(|n| format!(" [{n}]"))); t.push(".".into()); diff --git a/crates/b2-core/src/mv.rs b/crates/b2-core/src/mv.rs index e7d29e7..c9b6cdc 100644 --- a/crates/b2-core/src/mv.rs +++ b/crates/b2-core/src/mv.rs @@ -1,41 +1,19 @@ //! Move / rename a note, a resource or a folder, and repair inbound links. //! -//! **A move is where "rename keeps every backlink resolving" is earned.** Identity is -//! the vault-relative path (ADR-0003, L1/L3), so a move changes the moved member's -//! identity, and this module makes that a re-key rather than a break. Two halves, both -//! bounded by the moved set's backlink count: the human-facing copy — the link text in -//! every file linking *at* a moved member, rewritten in place — and the index: each moved -//! note re-keyed by one [`db::repoint_note_path`] whose `ON UPDATE CASCADE` FKs carry -//! chunks, centroid and outbound edges atomically, each moved resource by one -//! [`db::repoint_resource`], then a re-projection of the inbound sources so their edges -//! (`edges.dst_path` has no FK — it must be free to dangle) point at the new paths. A -//! moved note's **vectors are not touched at all**: content-addressed (ADR-0006), they -//! belong to the chunk text, which a move does not change. +//! Identity is the path (ADR-0003, L1/L3), so a move re-keys: it rewrites the link text in +//! every file linking at a moved member, repoints the index rows (`ON UPDATE CASCADE` +//! carries chunks, centroid and outbound edges), and re-projects the inbound sources, +//! whose `edges.dst_path` has no FK. Vectors are untouched (ADR-0006). //! -//! **One pipeline for every move.** `move_note`, `move_resource` and `move_dir` only -//! validate and build a [`MoveSet`] — the one rename, and the indexed members travelling -//! with it — and [`execute`] runs it. It is **Markdown-first**: rewrite the inbound text, -//! rename on disk, *then* re-project from the now-current Markdown. And bounded, not a -//! scan: [`db::inbound_edges_of`] names exactly the files to touch, so the cost is -//! O(inbound links). The link text is found by [`crate::link`]'s own scanner, so a move -//! reads links exactly as ingest does. Known gap: within one inbound file the rewrite is -//! keyed by a link's *written* text, so a second link with the same text that resolves -//! elsewhere (a note-relative Markdown link and a vault-root wikilink both written -//! `img.png`, naming two different files) is rewritten too. +//! Every move builds a [`MoveSet`] and runs [`execute`]: Markdown first, then re-project. +//! Cost is O(inbound links) via [`db::inbound_edges_of`]. Known gap: rewrites are keyed by +//! a link's written text, so two same-text links resolving to different files in one +//! inbound file are both rewritten. //! -//! **The vault half is all or nothing** (GH #230). Reindexing can't repair rewritten -//! link text (it projects whatever the Markdown says), so a failed move must leave every -//! file as it was. Three layers: the destination is checked before anything is written -//! ([`refuse_occupied`], [`refuse_file_ancestor`]); every rewrite is read and computed -//! in memory before the first write ([`plan_inbound`]); and the writes, the folders the -//! move creates and the rename run as one [`commit`] that undoes whatever it did if a -//! later step fails. Once the rename lands the vault is final: a failure re-keying or -//! re-projecting the index leaves correct Markdown, which `b2 reindex` projects. -//! -//! No process can undo a crash, so the one unguarded window is between the first -//! rewrite and the rename: the rewritten links already name the destination, the note -//! is still at its source. Re-running the same move finishes it (the rewrites are -//! already done, so only the rename is left). +//! The vault half is all or nothing (GH #230), since a reindex can't repair link text: the +//! destination is checked and every rewrite computed before the first write, then +//! [`commit`] undoes itself on failure. A crash between the first rewrite and the rename +//! is the one unguarded window; re-running the move finishes it. use crate::db::{self, InboundTarget}; use crate::error::{Error, Result}; @@ -49,35 +27,22 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; -/// What [`move_note`] did: the note's old and new vault-relative paths, the inbound -/// files whose link text was rewritten, and the total number of `[[…]]` targets -/// repaired across them. `to` is the note's identity after the move (L1). +/// What [`move_note`] did. `to` is the note's identity after the move (L1). #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct MoveReport { pub from: String, pub to: String, - /// Post-move vault-relative paths of the files whose link text was rewritten - /// (sorted, deduped) — the moved note itself under `to`, when it links to itself. - /// Empty when nothing linked to the moved note. + /// Post-move paths of the files whose link text was rewritten (sorted, deduped), + /// including the moved note itself when it self-links. pub rewrote: Vec, /// Total individual `[[…]]` link targets rewritten across `rewrote`. pub links_rewritten: usize, } -/// Move the note at `old_rel` to `new_rel_input`, rewriting every inbound -/// `[[oldpath|alias]]` link and re-keying the index. `old_rel` is the note's current -/// path (as the façade resolved it); `new_rel_input` is the raw destination the user -/// gave (a `.md` suffix is optional). -/// -/// Re-projection **re-embeds the inbound files** — their bodies changed — so the caller -/// must open the vault with the embedder the index was built with. The *moved* note -/// re-embeds nothing (ADR-0006). Errors with [`Error::MoveDestination`] for an invalid -/// destination and [`Error::MoveTargetExists`] rather than clobber. -/// -/// Only the `[[…]]` form is rewritten at a note. A Markdown-form link at one -/// (`[x](notes/a.md)`) keeps its text, but its source is re-projected like every inbound -/// source, so its edge is exactly what a rebuild would project — dangling, or resolved to -/// whatever note that text now names — rather than keep naming a path that is gone. +/// Move the note at `old_rel` to `new_rel_input` (`.md` optional), rewriting inbound +/// `[[…]]` links and re-keying the index. Inbound files re-embed, so the caller needs the +/// index's embedder. A Markdown-form link at a note keeps its text; its source is still +/// re-projected, so its edge matches what a rebuild would project. pub fn move_note(ctx: EmbedCtx, old_rel: &str, new_rel_input: &str) -> Result { let new_rel = pathspec::normalize_rel_md(new_rel_input).map_err(Error::MoveDestination)?; refuse_same_path(&new_rel, old_rel, "note")?; @@ -94,28 +59,21 @@ pub fn move_note(ctx: EmbedCtx, old_rel: &str, new_rel_input: &str) -> Result, /// Total individual link targets rewritten across `rewrote`. pub links_rewritten: usize, } -/// Move the resource at `old_rel` to `new_rel_input` — the note move minus the identity -/// step: rewrite every inbound link's authored text (both syntaxes, each keeping its own -/// relative-vs-root convention), move the file, update the inventory, re-project the -/// inbound notes. B2 never touches the resource's bytes; the move is path-only. Errors -/// mirror [`move_note`] (the façade owns [`Error::ResourceNotFound`]), plus -/// [`Error::MoveDestination`] for a `.md` destination: that path names a note, so the -/// file would stop being this resource (and a rebuild would index it as a note). +/// Move the resource at `old_rel` to `new_rel_input`, rewriting inbound links in both +/// syntaxes (each keeping its relative-vs-root convention). Path-only; the bytes are never +/// touched. A `.md` destination is refused: a rebuild would index it as a note. pub fn move_resource( ctx: EmbedCtx, old_rel: &str, @@ -145,40 +103,23 @@ pub fn move_resource( }) } -/// What [`move_dir`] did: the folder's old and new vault-relative paths, how many -/// **indexed** notes/resources travelled (unindexed files travel too — the whole -/// directory is renamed — but only indexed rows are counted), the files whose -/// link text was rewritten (reported at their **post-move** paths, sorted), and -/// the total link targets repaired. +/// What [`move_dir`] did. The counts are of indexed members; unindexed files travel too. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct DirMoveReport { pub from: String, pub to: String, pub moved_notes: usize, pub moved_resources: usize, - /// Post-move vault-relative paths of the files whose link text was rewritten - /// (sorted, deduped). Empty when no links referenced the moved set. + /// Post-move paths of the files whose link text was rewritten (sorted, deduped). pub rewrote: Vec, /// Total individual link targets rewritten across `rewrote`. pub links_rewritten: usize, } -/// Move/rename the whole directory `from_input` to `to_input`. One `fs::rename` moves -/// the directory — so **unindexed** files inside travel too — after every inbound link -/// at the moved set is rewritten, exactly as the per-file moves do: -/// -/// - wikilinks are vault-root-anchored, so links *between* co-moved notes are rewritten -/// just like links from outside the set; -/// - note-relative Markdown targets between co-moved files survive unchanged (a computed -/// replacement equal to the authored text is skipped); -/// - after the rename every moved note's `notes.path` is repointed **first**, then each -/// moved file re-projects — so path-based link resolution never depends on -/// re-projection order, the same reason full ingest is two-phase. -/// -/// Re-projection re-embeds only genuinely rewritten bodies, but still requires the real -/// embedder. Errors: [`Error::DirNotFound`], [`Error::MoveDestination`] (including a -/// destination inside the moved folder), [`Error::MoveTargetExists`] rather than merge -/// (with the case-only-rename carve-out on case-insensitive filesystems). +/// Move/rename the directory `from_input` to `to_input` with one `fs::rename`, after +/// rewriting inbound links as the per-file moves do. Wikilinks between co-moved notes are +/// rewritten (they are vault-root); note-relative Markdown links between them compute to +/// themselves and are skipped. Requires the real embedder. pub fn move_dir(ctx: EmbedCtx, from_input: &str, to_input: &str) -> Result { let conn = ctx.proj.conn; let from = pathspec::normalize_rel_dir(from_input).map_err(Error::MoveDestination)?; @@ -193,8 +134,7 @@ pub fn move_dir(ctx: EmbedCtx, from_input: &str, to_input: &str) -> Result = db::notes_under_dir(conn, &from)? .into_iter() @@ -228,17 +168,12 @@ pub fn move_dir(ctx: EmbedCtx, from_input: &str, to_input: &str) -> Result Self { Self { from: from.to_string(), @@ -265,8 +199,7 @@ impl MoveSet { } } - /// Where the note at `path` is after the move — its new path if it travels, - /// otherwise `path` itself. + /// Where the note at `path` is after the move. fn after<'a>(&'a self, path: &'a str) -> &'a str { match self .notes @@ -277,7 +210,6 @@ impl MoveSet { } } - /// Whether the note at `path` travels with this set. fn moves_note(&self, path: &str) -> bool { self.notes .binary_search_by(|(old, _)| old.as_str().cmp(path)) @@ -285,17 +217,14 @@ impl MoveSet { } } -/// What [`execute`] did: the rewritten files at their post-move paths (sorted), and -/// how many link targets it rewrote across them. +/// What [`execute`] did: rewritten files at post-move paths (sorted), and the link count. #[derive(Debug)] struct Moved { rewrote: Vec, links_rewritten: usize, } -/// Run one move: refuse an occupied or unreachable destination, plan every inbound -/// rewrite, commit the rewrites and the rename (all or nothing), re-key every moved -/// member, then re-project from the now-current Markdown. +/// Run one move: refuse, plan, commit (all or nothing), re-key, re-project. fn execute(ctx: EmbedCtx, set: &MoveSet) -> Result { let (conn, root) = (ctx.proj.conn, ctx.proj.root); let old_abs = root.join(&set.from); @@ -303,19 +232,14 @@ fn execute(ctx: EmbedCtx, set: &MoveSet) -> Result { refuse_occupied(&old_abs, &new_abs, &set.to)?; refuse_file_ancestor(root, &set.to)?; - // The graph names the bounded inbound set: for each edge pointing at a moved - // member, its source file and the exact link text (`dst_path_raw`) written there. - // Group by file into per-syntax target→replacement maps. A note is rewritten in - // its `[[…]]` form only, each link keeping its own `.md`-or-not convention; a - // resource is linked as `![[img.png]]` *or* `![](img.png)`, so its replacement - // feeds both, re-relativized against the source's **post-move** folder so a - // relative link between two co-moved files computes to itself and is skipped. + // Per-file, per-syntax target→replacement maps. A note is rewritten in its `[[…]]` + // form only; a resource in both, re-relativized against the source's post-move folder + // so a link between co-moved files computes to itself. let note_paths: Vec<&str> = set.notes.iter().map(|(old, _)| old.as_str()).collect(); let resource_paths: Vec<&str> = set.resources.iter().map(|r| r.from.as_str()).collect(); let (mut wiki, mut md) = (ByFile::new(), ByFile::new()); - // Every inbound source is re-projected below, whether or not its *text* changes: - // its edges name paths that are about to change, and `edges.dst_path` carries no FK - // to cascade (it must be free to be NULL — the dangling case, G5). + // Every inbound source is re-projected, text changed or not: `edges.dst_path` has no + // FK to cascade (it must be free to dangle, G5). let mut sources = BTreeSet::new(); for (target, e) in db::inbound_edges_of(conn, ¬e_paths, &resource_paths)? { let (replacement, both_forms) = match target { @@ -340,17 +264,12 @@ fn execute(ctx: EmbedCtx, set: &MoveSet) -> Result { sources.insert(e.src_path); } - // 1. Markdown first: rewrite inbound link text in place at the pre-move paths (a - // self-link or a co-moved linker included), then one rename moves the note, - // resource or folder — unindexed files in a folder travel for free — all or - // nothing. + // 1. Markdown first: rewrite inbound links at the pre-move paths, then rename. let plan = plan_inbound(root, &wiki, &md)?; commit(root, &plan.rewrites, &old_abs, &new_abs)?; - // 2. Re-key the index before anything re-projects, so path-based link resolution - // is independent of re-projection order (the same reason full ingest is - // two-phase). Old and new paths are disjoint — the destination didn't exist — so - // the UNIQUE(path) constraints can't trip. + // 2. Re-key before re-projecting, so resolution is order-independent. Old and new + // paths are disjoint, so UNIQUE(path) can't trip. for (old, new) in &set.notes { db::repoint_note_path(conn, old, new)?; } @@ -360,15 +279,12 @@ fn execute(ctx: EmbedCtx, set: &MoveSet) -> Result { .as_ref() .and_then(ingest::unix_mtime); if !db::repoint_resource(conn, &r.from, &r.to, r.class.as_str(), mtime)? { - // Not inventoried after all (an out-of-band change the index hasn't seen): - // inventory the file where it now is, as a reindex would. + // Not inventoried (an out-of-band change): inventory it where it now is. ingest::project_resource_file(conn, root, &r.to, r.class, true)?; } } - // 3. Re-project from the now-current Markdown: every moved note at its new path - // (refreshing its filename-derived title, mtime and outbound edges — an - // unchanged body reuses its vectors), then every inbound source that stayed put. + // 3. Re-project moved notes at their new paths, then inbound sources that stayed. for (_, new) in &set.notes { ingest::ingest_file(ctx, new)?; } @@ -388,17 +304,13 @@ fn execute(ctx: EmbedCtx, set: &MoveSet) -> Result { }) } -/// Each inbound file's authored-target → replacement map, keyed by the file's -/// vault-relative path. `BTreeMap` throughout, so the rewrite order — and the -/// `rewrote` list every report carries — is sorted and deterministic. +/// Each inbound file's target → replacement map. `BTreeMap` keeps the order deterministic. type ByFile = BTreeMap; /// One file's authored-target → replacement map for one link syntax. type Targets = BTreeMap; -/// Refuse a destination equal to the source. `subject` names the thing being -/// moved, so the message reads in the user's own nouns ("… is the note's current -/// path"). +/// Refuse a destination equal to the source. `subject` names the thing moved, for the message. fn refuse_same_path(new_rel: &str, old_rel: &str, subject: &str) -> Result<()> { if new_rel == old_rel { return Err(Error::MoveDestination(format!( @@ -408,10 +320,8 @@ fn refuse_same_path(new_rel: &str, old_rel: &str, subject: &str) -> Result<()> { Ok(()) } -/// Refuse an occupied destination rather than clobber it (the vault never -/// overwrites, data-model.md §1) — with the case-only-rename carve-out: on a -/// case-insensitive filesystem the destination "exists" because it *is* the -/// source ([`is_same_dirent`]). +/// Refuse an occupied destination rather than clobber it (data-model.md §1), except a +/// case-only rename on a case-insensitive filesystem ([`is_same_dirent`]). fn refuse_occupied(old_abs: &Path, new_abs: &Path, new_rel: &str) -> Result<()> { if new_abs.exists() && !is_same_dirent(old_abs, new_abs) { return Err(Error::MoveTargetExists(new_rel.to_string())); @@ -419,11 +329,8 @@ fn refuse_occupied(old_abs: &Path, new_abs: &Path, new_rel: &str) -> Result<()> Ok(()) } -/// Refuse a destination beneath a regular file (`blocked/a.md` when `blocked` is a -/// file): no rename can land there, and finding out *after* the inbound rewrites is -/// exactly the broken-links failure GH #230 reported. Walks the destination's -/// ancestors from the vault root down and stops at the first missing one — the -/// move creates everything below it. +/// Refuse a destination beneath a regular file, before any rewrite (GH #230). Stops at the +/// first missing ancestor; the move creates everything below it. fn refuse_file_ancestor(vault_root: &Path, new_rel: &str) -> Result<()> { for (i, _) in new_rel.match_indices('/') { let dir = &new_rel[..i]; @@ -440,8 +347,7 @@ fn refuse_file_ancestor(vault_root: &Path, new_rel: &str) -> Result<()> { Ok(()) } -/// One inbound file's planned rewrite: the bytes it holds now and the bytes it will -/// hold. `original` is kept so [`commit`] can put the file back. +/// One inbound file's planned rewrite. `original` lets [`commit`] put the file back. #[derive(Debug)] struct Rewrite { rel: String, @@ -458,10 +364,8 @@ struct Plan { links_rewritten: usize, } -/// Markdown first, for every move: read each inbound file and compute its rewritten -/// link text, writing nothing. `wiki` holds each file's `[[…]]` replacements and `md` -/// its `[…](…)` ones; a file whose rewrite changes nothing is left out of the plan. A -/// file that can't be read fails the move here, while the vault is still untouched. +/// Read each inbound file and compute its rewrite, writing nothing. An unreadable file +/// fails the move here, while the vault is untouched. fn plan_inbound(vault_root: &Path, wiki: &ByFile, md: &ByFile) -> Result { let none = Targets::new(); let mut rewrites = Vec::new(); @@ -491,14 +395,9 @@ fn plan_inbound(vault_root: &Path, wiki: &ByFile, md: &ByFile) -> Result { }) } -/// Rewrite every link in `raw` whose trimmed target is a key of its syntax's map — -/// `wiki` for `[[…]]`/`![[…]]`, `md` for `[…](…)`/`![…](…)` — to that key's replacement. -/// Only the target token changes: every other byte (the brackets, the `|alias`, the -/// link text, whitespace around the target) is preserved, and a target merely sharing a -/// prefix with a key is never touched. The links are the ones [`link::link_spans`] -/// finds — the scanner ingest projects edges from — matched by their written text (see -/// the module doc's known gap). Returns the rewritten text and the count of targets -/// replaced. +/// Replace each link target in `raw` found in its syntax's map (`wiki` for `[[…]]`, `md` +/// for `[…](…)`). Only the target token changes. Uses [`link::link_spans`], the scanner +/// ingest uses. Returns the text and the count replaced. fn rewrite_targets(raw: &str, wiki: &Targets, md: &Targets) -> (String, usize) { let mut out = String::with_capacity(raw.len()); let (mut copied, mut count) = (0usize, 0usize); @@ -519,7 +418,7 @@ fn rewrite_targets(raw: &str, wiki: &Targets, md: &Targets) -> (String, usize) { (out, count) } -/// One vault change [`commit`] made, and so must undo if a later step fails. +/// One vault change [`commit`] must undo if a later step fails. #[derive(Debug)] enum Done { /// `rewrites[i]` was opened for writing (and so truncated). @@ -528,13 +427,9 @@ enum Done { CreatedDir(PathBuf), } -/// Apply a move's vault writes as one unit: write every planned rewrite, create any -/// missing destination folders, then rename `old_abs` → `new_abs` (the one step that -/// moves the note, resource or folder). If any step fails, undo the earlier ones in -/// reverse and return the failure; the rename is last, so once it succeeds there is -/// nothing left to undo. A filesystem has no transaction, so an undo step can fail -/// too — then [`Error::MoveIncomplete`] names the files still holding a rewrite, and -/// carries the failure that started the undo, rather than pretending the vault is whole. +/// Apply a move's vault writes as one unit: rewrites, missing folders, then the rename +/// (last, so success leaves nothing to undo). On failure, undo in reverse. If the undo +/// fails too, [`Error::MoveIncomplete`] names the files still holding a rewrite. fn commit(vault_root: &Path, rewrites: &[Rewrite], old_abs: &Path, new_abs: &Path) -> Result<()> { let mut done = Vec::new(); let Err(err) = apply(rewrites, old_abs, new_abs, &mut done) else { @@ -561,8 +456,7 @@ fn apply(rewrites: &[Rewrite], old_abs: &Path, new_abs: &Path, done: &mut Vec = new_abs .ancestors() .skip(1) @@ -577,10 +471,8 @@ fn apply(rewrites: &[Rewrite], old_abs: &Path, new_abs: &Path, done: &mut Vec Vec { let mut unrestored = Vec::new(); for step in done.iter().rev() { @@ -608,8 +500,7 @@ fn undo(vault_root: &Path, rewrites: &[Rewrite], done: &[Done]) -> Vec { unrestored } -/// Split an authored link target at its first `#` into the path and the fragment — -/// which addresses a place *inside* the target, so a move carries it through verbatim. +/// Split a link target at its first `#`. A move carries the fragment through verbatim. fn split_fragment(authored: &str) -> (&str, Option<&str>) { match authored.split_once('#') { Some((base, fragment)) => (base, Some(fragment)), @@ -617,7 +508,6 @@ fn split_fragment(authored: &str) -> (&str, Option<&str>) { } } -/// `base` with the authored `#fragment` (if any) put back on. fn with_fragment(base: &str, fragment: Option<&str>) -> String { match fragment { Some(f) => format!("{base}#{f}"), @@ -625,10 +515,9 @@ fn with_fragment(base: &str, fragment: Option<&str>) -> String { } } -/// The replacement for a wikilink at a note now living at `new_path`, preserving the -/// link's own `.md`-or-not convention (Obsidian omits the extension; an authored `.md` -/// is kept) and any `#heading` fragment. Only a lowercase `.md` is dropped: that is the -/// suffix the resolver's `+ ".md"` ladder adds back, so any other spelling stays whole. +/// The new wikilink target for a note at `new_path`, keeping the link's `.md`-or-not +/// convention and fragment. Only a lowercase `.md` is dropped, the suffix the resolver +/// adds back. fn wiki_replacement(new_path: &str, authored: &str) -> String { let (base, fragment) = split_fragment(authored); let new_base = if base.ends_with(".md") { @@ -639,28 +528,20 @@ fn wiki_replacement(new_path: &str, authored: &str) -> String { with_fragment(new_base, fragment) } -/// The replacement for a link at a resource moving `old_path` → `new_path`, as -/// authored from a note in `src_dir` (its directory **after** the move, so a link -/// between two co-moved files computes to itself and is skipped). Two things -/// survive: the authored convention — a vault-root target stays vault-root, -/// anything else is re-relativized against `src_dir` — and a `#fragment` suffix, -/// carried through untouched. +/// The new target for a link at a moved resource, from a note in `src_dir` (post-move). +/// A vault-root target stays vault-root; anything else is re-relativized. Keeps the fragment. fn resource_replacement(authored: &str, old_path: &str, new_path: &str, src_dir: &str) -> String { let (base, fragment) = split_fragment(authored); let new_base = if base.trim() == old_path { - new_path.to_string() // authored vault-root — keep it vault-root + new_path.to_string() } else { - pathspec::relativize(src_dir, new_path) // authored note-relative — keep it relative + pathspec::relativize(src_dir, new_path) }; with_fragment(&new_base, fragment) } -/// Whether `a` and `b` name the **same directory entry** on disk — true only on -/// a case-insensitive filesystem (APFS default) for a case-only rename, where -/// `Path::exists` on the destination false-positives against the source itself. -/// `fs::canonicalize` returns the on-disk-case path, so the two canonicalize -/// equal iff they are one entry; any error (e.g. the path doesn't exist) means -/// "not the same entry" and the ordinary target-exists refusal stands. +/// Whether `a` and `b` are the same directory entry: a case-only rename on a +/// case-insensitive filesystem (APFS default). Any canonicalize error means "not the same". fn is_same_dirent(a: &Path, b: &Path) -> bool { match (fs::canonicalize(a), fs::canonicalize(b)) { (Ok(ca), Ok(cb)) => ca == cb, @@ -741,8 +622,6 @@ mod tests { assert_eq!(n, 2); } - /// Each syntax is rewritten from its own map, and a Markdown target keeps the - /// whitespace and link text around it. #[test] fn each_syntax_is_rewritten_from_its_own_map() { let wiki = targets(&[("img.png", "media/img.png")]); @@ -753,14 +632,11 @@ mod tests { "![[media/img.png|cap]] and ![alt]( ../media/img.png )\n" ); assert_eq!(n, 2); - // A map for one syntax never touches the other's links. let (out, n) = rewrite_targets("![alt](img.png)", &wiki, &Targets::new()); assert_eq!((out.as_str(), n), ("![alt](img.png)", 0)); } - /// The divergence one grammar closes: a stray `[[` on one line used to pair with - /// the next line's `]]` in the move's whole-file scan, hiding a link ingest had - /// projected — so the backlink dangled after the move. + /// The move uses ingest's scanner, so a stray `[[` can't hide the next line's link. #[test] fn a_stray_open_bracket_does_not_hide_the_next_lines_link() { let t = targets(&[("old", "new")]); @@ -770,14 +646,10 @@ mod tests { } // --- the replacement rules, direct (GH #134) ------------------------------ - // - // Pure functions, so they get pinned here rather than only through a whole-vault - // move. #[test] fn a_wikilink_keeps_its_own_md_convention() { - // Obsidian's bare form stays bare; an authored `.md` keeps its `.md`. Both - // land on the same note — the convention is the *link's*, not the vault's. + // The convention is the link's, not the vault's. assert_eq!( wiki_replacement("archive/memory.md", "concepts/memory"), "archive/memory" @@ -788,8 +660,6 @@ mod tests { ); } - /// A `[[note#heading]]` link resolves to the note (the fragment is stripped for - /// the lookup only), so its move keeps the heading it addresses. #[test] fn a_wikilink_keeps_its_heading_fragment() { assert_eq!( @@ -804,8 +674,7 @@ mod tests { #[test] fn a_resource_link_keeps_its_convention_and_its_fragment() { - // Authored vault-root (the target equals the resource's vault path) → stays - // vault-root, whoever links it. + // Authored vault-root stays vault-root. assert_eq!( resource_replacement( "assets/plan.pdf", @@ -815,7 +684,7 @@ mod tests { ), "docs/plan.pdf" ); - // Authored note-relative → re-relativized against the linking note's dir. + // Authored note-relative is re-relativized. assert_eq!( resource_replacement( "../assets/plan.pdf", @@ -825,8 +694,7 @@ mod tests { ), "../docs/plan.pdf" ); - // A `#fragment` survives verbatim on both routes — it addresses a place - // *inside* the resource, which a move never touches. + // A `#fragment` survives on both routes. assert_eq!( resource_replacement( "assets/plan.pdf#page=3", @@ -849,9 +717,7 @@ mod tests { #[test] fn a_relative_link_between_co_moved_files_computes_to_itself() { - // The folder move passes the source's **post-move** directory, so a link from - // `dir/note.md` to `dir/img.png` is unchanged by moving `dir/` — and an - // unchanged replacement is what the pipeline skips, leaving the file untouched. + // `src_dir` is the post-move directory, so the link is unchanged and skipped. assert_eq!( resource_replacement("img.png", "dir/img.png", "moved/img.png", "moved"), "img.png" @@ -874,11 +740,8 @@ mod tests { // --- the all-or-nothing commit (GH #230) ------------------------------------ // - // The façade suite (tests/mv.rs) drives a failed rename end to end. These pin the - // two failures it can't reach deterministically: a write that fails after an - // earlier one landed, and an undo that itself fails. + // Failures tests/mv.rs can't reach deterministically. - /// A planned rewrite of `rel` under `root`, from whatever is on disk now. fn planned(root: &Path, rel: &str, rewritten: &str) -> Rewrite { let abs = root.join(rel); Rewrite { diff --git a/crates/b2-core/src/note.rs b/crates/b2-core/src/note.rs index c0e9239..7978ae5 100644 --- a/crates/b2-core/src/note.rs +++ b/crates/b2-core/src/note.rs @@ -1,44 +1,31 @@ //! Lossless note parsing and the surgical frontmatter/body splices. //! -//! A note is optional YAML frontmatter followed by a Markdown body. To make -//! `parse -> serialize -> parse` byte-identical, a [`ParsedNote`] keeps the **raw text -//! verbatim** and records only the byte spans of the frontmatter; serialization returns -//! the raw bytes. Every mutation here is a surgical splice performed on the human's -//! explicit command (ADR-0004) — appending a `b2_relations:` entry, replacing the body, -//! replacing the frontmatter block. The queryable fields come from a read-only YAML parse -//! and are never used to re-serialize. A `b2id:` key an older B2 left behind is now an -//! ordinary unknown key: never parsed, never rewritten, round-tripped verbatim. +//! A [`ParsedNote`] keeps the raw text verbatim plus the frontmatter's byte spans, so +//! serializing is byte-identical. Every mutation is a splice on an explicit command +//! (ADR-0004). Parsed fields are read-only and never re-serialized. use crate::error::{Error, Result}; use yaml_rust2::{Yaml, YamlLoader}; -/// The frontmatter fields B2 reads: the ones a reader displays (`type`, `created`, -/// `updated`, `tags`) and B2's own `b2_relations:`. Extraction is best-effort: -/// unparseable frontmatter still round-trips (raw is preserved); the fields just come -/// back empty. Every other key — `title`, `description`, `aliases`, your own — is -/// round-tripped verbatim and read by nothing (data-model.md §1). +/// The frontmatter fields B2 reads, best-effort: unparseable YAML just yields empty +/// fields. Every other key round-trips verbatim, read by nothing (data-model.md §1). #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct NoteFields { pub r#type: Option, pub created: Option, pub updated: Option, pub tags: Vec, - /// Raw `b2_relations:` entries (typed-link strings, §2) — B2's namespaced - /// frontmatter home for typed edges, the only place a verb/explanation lives. - /// Parsed into `origin=frontmatter` edges at ingest. (A generic `relations:` - /// key is NOT read — it is another tool's, preserved verbatim like any - /// unknown key; data-model §1.) + /// Raw `b2_relations:` entries (§2), the only home of typed edges. A generic + /// `relations:` key is another tool's and is not read (data-model §1). pub relations: Vec, } /// Byte spans of a frontmatter block within the raw text (fences excluded). #[derive(Debug, Clone, Copy)] struct Frontmatter { - /// First byte of the YAML content (right after the opening `---` line). content_start: usize, - /// First byte of the closing `---` line (YAML content ends here). + /// First byte of the closing `---` line. content_end: usize, - /// First byte of the body (right after the closing `---` line's newline). body_start: usize, } @@ -51,8 +38,7 @@ pub struct ParsedNote { fm_readable: bool, } -/// Parse `raw` into a [`ParsedNote`]. Never fails: text with no/!invalid -/// frontmatter is still represented (and still round-trips). +/// Parse `raw` into a [`ParsedNote`]. Never fails: invalid frontmatter still round-trips. pub fn parse(raw: &str) -> ParsedNote { let fm = detect_frontmatter(raw); let (fields, fm_readable) = match &fm { @@ -68,19 +54,16 @@ pub fn parse(raw: &str) -> ParsedNote { } impl ParsedNote { - /// The note serialized — byte-identical to what was parsed, plus whatever - /// splice the caller asked for. + /// The note serialized: the parsed bytes plus any splices. pub fn as_str(&self) -> &str { &self.raw } - /// The extracted, queryable frontmatter fields. pub fn fields(&self) -> &NoteFields { &self.fields } - /// The body (everything after the frontmatter; the whole text if there is - /// none). This is what gets hashed and, later, chunked. + /// Everything after the frontmatter: what gets hashed and chunked. pub fn body(&self) -> &str { match &self.fm { Some(f) => &self.raw[f.body_start..], @@ -88,32 +71,20 @@ impl ParsedNote { } } - /// The raw frontmatter YAML **verbatim** — the text between the `---` fences - /// (fences excluded), exactly as on disk — or `None` when the note has no - /// frontmatter block. Byte-honest like [`body`](Self::body): the actual bytes, - /// not a re-serialization of the parsed [`fields`](Self::fields), so keys B2 - /// doesn't model (`b2_relations:`, `aliases:`, custom keys) show as written. Powers - /// the Desktop UI's frontmatter drawer (crates/b2-desktop/CLAUDE.md). + /// The frontmatter YAML verbatim, fences excluded, or `None` with no block. Feeds the + /// desktop's frontmatter drawer. pub fn frontmatter(&self) -> Option<&str> { self.fm.map(|f| &self.raw[f.content_start..f.content_end]) } - /// Whether the frontmatter block reads as YAML metadata: `true` for no block, an - /// empty one, or one that parses to a key/value mapping; `false` when it is malformed - /// or isn't a mapping — the raw bytes still round-trip verbatim, but - /// [`fields`](Self::fields) came back empty-handed. Surfaced so an adapter can warn - /// rather than silently show blank metadata (GH #79); never a block, since the bytes - /// stay the human's to fix. + /// Whether the frontmatter reads as a YAML mapping (or is absent/empty). `false` means + /// [`fields`](Self::fields) came back empty, so an adapter can warn (GH #79). pub fn frontmatter_readable(&self) -> bool { self.fm_readable } - /// Replace the note's **body** with `new_body`, verbatim — the byte-honest - /// splice behind `Vault::write`. Everything up to - /// `body_start` (the frontmatter block and its fences — every byte, including - /// keys B2 doesn't model) is preserved *by construction*; a note with no - /// frontmatter is replaced wholesale (its body **is** the file). No newline - /// normalization or trimming: the editor buffer is the user's text. + /// Replace the body with `new_body` verbatim (`Vault::write`). The frontmatter block is + /// preserved byte for byte. No newline normalization: the buffer is the user's text. pub fn replace_body(&mut self, new_body: &str) { match self.fm { Some(f) => { @@ -125,21 +96,13 @@ impl ParsedNote { self.raw.push_str(new_body); } } - // Re-derive spans + fields from the mutated text so state stays exact - // (the same discipline as add_relation/replace_frontmatter; a body could even - // introduce a frontmatter block if it starts with `---`). + // A body starting with `---` could even introduce a frontmatter block. self.reparse(); } - /// Replace the note's **frontmatter YAML** with `new_yaml`, verbatim — the byte-honest - /// splice behind `Vault::write_frontmatter` (GH #79). Only the bytes between the - /// fences change; the fences and every body byte are preserved by construction. A note - /// with no frontmatter gains a block; an empty `new_yaml` removes the block entirely, - /// exactly as a hand-edit deleting it would. The one mechanical touch: a non-empty - /// `new_yaml` missing its final newline gains one. - /// - /// A `new_yaml` containing a top-level `---` line would end the block early and shift - /// the following bytes into the body; the façade op refuses that input before splicing. + /// Replace the frontmatter YAML with `new_yaml` verbatim (`Vault::write_frontmatter`, + /// GH #79). No block gains one; empty `new_yaml` removes it. A missing final newline is + /// added. The façade refuses a top-level `---` line, which would end the block early. pub fn replace_frontmatter(&mut self, new_yaml: &str) { let mut block = new_yaml.to_string(); if !block.is_empty() && !block.ends_with('\n') { @@ -156,12 +119,9 @@ impl ParsedNote { self.reparse(); } - /// Append a typed-link `spec` (e.g. `contradicts [[path|title]] — why`) to the - /// frontmatter `b2_relations:` list, creating the block (or the whole frontmatter) - /// if absent. **Frontmatter only — never the body** (data-model §0). The single - /// surgical insertion preserves every other byte; the new item is YAML-quoted so - /// `[[`, `|`, `:` are safe. Errors on a pre-existing flow-style `b2_relations:` - /// rather than risk corrupting it. + /// Append a typed-link `spec` to `b2_relations:`, creating the list or block if absent. + /// Frontmatter only, never the body (data-model §0). Errors on a flow-style list rather + /// than risk corrupting it. pub fn add_relation(&mut self, spec: &str) -> Result<()> { let quoted = yaml_quote(spec); match self.fm { @@ -180,8 +140,7 @@ impl ParsedNote { Ok(()) } - /// Re-derive spans + fields + readability from the mutated `raw` so state - /// stays exact — every mutator's closing step. + /// Re-derive state from the mutated `raw`; every mutator's last step. fn reparse(&mut self) { let reparsed = parse(&self.raw); self.fm = reparsed.fm; @@ -190,9 +149,7 @@ impl ParsedNote { } } -/// Locate a frontmatter block: an opening `---` line at the very top and the next -/// `---` line. Returns `None` if the first line isn't `---` or no closing fence is -/// found (in which case the whole text is body). +/// Locate a frontmatter block: `---` as the first line, up to the next `---` line. fn detect_frontmatter(raw: &str) -> Option { let first_nl = raw.find('\n')?; if raw[..first_nl].trim_end_matches('\r') != "---" { @@ -229,12 +186,8 @@ fn detect_frontmatter(raw: &str) -> Option { } } -/// Extract the projected fields from a frontmatter region, plus whether the -/// region *read* as YAML metadata (the [`ParsedNote::frontmatter_readable`] -/// flag): `true` for an empty/comments-only block (nothing to read, nothing -/// wrong) or a key/value mapping; `false` when the YAML fails to parse or parses -/// to a non-mapping (a bare scalar, a list) — the fields then come back empty -/// while the raw bytes stay the human's to fix. +/// Extract the fields from a frontmatter region, plus +/// [`ParsedNote::frontmatter_readable`]. fn extract_fields(yaml: &str) -> (NoteFields, bool) { let mut f = NoteFields::default(); let mut readable = false; @@ -251,17 +204,10 @@ fn extract_fields(yaml: &str) -> (NoteFields, bool) { } } } - // GH #75's raw `b2id:` line scan lived here: stamping was gated on the id being - // *definitively* absent, and a parse failure reading as "absent" made ingest - // stamp a fresh line on every pass — the file grew forever and the note's - // identity churned. With nothing stamped (GH #170) there is no write to gate, so - // malformed frontmatter now costs exactly what it should: empty projected fields, - // raw bytes untouched, and a warning through `frontmatter_readable`. (f, readable) } -/// Read a scalar as text, tolerating YAML that typed it as a number/bool (e.g. a -/// date-looking `created:` or a numeric value). +/// A scalar as text, even when YAML typed it as a number or bool. fn scalar_to_string(y: &Yaml) -> Option { match y { Yaml::String(s) => Some(s.clone()), @@ -272,7 +218,6 @@ fn scalar_to_string(y: &Yaml) -> Option { } } -/// Read a YAML sequence of strings; empty for a missing key or a non-sequence. fn string_list(y: &Yaml) -> Vec { match y.as_vec() { Some(items) => items @@ -283,11 +228,8 @@ fn string_list(y: &Yaml) -> Vec { } } -/// Locate where to insert a new `b2_relations:` block item, scanning the frontmatter -/// region. Returns `Some((byte_offset, indent))` to insert `"{indent}- …\n"` (the -/// indent matches existing items, or 2 spaces for a fresh/empty block), or `None` -/// if there is no `b2_relations:` key (the caller then creates one). Errors on a -/// flow-style `b2_relations:` (e.g. `b2_relations: [a, b]`) — not safely appendable. +/// Where to insert a new `b2_relations:` item: `Some((offset, indent))`, or `None` with no +/// key. Errors on a flow-style value, which isn't safely appendable. fn relations_insertion(raw: &str, fm: &Frontmatter) -> Result> { let region = &raw[fm.content_start..fm.content_end]; let mut pos = fm.content_start; @@ -303,7 +245,7 @@ fn relations_insertion(raw: &str, fm: &Frontmatter) -> Result Result Result String { let name = crate::pathspec::file_name(path); match name.get(..name.len().saturating_sub(".md".len())) { @@ -334,10 +272,7 @@ pub fn display_title(path: &str) -> String { } } -/// YAML double-quote a string (escaping `\` and `"`), so a value with `[[`, `|`, -/// `:` (a typed-link spec, or a new note's `title`) is always a safe scalar. -/// `pub(crate)` so [`crate::add`] renders a new note's frontmatter with the same -/// quoting the relation-append path uses. +/// YAML double-quote a string, so `[[`, `|` and `:` are safe. Shared with [`crate::add`]. pub(crate) fn yaml_quote(s: &str) -> String { format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) } @@ -350,9 +285,6 @@ mod tests { fn frontmatter_returns_raw_yaml_verbatim_between_the_fences() { let raw = "---\nb2id: 01ABC\ntitle: Foo\nb2_relations:\n - references [[x]]\n---\nbody\n"; let note = parse(raw); - // The exact bytes on disk, not a re-serialization — `b2_relations:` (a key - // the projected fields flatten) survives verbatim, and so does the legacy - // `b2id:` line, which since GH #170 is an unknown key like any other. assert_eq!( note.frontmatter(), Some("b2id: 01ABC\ntitle: Foo\nb2_relations:\n - references [[x]]\n") @@ -360,16 +292,13 @@ mod tests { assert_eq!(note.body(), "body\n"); } - /// A vault indexed by an older B2 still has `b2id:` lines in it, and W5 is the - /// whole migration story: the key is not read, not rewritten, not removed — - /// which is what makes "delete `.b2/` and reindex" a complete upgrade. + /// Legacy `b2id:` lines are never read or rewritten (W5, GH #170). #[test] fn a_legacy_b2id_line_is_an_ordinary_unknown_key() { let raw = "---\nb2id: 01JMEM0000000000000000000A\ntype: concept\n---\nThe body.\n"; let mut note = parse(raw); assert_eq!(note.as_str(), raw, "round-trips byte-identically"); assert_eq!(note.fields().r#type.as_deref(), Some("concept")); - // A body save splices only the body, so the line survives that too. note.replace_body("Rewritten.\n"); assert_eq!( note.as_str(), @@ -385,20 +314,15 @@ mod tests { #[test] fn frontmatter_is_some_empty_for_an_empty_block() { - // Fences with nothing between them: the block exists (Some) but is empty — - // distinct from no frontmatter at all (None). let note = parse("---\n---\nbody\n"); assert_eq!(note.frontmatter(), Some("")); } #[test] fn frontmatter_readable_reflects_whether_the_yaml_reads_as_metadata() { - // No block, an empty block, and a mapping all read fine. assert!(parse("just a body\n").frontmatter_readable()); assert!(parse("---\n---\nbody\n").frontmatter_readable()); assert!(parse("---\ntitle: X\ntags: [x]\n---\nbody\n").frontmatter_readable()); - // Malformed YAML and a non-mapping block don't — the fields come back - // empty while the bytes round-trip, so the human should be told. let broken = parse("---\ntitle: \"unclosed\ntags: [a\n---\nbody\n"); assert!(!broken.frontmatter_readable()); assert_eq!(broken.frontmatter(), Some("title: \"unclosed\ntags: [a\n")); @@ -419,8 +343,7 @@ mod tests { #[test] fn replace_frontmatter_adds_the_missing_final_newline() { - // The one mechanical touch: without it the closing fence would join the - // last YAML line and the block would never close. + // Otherwise the closing fence would join the last YAML line. let mut note = parse("---\nb2id: 01A\n---\nbody\n"); note.replace_frontmatter("b2id: 01A\ntitle: X"); assert_eq!(note.as_str(), "---\nb2id: 01A\ntitle: X\n---\nbody\n"); @@ -428,11 +351,9 @@ mod tests { #[test] fn replace_frontmatter_creates_and_removes_the_block() { - // No block + non-empty YAML → fences created around it. let mut note = parse("only a body\n"); note.replace_frontmatter("b2id: 01A\n"); assert_eq!(note.as_str(), "---\nb2id: 01A\n---\nonly a body\n"); - // Non-empty → empty removes the block entirely, fences included. note.replace_frontmatter(""); assert_eq!(note.as_str(), "only a body\n"); assert_eq!(note.frontmatter(), None); @@ -445,12 +366,9 @@ mod tests { "spaced-repetition" ); assert_eq!(display_title("memory.md"), "memory"); - // Case-insensitive extension; nested folders drop away. assert_eq!(display_title("a/b/Read Me.MD"), "Read Me"); - // A dotted stem keeps every dot but the extension. assert_eq!(display_title("notes/2026.07.14-log.md"), "2026.07.14-log"); - // Non-`.md` or extension-only names are returned whole (never used for a - // real note, but the helper must not mangle them). + // Returned whole. assert_eq!(display_title("data.csv"), "data.csv"); assert_eq!(display_title(".md"), ".md"); } diff --git a/crates/b2-core/src/pathspec.rs b/crates/b2-core/src/pathspec.rs index 5d0edcf..024626f 100644 --- a/crates/b2-core/src/pathspec.rs +++ b/crates/b2-core/src/pathspec.rs @@ -1,38 +1,17 @@ -//! Vault-relative path algebra — the one place B2 spells out what a vault path is and how -//! paths relate. A vault path is `/`-separated and relative to the vault root; it is the -//! identity of the note or resource at it (L1, L3), so every op that validates, compares -//! or rewrites one goes through here rather than re-deriving the rules. -//! -//! Three concerns: validating a user-supplied destination ([`normalize_rel`] and its -//! folder/note variants — error-type-free, returning `Err(reason)` so each op maps it onto -//! its own [`crate::Error`] variant); the hidden-path predicate ([`is_hidden`]); and the -//! pure string algebra over paths that are already vault-relative ([`file_name`], -//! [`parent_dir`], [`extension`], [`is_under`], [`rebase`], [`relativize`]). +//! Vault-relative path algebra. A vault path is `/`-separated, relative to the root, and +//! the identity of what it names (L1, L3), so every op validates and compares paths here. +//! Validators return `Err(reason)` for each op to map onto its own [`crate::Error`]. -/// Whether this entry's *name* is dot-prefixed — the hidden-path predicate shared by the -/// folder walk, the ingest walk, and [`normalize_rel`]. **Hidden means hidden** (GH #136): -/// a dot-prefixed name is not vault material whatever its extension, so a `.scratch.md` -/// is skipped exactly as `.DS_Store` is. -/// -/// Asked of the name's **bytes**, not a decoded `&str`: `to_str` answers `None` for a name -/// UTF-8 rejects, which would make `.draft-\xFF.md` *not hidden* and route it to the note -/// collector, where the lossy path fails to reopen and surfaces as a bogus "file no longer -/// exists" skip. A leading `.` is ASCII and `OsStr`'s encoding is ASCII-compatible, so the -/// byte test is exact on every platform. +/// Whether the entry's name is dot-prefixed, whatever its extension (GH #136). Tested on +/// bytes, not `to_str`, so a non-UTF-8 name like `.draft-\xFF.md` is still hidden. pub(crate) fn is_hidden(path: &std::path::Path) -> bool { path.file_name() .is_some_and(|n| n.as_encoded_bytes().starts_with(b".")) } -/// Normalize + validate `input` into a vault-relative path of any file kind: trim, switch -/// backslashes to `/` (the index keeps one separator convention), and reject an empty, -/// absolute, vault-escaping or **hidden** path, returning the reason as `Err(String)`. The -/// extension is left exactly as given. -/// -/// The hidden check is [`is_hidden`]'s rule over every segment, and it lives on the base -/// validator rather than the directory variant alone (GH #136): the walk indexes no -/// dot-prefixed member of any kind, so a destination b2 would create and then never see is -/// a silent fs/index desync. +/// Normalize and validate a vault-relative path: trim, `\` → `/`, and reject empty, +/// absolute, escaping or hidden paths. Hidden is refused for every kind, since the walk +/// would never see what b2 created (GH #136). pub(crate) fn normalize_rel(input: &str) -> Result { let s = input.trim().replace('\\', "/"); if s.is_empty() { @@ -52,34 +31,27 @@ pub(crate) fn normalize_rel(input: &str) -> Result { Ok(s) } -/// Normalize + validate `input` into a vault-relative *directory* path — the -/// folder variant of [`normalize_rel`]: same checks, plus a trailing `/` trimmed -/// so `notes/` and `notes` name the same folder. +/// [`normalize_rel`] for a folder: a trailing `/` is trimmed. pub(crate) fn normalize_rel_dir(input: &str) -> Result { normalize_rel(input.trim().trim_end_matches('/')) } -/// Normalize + validate `input` into a vault-relative `.md` path — the note -/// variant of [`normalize_rel`]: same checks, plus `.md` appended if omitted. +/// [`normalize_rel`] for a note: `.md` appended if omitted. pub(crate) fn normalize_rel_md(input: &str) -> Result { let s = normalize_rel(input)?; Ok(if is_md(&s) { s } else { format!("{s}.md") }) } -/// A vault path's last segment — the file (or folder) name. pub(crate) fn file_name(path: &str) -> &str { path.rsplit('/').next().unwrap_or(path) } -/// A vault path's folder: everything before the last `/`, and `""` at the vault root — -/// the base a note-relative Markdown target is resolved and re-relativized against. +/// A vault path's folder; `""` at the root. pub(crate) fn parent_dir(path: &str) -> &str { path.rsplit_once('/').map_or("", |(dir, _)| dir) } -/// The extension of a vault path's file name, as written (case kept): the text after -/// the name's last `.`. `None` when there is none — no `.`, a trailing `.`, or only a -/// leading one (a dotfile's name is all stem). +/// The file name's extension, case kept. A dotfile's name is all stem. pub(crate) fn extension(path: &str) -> Option<&str> { match file_name(path).rsplit_once('.') { Some((stem, ext)) if !stem.is_empty() && !ext.is_empty() => Some(ext), @@ -87,29 +59,25 @@ pub(crate) fn extension(path: &str) -> Option<&str> { } } -/// Whether a vault path names a note by its shape: its file name's extension is `md`, -/// in any case (`Foo.MD` is a note, as the walk classifies it). +/// Whether the extension is `md`, in any case. pub(crate) fn is_md(path: &str) -> bool { extension(path).is_some_and(|e| e.eq_ignore_ascii_case("md")) } -/// Whether `path` lies strictly inside the folder `dir` (at any depth). A folder is not -/// under itself, and a prefix-sharing sibling (`docs2` for `docs`) is not under it. +/// Whether `path` lies strictly inside the folder `dir`, at any depth. pub(crate) fn is_under(path: &str, dir: &str) -> bool { path.strip_prefix(dir) .is_some_and(|rest| rest.starts_with('/')) } -/// `path` carried by moving the folder `from` to `to`: the same place under `to`, or -/// `None` when `path` is not under `from` ([`is_under`]). +/// `path` after moving the folder `from` to `to`; `None` when not under `from`. pub(crate) fn rebase(path: &str, from: &str, to: &str) -> Option { let rest = path.strip_prefix(from)?.strip_prefix('/')?; Some(format!("{to}/{rest}")) } -/// Join a relative `target` onto `base_dir` (both vault-relative, `/`-separated), -/// normalizing `.` and `..` segments. `None` when the target escapes the vault -/// root — such a path can never resolve. The inverse of [`relativize`]. +/// Join a relative `target` onto `base_dir`, normalizing `.` and `..`. `None` when it +/// escapes the vault. The inverse of [`relativize`]. pub(crate) fn join_relative(base_dir: &str, target: &str) -> Option { let mut segments: Vec<&str> = if base_dir.is_empty() { Vec::new() @@ -128,9 +96,7 @@ pub(crate) fn join_relative(base_dir: &str, target: &str) -> Option { (!segments.is_empty()).then(|| segments.join("/")) } -/// The relative path from the folder `base_dir` (`""` = the vault root) to the vault -/// path `to_path`: the shared leading folders dropped, one `..` per remaining `base_dir` -/// segment — the inverse of resolving a note-relative Markdown target. +/// The relative path from folder `base_dir` (`""` = root) to `to_path`. pub(crate) fn relativize(base_dir: &str, to_path: &str) -> String { let base: Vec<&str> = if base_dir.is_empty() { Vec::new() @@ -155,8 +121,6 @@ mod tests { assert_eq!(normalize_rel_md("notes/foo.md").unwrap(), "notes/foo.md"); } - /// An authored `.MD` is already a note path: the check is on the file name's - /// extension in any case, so no second `.md` is stacked on. #[test] fn an_uppercase_md_extension_is_kept_not_doubled() { assert_eq!(normalize_rel_md("Foo.MD").unwrap(), "Foo.MD"); @@ -253,14 +217,7 @@ mod tests { assert!(normalize_rel_dir("../up").is_err()); } - /// A filename is bytes, not text. `to_str` answers `None` for a name UTF-8 rejects, - /// which made `.draft-\xFF.md` read as *not* hidden and routed it into the note - /// collector — where the lossy path names no file, so every pass reported a bogus - /// "file no longer exists" skip. - /// - /// Asserted on the predicate rather than through a real file on purpose: `read_dir` - /// can hand us any bytes the filesystem holds, but APFS refuses to *create* such a - /// name, so a fixture that writes one cannot run on the platform B2 ships on. + /// Tested on the predicate, not a real file: APFS refuses to create such a name. #[cfg(unix)] #[test] fn a_non_utf8_dot_prefixed_name_is_hidden() { @@ -271,14 +228,11 @@ mod tests { let undecodable = |bytes: &[u8]| is_hidden(Path::new(OsStr::from_bytes(bytes))); assert!(undecodable(b"/vault/.draft-\xFF.md")); assert!(undecodable(b"/vault/.\xFF")); - // …and the same bytes without the leading dot stay ordinary vault material, - // so the fix widens what counts as hidden by exactly the leading dot. + // Without the leading dot, not hidden. assert!(!undecodable(b"/vault/draft-\xFF.md")); } - /// GH #136: b2 indexes no dot-prefixed member, so no authoring destination may - /// name one — folder, resource, or note alike. An interior dot (`a.b.md`) is - /// not hidden; only a *leading* one is. + /// GH #136. Only a leading dot hides; an interior one (`a.b.md`) does not. #[test] fn every_destination_form_refuses_a_hidden_segment() { assert!(normalize_rel_dir(".b2").is_err()); diff --git a/crates/b2-core/src/relation.rs b/crates/b2-core/src/relation.rs index 44afab8..2256fdd 100644 --- a/crates/b2-core/src/relation.rs +++ b/crates/b2-core/src/relation.rs @@ -1,14 +1,12 @@ -//! The relation vocabulary (ADR-0010): a closed three-verb stance core — `references` -//! (neutral), `supports` (for), `contradicts` (against) — with display-only inverse -//! labels, plus a tolerated tail kept verbatim. The core encodes the one thing embedding -//! similarity cannot infer: stance. Adding a verb here is the whole change. +//! The relation vocabulary (ADR-0010): a closed stance core (`references`, `supports`, +//! `contradicts`) with display-only inverse labels, plus a tail kept verbatim. Stance is +//! what similarity cannot infer. Adding a verb here is the whole change. /// A core relation verb and its display metadata. pub struct CoreVerb { pub verb: &'static str, - /// The label shown for an *inbound* edge of this type (display only — the - /// edge is stored once, directed). The symmetric verb, `contradicts`, is its own - /// inverse (data-model.md §2). + /// The display label for an inbound edge; edges are stored once, directed + /// (data-model.md §2). pub inverse: &'static str, } @@ -48,24 +46,19 @@ pub fn inverse_label(verb: &str) -> &str { mod tests { use super::*; - /// The core is *closed* and its membership is what `b2 link` validates against - /// (`Vault::link` refuses a non-core verb). A verb silently joining or leaving - /// it would change the typing palette, so pin the set itself. + /// `b2 link` validates against the core, so pin its membership. #[test] fn the_core_is_exactly_the_three_stance_verbs() { let verbs: Vec<&str> = CORE.iter().map(|c| c.verb).collect(); assert_eq!(verbs, vec!["references", "supports", "contradicts"]); assert!(verbs.iter().all(|v| is_core(v))); - // The tolerated tail is stored verbatim but is never *core*. for tail in ["inspired-by", "supersedes", "Supports", ""] { assert!(!is_core(tail), "{tail:?} must not be a core verb"); assert!(core(tail).is_none()); } } - /// Inverse labels are display-only (the edge is stored once, directed): each - /// directed core verb has its own, and a tail verb falls back to itself rather - /// than gaining an invented "-by" form. + /// A tail verb falls back to itself rather than an invented "-by" form. #[test] fn inverse_labels_cover_the_directed_core_and_the_tail() { assert_eq!(inverse_label("references"), "referenced-by"); @@ -73,9 +66,6 @@ mod tests { assert_eq!(inverse_label("inspired-by"), "inspired-by"); } - /// Stance is the one thing embedding similarity cannot infer, and `contradicts` - /// is the only core verb that reads the same from both ends: its inverse is - /// itself, and no other core verb's is. #[test] fn only_contradicts_is_its_own_inverse() { assert_eq!(inverse_label("contradicts"), "contradicts"); diff --git a/crates/b2-core/src/resource.rs b/crates/b2-core/src/resource.rs index 8480264..02d24bd 100644 --- a/crates/b2-core/src/resource.rs +++ b/crates/b2-core/src/resource.rs @@ -1,11 +1,9 @@ //! Resource classification and document-kind dispatch. //! -//! Class is decided by **extension only** — deterministic, no content sniffing, so a -//! mislabeled file degrades gracefully. The table is closed with -//! [`ResourceClass::Binary`] as the total fallback, so *every* file classifies. +//! Class is decided by extension only, no content sniffing. [`ResourceClass::Binary`] is +//! the fallback, so every file classifies. -/// The closed class table (data-model.md §10). Everything that is not a note maps to -/// exactly one of these; `Binary` catches all the rest. +/// The closed class table (data-model.md §10) for everything that is not a note. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ResourceClass { Text, @@ -29,10 +27,8 @@ impl ResourceClass { } } - /// Classify a vault-relative path. `None` means the file is a **note** - /// (`.md`) and belongs to the note pipeline, not the resource inventory. - /// Extensions are case-insensitive and read off the file name (a dotted folder - /// lends none); no extension → `Binary`. + /// Classify a vault-relative path; `None` means a note (`.md`). Case-insensitive, + /// read off the file name; no extension is `Binary`. pub fn of_path(path: &str) -> Option { let ext = crate::pathspec::extension(path) .map(str::to_ascii_lowercase) @@ -53,9 +49,8 @@ impl ResourceClass { } } -/// Which arm of the vault an argument names — the pure dispatch rule locked in -/// data-model.md §10, kept in core so the CLI and the desktop -/// can never drift on it. +/// Which arm of the vault an argument names (data-model.md §10). In core so the adapters +/// can't drift. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DocKind { /// A note ref: a path, with or without the `.md`. @@ -64,12 +59,9 @@ pub enum DocKind { Resource, } -/// Dispatch a document reference — an adapter argument or a link target — to the note or -/// resource arm by the reference's own shape, never by DB state: **an extension other than -/// `md` means resource; `.md` or no extension means note.** Extensionless covers the -/// wikilink habit (`concepts/memory`), the only extra form a note ref takes (ADR-0003). -/// Known limit, accepted: an extensionless *file* (`Makefile`) dispatches as a note ref — -/// it is still walked, inventoried, and reachable through surfaces that know its kind. +/// Dispatch a reference by its shape, never DB state: an extension other than `md` is a +/// resource; `.md` or none is a note (wikilinks omit it, ADR-0003). Accepted limit: an +/// extensionless file (`Makefile`) dispatches as a note ref. pub fn doc_kind(arg: &str) -> DocKind { match crate::pathspec::extension(arg) { Some(ext) if !ext.eq_ignore_ascii_case("md") => DocKind::Resource, @@ -83,7 +75,7 @@ mod tests { #[test] fn classification_is_total_and_extension_only() { - // (path, expected) — parameterized over the class table; None = note. + // None = note. let cases: &[(&str, Option)] = &[ ("notes/a.md", None), ("NOTES/A.MD", None), diff --git a/crates/b2-core/src/rm.rs b/crates/b2-core/src/rm.rs index 03c64ea..34963b3 100644 --- a/crates/b2-core/src/rm.rs +++ b/crates/b2-core/src/rm.rs @@ -1,16 +1,9 @@ -//! Delete a note / resource / folder — the destructive complement of [`crate::mv`]. +//! Delete a note, resource or folder: the files leave the disk, the rows leave the +//! index, and inbound links dangle, never rewritten. The result must equal an external +//! `rm` plus a full reindex (S3). //! -//! A delete is "a move minus the destination": the files leave the disk, the projection -//! rows leave the index, and inbound links **dangle** — never rewritten, since there is -//! nothing to rewrite them to — exactly the state an external `rm` plus a full reindex -//! produces. That equivalence is the correctness bar (S3). -//! -//! The single-note projection paths never prune, so every delete runs one pipeline -//! ([`delete_set`]): read the inbound linkers off the graph, remove from disk, drop the -//! rows directly, then re-project the **surviving** linkers — their edges re-derive -//! against the pruned tables and re-key to the dangling edge id a rebuild would derive. -//! Bodies are untouched, so re-projection re-chunks nothing and the ops are -//! **model-free**. +//! Every delete runs [`delete_set`], which re-projects the surviving linkers. Bodies are +//! untouched, so the ops are model-free. use crate::db; use crate::error::{Error, Result}; @@ -22,21 +15,16 @@ use std::fs; use std::io::ErrorKind; use std::path::Path; -/// What [`delete_note`] did: the deleted note's path — its identity (L1) — and the -/// surviving files whose links at it now dangle (sorted, deduped; empty when nothing -/// linked here). +/// What [`delete_note`] did: the deleted path (L1) and the files whose links now dangle. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct DeleteReport { pub path: String, - /// Vault-relative paths of the files whose links at the deleted note now - /// dangle. They are re-projected, never rewritten — the link text stays as - /// authored and surfaces as an unresolved link (GH #12). + /// Files whose links at the deleted note now dangle, sorted. Re-projected, never + /// rewritten (GH #12). pub dangled: Vec, } -/// What [`delete_resource`] did — the resource sibling of [`DeleteReport`]; the two -/// now carry the same fields, since notes and resources share one identity model -/// (L3, data-model.md §10). +/// What [`delete_resource`] did (L3, data-model.md §10). #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ResourceDeleteReport { pub path: String, @@ -44,24 +32,19 @@ pub struct ResourceDeleteReport { pub dangled: Vec, } -/// What [`delete_dir`] did: the deleted folder, how many **indexed** -/// notes/resources died with it (unindexed files inside are removed too — the -/// whole directory goes), and the surviving files whose links now dangle. +/// What [`delete_dir`] did: the folder, how many indexed notes and resources went with +/// it, and the files whose links now dangle. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct DirDeleteReport { pub dir: String, pub deleted_notes: usize, pub deleted_resources: usize, - /// See [`DeleteReport::dangled`] — only files *outside* the deleted folder - /// (a linker inside it died with the folder). + /// See [`DeleteReport::dangled`]; only files outside the folder. pub dangled: Vec, } -/// Delete the note at `rel`: file off disk, projection rows off the index -/// (chunks/FTS/centroid/outbound edges cascade with the `notes` row), then -/// re-project the inbound linkers so their edges re-dangle. The façade resolved the ref. -/// The note's chunk **vectors** deliberately do not cascade — content-addressed and -/// possibly shared (ADR-0006), they are collected by the whole-vault pass. +/// Delete the note at `rel` (already resolved). Vectors stay for the whole-vault pass +/// to collect, as they may be shared (ADR-0006). pub fn delete_note(ctx: ProjectionCtx, rel: &str) -> Result { let dangled = delete_set(ctx, &[rel], &[], || { remove_file_if_present(&ctx.root.join(rel)) @@ -72,11 +55,7 @@ pub fn delete_note(ctx: ProjectionCtx, rel: &str) -> Result { }) } -/// Delete the resource at `rel` — the note delete minus the identity step: file -/// off disk, inventory row off the index (inbound edges' `dst_resource_path` is -/// `ON DELETE SET NULL`), then re-project the inbound linkers so their edges -/// re-key to the raw-path (dangling) ids a rebuild derives. The façade checked the -/// inventory, as it does for every resource op. +/// Delete the resource at `rel` (already inventory-checked). pub fn delete_resource(ctx: ProjectionCtx, rel: &str) -> Result { let dangled = delete_set(ctx, &[], &[rel], || { remove_file_if_present(&ctx.root.join(rel)) @@ -87,15 +66,10 @@ pub fn delete_resource(ctx: ProjectionCtx, rel: &str) -> Result Result { - // The UI only sends tree-derived paths, so an invalid input (empty, absolute, - // escaping, a dotfolder) is refused as "no such folder" rather than growing a - // delete-specific destination error. + // The UI only sends tree-derived paths, so invalid input reads as "no such folder". let dir = pathspec::normalize_rel_dir(dir_input) .map_err(|_| Error::DirNotFound(dir_input.trim().to_string()))?; let abs = ctx.root.join(&dir); @@ -120,11 +94,8 @@ pub fn delete_dir(ctx: ProjectionCtx, dir_input: &str) -> Result Result<()> { match fs::remove_file(abs) { Ok(()) => Ok(()), diff --git a/crates/b2-core/src/search.rs b/crates/b2-core/src/search.rs index badfec4..4b1035b 100644 --- a/crates/b2-core/src/search.rs +++ b/crates/b2-core/src/search.rs @@ -19,35 +19,24 @@ pub struct Hit { pub note_path: String, /// Higher is better (RRF score for hybrid; negated distance for vector-only). pub score: f64, - /// Which signals ranked this chunk, and how far its vector really sits from - /// the query — the absolute readings RRF discards (invariants.md D2). + /// The absolute readings RRF discards (invariants.md D2). pub provenance: HitProvenance, } -/// Per-hit provenance carried **beside** the fused order, never folded into it -/// (ADR-0015, GH #201). -/// -/// RRF reduces each list to integer ranks and adds reciprocals, which is exactly what -/// makes a fused score read as confidence: rank 1 of a list that should have been empty -/// scores identically to rank 1 of a list full of real matches. These are the absolute -/// signals that reduction throws away. Recording them changes no ordering. +/// Per-hit provenance carried beside the fused order, never folded into it (ADR-0015, +/// GH #201). RRF's ranks can't tell a real rank 1 from one in a list that should have been +/// empty; these absolute signals can. #[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct HitProvenance { - /// 0-based rank in the BM25 list; `None` = the lexical half never ranked - /// this chunk — the "dense-only hit" a nonsense query serves `limit` of. + /// 0-based rank in the BM25 list; `None` = a dense-only hit. pub bm25_rank: Option, - /// 0-based rank in the dense list; `None` = the vector half never ranked it, - /// or never ran (a projected-but-unembedded vault). + /// 0-based rank in the dense list; `None` = not ranked, or no vectors yet. pub vector_rank: Option, - /// L2 distance from the query vector to this chunk's, in - /// [`db::vector_search`]'s units — `cosine = 1 - d²/2`, the model's vectors - /// being unit-length. `None` whenever `vector_rank` is. + /// L2 distance to the query vector (`cosine = 1 - d²/2` for unit vectors). pub distance: Option, } -/// Build the per-chunk provenance map for a fusion over `(bm25, vector)`, the -/// dense list carrying its distances. Chunks absent from a list simply carry -/// `None` for it — absence *is* the reading. +/// The per-chunk provenance map for a fusion over `(bm25, vector)`. fn provenance_of(bm25: &[i64], vector: &[(i64, f32)]) -> HashMap { let mut map: HashMap = HashMap::new(); for (rank, &id) in bm25.iter().enumerate() { @@ -61,17 +50,11 @@ fn provenance_of(bm25: &[i64], vector: &[(i64, f32)]) -> HashMap], k: usize) -> Vec<(i64, f64)> { let mut scores: HashMap = HashMap::new(); for list in ranked_lists { @@ -94,10 +77,8 @@ pub fn rrf_fuse(ranked_lists: &[Vec], k: usize) -> Vec<(i64, f64)> { out } -/// BM25 keyword search over chunk text -> chunk ids, best first. The raw query is -/// sanitized into a safe FTS5 expression first ([`fts5_query`]): natural-language -/// punctuation is FTS5 *syntax* and would otherwise raise a parse error. A query with -/// no usable terms yields no hits; the vector half still runs. +/// BM25 keyword search over chunk text -> chunk ids, best first. The query is sanitized +/// first ([`fts5_query`]), since punctuation is FTS5 syntax. pub fn keyword_search(conn: &rusqlite::Connection, query: &str, limit: usize) -> Result> { let match_expr = fts5_query(query); if match_expr.is_empty() { @@ -111,20 +92,15 @@ pub fn keyword_search(conn: &rusqlite::Connection, query: &str, limit: usize) -> Ok(rows.collect::>>()?) } -/// Turn arbitrary user text into a safe FTS5 `MATCH` expression: extract -/// alphanumeric terms, wrap each as a double-quoted string literal (so nothing in -/// the input is interpreted as FTS5 operators), and OR them for keyword recall — -/// the vector half supplies semantics, so the keyword half should be forgiving. -/// Returns an empty string when the query has no usable terms. +/// Arbitrary text as a safe FTS5 `MATCH` expression: alphanumeric terms, each quoted, +/// OR-ed for recall. Empty when there are no usable terms. pub fn fts5_query(raw: &str) -> String { let terms: Vec = query_terms(raw).iter().map(|t| quoted(t)).collect(); terms.join(" OR ") } -/// The terms [`fts5_query`] ORs, in query order, before quoting: every maximal -/// alphanumeric run. Factored out because the evidence reading must judge **exactly** -/// the terms the `MATCH` expression searches for — a second tokenizer here would let -/// the two disagree about what the query even said. +/// The terms [`fts5_query`] ORs, unquoted. Shared so the evidence reading judges exactly +/// the terms searched for. pub fn query_terms(raw: &str) -> Vec { raw.split(|c: char| !c.is_alphanumeric()) .filter(|t| !t.is_empty()) @@ -132,78 +108,47 @@ pub fn query_terms(raw: &str) -> Vec { .collect() } -/// One term as a double-quoted FTS5 string literal, so nothing in it is read as -/// an operator. Internal quotes cannot occur (the split drops them) but are -/// doubled defensively regardless. +/// One term as a quoted FTS5 string literal. Quotes can't occur but are doubled anyway. fn quoted(term: &str) -> String { format!("\"{}\"", term.replace('"', "\"\"")) } -/// One query term's lexical reading: the term as written, and its **document -/// frequency** — how many chunks match it alone. +/// One query term and its document frequency. #[derive(Debug, Clone, PartialEq)] pub struct TermEvidence { - /// The term the sanitizer extracted (unstemmed; FTS5 stems it on the way in, - /// so `df` is the stemmed reading of this surface form). + /// The term as written; FTS5 stems it, so `df` is the stemmed reading. pub term: String, - /// Chunks matching this term. `0` = the vault has never seen the word, which - /// is the strongest lexical statement available: not "ranked low", *absent*. + /// Chunks matching this term. `0` = the vault has never seen the word. pub df: usize, } -/// The lexical half's **absolute** reading for one query (ADR-0015) — what the OR-ed -/// `MATCH` knows and then loses. -/// -/// Matching at all is not evidence: [`keyword_search`] ORs every term for recall, so a -/// phrase-shaped query matches on its function words — on the eval corpus the labelled -/// negatives match 68 of 70 chunks through `a` / `to` / `my` alone (GH #201). So neither -/// a raw hit count nor a raw best score is a lexical-anchor test. -/// -/// Document frequency is, and it is kept as a **weight**, not a bin: a term is worth its -/// IDF, so a word in most chunks counts for almost nothing and a word in none counts for -/// the most there is. That keeps the rule working on a vault whose jargon is another -/// vault's rare word (see [`LexicalEvidence::idf`]). +/// The lexical half's absolute reading for one query (ADR-0015). Matching at all is not +/// evidence: OR-ed terms match on function words (GH #201). Each term is weighted by IDF +/// instead, so common words count for almost nothing. #[derive(Debug, Clone, PartialEq)] pub struct LexicalEvidence { - /// Chunks in the index — the scale every term's weight is read against. + /// Chunks in the index. pub chunk_total: usize, /// Every distinct term the query contributed, in first-appearance order. pub terms: Vec, } impl LexicalEvidence { - /// A term's inverse document frequency in this vault: `ln((chunks + 1) / (df + 1))`. - /// - /// This is the whole stopword policy, and it is **continuous** — no ceiling anywhere - /// in it. The `+1`s keep both ends finite: a `df` of zero is the reading this rule - /// most depends on, and an empty vault must not divide by zero. - /// - /// A ceiling was tried first and **failed its transfer check** (GH #201): a "term in - /// <= 10% of chunks is content" rule is scale-free in the vault's *size* but not its - /// *topical concentration*, so on the 15-chunk single-domain fixture `drone` (df 3) - /// and `comb` (df 7) were classed stopwords in a vault about beekeeping and the bar - /// cut 3 of 15 answerable queries. Weighting has no bin to put them on the wrong - /// side of. + /// A term's IDF in this vault: `ln((chunks + 1) / (df + 1))`. The whole stopword policy, + /// continuous by design: a df ceiling failed on topically concentrated vaults (GH #201). + /// The `+1`s keep `df = 0` and an empty vault finite. pub fn idf(&self, df: usize) -> f64 { ((self.chunk_total as f64 + 1.0) / (df as f64 + 1.0)).ln() } - /// How much of the query's **content** the vault actually holds: the share of its - /// total term IDF carried by terms with `df >= 1`. - /// - /// Weighted, not counted, and that is what makes it a content test. Function words - /// weigh nothing, so they neither grant coverage nor dilute it; words the vault has - /// never seen weigh the *most*, so a query is judged mostly on the words that would - /// have mattered. `None` when no term carries any weight at all — that is *no - /// reading*, not a coverage of zero, and the cosine half then decides alone. + /// The share of the query's total IDF carried by terms the vault holds (`df >= 1`). + /// `None` when no term has any weight: no reading, not zero coverage. pub fn term_coverage(&self) -> Option { let total: f64 = self.terms.iter().map(|t| self.idf(t.df)).sum(); if total <= f64::EPSILON { return None; } - // Folded from an explicit `0.0` rather than `.sum()`, whose additive identity - // for `f64` is `-0.0`: an empty present-set is this function's most common - // reading, and a coverage of `-0.0` prints as `-0.00`. + // Not `.sum()`: its `f64` identity is `-0.0`, which prints as `-0.00`. let present: f64 = self .terms .iter() @@ -213,24 +158,14 @@ impl LexicalEvidence { Some(present / total) } - /// Whether the query has a **lexical anchor** — ADR-0015's lexical half. Coverage, - /// not presence: an off-topic query shares one word with almost any vault, so what - /// separates a query the vault has material for is how much of the query's own - /// weight the vault carries. + /// Whether the query has a lexical anchor (ADR-0015): coverage, not mere presence. pub fn anchored(&self, min_term_coverage: f64) -> bool { self.term_coverage().is_some_and(|c| c >= min_term_coverage) } } -/// Read the lexical evidence for `query`: every term's document frequency, over the -/// same sanitized terms [`keyword_search`] searches for. One `count(*)` per distinct -/// term — the probe FTS5 has no cheaper form of without an `fts5vocab` shadow table. -/// -/// **Distinct means distinct to FTS5, not to `str::eq`** (PR #205 review). `chunks_fts` -/// is tokenized, so `Memory`, `memory` and `memories` are one token and match the same -/// chunks; counting them separately would let a query weigh one piece of evidence twice, -/// which changes verdicts rather than numbers (a measured pair straddled the shipped -/// 0.20 bar). So the fold is onto [`fts_tokens`]' reading, which is the index's own. +/// Read the lexical evidence for `query`: one `count(*)` per distinct term. Distinct to +/// FTS5 ([`fts_tokens`]), not `str::eq`, so `Memory`/`memories` count once (PR #205). pub fn lexical_evidence(conn: &rusqlite::Connection, query: &str) -> Result { let chunk_total: usize = conn .query_row("SELECT count(*) FROM chunks", [], |r| r.get::<_, i64>(0))? @@ -241,8 +176,7 @@ pub fn lexical_evidence(conn: &rusqlite::Connection, query: &str) -> Result = Vec::new(); let mut seen: Vec = Vec::new(); for (term, token) in raw.into_iter().zip(tokens) { - // A repeated term is one piece of evidence, not two — and the first spelling is - // shown, since the token is an index-internal form no reader typed. + // Count a repeated token once, showing its first spelling. if seen.contains(&token) { continue; } @@ -260,24 +194,15 @@ pub fn lexical_evidence(conn: &rusqlite::Connection, query: &str) -> Result Result> { if terms.is_empty() { return Ok(Vec::new()); } - // Named for the tokenizer it was built with, so a mid-session `rebuild_fts` - // swap lands on a different table instead of reusing a stale one. + // Named for its tokenizer, so a mid-session swap doesn't reuse a stale table. let tokenizer = db::index_tokenizer(conn)?; let table = format!("b2_qtok_{}", tokenizer.sql().replace(' ', "_")); conn.execute_batch(&format!( @@ -298,8 +223,7 @@ fn fts_tokens(conn: &rusqlite::Connection, terms: &[String]) -> Result = terms.to_vec(); let mut stmt = conn.prepare(&format!( "SELECT doc, term FROM temp.{table}_v ORDER BY doc, offset" @@ -323,36 +247,20 @@ fn fts_tokens(conn: &rusqlite::Connection, terms: &[String]) -> Result Option { let base = model_id.split_once('@').map_or(model_id, |(id, _)| id); match base { @@ -362,69 +270,43 @@ impl EvidenceBar { } } -/// The evidence bar for the shipped model, `BAAI/bge-base-en-v1.5` (ADR-0015). -/// -/// Read from the search evidence bake-off and re-derived on every `make eval` run rather -/// than quoted here — deliberately no numbers in this comment, which is the mistake -/// GH #187 named, where a floor's docstring froze the day's readings. -/// -/// Where the margin lives is worth knowing, though. The lexical half does nearly all the -/// work and the cosine half is a **thin backstop** for what it drops: ask the lexical -/// half for a stricter coverage and the cosine window collapses, because the queries it -/// then has to rescue are exactly the ones with the weakest semantic evidence too. So -/// the safe band is a joint one and the two are placed inside it together, never tuned -/// one at a time. Both sit toward the **serving** side of their windows, because the -/// errors are not symmetric: serving a weak result costs a little trust, and cutting a -/// real one is the tripwire ADR-0015 asserts at zero. +/// The evidence bar for `BAAI/bge-base-en-v1.5` (ADR-0015), re-derived on every `make eval` +/// (no numbers here, GH #187). The two values form a joint band: tune them together, never +/// one at a time. Both lean toward serving, since cutting a real result is the worse error. pub const BGE_BASE_EVIDENCE_BAR: EvidenceBar = EvidenceBar { min_term_coverage: 0.20, min_cos: 0.54, }; -/// The query-level evidence reading behind a search (ADR-0015): the two absolute signals -/// a surface needs to say whether it vouches for anything at all. Assembled by a caller -/// that wants a verdict, from a [`Retrieval`]'s `best_cos` and a [`lexical_evidence`] read. +/// The query-level evidence behind a search (ADR-0015), from a [`Retrieval`]'s `best_cos` +/// and a [`lexical_evidence`] read. #[derive(Debug, Clone, PartialEq)] pub struct QueryEvidence { - /// The lexical half's reading — see [`LexicalEvidence`]. pub lexical: LexicalEvidence, - /// The dense half's — see [`Retrieval::best_cos`]. + /// See [`Retrieval::best_cos`]. pub best_cos: Option, } impl QueryEvidence { - /// ADR-0015's query-level verdict: does the vault hold **positive evidence** for - /// this query — a lexical anchor, *or* semantic proximity clearing `bar`? - /// - /// Two independent signals, which is what keeps this out of the single-population - /// trap GH #196 measured: a topic query in a vault full of that topic carries lexical - /// evidence, semantic evidence, or both, while nonsense carries neither. A - /// one-signal test could not tell *nothing matches* from *everything matches*. + /// Whether the vault holds positive evidence: a lexical anchor or semantic proximity + /// clearing `bar` (ADR-0015). Two independent signals, since one can't tell "nothing + /// matches" from "everything matches" (GH #196). pub fn vouched(&self, bar: EvidenceBar) -> bool { self.lexical.anchored(bar.min_term_coverage) || self.best_cos.is_some_and(|c| c >= bar.min_cos) } } -/// A retrieval's fused order plus the one absolute signal that comes free with it: the -/// dense half's best cosine, the head of a list the call has already produced. -/// -/// The **lexical** half of the evidence is deliberately not here. It costs a `count(*)` -/// per distinct term and is read by [`lexical_evidence`] only where a caller wants a -/// verdict — every search would otherwise pay for a reading nothing consumes, worst on -/// the keyword-only fallback, which has no vector scan to hide the cost behind. +/// A retrieval's fused order plus the dense half's best cosine, which comes free. The +/// lexical evidence costs a query per term, so [`lexical_evidence`] reads it on demand. #[derive(Debug, Clone, PartialEq)] pub struct Retrieval { pub hits: Vec, - /// Best cosine between the query vector and any scanned chunk vector: the - /// dense half's strongest absolute claim about *this* query, as opposed to - /// the rank it always has. `None` on a projected-but-unembedded vault, where - /// there is no dense half and BM25's own emptiness is already honest. + /// Best cosine between the query and any chunk vector. `None` on an unembedded vault. pub best_cos: Option, } impl Retrieval { - /// No hits and no dense reading: what a zero `limit` asks for. fn empty() -> Self { Self { hits: Vec::new(), @@ -433,28 +315,15 @@ impl Retrieval { } } -/// How wide a pool to pull from each signal before fusing (qmd keeps ~30). -/// -/// `pub(crate)` because a *measurement* needs it: the façade's two candidate-pool -/// helpers compose it with each view's headroom to state the total per-signal depth -/// that view reaches (GH #141). It is this 5x multiplication that makes one unit of -/// façade headroom cost five candidates, and so makes headroom a **ranking** choice -/// rather than a plumbing one (GH #142). -/// -/// Saturating, like the façade's own widenings: `limit` is user input and the two -/// widenings compose, so a caller can reach a product that overflows `usize` — which -/// release builds *wrap*, turning an absurd ask into a silently tiny pool. Saturation -/// degrades honestly: an unreachable pool is just every candidate there is. +/// How wide a pool to pull from each signal before fusing (qmd keeps ~30). `pub(crate)` +/// for the façade's candidate-pool measurements (GH #141, #142). Saturating: `limit` is +/// user input, and a wrapped product would silently shrink the pool. pub(crate) fn pool_size(limit: usize) -> usize { limit.saturating_mul(5).max(30) } -/// Keyword-only search: BM25 over `chunks_fts`, resolved to notes — the fallback that -/// makes a **projected-but-unembedded** vault searchable with no query embedding and no -/// model. Scores are the RRF of the single BM25 list, so they live on the same scale as -/// [`hybrid_search`]'s. This path was always honest about zero, and the [`Retrieval`] -/// says so with `best_cos: None`: there is no dense half here to overrule the lexical -/// half's silence (ADR-0015). +/// Keyword-only search, the fallback for an unembedded vault. Scores are single-list RRF, +/// on [`hybrid_search`]'s scale; `best_cos` is `None` (ADR-0015). pub fn keyword_only_search( conn: &rusqlite::Connection, query: &str, @@ -475,9 +344,7 @@ pub fn keyword_only_search( } /// Hybrid search: BM25 ⊕ vector(query) -> RRF -> top `limit`, resolved to notes -/// (ADR-0008). A `limit` of 0 returns before the query is embedded — with the real model -/// that call is the expensive part, and asking for no results should cost none. The same -/// guard opens [`keyword_only_search`]. +/// (ADR-0008). A `limit` of 0 returns before the costly query embedding. pub fn hybrid_search( conn: &rusqlite::Connection, embedder: &dyn Embedder, @@ -500,10 +367,8 @@ pub fn hybrid_search( fuse(conn, bm25, &dense, limit) } -/// The shared tail of both retrievals: fuse the BM25 list with the dense one (if any) -/// by RRF, keep each chunk's provenance, resolve the first `limit` to notes, and carry -/// the dense half's absolute reading. With no dense list this is the single-list RRF -/// the keyword-only fallback has always served, and `best_cos` is `None`. +/// The shared tail of both retrievals: RRF-fuse, keep provenance, resolve the first +/// `limit` to notes. Dense goes last, so it breaks ties. fn fuse( conn: &rusqlite::Connection, bm25: Vec, @@ -511,8 +376,7 @@ fn fuse( limit: usize, ) -> Result { let provenance = provenance_of(&bm25, dense); - // The dense list is nearest-first, so its head *is* the best cosine — the absolute - // reading RRF is about to reduce to "rank 0". + // The dense list is nearest-first, so its head is the best cosine. let best_cos = dense.first().map(|&(_, d)| cosine_of_distance(d)); let mut lists = vec![bm25]; if !dense.is_empty() { @@ -524,23 +388,16 @@ fn fuse( }) } -/// Cosine similarity from an L2 distance between **unit** vectors: `cos = 1 - d²/2`. -/// bge normalizes its output, so this is an identity rather than an approximation — and -/// it is what puts the dense half's absolute reading on the one scale a bar can be -/// written in. +/// Cosine from an L2 distance between unit vectors: `cos = 1 - d²/2` (exact, as bge +/// normalizes its output). pub fn cosine_of_distance(distance: f32) -> f64 { let d = distance as f64; 1.0 - (d * d) / 2.0 } -/// Vector-only search: the dense half of [`hybrid_search`] alone. -/// -/// **An ablation instrument, not a product surface** (ADR-0013, GH #158): the eval scores -/// it beside bm25-only and hybrid on every run, so fusion has a measured single-signal -/// baseline to answer to. Scores are negated L2 distance (closer = higher), the same -/// convention as discovery's, and **not** commensurable with RRF scores. A -/// vault with no embedding space yields no hits — there are no vectors to scan, and -/// pretending otherwise would rank on nothing. +/// Vector-only search: the dense half of [`hybrid_search`] alone. An eval ablation +/// instrument, not a product surface (ADR-0013, GH #158). Scores are negated L2 distance, +/// not comparable with RRF scores. pub fn vector_only_search( conn: &rusqlite::Connection, embedder: &dyn Embedder, @@ -566,16 +423,9 @@ pub fn vector_only_search( resolve_hits(conn, ranked, &provenance, limit) } -/// The shared tail of the retrieval entry points: resolve the ranked -/// `(chunk_id, score)` list to [`Hit`]s, best first, keeping the first `limit` **that -/// still resolve** to a note. -/// -/// One pass over the whole ranking, stopping at `limit` — an unresolved chunk is skipped -/// over, never charged against the budget (GH #137). A chunk can only fail to resolve -/// when a concurrent reindex replaced its note's chunks mid-query, the posture C1 -/// promises. Taking the top `limit` *first* would let a dead chunk hold a slot a live -/// candidate could fill, so search would quietly under-fill during that window; steady -/// state is unchanged. Per-hit resolution is fine here because the loop stops at `limit`. +/// Resolve a ranked `(chunk_id, score)` list to the first `limit` [`Hit`]s that still +/// resolve to a note. An unresolved chunk (a concurrent reindex, C1) is skipped, not +/// charged against `limit` (GH #137). fn resolve_hits( conn: &rusqlite::Connection, fused: Vec<(i64, f64)>, @@ -584,9 +434,7 @@ fn resolve_hits( ) -> Result> { let mut hits = Vec::new(); for (chunk_id, score) in fused { - // Tested *before* the push: an after-the-push test can never fire at a limit - // the loop starts below (`limit == 0`), which would turn "give me nothing" into - // "give me the whole ranking". + // Before the push, so `limit == 0` returns nothing. if hits.len() == limit { break; } diff --git a/crates/b2-core/src/snippet.rs b/crates/b2-core/src/snippet.rs index 1f599f3..c2261e2 100644 --- a/crates/b2-core/src/snippet.rs +++ b/crates/b2-core/src/snippet.rs @@ -19,24 +19,20 @@ fn head_snippet(flat: &str) -> String { } } -/// Collapse a chunk's text to a single-line, length-bounded snippet (its head). Used -/// where there is no query to center on (e.g. `similar`'s evidence passage). +/// A chunk's head as a single-line, length-bounded snippet, where there is no query. pub(crate) fn snippet(text: &str) -> String { head_snippet(&flatten(text)) } -/// Like [`snippet`] but windows the excerpt around the first query-term match, so a -/// section-sized chunk still surfaces the matched text instead of only its head. -/// Falls back to the head when no term matches or the match is already in view — a -/// pure vector hit keeps the head. +/// Like [`snippet`] but windowed around the first query-term match. Falls back to the +/// head when nothing matches or the match is already in view. pub(crate) fn query_snippet(text: &str, query: &str) -> String { let flat = flatten(text); if flat.chars().count() <= SNIPPET_CHARS { return flat; } let lower = flat.to_lowercase(); - // The same tokenizer the lexical half matched with, so the window lands on a term - // that actually ranked the hit. + // The lexical half's tokenizer, so the window lands on a term that ranked the hit. let match_pos = crate::search::query_terms(query) .iter() .filter(|t| t.chars().count() >= 2) @@ -51,8 +47,7 @@ pub(crate) fn query_snippet(text: &str, query: &str) -> String { return head_snippet(&flat); }; let chars: Vec = flat.chars().collect(); - // `pos` indexes the lowercased text, whose length can differ from `flat` for - // exotic Unicode; clamp so the slice below can never go out of range. + // Lowercasing can change length for exotic Unicode; clamp so the slice stays in range. let start = (pos - LEAD).min(chars.len()); let end = (start + SNIPPET_CHARS).min(chars.len()); let mut out = String::from("…"); diff --git a/crates/b2-core/src/vault.rs b/crates/b2-core/src/vault.rs index 0955449..7e59086 100644 --- a/crates/b2-core/src/vault.rs +++ b/crates/b2-core/src/vault.rs @@ -1,13 +1,10 @@ -//! The `Vault` façade — B2's one typed core API and the only surface the two dumb -//! adapters call (ADR-0012). It owns the connection and the injected embedder, and -//! exposes only what the shipped commands need; add an operation when a command -//! needs it, never speculatively. +//! The `Vault` façade: B2's one typed core API and the only surface the adapters call +//! (ADR-0012). It owns the connection and the injected embedder. Add an operation when a +//! command needs it, never speculatively. //! -//! A vault is one portable folder: the index lives under `/.b2/` (ADR-0002). -//! [`open`](Vault::open) defaults to the deterministic [`FakeEmbedder`]; -//! [`open_with_embedder`](Vault::open_with_embedder) wires the real model -//! (ADR-0005). Under the fake, `search`'s BM25 half is still real but the vector -//! half is not semantic — callers must not overstate it. +//! The index lives under `/.b2/` (ADR-0002). [`open`](Vault::open) uses the +//! deterministic [`FakeEmbedder`], whose vector half is not semantic; +//! [`open_with_embedder`](Vault::open_with_embedder) wires the real model (ADR-0005). use crate::add; use crate::chunk::ChunkConfig; @@ -34,8 +31,7 @@ mod views; pub use views::*; -// Report types the façade returns — re-exported so adapters name them through the -// one typed contract. +// Report types the façade returns, re-exported for the adapters. pub use crate::add::AddReport; pub use crate::dirs::DirCreateReport; pub use crate::import::ImportReport; @@ -48,51 +44,33 @@ pub use crate::rm::{DeleteReport, DirDeleteReport, ResourceDeleteReport}; /// brings its own (768); a model or dim swap re-embeds on `reindex` (ADR-0007). const EMBED_DIM: usize = 64; -/// Headroom [`Vault::search_chunks`] keeps over `limit`. A small constant, because -/// façade headroom is multiplied: each unit buys [`search::pool_size`] more -/// candidates from each signal, and candidate width changes answers (GH #142). +/// Headroom [`Vault::search_chunks`] keeps over `limit`. Small, because each unit buys +/// [`search::pool_size`] more candidates per signal, and width changes answers (GH #142). const TORN_READ_HEADROOM: usize = 2; -/// [`Vault::search`]'s hit pool: **3×**, and load-bearing. Its results are -/// note-level, so chunks sharing a note collapse onto that note's best one; without -/// the headroom an ordinary query under-fills `limit` distinct notes. The same -/// headroom absorbs a chunk whose note row vanished mid-query (the -/// concurrent-reindex window C1 allows). Dedup scales with `limit`, so this does too. +/// [`Vault::search`]'s hit pool: 3× `limit`. Chunks of one note collapse to its best, +/// so without headroom a query under-fills `limit` notes. Also absorbs a note row +/// vanishing mid-query (C1). fn note_hit_pool(limit: usize) -> usize { limit.saturating_mul(3) } -/// [`Vault::search_chunks`]'s hit pool: `limit` + [`TORN_READ_HEADROOM`]. No dedup -/// here — this is the un-deduped passage view — so the only hit it drops is one -/// whose `chunk_detail` lookup missed on a torn read, a bounded window a constant -/// covers (GH #137). -/// -/// Deliberately *not* `search`'s 3× (GH #142): RRF scores `Σ 1/(k + rank + 1)`, so a -/// wider pool returns different answers, not merely more of them — 10 of 10 probes -/// changed their top-4 passages across exactly that step (`make stability`). Wider -/// may well be better; the labelled corpus is too small to price it (GH #141), so -/// width stays at the conservative setting until an eval can (ADR-0013). +/// [`Vault::search_chunks`]'s hit pool: `limit` + [`TORN_READ_HEADROOM`], covering only +/// torn reads (GH #137). Not 3×: under RRF a wider pool changes answers, and the eval +/// cannot yet price that (GH #141, #142, ADR-0013). fn chunk_hit_pool(limit: usize) -> usize { limit.saturating_add(TORN_READ_HEADROOM) } -/// Candidates **each retrieval signal** pulls for a `limit`-sized -/// [`Vault::search`]: the façade asks retrieval for [`note_hit_pool`] hits, and each -/// signal widens that again before the two lists are fused (ADR-0008). -/// -/// Public because a *measurement* needs it. A corpus with no more chunks than this -/// truncates neither candidate list, so its scores are invariant under candidate -/// width — a pool change reads as "no change" there while moving real-vault results. -/// The eval harness prints that blindness rather than let a reader trust a number -/// that could not have moved (GH #141). ([`search::RRF_K`] re-weights the *same* -/// lists, so it reorders even a tiny corpus.) +/// Candidates each retrieval signal pulls for a `limit`-sized [`Vault::search`] +/// (ADR-0008). Public for the eval: a corpus smaller than this is blind to pool-width +/// changes, and the harness says so (GH #141). pub fn note_candidate_pool(limit: usize) -> usize { search::pool_size(note_hit_pool(limit)) } -/// [`note_candidate_pool`] for the passage view, over [`chunk_hit_pool`]. Always -/// the narrower of the two, which makes it the threshold a *blindness* claim must -/// clear: under it, no number a run prints can move with candidate width (GH #141). +/// [`note_candidate_pool`] for the passage view. The narrower of the two, so the +/// threshold for the blindness claim (GH #141). pub fn chunk_candidate_pool(limit: usize) -> usize { search::pool_size(chunk_hit_pool(limit)) } @@ -102,36 +80,27 @@ pub fn chunk_candidate_pool(limit: usize) -> usize { pub struct Vault { root: PathBuf, conn: Connection, - /// Injected through the seam (ADR-0005): the adapters wire the real model, - /// `open` defaults to `FakeEmbedder`. + /// Injected through the seam (ADR-0005). embedder: Box, - /// The vault's one chunking policy, held here rather than re-defaulted per - /// call, so every path that chunks cuts identically and `incremental ≡ full - /// rebuild` holds by construction. Across a `set_chunk_config` change that - /// guarantee is doc-enforced instead: the change must pair with - /// `project(force)`. The retrieval eval is the only client that overrides it. + /// The one chunking policy, held here so every path cuts identically and + /// incremental ≡ full rebuild. chunk_config: ChunkConfig, } impl Vault { - /// Open the vault rooted at `vault_root` with the deterministic [`FakeEmbedder`] - /// — the default for tests/dev. Creating `/.b2/` if absent; idempotent. + /// Open the vault at `vault_root` with the deterministic [`FakeEmbedder`], creating + /// `/.b2/` if absent. pub fn open(vault_root: &Path) -> Result { Self::open_with_embedder(vault_root, Box::new(FakeEmbedder::new(EMBED_DIM))) } - /// Open the vault with a caller-supplied embedder — the seam the adapters use to - /// inject the real candle model while tests keep the fake. + /// Open the vault with a caller-supplied embedder. /// - /// `open` **never mutates the embedding space** (ADR-0007): shaping the vector - /// tables and any re-embed happen only on `reindex`, so changing the configured - /// model can never silently wipe vectors on the next command. `search` fails - /// fast on a mismatch instead. + /// Never mutates the embedding space (ADR-0007): only `reindex` does, so a model + /// change can't silently wipe vectors. `search` fails fast on a mismatch instead. pub fn open_with_embedder(vault_root: &Path, embedder: Box) -> Result { - // Every façade op opens a `tracing` span (target `b2::vault`), so each op - // reports its own duration and the per-query `b2::sqlite` events carry which - // op they ran under. Inert until an adapter installs a subscriber, so the - // core still reads no clock of its own. + // Every façade op opens a `b2::vault` span, so `b2::sqlite` events carry their + // op. Inert until an adapter installs a subscriber, so the core reads no clock. let _op = tracing::debug_span!(target: "b2::vault", "open").entered(); // `Connection::open` creates the DB file but not its parent. fs::create_dir_all(vault_root.join(".b2"))?; @@ -144,63 +113,40 @@ impl Vault { }) } - /// The vault's **projection context** — the `(conn, root, cfg)` bundle every - /// write-side op threads, built once here rather than per call site (GH #134). - /// Handing an op this rather than [`embed_ctx`](Self::embed_ctx) is what *makes* - /// it model-free: it carries no embedder, so `rm`/`create_note`/`write` cannot - /// embed even by mistake. + /// The projection context every write-side op threads (GH #134). It carries no + /// embedder, so an op given this cannot embed. fn ctx(&self) -> ingest::ProjectionCtx<'_> { ingest::ProjectionCtx::new(&self.conn, &self.root, &self.chunk_config) } - /// [`ctx`](Self::ctx) plus the injected embedder — for the ops that re-embed what - /// they touch (`reindex`, `add`, `link`, `mv`). + /// [`ctx`](Self::ctx) plus the embedder, for ops that re-embed what they touch. fn embed_ctx(&self) -> ingest::EmbedCtx<'_> { ingest::EmbedCtx::new(self.ctx(), self.embedder.as_ref()) } - /// Override the vault's chunking policy. Every subsequent op that chunks cuts - /// with it, so the index stays self-consistent. The client is the out-of-CI - /// retrieval eval (ADR-0013), which sweeps chunker levers in one process; the - /// shipped adapters never call it. It does not re-chunk by itself — pair it with - /// `project(force)`. + /// Override the chunking policy, for the retrieval eval (ADR-0013). Does not + /// re-chunk: pair it with `project(force)`. pub fn set_chunk_config(&mut self, cfg: ChunkConfig) { self.chunk_config = cfg; } - /// Rebuild the FTS index over the **same stored chunk text** with a different - /// tokenizer — the eval harness's lexical-half lever (ADR-0013), not an adapter - /// surface, so the GH #157 stemmer A/B can flip it without re-chunking or - /// re-embedding. The choice is recorded nowhere durable: the index is disposable, - /// and a `reindex` into a fresh `.b2/` restores the shipped default. + /// Rebuild the FTS index with a different tokenizer: the eval's lexical lever + /// (ADR-0013, GH #157). Not recorded; a fresh `.b2/` restores the default. pub fn rebuild_fts(&self, tokenizer: db::FtsTokenizer) -> Result<()> { db::rebuild_fts(&self.conn, tokenizer) } - /// Re-project every `.md` note under the vault root into the index (Flow ①): - /// notes, chunks (+embeddings), and the typed graph. Writes nothing to the vault. - /// **Incremental** — a note whose body is unchanged reuses its vectors rather - /// than re-embedding (see [`reindex_with_progress`](Self::reindex_with_progress) - /// to force a full re-embed or observe progress). + /// Re-project every note into the index and embed it (Flow ①). Incremental; writes + /// nothing to the vault. pub fn reindex(&self) -> Result { self.reindex_with_progress(false, &mut |_| ControlFlow::Continue(())) } - /// [`reindex`](Self::reindex) with the three knobs its adapters need: `force` - /// re-chunks every note even if unchanged; `on_progress` fires after each embed - /// batch so a slow reindex shows a live line instead of looking frozen; and - /// returning [`ControlFlow::Break`] from it **cooperatively cancels** the embed - /// phase at that batch boundary while projection still completes, leaving a - /// consistent, resumable index ([`ReindexReport::cancelled`]). The desktop maps - /// its cancel flag to `Break`; the CLI always returns `Continue`. + /// [`reindex`](Self::reindex) with knobs: `force` re-chunks every note; `on_progress` + /// fires after each embed batch, and returning [`ControlFlow::Break`] cancels the + /// embed phase there, leaving a consistent, resumable index. /// - /// **`force` re-chunks; whether it re-*embeds* is content's to decide.** Vectors - /// are keyed by chunk text (ADR-0006), so forcing a rebuild over unchanged notes - /// finds every vector already stored and reports `embedded: 0` truthfully. Where - /// `force` is actually reached for — a chunker-policy change — the chunk text - /// moves, the hashes miss, and the model runs on exactly what changed. It no - /// longer repairs a *damaged* stored vector; the index is disposable, so deleting - /// `.b2/` is the answer there. + /// `force` re-embeds only chunk text that changed (ADR-0006). pub fn reindex_with_progress( &self, force: bool, @@ -228,13 +174,9 @@ impl Vault { }) } - /// The **projection pass** alone: re-project every `.md` note into - /// `notes`/`chunks`(+FTS)/`edges` with **no model and no vector work**, and no - /// write to the vault. After it returns the file tree lists, notes open, keyword - /// search answers and the graph resolves; only vectors — and thus `similar` and - /// semantic ranking — wait for [`embed`](Self::embed). `force` re-chunks every - /// note, so `project(force)` + `embed` is a full rebuild, costing model calls - /// only where chunk text genuinely moved (ADR-0006). + /// The projection pass alone: model-free, no vault write. Everything but vectors + /// works afterwards; those wait for [`embed`](Self::embed). `force` re-chunks every + /// note. pub fn project(&self, force: bool) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "project", force).entered(); let outcome = ingest::project_vault(self.ctx(), force)?; @@ -247,13 +189,9 @@ impl Vault { }) } - /// The **embed pass** alone: fill a vector for every chunk that lacks one. The - /// pending set is derived from the index itself, so this needs no prior - /// [`project`](Self::project) call in the same process and heals any interruption - /// — a cancelled embed, a crash between passes — by embedding exactly what is - /// still missing. Progress and cooperative cancel behave as in - /// [`reindex_with_progress`](Self::reindex_with_progress), and progress is - /// determinate from the first batch. + /// The embed pass alone: fill a vector for every chunk that lacks one. The pending + /// set comes from the index, so any interruption heals here. Progress and cancel as + /// in [`reindex_with_progress`](Self::reindex_with_progress). pub fn embed( &self, on_progress: &mut dyn FnMut(ingest::ReindexProgress) -> ControlFlow<()>, @@ -266,10 +204,7 @@ impl Vault { }) } - /// Preview a reindex (`reindex --dry-run`): what [`reindex`](Self::reindex) - /// **would** index and (re)embed, with no write of any kind — not to the - /// Markdown, the index, or the vectors. `force` previews a re-chunk of every - /// note. A pure read, so it needs no model. + /// Preview a reindex (`reindex --dry-run`) with no write of any kind. Model-free. pub fn plan_reindex(&self, force: bool) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "plan_reindex", force).entered(); let plan = ingest::plan_reindex(&self.conn, &self.root, force)?; @@ -279,19 +214,15 @@ impl Vault { }) } - /// Active neighbors of the note referenced by `note_ref` (a path, with or - /// without the `.md`), each resolved to the other note's path + title. Errors - /// with [`Error::NoteNotFound`] for an unknown ref — distinct from a found note - /// with no neighbors, which is an empty list. + /// Active neighbors of `note_ref` (a path, `.md` optional). + /// [`Error::NoteNotFound`] for an unknown ref. pub fn neighbors(&self, note_ref: &str) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "neighbors", note = note_ref).entered(); let path = self.resolve_ref(note_ref)?; self.neighbors_of(&path) } - /// The active neighbors of an already-resolved path. Shared by - /// [`neighbors`](Self::neighbors) and [`explain`](Self::explain), so the two - /// present the same edge shape. + /// The active neighbors of an already-resolved path. fn neighbors_of(&self, note_path: &str) -> Result> { let mut out = Vec::new(); for n in graph::neighbors(&self.conn, note_path)? { @@ -314,8 +245,7 @@ impl Vault { Ok(out) } - /// The resource links of an already-resolved path — every outbound edge at a - /// non-`.md` file, resolved with its inventory class for display (GH #22). + /// The outbound resource links of an already-resolved path (GH #22). fn resource_links_of(&self, note_path: &str) -> Result> { Ok(db::outbound_resource_edges(&self.conn, note_path)? .into_iter() @@ -331,8 +261,7 @@ impl Vault { .collect()) } - /// The unresolved (dangling) outbound links of an already-resolved path, shared - /// by [`explain`](Self::explain) and [`unresolved_links`](Self::unresolved_links). + /// The dangling outbound links of an already-resolved path. fn unresolved_of(&self, note_path: &str) -> Result> { Ok(graph::unresolved_outbound(&self.conn, note_path)? .into_iter() @@ -345,10 +274,8 @@ impl Vault { .collect()) } - /// The unresolved (dangling) outbound links of `note_ref`: links it authored - /// that resolve to no note and no resource — a `[[Hermes]]` naming a *folder*, or - /// a typo (GH #12). A pure graph read; [`Error::NoteNotFound`] for an unknown - /// ref, an empty list for a note whose every link resolves. + /// The dangling outbound links of `note_ref`: links resolving to no note or + /// resource, such as a folder name or a typo (GH #12). pub fn unresolved_links(&self, note_ref: &str) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "unresolved_links", note = note_ref) .entered(); @@ -356,11 +283,8 @@ impl Vault { self.unresolved_of(&path) } - /// Explain a note's connections (`b2 explain`): the note resolved to its - /// identity + title, every active typed edge and its "why", its outbound - /// **resource** links (GH #22), and any **unresolved** outbound links — surfaced, - /// not dropped (GH #12). A pure graph read; [`Error::NoteNotFound`] for an - /// unknown ref, empty vectors for a note with no edges. + /// Explain a note's connections (`b2 explain`): typed edges with their "why", + /// resource links (GH #22) and unresolved links (GH #12). pub fn explain(&self, note_ref: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "explain", note = note_ref).entered(); let path = self.resolve_ref(note_ref)?; @@ -377,11 +301,8 @@ impl Vault { }) } - /// Read a note for display: resolve `note_ref` to its file and return the note's - /// **raw Markdown body from disk** (the source of truth, not the index - /// projection) plus the frontmatter metadata worth showing a reader. A pure, - /// model-free read, and ref resolution lives here so the adapter never touches - /// the filesystem itself. [`Error::NoteNotFound`] for an unknown ref. + /// Read a note for display: the raw Markdown from disk (not the index) plus its + /// frontmatter metadata. [`Error::NoteNotFound`] for an unknown ref. pub fn read(&self, note_ref: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "read", note = note_ref).entered(); let path = self.resolve_ref(note_ref)?; @@ -389,8 +310,8 @@ impl Vault { let revision = revision_of(&raw); let parsed = note::parse(&raw); let fields = parsed.fields(); - // The display title is the filename (the frontmatter `title:` is inert), - // derived from the path here so even a not-yet-reindexed note shows one. + // The title is the filename (frontmatter `title:` is inert), so even an + // unindexed note has one. let title = Some(note::display_title(&path)); Ok(NoteView { path, @@ -406,40 +327,25 @@ impl Vault { }) } - /// Save a note's **body** — the editing surface's body write op - /// ([`write_frontmatter`](Self::write_frontmatter) is its frontmatter sibling). - /// Markdown-first and **model-free**: validate that the file on disk still hashes - /// to `base_revision` (else [`Error::WriteConflict`] and nothing is written), - /// splice `body` in verbatim after the untouched frontmatter, write, and - /// re-project the note. A changed body's stale vectors are cleared and join the - /// pending set for any later [`embed`](Self::embed), so saving works with no - /// model provisioned. + /// Save a note's body, model-free. Refuses with [`Error::WriteConflict`] unless the + /// file still hashes to `base_revision`; frontmatter bytes are untouched. /// - /// Returns the **new revision**, hashing the *final* on-disk bytes — which the - /// editor chains its next save on: sequential saves never self-conflict, and only - /// an external write trips the guard. + /// Returns the new revision of the on-disk bytes, which the editor chains its next + /// save on, so only an external write trips the guard. pub fn write(&self, note_ref: &str, body: &str, base_revision: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "write", note = note_ref).entered(); - // The byte-honest splice: frontmatter bytes untouched. self.save_guarded(note_ref, base_revision, |parsed| { parsed.replace_body(body); Ok(()) }) } - /// Save a note's **frontmatter** — [`write`](Self::write)'s sibling (GH #79), - /// with the same shape: validate the content-hash `base_revision`, splice - /// `frontmatter` in verbatim between the fences (every body byte preserved by - /// construction), write, re-project **model-free**. An unchanged body keeps its - /// chunks and vectors, so a frontmatter save never re-embeds. + /// Save a note's frontmatter verbatim: [`write`](Self::write)'s sibling (GH #79). + /// Body bytes are untouched, so it never re-embeds. /// - /// **One** refusal, before any byte reaches disk: a top-level `---` line - /// ([`Error::Frontmatter`]) would close the block early and shift the rest into - /// the body, and the body is not this op's to change. Everything else in the - /// block is the human's: malformed YAML saves fine — it round-trips verbatim, - /// projects best-effort, and surfaces through - /// [`NoteView::frontmatter_readable`], exactly as the same edit made in an - /// external editor would. + /// The one refusal is a `---` line ([`Error::Frontmatter`]), which would end the + /// block early. Malformed YAML saves fine and surfaces through + /// [`NoteView::frontmatter_readable`]. pub fn write_frontmatter( &self, note_ref: &str, @@ -462,13 +368,9 @@ impl Vault { }) } - /// The shared shape of the two editor saves: resolve, check that the file on disk - /// still hashes to `base_revision` (else [`Error::WriteConflict`], nothing written), - /// apply `splice` (which may refuse, also before any byte reaches disk), write, and - /// re-project **model-free** through the ordinary path. - /// - /// The returned revision is read back from disk rather than assumed: the token the - /// editor chains its next save on must describe the file, not our intent. + /// The shared editor save: check `base_revision`, apply `splice` (which may refuse), + /// write, re-project model-free. The returned revision is read back from disk, so it + /// describes the file rather than our intent. fn save_guarded( &self, note_ref: &str, @@ -492,10 +394,7 @@ impl Vault { }) } - /// Every indexed note as a lightweight [`NoteSummary`], ordered by `path` — what - /// the file tree is built from. A pure model-free read: the tree shows exactly - /// the notes the index knows, and every one is [`read`](Self::read)-resolvable, - /// so a click always opens. A never-reindexed vault lists nothing, no error. + /// Every indexed note as a [`NoteSummary`], path-ordered, for the file tree. pub fn list_notes(&self) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "list_notes").entered(); Ok(db::all_notes(&self.conn)? @@ -504,10 +403,7 @@ impl Vault { .collect()) } - /// Every inventoried resource as a lightweight [`ResourceSummary`], ordered by - /// `path` — the file tree's resource half, a sibling of - /// [`list_notes`](Self::list_notes) rather than a widened union; the adapters - /// compose the tree. + /// Every inventoried resource as a [`ResourceSummary`], path-ordered. pub fn list_resources(&self) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "list_resources").entered(); Ok(db::list_resources(&self.conn)? @@ -521,50 +417,34 @@ impl Vault { .collect()) } - /// Every folder in the vault (vault-relative, sorted, empty ones included) — the - /// file tree's structure half. Read **live off the filesystem, never the index**: - /// folders are user-authored structure with no derived data, so the walk itself - /// is the projection and the tree stays one-to-one with disk. Dot-folders are - /// skipped, as in the ingest walk. Works on a never-reindexed vault. + /// Every folder in the vault (sorted, empty ones included, dot-folders skipped). + /// Read live off the filesystem, never the index, so the tree matches disk. pub fn list_dirs(&self) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "list_dirs").entered(); dirs::list_dirs(&self.root) } - /// Read a resource's **bytes** — [`read`](Self::read)'s non-note sibling, for the - /// viewers an adapter shows in place of the fallback card (an image, today). - /// - /// Inventory-checked *before* the filesystem is touched, the same posture the - /// desktop's *Open in system default* takes: the path must name a row the walk put - /// in `resources`, so this can never be talked into "read any file this process can - /// reach" by a link a note authored. Errors with [`Error::ResourceNotFound`] - /// otherwise — including for a note, which is not resource inventory. + /// Read a resource's bytes, for an adapter's viewer. /// - /// Reads whole, because every caller wants the whole file; a resource too large for - /// a viewer is the *adapter's* judgement (it knows the size from - /// [`explain_resource`](Self::explain_resource) before asking), not a rule the - /// engine imposes. + /// Inventory-checked before touching the filesystem, so a note's link can never + /// read an arbitrary file; [`Error::ResourceNotFound`] otherwise. Size limits are + /// the adapter's call. pub fn read_resource_bytes(&self, path: &str) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "read_resource_bytes", path).entered(); self.require_resource(path)?; Ok(fs::read(self.root.join(path))?) } - /// The absolute path of an inventoried resource, for an adapter that hands the file - /// to the OS (*Open in system default*). Inventory-checked like - /// [`read_resource_bytes`](Self::read_resource_bytes), so a path a note authored can - /// never name a file outside the vault's resources. [`Error::ResourceNotFound`] - /// otherwise. + /// The absolute path of an inventoried resource, for handing to the OS. + /// Inventory-checked like [`read_resource_bytes`](Self::read_resource_bytes). pub fn resource_path(&self, path: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "resource_path", path).entered(); self.require_resource(path)?; Ok(self.root.join(path)) } - /// The fallback card's data for one resource: inventory metadata plus the - /// backlinks panel, straight off the materialized graph. `path` is vault-relative - /// (the adapters dispatch here via [`crate::resource::doc_kind`]); errors with - /// [`Error::ResourceNotFound`] when it is not inventoried. + /// The fallback card's data for one resource: inventory metadata plus backlinks. + /// [`Error::ResourceNotFound`] when not inventoried. pub fn explain_resource(&self, path: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "explain_resource", path).entered(); let detail = self.require_resource(path)?; @@ -588,10 +468,7 @@ impl Vault { }) } - /// Move/rename a resource — the note move minus the identity step: rewrite every - /// inbound link's authored text (each syntax keeping its own relative-vs-root - /// convention), move the file, update the inventory, re-project the touched - /// notes. Errors with [`Error::ResourceNotFound`]; destination errors mirror + /// Move or rename a resource, rewriting inbound links. Errors mirror /// [`move_note`](Self::move_note). pub fn move_resource(&self, path: &str, to: &str) -> Result { let _op = @@ -600,19 +477,12 @@ impl Vault { mv::move_resource(self.embed_ctx(), path, to) } - /// Hybrid search (BM25 ⊕ vector → RRF) resolved to notes, best first, capped at - /// `limit` *notes*: chunk hits dedup to the highest-scoring chunk per note, so - /// one note never appears twice (ADR-0008). + /// Hybrid search (BM25 ⊕ vector → RRF) resolved to at most `limit` distinct notes, + /// best first (ADR-0008). /// - /// **Keyword-first fallback:** when the vector space does not exist yet — a - /// projected-but-unembedded vault — this runs BM25-only rather than returning - /// nothing, so a vault is searchable the moment [`project`](Self::project) - /// finishes. A never-indexed vault yields no hits and no error; callers read - /// [`embed_status`](Self::embed_status) to present keyword-only results honestly. - /// - /// A `limit` of 0 short-circuits ahead of [`retrieve`](Self::retrieve), so it - /// costs no query embedding and no [`Error::ModelMismatch`] either: that guard - /// exists to stop *wrong results*, and there are none to be wrong about. + /// With no vector space yet it runs BM25-only; callers read + /// [`embed_status`](Self::embed_status) to say so. A `limit` of 0 returns early, + /// without [`Error::ModelMismatch`]: there are no results to be wrong. pub fn search(&self, query: &str, limit: usize) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "search", query, limit).entered(); if limit == 0 { @@ -622,13 +492,9 @@ impl Vault { self.note_results(hits, query, limit) } - /// [`search`](Self::search)'s dense half alone — vector KNN resolved to notes, - /// deduped, best first. **The eval harness's ablation instrument** (ADR-0013, - /// GH #158), not an adapter surface: scoring it beside bm25-only and hybrid is - /// what gives fusion a measured single-signal baseline. Same model-mismatch - /// fail-fast as `search`, but a projected-but-unembedded vault returns no hits — - /// where `search` honestly *falls back* to keywords, an ablation that quietly did - /// the same would be measuring the wrong signal. + /// [`search`](Self::search)'s dense half alone: the eval's ablation arm (ADR-0013, + /// GH #158). An unembedded vault returns no hits rather than falling back, or the + /// ablation would measure the wrong signal. pub fn search_vector_only(&self, query: &str, limit: usize) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "search_vector_only", query, limit).entered(); @@ -648,46 +514,22 @@ impl Vault { self.note_results(hits, query, limit) } - /// [`search`](Self::search) with the **evidence behind it** (ADR-0015): the same - /// served results, plus the two absolute signals RRF discards and the verdict - /// they produce. - /// - /// Flow ② could not answer *zero* as first shipped — the dense half always has k - /// nearest, *nearest* is a fact about the vault rather than evidence about the - /// query, and RRF keeps only ranks — so a nonsense query served `limit` - /// confident-looking results. This read is what a surface consults to say **"no - /// matches"** honestly. It judges nothing itself: the results come back whole and - /// in order, and what a surface *vouches for* is the surface's decision to make. + /// [`search`](Self::search) plus the evidence RRF discards and its verdict + /// (ADR-0015), so a surface can say "no matches" honestly. Results come back whole; + /// what to vouch for is the surface's call. /// - /// `limit` caps the rows and nothing else — the evidence is a fact about the - /// query and the vault, not about how many results were asked for. `vouched` is - /// `None` when the active embedder has no calibrated bar (a fake space has no - /// geometry to hold a cosine bar over, and an uncalibrated model's bar would be - /// another model's constant); a caller that gets `None` never guesses one. - /// - /// [`search_evidence_excluding`](Self::search_evidence_excluding) is this same - /// read minus a caller-named set of notes — the follow-up-search form. + /// `limit` caps rows only. `vouched` is `None` when the embedder has no calibrated + /// bar; a caller never guesses one. pub fn search_evidence(&self, query: &str, limit: usize) -> Result { self.search_evidence_excluding(query, limit, &[]) } - /// [`search_evidence`](Self::search_evidence) minus a **caller-named** set of notes: - /// hits on `exclude` paths are dropped before note resolution, so the served rows - /// backfill from the same ranking with the next fresh notes. This is the - /// follow-up-search form an agent loop passes its already-inspected paths to — a - /// re-query that hands back the same head reads as progress and is none. - /// - /// The subtraction is the caller's, never the verdict's: `vouched`, the terms and - /// `best_cos` still read the whole vault, because the evidence is a fact about the - /// query and the vault (ADR-0015), not about what the caller has already read. The - /// retrieval pool is also unchanged — width is a ranking choice priced by the eval, - /// not plumbing (GH #141/#142) — so exclusion spends the same headroom note-dedup - /// does, and a heavily-excluded query may serve fewer than `limit` rows: the - /// ranking's head is spent, and the honest next move is a refined query, not a - /// deeper page. + /// [`search_evidence`](Self::search_evidence) minus the `exclude` paths, for an agent's + /// follow-up search. Rows backfill from the same ranking. /// - /// Paths match exactly as a result served them — a note's identity is its - /// vault-relative path (L1) — and an unknown path excludes nothing. + /// The verdict, terms and `best_cos` still read the whole vault (ADR-0015). The pool + /// is unchanged (GH #141/#142), so a heavily-excluded query may serve fewer than + /// `limit` rows. Paths match exactly (L1). pub fn search_evidence_excluding( &self, query: &str, @@ -702,19 +544,12 @@ impl Vault { excluded = exclude.len() ) .entered(); - // `limit.max(1)` where `search` would short-circuit: a zero limit caps the - // *rows*, and the question this read answers is about the evidence. Going - // through `retrieve(0)` would skip the dense scan and hand back - // `best_cos: None` — which reads as "no embedding space" — so a verdict taken - // from it would be the lexical half wearing both halves' name. + // `limit.max(1)`: `retrieve(0)` would skip the dense scan and report + // `best_cos: None`, which reads as "no embedding space". let mut retrieval = self.retrieve(query, note_hit_pool(limit.max(1)))?; - // The subtraction drops hits only. `best_cos` is a scalar the retrieval - // already carries, so the vault's nearest chunk still reports even when the - // caller has already read the note it belongs to. + // Drops hits only; `best_cos` still reports the vault's nearest chunk. retrieval.hits.retain(|h| !exclude.contains(&h.note_path)); - // The lexical half is read here rather than inside retrieval: it costs a - // `count(*)` per distinct query term, and only a caller that wants a verdict - // should pay for it. + // Read here, not in retrieval: a `count(*)` per term only a verdict needs. let evidence = search::QueryEvidence { lexical: search::lexical_evidence(&self.conn, query)?, best_cos: retrieval.best_cos, @@ -747,9 +582,7 @@ impl Vault { }) } - /// [`resolve_note_hits`](Self::resolve_note_hits) without the provenance — the - /// shared tail of [`search`](Self::search) and - /// [`search_vector_only`](Self::search_vector_only). + /// [`resolve_note_hits`](Self::resolve_note_hits) without the provenance. fn note_results( &self, hits: Vec, @@ -763,9 +596,7 @@ impl Vault { .collect()) } - /// The note-resolution tail of every note-level search: dedup chunk hits to their - /// best-scoring note, resolve path + title + query-windowed snippet, stop at - /// `limit`. + /// Dedup chunk hits to their best note, resolve each with a snippet, stop at `limit`. fn resolve_note_hits( &self, hits: Vec, @@ -780,9 +611,7 @@ impl Vault { if out.iter().any(|(r, _)| r.path == hit.note_path) { continue; // note already represented by a higher-scoring chunk } - // The chunk or its note can only be missing on a torn read; drop the hit - // rather than name a note that is gone — the pool has headroom to backfill - // it (GH #137). + // Missing only on a torn read; the pool's headroom backfills (GH #137). let Some(found) = db::chunk_hit(&self.conn, hit.chunk_id)? else { continue; }; @@ -799,11 +628,7 @@ impl Vault { Ok(out) } - /// [`search`](Self::search) at **chunk** granularity: the top `limit` ranked - /// chunks resolved to their note + heading breadcrumb + full text, with **no note - /// dedup** — one note may appear several times when several of its passages rank. - /// Same retrieval, same fallback, same fail-fast (see [`ChunkSearchResult`] for - /// who consumes this). Retrieves a narrower pool than [`search`](Self::search) — + /// [`search`](Self::search) at chunk granularity, with no note dedup. Narrower pool; /// see [`chunk_hit_pool`]. pub fn search_chunks(&self, query: &str, limit: usize) -> Result> { let _op = @@ -816,9 +641,7 @@ impl Vault { if out.len() == limit { break; } - // The lookup misses only on a torn read; drop such a hit rather than emit a - // half-resolved one — a rank slot with an empty path would read as a real - // result. The pool backfills it (GH #137). + // Missing only on a torn read; the pool backfills (GH #137). let Some(found) = db::chunk_hit(&self.conn, hit.chunk_id)? else { continue; }; @@ -832,11 +655,8 @@ impl Vault { Ok(out) } - /// The shared retrieval core of [`search`](Self::search), - /// [`search_chunks`](Self::search_chunks) and - /// [`search_evidence_excluding`](Self::search_evidence_excluding): hybrid when the embedding space - /// exists (failing fast on a model mismatch), BM25-only on a - /// projected-but-unembedded vault. + /// Shared retrieval: hybrid when the embedding space exists (failing fast on a model + /// mismatch), else BM25-only. fn retrieve(&self, query: &str, pool: usize) -> Result { if db::embedding_space_exists(&self.conn)? { self.ensure_query_space_matches()?; @@ -846,10 +666,8 @@ impl Vault { } } - /// The model-identity guard every query-embedding read shares: the stored vectors - /// must have been produced by the active embedder, or the query vector is - /// incomparable with them and results would be silently wrong (ADR-0007). The fix - /// is a `reindex`. + /// Refuse a query embedding when the stored vectors came from another embedder: the + /// results would be silently wrong (ADR-0007). fn ensure_query_space_matches(&self) -> Result<()> { if let Some((indexed_model, indexed_dim)) = db::recorded_embedder(&self.conn)? { if indexed_model != self.embedder.model_id() || indexed_dim != self.embedder.dim() { @@ -862,31 +680,23 @@ impl Vault { Ok(()) } - /// The vault's embedding coverage as an [`EmbedStatus`] — the honest "N/M - /// embedded" read (#26). A **pure model-free count**, so an adapter can flag - /// results "keyword-only for now" rather than silently under-rank while a vault - /// embeds behind the first tree paint. + /// The vault's embedding coverage (#26), so an adapter can flag results as + /// keyword-only while a vault embeds. Model-free. pub fn embed_status(&self) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "embed_status").entered(); let (embedded, total) = db::embed_progress(&self.conn)?; Ok(EmbedStatus { embedded, total }) } - /// The notes most semantically similar to `note_ref` that are **not already - /// connected** to it — [`discover::candidates`] surfaced directly: vector KNN over - /// the stored embeddings, minus the anchor's 1-hop neighbours, ranked - /// nearest-best-passage first. **The ranked list is what is served** (ADR-0014): - /// no statistic gates it, `limit` is a cap that under-fills only for want of - /// scorable notes, and an empty result means the candidate set is genuinely empty, - /// never a verdict that nothing relates. Each row carries path + title, the - /// passage that made it similar, and — on a real-embedded space with a large - /// enough population — the `z` the strength band derives from. A **pure read over - /// stored vectors, no model call**. [`Error::NoteNotFound`] for an unknown ref. + /// The notes most similar to `note_ref` that are not already linked to it, with the + /// passage that made each similar. No model call. + /// + /// The ranked list is served as is (ADR-0014): no statistic gates it, and empty means + /// no candidates, not that nothing relates. `z` is set only when it means something. pub fn similar(&self, note_ref: &str, limit: usize) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "similar", note = note_ref, limit).entered(); - // Never claim a statistic over a fake-embedded space (`grades`). Grading changes - // what the rows carry, never which rows exist (ADR-0014). + // Grading changes what rows carry, never which rows exist (ADR-0014). let grade = self.grades()?; self.refuse_resource_anchor(note_ref)?; let anchor = self.resolve_ref(note_ref)?; @@ -907,14 +717,9 @@ impl Vault { Ok(out) } - /// **Explain** one note against an anchor's *Similar & unlinked* list (GH #236): - /// where it stands (ranked, past the list, linked, not embedded, not shortlisted), - /// its z and whole-note rank, the field's z population, every passage pair graded on - /// the card's yardstick, and the neighbors both notes share. `limit` is the list - /// length the surface showed, so the rank and z are the card's. Any note can be - /// explained, not only a served one. A pure read over stored vectors, no model call. - /// [`Error::NoteNotFound`] for an unknown ref on either side; - /// [`Error::ResourceUnsupported`] for a resource anchor, as [`similar`](Self::similar). + /// Explain one note against an anchor's Similar & unlinked list (GH #236): where it + /// stands, its z and rank, graded passage pairs, and shared neighbors. `limit` is the + /// list length shown, so rank and z match the card. Any note works. No model call. pub fn explain_similar( &self, anchor_ref: &str, @@ -952,8 +757,7 @@ impl Vault { for g in ex.pairs { let anchor_side = self.passage(g.pair.anchor_chunk_id)?; let candidate_side = self.passage(g.pair.candidate_chunk_id)?; - // A chunk row that vanished mid-read (a concurrent reindex, C1) is skipped - // rather than shown half-empty. + // A chunk that vanished mid-read (C1) is skipped, not shown half-empty. let (Some(a), Some(c)) = (anchor_side, candidate_side) else { continue; }; @@ -983,9 +787,8 @@ impl Vault { }) } - /// Whether this vault's discovery is graded: never over a fake-embedded space, whose - /// hash vectors have no semantic geometry for a z to describe. Judged by the RECORDED - /// identity (the space being searched), not the injected embedder. + /// Whether discovery is graded: never over a fake-embedded space. Judged by the + /// recorded embedder (the space searched), not the injected one. fn grades(&self) -> Result { Ok(!matches!( db::recorded_embedder(&self.conn)?, @@ -1022,16 +825,11 @@ impl Vault { .collect()) } - /// Commit a typed connection `src --type--> dst` (`b2 link`, flow ③): append a - /// typed-link string to the **source note's frontmatter `b2_relations:`** — never - /// the body (ADR-0010) — and re-project it as an `origin='frontmatter'` active - /// edge. Both ends resolve by path. `edge_type` must be a **core** verb; a - /// non-core one errors with [`Error::InvalidRelation`] rather than silently store - /// a typo. **Idempotent:** an existing `(src, dst, type)` edge writes nothing - /// (`created: false`). + /// Commit a typed connection `src --type--> dst` (`b2 link`, flow ③) to the source's + /// frontmatter `b2_relations:`, never the body (ADR-0010). `edge_type` must be a core + /// verb ([`Error::InvalidRelation`]). Idempotent: an existing edge writes nothing. /// - /// Re-projection re-reads the source note, so the adapters open the vault with the - /// same embedder the index was built with, as for `add`/`mv`. + /// Re-projection embeds, so adapters open with the index's embedder. pub fn link( &self, src_ref: &str, @@ -1049,7 +847,7 @@ impl Vault { } let src_path = self.resolve_ref(src_ref)?; let dst_full = self.resolve_ref(dst_ref)?; - // The link path drops the `.md` Obsidian omits (matches how `[[links]]` are written). + // Drop the `.md`, as `[[links]]` are written. let dst_path = dst_full .strip_suffix(".md") .unwrap_or(&dst_full) @@ -1065,8 +863,7 @@ impl Vault { }); } - // The spec targets the dst's path. A note's title is its filename, so a bare - // `[[path]]` already reads as the title — B2 writes no alias. + // The title is the filename, so B2 writes no alias. let spec = link::render_relation(edge_type, &dst_path, explanation); // 1. Markdown first: append to frontmatter b2_relations: (never the body, §0). @@ -1075,8 +872,7 @@ impl Vault { parsed.add_relation(&spec)?; fs::write(&abs, parsed.as_str())?; - // 2. Re-project the source note so the edge re-materializes from the Markdown - // as origin='frontmatter' — a projection of the line just written. + // 2. Re-project, so the edge materializes from the line just written. ingest::ingest_file(self.embed_ctx(), &src_path)?; Ok(LinkReport { @@ -1087,76 +883,53 @@ impl Vault { }) } - /// Move/rename the whole folder `from` to `to` (both vault-relative; a trailing - /// `/` is tolerated). One `fs::rename` moves the directory — unindexed files - /// inside travel too — after every inbound link at the moved set (including - /// vault-root wikilinks *between* co-moved notes) is rewritten; the index then - /// re-projects and the graph never breaks (each moved note's rows re-key to its - /// new path, `ON UPDATE CASCADE`). Errors with [`Error::DirNotFound`], - /// [`Error::MoveDestination`] (including a destination inside the moved folder), - /// or [`Error::MoveTargetExists`] rather than merge into an existing entry. + /// Move or rename the folder `from` to `to` (trailing `/` tolerated), rewriting every + /// inbound link first, including links between co-moved notes. Refuses to merge into + /// an existing entry ([`Error::MoveTargetExists`]). /// - /// Rewriting an inbound file changes its body, so this **re-embeds** those files: - /// the adapters open with the real model for a dir move. + /// Rewritten files re-embed, so adapters open with the real model. pub fn move_dir(&self, from: &str, to: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "mv_dir", from, to).entered(); mv::move_dir(self.embed_ctx(), from, to) } - /// Move/rename the note `note_ref` to `to` (a vault-relative path, `.md` - /// optional), rewriting every inbound `[[oldpath|alias]]` link to the new path and - /// re-projecting. The graph never breaks — the note's rows re-key to the new path - /// (`ON UPDATE CASCADE`), so neighbors and backlinks show the same set before and - /// after and not one chunk is re-embedded (ADR-0003, ADR-0006); the human-readable - /// link text is repaired alongside. Errors with [`Error::NoteNotFound`], - /// [`Error::MoveDestination`] or [`Error::MoveTargetExists`]. + /// Move or rename the note `note_ref` to `to` (`.md` optional), rewriting every + /// inbound link. The note's rows re-key, so the graph never breaks and its chunks + /// are not re-embedded (ADR-0003, ADR-0006). /// - /// Rewriting an inbound file changes its body, so this **re-embeds** those files: - /// the adapters open with the real model for `mv`. + /// Rewritten inbound files re-embed, so adapters open with the real model. pub fn move_note(&self, note_ref: &str, to: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "mv", from = note_ref, to).entered(); let old_rel = self.resolve_ref(note_ref)?; mv::move_note(self.embed_ctx(), &old_rel, to) } - /// Delete the note `note_ref`: the file leaves the disk, its projection rows leave - /// the index, and every inbound link at it **dangles** — never rewritten, - /// surfacing as unresolved (GH #12) — exactly the state an external `rm` plus a - /// full reindex produces. **Model-free**: no body changes, so the inbound - /// re-projection touches no vectors. [`Error::NoteNotFound`] for an unknown ref. + /// Delete the note `note_ref` and its rows. Inbound links dangle, never rewritten + /// (GH #12), as after an external `rm` and a reindex. Model-free. pub fn delete_note(&self, note_ref: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "rm", note = note_ref).entered(); let rel = self.resolve_ref(note_ref)?; rm::delete_note(self.ctx(), &rel) } - /// [`delete_note`](Self::delete_note)'s resource sibling — same posture (file - /// off disk, inventory row off the index, inbound links dangle, model-free). - /// Errors with [`Error::ResourceNotFound`] for a path not in the inventory. + /// [`delete_note`](Self::delete_note) for a resource. pub fn delete_resource(&self, path: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "rm_resource", path).entered(); self.require_resource(path)?; rm::delete_resource(self.ctx(), path) } - /// Delete the whole folder `dir` (vault-relative) and everything inside it — - /// one `fs::remove_dir_all`, so unindexed files inside go too — then every - /// contained note's/resource's rows; surviving linkers outside the folder - /// dangle, as for [`delete_note`](Self::delete_note). Model-free. Errors with - /// [`Error::DirNotFound`] for a missing (or invalid) folder. + /// Delete the folder `dir` and everything in it, unindexed files included. Links + /// from outside dangle, as for [`delete_note`](Self::delete_note). Model-free. pub fn delete_dir(&self, dir: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "rm_dir", dir).entered(); rm::delete_dir(self.ctx(), dir) } - /// Create a new note (`b2 add`): write `path` with a minimal valid frontmatter (an - /// optional `title`, today's `created`) and `content` as its body, then project it - /// — the note is immediately searchable and in the graph. Nothing is added beyond - /// what the template wrote, so it stays fully reconstructible from Markdown. - /// Errors with [`Error::AddDestination`] or [`Error::AddTargetExists`] rather than - /// clobber an existing file. + /// Create a new note (`b2 add`) with minimal frontmatter and `content` as its body, + /// then project it. Never clobbers ([`Error::AddTargetExists`]). /// - /// Projection **embeds** the new note, so the CLI opens with the real model. + /// Projection embeds, so the CLI opens with the real model. pub fn add_note( &self, path: &str, @@ -1168,81 +941,55 @@ impl Vault { add::add_note(self.embed_ctx(), path, title, content, &created) } - /// Create a new, empty note **model-free** — the desktop's New-note action, the - /// create sibling of [`write`](Self::write). Same minimal frontmatter as - /// [`add_note`](Self::add_note) (no title — a note's display title is its - /// filename), projected with **no embedder touched**, so creation works with no - /// model provisioned and a fake-opened vault can never write foreign vectors into - /// a real embedding space (ADR-0007). The chunks join the pending set any later - /// [`embed`](Self::embed) heals; an empty body has nothing to embed anyway. Same - /// refusals as `add_note`. + /// Create a new, empty note model-free (the desktop's New note). No embedder, so a + /// fake-opened vault can't write foreign vectors into a real space (ADR-0007). pub fn create_note(&self, path: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "create", path).entered(); let created = self.today()?; add::create_note(self.ctx(), path, None, None, &created) } - /// Import a file the human already has into the vault folder `dir` (`""` for the - /// root) under `file_name`, from its **bytes** — the desktop's drag-onto-the-tree - /// gesture, where the OS hands the webview content rather than a path. A `.md` - /// lands as a note (projected); anything else as a resource (one inventory row), - /// exactly as the vault walk would have classified it. The bytes are written - /// **verbatim** — B2 authors nothing here, unlike [`add_note`](Self::add_note), - /// which mints a document. **Model-free**, like [`create_note`](Self::create_note). + /// Import a file's bytes verbatim into folder `dir` (`""` for the root) as + /// `file_name`: the desktop's drag onto the tree. Classified as the walk would. + /// Model-free. /// - /// Errors with [`Error::ImportDestination`] for an invalid name or folder/name - /// pair, and [`Error::ImportTargetExists`] rather than clobber an existing file: - /// the destination path *is* the arriving note's identity (ADR-0003), so refusing - /// an occupied one is the whole of the collision story. If projection fails the - /// placed file is removed again — an import either lands and indexes, or leaves - /// nothing behind. + /// Never clobbers ([`Error::ImportTargetExists`]): the path is the identity + /// (ADR-0003). If projection fails the file is removed again. pub fn import_file(&self, dir: &str, file_name: &str, bytes: &[u8]) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "import", dir, file_name).entered(); import::import_bytes(self.ctx(), dir, file_name, bytes) } - /// [`import_file`](Self::import_file) from a **path** instead of bytes — the same - /// op for the same gesture's keyboard half (an OS file picker, which yields - /// paths), keeping the file's own name. Same refusals, plus - /// [`Error::ImportDestination`] for a source that is a folder or has no file name. + /// [`import_file`](Self::import_file) from a path (an OS file picker), keeping the + /// file's name. pub fn import_path(&self, dir: &str, source: &Path) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "import_path", dir).entered(); import::import_path(self.ctx(), dir, source) } - /// Create the folder `dir` (missing parents included, an occupied target refused) - /// — the desktop's New-folder action, the structure sibling of - /// [`create_note`](Self::create_note). A folder is user-authored structure, so - /// this writes the filesystem and touches **nothing else**: no index rows exist - /// for a folder (see [`list_dirs`](Self::list_dirs)), and it is real to Finder, - /// the CLI and a sync the moment this returns. Errors with - /// [`Error::DirDestination`] or [`Error::DirTargetExists`]. + /// Create the folder `dir`, parents included. Touches only the filesystem: folders + /// have no index rows (see [`list_dirs`](Self::list_dirs)). pub fn create_dir(&self, dir: &str) -> Result { let _op = tracing::debug_span!(target: "b2::vault", "mkdir", dir).entered(); dirs::create_dir(&self.root, dir) } - /// Today's date (`YYYY-MM-DD`) from **SQLite** — the same clock that stamps - /// `indexed_at`, so `b2-core` needs no wall-clock crate and the façade stays the - /// determinism boundary. The vault convention for a note's `created:` field. + /// Today's date (`YYYY-MM-DD`) from SQLite, so `b2-core` reads no wall clock itself. fn today(&self) -> Result { Ok(self .conn .query_row("SELECT strftime('%Y-%m-%d','now')", [], |r| r.get(0))?) } - /// The inventory row for `path`, or [`Error::ResourceNotFound`]: the one existence - /// check every resource op makes before touching the filesystem, so a path a note - /// authored can never talk B2 into reading or moving a file outside the inventory. + /// The inventory row for `path`, or [`Error::ResourceNotFound`]. Every resource op + /// checks this first, so a note's link can't reach a file outside the inventory. fn require_resource(&self, path: &str) -> Result { db::resource_detail(&self.conn, path)? .ok_or_else(|| Error::ResourceNotFound(path.to_string())) } - /// Discovery's anchor guard. A resource anchor is honest, not silent: resources - /// become discoverable once they have chunks + centroids. Until then an inventoried - /// resource errs "not yet" ([`Error::ResourceUnsupported`]) — never an empty result — - /// and an unknown path falls through to the caller's not-found. + /// Discovery's anchor guard: an inventoried resource errs + /// [`Error::ResourceUnsupported`] rather than return an empty result. fn refuse_resource_anchor(&self, anchor_ref: &str) -> Result<()> { if crate::resource::doc_kind(anchor_ref) == crate::resource::DocKind::Resource && db::resource_detail(&self.conn, anchor_ref)?.is_some() @@ -1252,19 +999,15 @@ impl Vault { Ok(()) } - /// Resolve a note reference to the indexed note's vault-relative path — which *is* - /// its identity (ADR-0003), so this is one canonicalization rather than the - /// two-step "is it an id? is it a path?" it replaced. The ref may be written with - /// or without the `.md`, as links are. The [`Error::NoteNotFound`] carries the - /// caller's original `note_ref`, so the error reads as they typed it. + /// Resolve a note reference (`.md` optional) to its vault-relative path, its + /// identity (ADR-0003). The error carries `note_ref` as typed. fn resolve_ref(&self, note_ref: &str) -> Result { db::resolve_link_target(&self.conn, note_ref)? .ok_or_else(|| Error::NoteNotFound(note_ref.to_string())) } } -/// A file's save-guard revision: blake3 of its raw bytes. One fn, so `read` -/// (capture) and `write` (validate + return) can never drift. +/// A file's save-guard revision: blake3 of its raw bytes. fn revision_of(raw: &str) -> String { blake3::hash(raw.as_bytes()).to_hex().to_string() } diff --git a/crates/b2-core/src/vault/chat.rs b/crates/b2-core/src/vault/chat.rs index 689ad88..b9cbc73 100644 --- a/crates/b2-core/src/vault/chat.rs +++ b/crates/b2-core/src/vault/chat.rs @@ -1,7 +1,6 @@ -//! Flow ④ on the façade: grounded chat ([`Vault::ask`]) and the one tool-using turn, -//! **Why?** on a *Similar & unlinked* card ([`Vault::why_similar`], ADR-0022). The prompt -//! text and tool schemas live in [`crate::chat`]; this is the orchestration over the -//! façade's own reads. Chat is a reader: nothing model-derived is stored. +//! Flow ④ on the façade: grounded chat ([`Vault::ask`]) and the tool-using Why? turn +//! ([`Vault::why_similar`], ADR-0022). Prompts and tool schemas live in [`crate::chat`]. +//! Chat is a reader: nothing model-derived is stored. use super::{AnswerView, Citation, ToolUseView, Vault}; use crate::chat; @@ -13,26 +12,12 @@ use crate::snippet::snippet; use std::ops::ControlFlow; impl Vault { - /// Flow ④ — grounded chat over the vault: condense → retrieve → assemble → - /// stream → cite, orchestrated here over the core logic in [`crate::chat`]. The - /// provider is injected **per call**: chat is its sole consumer and, unlike the - /// embedder, it carries no index identity (contrast ADR-0007), so nothing about - /// it belongs on the open vault. + /// Flow ④, grounded chat: condense → retrieve → stream → cite. The provider is + /// injected per call: unlike the embedder it carries no index identity (ADR-0007). /// - /// - **Condense** (multi-turn only): one provider call rewrites the follow-up - /// into a standalone retrieval query; on failure it degrades to the raw - /// question, so that step can never break chat. - /// - **Retrieve**: [`search_chunks`](Self::search_chunks) at - /// [`chat::ASK_PASSAGES`], holding `search`'s posture — chat is a reader. - /// - **Stream**: tokens flow up through `on_token` as they arrive; returning - /// `ControlFlow::Break(())` cancels at token granularity and the result reports - /// it ([`AnswerView::cancelled`]). - /// - **Cite**: distinct `[n]` markers resolve to `(path, excerpt)`; a - /// hallucinated marker resolves to nothing, and the text is never rewritten. - /// - /// Nothing model-derived is stored anywhere, and history is the caller's, - /// session-only. Errors: retrieval as `search` raises it; a failed *answer* call - /// as [`Error::Llm`](crate::Error::Llm). + /// A failed condense falls back to the raw question. `on_token` returning `Break` + /// cancels. A hallucinated `[n]` resolves to nothing; the text is never rewritten. + /// A failed answer call is [`Error::Llm`](crate::Error::Llm). pub fn ask( &self, llm: &dyn LlmProvider, @@ -70,39 +55,19 @@ impl Vault { self.stream_answer(llm, &req, on_token) } - /// **Why was this suggested?** — the chat answer behind one *Similar & unlinked* - /// row: explain, grounded and cited, why `candidate_ref` surfaced for `anchor_ref`. - /// - /// A **tool-using** turn (ADR-0022). The model is offered B2's read-only tools - /// ([`chat::why_tools`]: `b2_passage_pairs`, `b2_similar`, `b2_neighbors`, `b2_read`) - /// and makes the lookups itself; each call is one read on this façade, run here and - /// replayed to the model with its result. Every argument defaults to this turn's - /// pair, so `{}` is always a valid call. Three things bound that: - /// - /// - **Round 1 is the lookup round.** Its text is never streamed: a model that - /// answers there answered from nothing. If it calls no tool — or the call fails, - /// most often a model with no tool support — the turn degrades to - /// [`chat::build_why_request`]: B2 makes the pair lookup and hands the evidence over - /// in one plain grounded request. If it calls tools but skips the pair lookup, B2 - /// appends that call itself (`seeded`), because the matched pairs are what the row - /// was ranked on and no explanation is written without them. - /// - **The loop is bounded** at [`chat::MAX_TOOL_ROUNDS`] model calls and - /// [`chat::MAX_CALLS_PER_ROUND`] tool calls each, and the last round offers no - /// tools, so it can only answer. - /// - **The row's position rides in the prompt** — rank and strength from - /// [`similar`](Self::similar) at the `limit` the surface showed, so the explanation - /// describes the card that was clicked. + /// Why? on a Similar & unlinked row: a grounded, cited explanation of why + /// `candidate_ref` surfaced for `anchor_ref`. A tool-using turn (ADR-0022) over + /// [`chat::why_tools`]; every argument defaults to this pair, so `{}` is valid. /// - /// One consequence of the hidden first round: a cancel lands when the first visible - /// token arrives, not during the lookups. + /// - Round 1 is the lookup round and is never streamed. No tool call, or a failed + /// one, degrades to [`chat::build_why_request`]. If the pair lookup was skipped, + /// B2 makes it itself (`seeded`): the row was ranked on those pairs. + /// - At most [`chat::MAX_TOOL_ROUNDS`] rounds of [`chat::MAX_CALLS_PER_ROUND`] calls; + /// the last round offers no tools. + /// - The row's rank and strength at `limit` ride in the prompt. /// - /// A tool call is model output, so it is untrusted: an unknown tool, malformed - /// arguments or an unknown note is answered with an `error:` *result* the model can - /// read, never a failed turn. Every passage a tool hands over is numbered once on one - /// ledger, and the answer's `[n]` markers resolve against it. Streaming, cancellation - /// and the [`Error::Llm`] normalization are [`ask`](Self::ask)'s — chat is a reader - /// here too, and nothing is stored. [`Error::NoteNotFound`] for an unknown ref on - /// either side. + /// Tool calls are untrusted: a bad one gets an `error:` result, never a failed turn. + /// Passages are numbered once on one ledger, which `[n]` resolves against. pub fn why_similar( &self, llm: &dyn LlmProvider, @@ -122,7 +87,7 @@ impl Vault { let anchor = self.resolve_ref(anchor_ref)?; let candidate = self.resolve_ref(candidate_ref)?; - // The row as the surface showed it: same call, same limit, so the same rank + z. + // Same call and limit as the surface, so the same rank and z. let served = self.similar(&anchor, limit)?; let row = served.iter().position(|s| s.path == candidate); let mut desk = ToolDesk { @@ -156,9 +121,7 @@ impl Vault { if round == chat::MAX_TOOL_ROUNDS { req.tools.clear(); // nothing left to do but answer } - // Round 1 is the lookup round, and its text is never shown: a model that - // answers there has answered from nothing, and one that calls tools says at - // most "let me check". From round 2 on, tokens stream as they arrive. + // Round 1's text is never shown: an answer there is from nothing. let completion = if round == 1 { llm.complete(&req, &mut |_| ControlFlow::Continue(())) } else { @@ -173,11 +136,10 @@ impl Vault { return self.why_handoff(llm, &mut facts, desk, tools_used, on_token); } Ok(c) => c, - // Never degraded: a reply past the tool-call cap is a broken or hostile - // server, and a quiet handoff would hide it. + // Never degraded: past the cap is a broken or hostile server. Err(e @ Error::ToolCallLimit { .. }) => return Err(e), - // The lookup round failed: most often a model with no tool support. - // If the server is simply down, the handoff fails the same way, honestly. + // Most often a model with no tool support; a down server fails the + // handoff too. Err(e) if round == 1 => { tracing::debug!( target: "b2::chat", @@ -217,9 +179,7 @@ impl Vault { }); req.exchanges.push(ToolExchange { call, result }); } - // Whatever the model chose to look up, the matched pairs for *this* pair are - // what the row was ranked on. If the lookup round skipped them, B2 makes that - // call itself, so no explanation is written without them on the desk. + // The row was ranked on this pair's matches, so B2 seeds them if skipped. if round == 1 && desk.passages.is_empty() { let seeded = self.seed_pairs(&mut desk, &facts)?; tools_used.push(ToolUseView { @@ -262,9 +222,8 @@ impl Vault { }) } - /// The degrade of a tool-using why-turn: B2 makes the pair lookup itself and hands - /// the evidence over in one plain grounded request — for a model with no tool - /// support, and for one that was offered tools and used none. + /// The why-turn's degrade: B2 makes the pair lookup and sends one plain grounded + /// request. fn why_handoff( &self, llm: &dyn LlmProvider, @@ -287,10 +246,8 @@ impl Vault { Ok(view) } - /// Run one tool call for the model and render its result as text. Infallible by - /// design: the call is model output, so every failure — an unknown tool, arguments - /// that aren't a JSON object, a note that doesn't exist, an index error — becomes an - /// `error:` result the model reads, and the turn carries on. + /// Run one tool call and render its result as text. Infallible: the call is model + /// output, so every failure becomes an `error:` result the model reads. fn run_tool(&self, desk: &mut ToolDesk, facts: &chat::WhyFacts, call: &ToolCall) -> String { let args: serde_json::Value = if call.arguments.trim().is_empty() { serde_json::json!({}) @@ -321,8 +278,7 @@ impl Vault { self.tool_neighbors(¬e) } chat::TOOL_READ => { - // The open note is already on the human's screen; the one worth reading - // by default is the suggestion. + // The anchor is already on screen; default to the suggestion. let note = note.unwrap_or_else(|| facts.candidate_path.clone()); let offset = args.get("offset").and_then(|v| v.as_u64()).unwrap_or(0); self.tool_read(desk, ¬e, offset as usize) @@ -336,9 +292,8 @@ impl Vault { }) } - /// `b2_passage_pairs`: the matched pairs between two notes, then each passage they - /// name that the ledger had not handed over yet. Also returns the pairs over the - /// ledger's numbering, which the seeded call's facts need. + /// `b2_passage_pairs`: the matched pairs between two notes and any passages new to the + /// ledger. Also returns the pairs in ledger numbering. fn tool_passage_pairs( &self, desk: &mut ToolDesk, @@ -350,8 +305,7 @@ impl Vault { for pair in discover::passage_pairs(&self.conn, note, candidate, chat::WHY_PAIRS)? { let a = self.ledger_marker(desk, pair.anchor_chunk_id, note, &mut fresh)?; let c = self.ledger_marker(desk, pair.candidate_chunk_id, candidate, &mut fresh)?; - // A miss means a torn read against a concurrent reindex; skip the pair rather - // than name a passage with no text. + // A miss is a torn read (C1); skip the pair. if let (Some(a), Some(c)) = (a, c) { pairs.push((a, c, pair.score)); } @@ -472,8 +426,7 @@ impl Vault { Ok(Some(desk.passages.len())) } - /// The text of the passages a tool call handed over for the first time. A passage - /// already on the ledger is named by its marker only — the model has its text. + /// The text of passages handed over for the first time. fn fresh_blocks(&self, desk: &ToolDesk, fresh: &[usize]) -> Vec { fresh .iter() @@ -486,9 +439,7 @@ impl Vault { .collect() } - /// The shared tail of [`ask`](Self::ask) and the handoff half of - /// [`why_similar`](Self::why_similar): stream the completion, then resolve the - /// answer's `[n]` markers against the request's own passages. + /// Stream the completion, then resolve `[n]` against the request's passages. fn stream_answer( &self, llm: &dyn LlmProvider, @@ -505,10 +456,8 @@ impl Vault { } } -/// The state one tool-using turn carries between calls: the list length the surface -/// showed, and the **passage ledger** — every passage a tool has handed the model, -/// numbered once in first-seen order, which is what `[n]` resolves against. Whose turn it -/// is (what `{}` arguments default to) rides on the turn's [`chat::WhyFacts`]. +/// A tool-using turn's state: the surface's list length and the passage ledger, every +/// passage handed to the model numbered once in first-seen order. struct ToolDesk { limit: usize, /// Parallel to `passages`: the chunk behind each, so a passage is never numbered twice. @@ -516,17 +465,15 @@ struct ToolDesk { passages: Vec, } -/// B2's own pair lookup: the call as the model would have made it, its result text, and -/// the pairs over the ledger's numbering. +/// B2's own pair lookup, as the model would have made it. struct SeededLookup { call: ToolCall, result: String, pairs: Vec, } -/// Normalize a provider failure: the trait returns the crate-wide `Result`, but every -/// failure of a model call is a failed model call, and adapters match [`Error::Llm`] for -/// the "can't reach the model server" message. Enforced here, not hoped for. +/// Normalize a provider failure to [`Error::Llm`], which adapters match for the +/// "can't reach the model server" message. fn llm_error(e: Error) -> Error { match e { Error::Llm(_) | Error::ToolCallLimit { .. } => e, @@ -539,8 +486,7 @@ fn cite(answer: &str, passages: &[ContextPassage]) -> Vec { chat::cited_markers(answer, passages.len()) .into_iter() .filter_map(|marker| { - // 1-based marker to 0-based passage; skip rather than index, so even a - // broken invariant degrades to a missing citation. + // 1-based marker; a broken invariant degrades to a missing citation. let p = passages.get(marker.checked_sub(1)?)?; Some(Citation { marker, diff --git a/crates/b2-core/src/vault/views.rs b/crates/b2-core/src/vault/views.rs index adfab12..460ec99 100644 --- a/crates/b2-core/src/vault/views.rs +++ b/crates/b2-core/src/vault/views.rs @@ -10,34 +10,27 @@ use super::Vault; #[cfg(doc)] use crate::{graph, note}; -/// What `reindex` did: notes projected, and how many were actually (re)embedded -/// (the rest reused their vectors). It reports no vault writes because there are -/// none — a reindex reads (ADR-0004). -/// -/// `cancelled` marks a cooperative cancel of the embed phase; the counts then -/// describe the partial work truthfully and the index is still consistent (keyword -/// and graph complete, a prefix embedded). Always `false` for -/// [`reindex`](Vault::reindex). +/// What `reindex` did: notes projected and how many were (re)embedded. No vault writes +/// to report: a reindex reads (ADR-0004). After `cancelled`, the counts describe the +/// partial work. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ReindexReport { pub indexed: usize, pub embedded: usize, pub cancelled: bool, - /// Files skipped as unreadable this run — one bad file never fails the pass. + /// Files skipped as unreadable; one bad file never fails the pass. pub skipped: Vec, - /// Ghost rows pruned: notes whose files were deleted outside b2 (#31), so - /// incremental equals a from-scratch rebuild. + /// Ghost rows pruned: notes whose files were deleted outside b2 (#31). pub notes_pruned: usize, pub resources_indexed: usize, pub resources_pruned: usize, } -/// What [`project`](Vault::project) did — the model-free half of a reindex. No -/// embed counts: projection never touches vectors. +/// What [`project`](Vault::project) did: the model-free half of a reindex. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ProjectReport { pub indexed: usize, - /// Files skipped as unreadable this pass, so an adapter can say which and why. + /// Files skipped as unreadable. pub skipped: Vec, /// Ghost rows pruned: notes whose files were deleted outside b2 (#31). pub notes_pruned: usize, @@ -45,46 +38,35 @@ pub struct ProjectReport { pub resources_pruned: usize, } -/// What [`embed`](Vault::embed) did — the model-bound half of a reindex: notes -/// whose missing vectors were filled, and whether a cooperative cancel cut the pass -/// short (the counts stay truthful, and a re-run embeds exactly the remainder). +/// What [`embed`](Vault::embed) did: notes whose missing vectors were filled, and whether +/// a cancel cut the pass short. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] pub struct EmbedReport { pub embedded: usize, pub cancelled: bool, } -/// The vault's embedding coverage — the honest "N/M embedded" signal (#26). -/// Model-free: a pure count over the projection, so an adapter can say -/// "keyword-only for now" precisely without loading a model. `embedded < total` -/// means [`search`](Vault::search) is running keyword-first over the remainder; -/// `embedded == 0` is a fully keyword-only vault. +/// The vault's embedding coverage, "N/M embedded" (#26). `embedded < total` means +/// [`search`](Vault::search) is keyword-only over the remainder. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] pub struct EmbedStatus { - /// Notes with no chunk awaiting a vector. A note with **no chunks at all** — an empty - /// body — counts here: there is no vector it could be waiting for, so it must be able - /// to reach the numerator (see [`crate::db::embed_progress`]). + /// Notes with no chunk awaiting a vector, empty notes included + /// (see [`crate::db::embed_progress`]). pub embedded: usize, /// Every projected note (the denominator). pub total: usize, } -/// What a reindex **would** do — the `reindex --dry-run` preview, computed -/// read-only. The `would_*` keys are the honesty signal: this is a forecast. -/// -/// It forecasts work and nothing else. The dry-run's old columns (which notes would -/// be stamped, which files collide) existed because a real run wrote to the vault; -/// it no longer does (ADR-0004), so only the embedding is left to size. +/// What a reindex would do: the `reindex --dry-run` forecast, computed read-only. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ReindexPlan { - /// Notes a real reindex would project (every `.md` file the walk collects). + /// Notes a real reindex would project. pub would_index: usize, /// …of which this many would be (re)embedded. pub would_embed: usize, } -/// One neighbor of a note, resolved for display: the note at the other end of an -/// edge, with its path + title, so the adapter stays a dumb printer. +/// One neighbor of a note, resolved for display. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct NeighborView { /// The other note's vault-relative path — its identity (ADR-0003). @@ -103,11 +85,8 @@ pub struct NeighborView { pub created: Option, } -/// One outbound link a note authors at a **resource** (an image, a PDF — any -/// non-`.md` vault file), resolved for display. Surfaced on [`ExplainView`] so a -/// note's file links are visible from the note's side, not only as the resource's -/// backlinks (GH #22). Distinct from [`NeighborView`]: a resource has no title and -/// authors no edges, so these are always outbound. +/// One outbound link from a note to a resource (any non-`.md` file), resolved for +/// display (GH #22). Always outbound: resources author no edges. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ResourceLinkView { pub path: String, @@ -122,11 +101,8 @@ pub struct ResourceLinkView { pub explanation: Option, } -/// One outbound link a note authored that resolves to **nothing** — no note and no -/// resource at its target (a `[[Hermes]]` naming a *folder*, or a typo). A note is -/// one `.md` file, so a folder is never a valid target; rather than drop such a link -/// B2 surfaces it as unresolved, so it reads as broken rather than missing (GH #12). -/// It has no `path` — that is the whole point. +/// One outbound link that resolves to no note or resource (a folder name, a typo), +/// surfaced so it reads as broken rather than missing (GH #12). #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct UnresolvedLink { /// The target exactly as written in the Markdown (`[[target]]`). @@ -136,26 +112,20 @@ pub struct UnresolvedLink { pub explanation: Option, } -/// A note's full connection picture for `b2 explain`: the note itself, every active -/// connection with its "why", its outbound resource links, and any unresolved -/// outbound links. A thin header over [`NeighborView`] — it reuses the per-edge -/// shape `neighbors` returns rather than a parallel one. +/// A note's full connection picture for `b2 explain`. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ExplainView { pub path: String, pub title: Option, /// Outbound edges first, then inbound (as [`graph::neighbors`] orders them). pub connections: Vec, - /// Outbound links at resources — the third target kind (GH #22). + /// Outbound links at resources (GH #22). pub resources: Vec, /// Outbound links that resolved to nothing (GH #12). pub unresolved: Vec, } -/// A note's content + display metadata for a reader. Carries the note's identity, -/// the frontmatter fields worth showing a human, and the **raw Markdown body read -/// from disk** (the source of truth, not the projection) so an adapter renders -/// Markdown itself. A pure read — no embedding. +/// A note's raw Markdown from disk (not the projection) plus display metadata. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct NoteView { pub path: String, @@ -166,33 +136,25 @@ pub struct NoteView { pub tags: Vec, /// The note's Markdown body (frontmatter stripped), verbatim from disk. pub body: String, - /// The raw frontmatter YAML **verbatim** (the text between the `---` fences, - /// fences excluded), not a re-serialization of the projected fields above — so - /// `b2_relations:` and any keys B2 doesn't model show as written. + /// The frontmatter YAML verbatim, fences excluded, so unmodelled keys show as + /// written. pub frontmatter: Option, - /// Whether that block *reads* as YAML metadata - /// ([`note::ParsedNote::frontmatter_readable`]): `false` means the raw bytes - /// above are shown verbatim but the projected fields came back empty. Every - /// read passes through here, so an external hand-edit surfaces the same warning - /// as an in-app save (GH #79). + /// Whether that block parses as YAML ([`note::ParsedNote::frontmatter_readable`]); + /// `false` means the projected fields are empty (GH #79). pub frontmatter_readable: bool, - /// blake3 of the **raw file bytes** at read time — the save-guard token - /// [`write`](Vault::write) validates, so a save can never silently clobber an - /// external edit. Whole-file, so *any* out-of-band change conflicts honestly. + /// blake3 of the whole file at read time: the save-guard token + /// [`write`](Vault::write) validates, so a save never clobbers an external edit. pub revision: String, } -/// One note's identity for a listing — `path` + `title`, with **no body**: enough -/// to show and open a note, cheap enough to fetch the whole vault at once. The body -/// is a separate [`read`](Vault::read) when a note is opened. +/// One note's identity for a listing, with no body. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct NoteSummary { pub path: String, pub title: Option, } -/// One resource's identity for the file tree — the per-kind sibling of -/// [`NoteSummary`], never a union type. +/// One resource's identity for the file tree. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ResourceSummary { pub path: String, @@ -233,36 +195,25 @@ pub struct SearchResult { pub snippet: String, } -/// A search's evidence reading — [`Vault::search_evidence`]'s return, and what a -/// surface needs to decide what it vouches for (ADR-0015). +/// A search's evidence reading, for a surface to decide what it vouches for (ADR-0015). #[derive(Debug, Clone, PartialEq, Serialize)] pub struct SearchEvidenceView { - /// The served results, whole and in fused order — evidence never reorders or - /// removes a row. + /// The served results, whole and in fused order. pub results: Vec, - /// Does the vault hold positive evidence for this query at the active model's - /// bar? `None` = no calibrated bar for this embedder, so no verdict is offered - /// rather than one guessed. + /// Whether the vault holds evidence for this query at the model's bar. `None`: no + /// calibrated bar for this embedder, so no verdict. pub vouched: Option, - /// Chunks in the index — the scale every term's weight is read against. + /// Chunks in the index. pub chunk_total: usize, /// Every query term with its document frequency, in query order. pub terms: Vec, - /// Best cosine between the query and any chunk vector — the dense half's - /// absolute claim. `None` on a projected-but-unembedded vault. + /// Best cosine between the query and any chunk. `None` on an unembedded vault. pub best_cos: Option, } -/// One served result with the provenance RRF discarded — which lists ranked its -/// chunk, and how near its vector actually was. -/// -/// The query-level verdict on [`SearchEvidenceView`] is what ADR-0015's "no matches" -/// rests on; **this** is what the per-hit tail bake-off is argued from (GH #206). -/// That bake-off has run — the labels carry the per-hit depth (`tail_relevant`) — -/// and **no tail fold shipped**: the fused order is not an evidence order, so every -/// admissible prefix cut proved vacuous, and the tail complaint is ordering work -/// (the reranker seam), not disclosure work. This stays an instrument reading, -/// re-priced on every `make eval` run. +/// One served result with the provenance RRF discarded: which lists ranked its chunk, +/// and how near its vector was. An instrument reading for `make eval`; no per-hit cut +/// shipped (GH #206). #[derive(Debug, Clone, PartialEq, Serialize)] pub struct EvidencedResult { #[serde(flatten)] @@ -282,22 +233,16 @@ pub struct QueryTermView { pub term: String, /// Chunks matching this term alone; `0` = the vault has never seen the word. pub df: usize, - /// This term's weight in the coverage reading — `ln((chunks+1)/(df+1))`; near - /// zero for a ubiquitous word, largest for one the vault has never seen. + /// This term's weight, `ln((chunks+1)/(df+1))`. pub idf: f64, } -/// One **chunk-level** search hit — the sub-note view of [`search`](Vault::search). -/// Same retrieval, but ranked chunks are returned as-is instead of deduped up to -/// notes, so a caller can see *which passage* matched and at what rank. The client -/// is the out-of-CI retrieval eval (ADR-0013): note-rank scoring is blind to -/// sub-note quality, which is exactly what chunking levers move. Carries the chunk's -/// **full text**, which the eval's containment scoring anchors on. +/// One chunk-level search hit, not deduped to notes, with the chunk's full text. For +/// the retrieval eval, whose chunking levers note-rank scoring can't see (ADR-0013). #[derive(Debug, Clone, PartialEq, Serialize)] pub struct ChunkSearchResult { pub path: String, - /// The chunk's heading breadcrumb (`"Fermentation > Vegetables"`), when the - /// chunker recorded one. + /// The chunk's heading breadcrumb (`"Fermentation > Vegetables"`), if any. pub heading_path: Option, /// Fused relevance score; higher is better. pub score: f64, @@ -305,106 +250,79 @@ pub struct ChunkSearchResult { pub text: String, } -/// The answer to one grounded-chat ask — flow ④'s display view: the model's -/// streamed text with its `[n]` citation markers resolved back to the vault. +/// One grounded-chat answer (flow ④), with its `[n]` markers resolved to the vault. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct AnswerView { - /// The answer, verbatim as streamed — including any marker that did *not* - /// resolve. Model output is untrusted content, but it is never rewritten here; - /// an unmatched marker is simply absent from `citations`. + /// The answer verbatim as streamed, never rewritten; an unresolved marker is just + /// absent from `citations`. pub answer: String, - /// The resolved citations, ascending by marker; one entry per **distinct** - /// marker that names a real passage. + /// One citation per distinct marker naming a real passage, ascending. pub citations: Vec, - /// `true` when the caller's callback broke the stream mid-answer: `answer` then - /// holds the partial text honestly, and citations resolve over what arrived. + /// The caller broke the stream; `answer` holds the partial text. pub cancelled: bool, - /// The B2 tools that ran to produce this answer, in order — empty for a plain - /// [`ask`](Vault::ask), which retrieves once and offers the model none. Serialized - /// only when present, so existing JSON consumers see no change. + /// The B2 tools that ran, in order; empty for a plain [`ask`](Vault::ask). #[serde(skip_serializing_if = "Vec::is_empty")] pub tools: Vec, } -/// One tool run during a tool-using chat turn — shown beside the answer so a human can -/// see what the explanation was built from. +/// One tool run during a chat turn, shown beside the answer. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ToolUseView { /// The tool's name (`b2_passage_pairs`, `b2_read`, …). pub name: String, - /// The call's arguments, as JSON text — the model's own, verbatim, for a call it made. + /// The call's arguments as JSON text, verbatim. pub arguments: String, - /// `true` for the lookup B2 made itself before asking the model anything; `false` - /// for a call the model chose. + /// `true` for B2's own lookup before asking the model; `false` for the model's call. pub seeded: bool, } -/// One resolved `[n]` citation: the passage's note, plus a one-line excerpt of the -/// cited passage as display evidence. +/// One resolved `[n]` citation: the note and an excerpt of the passage. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Citation { /// The marker as it appears in the answer text (1-based passage number). pub marker: usize, - /// Vault-relative path of the cited note — its identity (ADR-0003), and what an - /// adapter opens on click. A note renamed between answer and click goes stale, - /// exactly as any path handle does. + /// The cited note's path (ADR-0003). pub path: String, /// A one-line excerpt of the cited passage (its head, length-bounded). pub excerpt: String, } -/// One semantically-similar candidate for `b2 similar`: a note near the anchor in -/// embedding space that is **not** already connected to it, resolved for display -/// with the passage that made it similar. The machine finds the candidate, the human -/// decides whether to `link` it (ADR-0009). +/// One `b2 similar` candidate: a note near the anchor, not linked to it, with the +/// passage that made it similar. The human decides whether to link it (ADR-0009). #[derive(Debug, Clone, PartialEq, Serialize)] pub struct SimilarView { pub path: String, pub title: Option, /// Best chunk-pair similarity to the anchor; higher is nearer. pub score: f64, - /// A one-line excerpt of the candidate chunk that achieved `score` — the - /// evidence for *why* it surfaced. + /// An excerpt of the candidate chunk that achieved `score`. pub evidence: String, - /// How far this candidate stands above the anchor's own candidate population — - /// its best-passage z (GH #192), and the one honest input for a displayed - /// *strength* band (GH #150). Non-increasing down the row order, so the band - /// never contradicts the ranking. It **gates nothing** (ADR-0014): the band is a - /// within-list grading, never a verdict on existence. The unit is load-bearing — - /// a band calibrated in the retired centroid unit grades every card down - /// (GH #182) — so a surface reads its landmarks off `make eval`'s calibration - /// block. `None` when no statistic was computed (a fake-embedded space, a pool - /// under the statistics minimum, or zero variance), which is the adapters' cue - /// to say the list is *ungraded* rather than let bare cards read as uniformly - /// weak; serialized only when present, so older JSON consumers see no change. + /// The best-passage z over the anchor's candidate population (GH #192), the input + /// for the strength band (GH #150). Non-increasing down the list; gates nothing + /// (ADR-0014). Band landmarks come from `make eval` (GH #182). `None` when + /// ungraded, which adapters must say. #[serde(skip_serializing_if = "Option::is_none")] pub z: Option, } -/// **Explain** for a *Similar & unlinked* card (GH #236): where one note stands in an -/// anchor's discovery field, and the passage pairs behind it. Model-free and read from -/// the same computation as [`Vault::similar`], so a served row's rank, z and best pair are -/// exactly the card's. Raw distances are never the point: every grade is a z on the same -/// yardstick as the strength band. +/// Explain for a Similar & unlinked card (GH #236): where a note stands in the anchor's +/// field and the passage pairs behind it. Same computation as [`Vault::similar`], so a +/// served row matches its card. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct SimilarExplainView { pub anchor: NoteSummary, pub candidate: NoteSummary, - /// The list length the explanation is read at: the surface's own, so `served` - /// answers "was this a card?". + /// The surface's list length, so `served` answers "was this a card?". pub limit: usize, pub standing: SimilarStanding, /// The candidate's z when it was scored in a graded field (the card's band input). pub z: Option, - /// The candidate's rank judged by whole-note average (stage 1), when it entered - /// stage 1. Far worse than a ranked standing's `rank` means one section carries the - /// match: a buried gem. + /// The whole-note (stage 1) rank. Far worse than `rank` means one section carries the + /// match. pub centroid_rank: Option, - /// Every scored note's z, nearest first: the field the band is relative to. Empty - /// when ungraded (a fake space, a small pool, no spread). + /// Every scored note's z, nearest first. Empty when ungraded. pub population: Vec, - /// One pair per candidate passage, nearest first, each matched to the anchor passage - /// nearest to it. Empty when either side has no vectors. + /// Each candidate passage with its nearest anchor passage, nearest first. pub pairs: Vec, /// Notes both sides already link with (either direction). pub shared_neighbors: Vec, @@ -418,12 +336,11 @@ pub enum SimilarStanding { SameNote, /// The anchor has no stored vectors yet: nothing to compare from. AnchorUnembedded, - /// Directly linked (either direction), so discovery leaves it out as already known. + /// Directly linked (either direction), so already known. Linked, /// The candidate has no stored vectors yet. Unembedded, - /// Its whole-note rank fell past the first-pass shortlist of `shortlist` notes, so - /// it was never scored passage by passage. + /// Its whole-note rank fell past the `shortlist`, so passages were never scored. NotShortlisted { shortlist: usize }, /// Scored: `rank` (1-based) of `of` scored notes; `served` when `rank <= limit`. Ranked { @@ -442,8 +359,7 @@ pub struct PassagePairView { pub score: f64, /// The pair's z on the card's yardstick, `None` when ungraded. pub z: Option, - /// Both passages hold the same text (a template, a copy): a perfect match that says - /// nothing about what the notes are about. + /// Both passages hold the same text (a template, a copy): a meaningless match. pub identical: bool, } @@ -456,18 +372,15 @@ pub struct PassageView { pub text: String, } -/// What [`write`](Vault::write) did: the saved note's path and the **new revision** -/// (blake3 of the final on-disk bytes) — the token the editor chains its next save -/// on, so sequential saves never self-conflict. +/// What [`write`](Vault::write) did: the saved path and the new revision the editor +/// chains its next save on. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct WriteReport { pub path: String, pub revision: String, } -/// What `b2 link` did: the committed typed edge, resolved for display. `created` is -/// `false` when the directed `(src, dst, type)` edge already existed, so nothing was -/// written (the command is idempotent). +/// What `b2 link` did. `created` is `false` when the edge already existed. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct LinkReport { pub src_path: String, diff --git a/crates/b2-embed/Cargo.toml b/crates/b2-embed/Cargo.toml index 2e415d0..e909a91 100644 --- a/crates/b2-embed/Cargo.toml +++ b/crates/b2-embed/Cargo.toml @@ -5,27 +5,19 @@ edition = "2021" rust-version = "1.96" description = "B2's real local embedder — candle-backed inference behind the b2-core Embedder seam" -# Heavy ML deps (candle + tokenizers) live *only* here, never in b2-core, so the -# core test suite stays fast, deterministic, and model-free (CLAUDE.md). The `b2` -# binary depends on this crate to inject the real model. +# Heavy ML deps live only here, never in b2-core, so the core suite stays model-free +# (CLAUDE.md). [dependencies] b2-core = { path = "../b2-core" } candle-core = "0.9" candle-nn = "0.9" candle-transformers = "0.9" -# Download seam for `b2 init`. `default-features = false` is load-bearing (GH #174): -# hf-hub's defaults are ["default-tls", "tokio", "ureq"], and the `tokio` one pulls -# reqwest → hyper → h2 → tokio — the whole async HTTP stack, compiled at opt-3 by the -# workspace's dependency profile, to serve an async API B2 never calls (we use -# `hf_hub::api::sync`). Dropping the defaults also drops `default-tls` → `native-tls`, -# which is the one *behavioural* change: hf-hub's sync agent falls back to ureq's own -# default TLS (rustls + webpki-roots) instead of the OS trust store. That is already -# what b2-llm speaks to cloud endpoints with, and what hf-hub's redirect-following -# agent used even before this — only the first hop was native-tls. Proxies are -# unaffected (hf-hub calls `try_proxy_from_env` either way). The trade is that a -# corporate MITM CA installed in the *system* store is no longer trusted here; -# `ureq/native-certs` is the escape hatch if that ever bites. +# Download seam for `b2 init`. No default features (GH #174): the `tokio` default pulls the +# whole async HTTP stack for an API B2 never calls (we use `hf_hub::api::sync`). This also +# drops `native-tls`, so TLS is ureq's rustls + webpki-roots rather than the OS trust store: +# a corporate MITM CA in the system store is not trusted. `ureq/native-certs` is the escape +# hatch. hf-hub = { version = "0.4", default-features = false, features = ["ureq"] } tokenizers = "0.21" serde = { version = "1.0.228", features = ["derive"] } @@ -34,31 +26,22 @@ toml = "0.8" dirs = "6.0" thiserror = "2.0.18" -# On macOS, link Apple's Accelerate (vDSP/BLAS) for candle's CPU matmuls. Without -# it candle falls back to a naive kernel that makes bge-base inference ~10-100× -# slower — the real bottleneck behind a slow `b2 reindex`, far more than batching. -# Cargo merges these features with the base deps above; on non-macOS the feature is -# simply absent (candle stays portable), so a Linux build is unaffected. +# On macOS, link Accelerate for candle's CPU matmuls; without it bge-base inference is +# ~10-100× slower. Other targets are unaffected. [target.'cfg(target_os = "macos")'.dependencies] candle-core = { version = "0.9", features = ["accelerate"] } candle-nn = { version = "0.9", features = ["accelerate"] } candle-transformers = { version = "0.9", features = ["accelerate"] } -# Research lever (GH #40): run the BERT forward pass on the Apple-Silicon Metal GPU instead -# of the CPU. `metal` is *additive* over `accelerate` (independent code paths — CPU mode still -# gets Accelerate BLAS, Metal mode uses the GPU), and the device is chosen at load via candle's -# `metal_is_available()` (== `cfg!(feature = "metal")`) with a graceful CPU fallback — see -# `model.rs::select_device`. Switching device is a *model swap* (the recorded model id gains an -# `@metal` tag), so a re-embed is forced and `search` fails fast rather than mixing vectors. -# **macOS-only:** the candle metal-kernels crate builds only on macOS; don't pass `--features -# metal` on Linux. Enable per-build: `cargo run -p b2-cli --features metal -- reindex`. +# Run the forward pass on the Metal GPU (GH #40), additive over `accelerate`, with a CPU +# fallback (`model.rs::select_device`). Switching device is a model swap (the model id gains +# `@metal`), so it forces a re-embed (ADR-0007). macOS-only: don't pass `--features metal` on +# Linux. [features] metal = ["candle-core/metal", "candle-nn/metal", "candle-transformers/metal"] [dev-dependencies] tempfile = "3.27.0" -# The eval harness's search evidence probe (GH #201) reads `chunks_fts` directly -# over the connection `b2_core::open` hands back, so the examples need rusqlite's -# *types* in scope. Dev-only, and feature-unified with b2-core's bundled build — -# this adds nothing to the `b2` binary's tree. +# The eval's search evidence probe (GH #201) reads `chunks_fts` directly. Dev-only, and +# feature-unified with b2-core's bundled build, so it adds nothing to the `b2` binary. rusqlite = "0.40.1" diff --git a/crates/b2-embed/examples/calibrate.rs b/crates/b2-embed/examples/calibrate.rs index 26b2237..cbe0f1e 100644 --- a/crates/b2-embed/examples/calibrate.rs +++ b/crates/b2-embed/examples/calibrate.rs @@ -1,10 +1,7 @@ -//! Real-vault discovery calibration — the instrument ADR-0014's Phase 0a promoted out of -//! GH #196's hand arithmetic. It runs against **any built vault**, needs no labels, and -//! prints the numbers every discovery-surfacing ruling has turned on: per-anchor pool -//! distributions, each anchor's leader cosine and z, what a z gate would serve versus what -//! always-serve does, and the strength bands the desktop would paint. This is process rule -//! 5 made mechanical — **a constant derived from a corpus's score distribution is invalid -//! until transfer-checked on a real vault**. +//! Real-vault discovery calibration (ADR-0014, GH #196). Runs against any built vault with +//! no labels, and prints per-anchor pool distributions, leader cosine and z, what a z gate +//! would serve against always-serve, and the strength bands. Process rule 5: a constant +//! derived from a corpus's score distribution is invalid until checked on a real vault. //! //! ```console //! make calibrate VAULT=$HOME/notes # per-anchor lines + the summary block @@ -15,29 +12,13 @@ //! make calibrate VAULT=$HOME/notes ARGS="--mutual-k 5" # replay the fold at a different depth //! ``` //! -//! Beside the retired z gate it replays the two **default-disclosure** candidates ADR-0014 -//! admits a fold from. The **mutual-k reciprocity fold** (GH #200): B is reciprocal for A -//! iff A ranks within B's own top `mutual_k`, and the replayed default view is the ranked -//! list's longest reciprocal prefix — a cut in the served order, never a filter that skips -//! rows. That bake-off has **ruled, and no fold ships**: the window is empty on both corpora, -//! and this instrument supplied the reading that generalized it — the same `k` is a different -//! rule on every vault (10 discloses 36% of cards on the orthogonal corpus, 91% on the dense -//! fixture, 98% on `fixtures/test-vault`). The **authored-edge reference bar** is replayed -//! beside it, and this is the only instrument that *can*: the rule calibrates from the score -//! distribution of the human's own committed edges, and both eval corpora are link-free by -//! construction. Unlike reciprocity that is a distributional constant, so this instrument is -//! not an aside for it — it is the whole of its evidence. +//! It also replays the two default-disclosure folds (GH #200): mutual-k reciprocity (ruled +//! out; the same `k` is a different rule on every vault) and the authored-edge reference +//! bar, which only a real vault with links can price. //! -//! **It is a pure read** — stored vectors only, no model call — so it runs in seconds on any -//! personal-scale vault and never perturbs what it measures. -//! -//! The z is **recomputed harness-side from the served scores**, kept independent of the -//! engine's own statistic so the instrument can also *check* it: where the engine ships a z -//! beside a candidate, the two are diffed and a drift reports as a fault. -//! -//! The replayed gate defaults to the constants ADR-0014 retired, so the acceptance reading -//! reproduces GH #196's finding on its reporting vault: 16 of 17 anchors dark. It is a -//! **simulation** — the shipped surface gates nothing. +//! A pure read of stored vectors, no model call. The z is recomputed from the served scores +//! and diffed against the engine's as a drift check. The replayed gate defaults to the +//! constants ADR-0014 retired; it is a simulation, and the shipped surface gates nothing. mod common; @@ -48,39 +29,32 @@ use rusqlite::Connection; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; -/// How deep each anchor's pool is read. A bar is judged against the population it -/// has to cut, so the read must cover the whole candidate set, not a pane's -/// prefix; anything a personal vault holds fits far under this. +/// How deep each anchor's pool is read: the whole candidate set, which a bar is judged +/// against. const SCAN_LIMIT: usize = 100_000; -/// The replayed gate is inert under this population — `discover.rs`'s statistics -/// guard, restated here because the replay must match the rule it prices. (The -/// strength bands the pane paints are [`common::BAND_STRONG_Z`]'s, shared with -/// `make eval`'s calibration block, where their values are re-measured.) +/// The replayed gate is inert under this population: `discover.rs`'s guard, restated. const MIN_POPULATION: usize = 12; -/// The z existence gate being replayed — GH #197 retired it from the engine; the -/// instrument keeps it (and any variant the flags name) priceable. +/// The z existence gate being replayed, retired from the engine by GH #197. struct GateSim { leader_z: f64, member_z: f64, } -/// One anchor's complete reading: its whole candidate pool in rank order, with -/// the harness-side z when the population carries a statistic. +/// One anchor's whole candidate pool in rank order, with the harness-side z. struct AnchorReading { path: String, /// (candidate path, cosine, engine z if shipped) in served (nearest-first) order. pool: Vec<(String, f64, Option)>, - /// Harness-recomputed z per pool entry — `None` when the population is under - /// [`MIN_POPULATION`] or has zero variance (no statistic exists). + /// Harness-recomputed z per pool entry; `None` under [`MIN_POPULATION`] or at zero + /// variance. z: Option>, } impl AnchorReading { fn from_candidates(path: &str, cands: &[SimilarView]) -> Self { let d2: Vec = cands.iter().map(|c| c.score * c.score).collect(); - // The harness's own z (never the engine's), gated at the replayed rule's - // population floor before the shared restatement's own inertness guard. + // The harness's own z, never the engine's. let z = (d2.len() >= MIN_POPULATION) .then(|| passage_z(&d2)) .flatten(); @@ -98,9 +72,8 @@ impl AnchorReading { self.pool.iter().map(|(_, cos, _)| *cos).collect() } - /// How many candidates the replayed z gate would serve, capped at `limit` — - /// 0 with the leader gate fired ("dark"), everything (to the cap) where the - /// pool is too small for a statistic (the rule's own inertness). + /// How many candidates the replayed gate serves, capped at `limit`: 0 when the leader + /// gate fires, everything when the pool has no statistic. fn gate_serves(&self, gate: &GateSim, limit: usize) -> usize { match &self.z { None => self.pool.len().min(limit), @@ -117,9 +90,7 @@ impl AnchorReading { } } - /// Band histogram over the top-`limit` cards always-serve shows: - /// (strong ●●●, clear ●●○, near ●○○), or `None` for an ungraded pool — - /// the A6 readout (do dense vaults compress every card into one band?). + /// Band histogram (strong, clear, near) over the top-`limit` cards; `None` when ungraded. fn bands(&self, limit: usize) -> Option<(usize, usize, usize)> { let z = self.z.as_ref()?; let (mut strong, mut clear, mut near) = (0, 0, 0); @@ -133,8 +104,7 @@ impl AnchorReading { Some((strong, clear, near)) } - /// Worst |engine z − recomputed z| across the pool — the drift check, biting - /// only where the engine shipped a z at all. + /// Worst |engine z − recomputed z| across the pool, where the engine shipped a z. fn recheck_delta(&self) -> f64 { let Some(z) = &self.z else { return 0.0 }; self.pool @@ -144,10 +114,8 @@ impl AnchorReading { .fold(0.0, f64::max) } - /// How many pool entries the drift check actually compared — both z's - /// present. Zero pairs must print as "nothing to cross-check", never as a - /// pass: a check that reports success without having run is the - /// advisory-but-exit-0 hole this repo's gates exist to close. + /// How many entries the drift check compared. Zero must print as "nothing to + /// cross-check", never as a pass. fn recheck_pairs(&self) -> usize { if self.z.is_none() { return 0; @@ -159,18 +127,10 @@ impl AnchorReading { } } -/// **Candidate 2** of GH #200's bake-off, replayed: an *authored-edge reference bar*. It -/// calibrates "what related looks like **in this vault**" from the one labelled population -/// every real vault carries — the score distribution of the human's committed edges — and -/// folds the default view at the longest prefix scoring at or above it. -/// -/// Priceable only where that population exists, which is why it lives here rather than in -/// `make eval`: **both eval corpora are link-free by construction**. Its pair score is the -/// same statistic discovery ranks on, computed over the *linked* pairs discovery never scores -/// (the 1-hop exclusion removes exactly them). The bar is the population's **lower quartile** -/// — a candidate at least as related as the weaker quarter of what this human already linked -/// — and the whole distribution prints beside it, because a quantile is a choice and a choice -/// printed as one number is an assumption. +/// GH #200's candidate 2: an authored-edge reference bar. It calibrates from the best-passage +/// cosines of the human's own linked pairs (absent from the link-free eval corpora) and +/// folds at the longest prefix at or above their lower quartile. The whole distribution +/// prints, since the quantile is a choice. struct EdgeBar { /// Authored edges whose pair could be scored (both notes embedded). n: usize, @@ -181,16 +141,13 @@ struct EdgeBar { } impl EdgeBar { - /// The bar itself — the lower quartile of the authored-edge cosines. + /// The lower quartile of the authored-edge cosines. fn bar(&self) -> f64 { self.q1 } - /// This vault's authored-edge pair cosines. Undirected and de-duplicated: an - /// edge read from both endpoints is one relation, and counting it twice - /// would weight the reciprocally-visible pairs double. `None` when the vault - /// has no scorable authored edge — the rule has no population, which is a - /// reading about the vault, not a failure of the instrument. + /// This vault's authored-edge pair cosines, undirected and de-duplicated. `None` when + /// the vault has no scorable authored edge. fn read(vault: &Vault, conn: &Connection) -> Result, Box> { let mut seen: HashSet<(String, String)> = HashSet::new(); let mut cos: Vec = Vec::new(); @@ -199,9 +156,7 @@ impl EdgeBar { if neighbors.is_empty() { continue; } - // Loaded once per source note and dropped with it: the population is - // read edge by edge rather than by caching the whole vault's - // vectors, so a large vault costs time here, never memory. + // Per source note, not cached, so a large vault costs time, not memory. let src: Vec> = b2_core::db::note_chunk_vectors(conn, ¬e.path)? .into_iter() .map(|(_, v)| v) @@ -219,10 +174,7 @@ impl EdgeBar { .into_iter() .map(|(_, v)| v) .collect(); - // Best-passage, the same statistic `similar` ranks on: the - // nearest chunk pair across the two notes. A pair with an - // unembedded side (or a dangling/resource target) scores nothing - // and drops out rather than entering the population as a zero. + // Best-passage, as `similar` ranks. An unembedded side drops out. let best = src .iter() .flat_map(|x| dst.iter().map(move |y| b2_core::embed::l2_sq(x, y))) @@ -246,8 +198,7 @@ impl EdgeBar { })) } - /// How many of `pool`'s leading candidates clear the bar — candidate 2's - /// fold, in the same prefix form candidate 1 takes. + /// How many of `pool`'s leading candidates clear the bar (a prefix fold). fn fold(&self, pool: &[(String, f64, Option)], limit: usize) -> usize { pool.iter() .take(limit) @@ -256,21 +207,14 @@ impl EdgeBar { } } -/// English function words, probed for **where they weigh** against a query's -/// content (GH #201). The lexical-anchor rule weighs each term by its IDF, so -/// its whole premise is that a vault's function words weigh near nothing beside -/// its subject words. That is a claim about a vault, not about English, and this -/// is where it is checked. +/// English function words (GH #201). The lexical-anchor rule assumes they weigh near nothing +/// in a vault's IDF; this checks that per vault. const FUNCTION_WORDS: [&str; 12] = [ "the", "and", "of", "to", "a", "in", "is", "it", "that", "for", "with", "on", ]; -/// Built-in nonsense queries — the **strict** half of D2's negative pile, and -/// the only half that can be built in: an off-topic *phrase* is off-topic -/// relative to a particular vault ("choreographing a ballroom waltz" is a -/// labelled negative on the eval corpus and would be a positive in a dancer's -/// vault), so the eval corpus is where those live. These are strings no vault -/// holds, which makes them vault-independent. +/// Built-in nonsense queries: the only vault-independent negatives, since an off-topic +/// phrase is only off-topic relative to a particular vault. const NONSENSE: [&str; 4] = [ "shjfasd", "vrelqip zonktar wembleforth", @@ -278,83 +222,66 @@ const NONSENSE: [&str; 4] = [ "zzzyqx", ]; -/// How many notes contribute a title-as-query positive. A cap rather than a -/// sample: the walk takes them in `list_notes` order, which is deterministic, so -/// a re-run on an unchanged vault reads the same queries. +/// How many notes contribute a title-as-query positive, in deterministic `list_notes` order. const MAX_TITLE_QUERIES: usize = 400; -/// One query's transfer reading — the same two absolute signals `make eval`'s -/// bake-off judges, read on a real vault instead of a labelled corpus. +/// One query's transfer reading: `make eval`'s two signals, on a real vault. struct SearchProbe { query: String, - /// Share of the query's term IDF this vault carries. `None` when nothing in - /// the query carries weight here. + /// Share of the query's term IDF this vault carries; `None` when no term has weight. coverage: Option, /// Dense top-1 cosine; `None` on a vault with no embedding space. best_cos: Option, - /// What the shipped bar would say — `true` = the default view vouches. + /// Whether the shipped bar vouches. vouched: bool, - /// For a title-as-query probe, the note the title came from — the one row - /// this bench can certify as relevant with no label (GH #206). `None` for - /// nonsense. + /// For a title query, the note it came from: the one row certified relevant without a + /// label (GH #206). own_path: Option, - /// The served list's per-hit provenance, in fused order — what the tail - /// families' constraints are read from. + /// The served list's per-hit provenance, in fused order. rows: Vec, } impl SearchProbe { - /// Where this probe's own note landed in the served list, 0-based — `None` - /// when retrieval never served it within the limit (or the probe has no own - /// note at all, which is every nonsense negative). + /// Where this probe's own note landed in the served list, 0-based. fn own_rank(&self) -> Option { let own = self.own_path.as_ref()?; self.rows.iter().position(|r| &r.path == own) } } -/// One served row's per-hit provenance (GH #206) — `EvidencedResult`, shorn of -/// the display fields this instrument never prints. +/// One served row's per-hit provenance (GH #206). struct TailRow { path: String, bm25_rank: Option, cos: Option, } -/// One tail family's edge: the tightest constant that still hides no own note, -/// and the served row that pins it there. -/// -/// Owned rather than borrowed out of the probe pile: the edge outlives the fold -/// that finds it — it is carried to both renderings — and a lifetime here would -/// buy nothing but the borrow. +/// One tail family's edge: the tightest constant that hides no own note, and the row that +/// pins it. struct TailEdge { value: f64, query: String, path: String, } -/// A row served above its own note carrying **no finite cosine** — the reading -/// that kills a cosine family outright rather than merely constraining it. +/// A row served above its own note with no finite cosine, which kills a cosine family. struct DeadRow { query: String, path: String, - /// Dense-only besides, which kills the two-signal family too: no lexical - /// rank and no finite cosine passes at any bar. + /// Also dense-only, which kills the two-signal family too. dense_only: bool, } -/// An own note the dense-only fold would hide: a row the lexical half never -/// ranked, served *above* the one row this bench certifies. +/// An own note the dense-only fold would hide, behind a row the lexical half never ranked. struct LexHidden { query: String, - /// 0-based, like every other rank this instrument reports (the text block - /// prints them +1). + /// 0-based, like every rank here (printed +1). own_rank: usize, dense_only_rank: usize, } -/// The per-hit tail families (GH #206) priced on this vault: how tight each -/// family's constant could be drawn before it hides a row the bench certifies. +/// The per-hit tail families (GH #206) priced on this vault: how tight each constant can be +/// before it hides an own note. struct TailReading { own_served: usize, own_missed: usize, @@ -368,15 +295,9 @@ struct TailReading { } impl TailReading { - /// `make eval`'s tail bake-off derives each family's admissible window from - /// the labelled corpora; this is the reading that says whether such a window - /// survives a real vault (process rule 5 — owed even by the parameterless - /// family, whose "lexical half never ranked it" signal is partly a fact - /// about pool depth against vault size). No labels here: the one served row - /// a title query certifies is its **own note**, so each family is priced on - /// what its constant would have to be to hide none of them — the tripwire - /// direction. The fold is a prefix cut (D1), so every row served above an - /// own note must pass the family's test too. + /// Whether `make eval`'s tail windows survive a real vault (process rule 5). Each + /// family is priced so it hides no title query's own note; the fold is a prefix cut + /// (D1), so every row above an own note must pass too. fn read(positives: &[SearchProbe]) -> Self { let mut r = Self { own_served: 0, @@ -397,8 +318,7 @@ impl TailReading { continue; }; r.own_served += 1; - // The drop family's reference is the list's own best served cosine — - // the whole list's, not the prefix's, matching how the fold would read. + // The whole list's best cosine, not the prefix's, as the fold reads it. let best = p .rows .iter() @@ -413,9 +333,7 @@ impl TailReading { }); } for row in prefix { - // Finiteness-filtered (PR #212 review): a NaN cosine is *no - // reading*, and `is_none_or` would otherwise let a NaN first row - // seed an edge. It lands in the dead arm, named, never skipped. + // A NaN is no reading and would seed an edge via `is_none_or` (PR #212). match row.cos.filter(|c| c.is_finite()) { None => { if r.cos_dead.is_none() { @@ -452,8 +370,7 @@ impl TailReading { } } -/// The half of the search bench that needs a calibrated bar: everything below -/// the model-free function-word reading, which is judged against one. +/// The half of the search bench that needs a calibrated bar. struct JudgedSearch { bar: b2_core::search::EvidenceBar, positives: Vec, @@ -461,30 +378,23 @@ struct JudgedSearch { tail: TailReading, } -/// The whole search-side reading, **computed once**. The text block and the -/// `--json` object are two renderings of this one value rather than two -/// computations of it, so a sweep that scripts the JSON and a human reading the -/// table cannot be looking at different numbers. +/// The whole search-side reading, computed once so the text and `--json` renderings can't +/// disagree. struct SearchReading { - /// The embedding space the reading was taken in — carried so the renderer - /// names the same model the bar was looked up for, never one passed beside it. + /// The model the bar was looked up for. model_id: String, - /// Chunks in the index — the scale every weight below is read against. chunk_total: usize, - /// The weight a word the vault has never seen would carry: the scale the - /// function words are read against, since that is what an absent *content* - /// word contributes to the same sum. + /// The weight of a word the vault lacks: the scale function words are read against. absent_idf: f64, /// Each [`FUNCTION_WORDS`] entry with the weight it carries here. function_words: Vec<(String, f64)>, - /// `None` when the active model has no calibrated bar — the piles need one - /// to be judged, so none of them was read. + /// `None` when the model has no calibrated bar. judged: Option, } impl SearchReading { - /// The heaviest function word's weight — the anchor test's premise, which - /// holds only where it is small beside [`Self::absent_idf`]. + /// The heaviest function word's weight; the anchor test needs it small beside + /// [`Self::absent_idf`]. fn heaviest(&self) -> f64 { self.function_words .iter() @@ -492,36 +402,23 @@ impl SearchReading { .fold(0.0_f64, f64::max) } - /// That weight as a share of an absent word's, in percent — the printed form. + /// That weight as a percentage of an absent word's. fn heaviest_share(&self) -> f64 { 100.0 * self.heaviest() / self.absent_idf.max(f64::EPSILON) } } -/// The **search evidence transfer check** (ADR-0015) — process rule 5's bench for the -/// query-level bar, which is a distributional constant and therefore invalid until a real -/// vault has answered for it. -/// -/// It needs no labels, and that is the point: the positives are each note's own **title**, a -/// query the vault demonstrably holds material for by construction, and the negatives are -/// [`NONSENSE`]. Neither is a hand-label, so running this on someone's notes costs them -/// nothing and the reading cannot be tuned by relabelling. -/// -/// It **can** see the tripwire direction — a bar that cuts queries a real vault holds -/// material for is the failure ADR-0014 punished. It **cannot** see the paraphrase case, -/// which needs judgement and is what the labelled corpus is for. Read the two together. +/// The search evidence transfer check (ADR-0015): process rule 5's bench for the query-level +/// bar. Label-free: positives are note titles, negatives are [`NONSENSE`]. It sees the +/// tripwire direction but not paraphrase, which is the labelled corpus's job. fn read_search_transfer( vault: &Vault, conn: &Connection, model_id: &str, limit: usize, ) -> Result> { - // The function-word reading is **model-free**: it is a fact about this - // vault's vocabulary, and it is the lexical anchor's whole premise. It is - // read before the bar lookup, and printed above it, because gating it behind - // a calibrated bar (as this did until the PR #205 review) left a - // fake-embedded vault printing nothing at all, which the recipe's own help - // text promised it would not. + // The function-word reading is model-free, so it is read before, and without, a + // calibrated bar (PR #205). let probe = b2_core::search::lexical_evidence(conn, &FUNCTION_WORDS.join(" "))?; let reading = SearchReading { model_id: model_id.to_string(), @@ -535,7 +432,6 @@ fn read_search_transfer( judged: None, }; - // Only the *verdicts* need a bar, so only they stop here. let Some(bar) = b2_core::search::EvidenceBar::for_model(model_id) else { return Ok(reading); }; @@ -546,12 +442,9 @@ fn read_search_transfer( let view = vault.search_evidence(query, limit)?; Ok(SearchProbe { query: query.to_string(), - // The engine's own term weights — `make eval`'s labelled bake-off is - // where the formula is independently restated and drift-checked. coverage: term_coverage(&view), best_cos: view.best_cos, - // The engine's own verdict, not a restatement of it: this bench - // prices what would actually ship. + // The engine's verdict: this bench prices what would ship. vouched: view.vouched.unwrap_or(true), own_path, rows: view @@ -595,8 +488,7 @@ fn read_search_transfer( }) } -/// The search reading as the text block — the human half of what -/// [`read_search_transfer`] measured. +/// The search reading as the text block. fn print_search_transfer(reading: &SearchReading) { println!(); println!("search evidence transfer check (D2 — process rule 5's bench for GH #201's bar)"); @@ -668,9 +560,7 @@ fn print_search_transfer(reading: &SearchReading) { if cut.len() > 10 { println!(" … and {} more", cut.len() - 10); } - // The closest calls, not a sample: a bar is placed against the queries that - // nearly miss, and on a vault of any size those are the ten weakest — the - // same reason the eval prints piles rather than means. + // The ten weakest, since a bar is placed against the near misses. let mut weakest: Vec<&SearchProbe> = positives.iter().collect(); weakest.sort_by(|a, b| { a.best_cos @@ -761,8 +651,6 @@ fn print_search_transfer(reading: &SearchReading) { None => bar_line("cos ≥ c", &tail.cos_edge, false), } match &tail.cos_dead { - // A dead row that is also dense-only kills the two-signal family too: - // no lexical rank and no finite cosine passes at no bar. Some(d) if d.dense_only => println!( " lex-or-cos ≥ c DEAD — {} → {} is dense-only with no finite cosine", truncate(&d.query, 32), @@ -781,8 +669,7 @@ fn print_search_transfer(reading: &SearchReading) { println!(" where the joint corpus edge also hides nothing here (process rule 5)"); } -/// A fixed-precision optional reading, or an em dash where there is none — the -/// one spelling of "no reading" the table uses. +/// A fixed-precision optional reading, or an em dash. fn fmt_opt(v: Option, places: usize) -> String { v.map(|c| format!("{c:.places$}")) .unwrap_or_else(|| "—".to_string()) @@ -827,14 +714,11 @@ fn run() -> Result<(), Box> { "usage: calibrate [--limit N] [--leader-z Z] [--member-z Z] [--mutual-k N] \ [--search] [--json]", )?; - // The reciprocity depth defaults to the simulated pane size, resolved after - // parsing so flag order can't matter. + // Resolved after parsing so flag order can't matter. let mutual_k = mutual_k_flag.unwrap_or(limit); - // A pure read over stored vectors, so the fake-embedder open is correct — the - // same posture as `b2 similar`. The recorded identity is read straight from - // the index, because it is what the vectors actually are, not what an - // injected embedder would be. + // A pure read of stored vectors, so the fake-embedder open is correct (as `b2 similar`). + // The model identity comes from the index: it is what the vectors actually are. let vault = Vault::open(&vault_root)?; let conn = b2_core::open(&vault_root.join(".b2").join("b2.sqlite"))?; let Some((model, dim)) = b2_core::db::recorded_embedder(&conn)? else { @@ -855,21 +739,15 @@ fn run() -> Result<(), Box> { for note in ¬es { let cands = vault.similar(¬e.path, SCAN_LIMIT)?; if cands.is_empty() { - // No stored vectors on the anchor, or nothing unlinked with vectors - // to compare against — a genuinely empty candidate set, named rather - // than averaged in as a zero. + // Named rather than averaged in as a zero. poolless.push(note.path.clone()); } else { readings.push(AnchorReading::from_candidates(¬e.path, &cands)); } } - // The mutual-k reciprocity fold, replayed (GH #200). Every note's own top `mutual_k` - // candidate paths come from the same full-depth pools just read, so reciprocity costs no - // extra discovery pass. A candidate with no pool of its own cannot reciprocate. The fold - // is the ranked list's longest reciprocal prefix, capped at `limit` — prefix form is - // ADR-0014's admissibility requirement, since a fold that skipped rank 2 to admit rank 5 - // would visibly disagree with the row order. + // The mutual-k reciprocity fold (GH #200), from the pools just read: the longest + // reciprocal prefix, capped at `limit`. Prefix form is ADR-0014's requirement. let top_of: HashMap<&str, HashSet<&str>> = readings .iter() .map(|r| { @@ -900,16 +778,11 @@ fn run() -> Result<(), Box> { }) .collect(); - // Candidate 2 of the same bake-off (GH #200): the authored-edge reference - // bar, priceable only where the human has committed edges — which is why it - // is measured here and not in `make eval` (both eval corpora are link-free - // by construction, so the rule has no population there). + // Candidate 2 (GH #200), the authored-edge reference bar. let edge_bar = EdgeBar::read(&vault, &conn)?; - // The search-side bench (GH #201), opt-in because it is the one part of this - // instrument that is **not** a pure read: judging the cosine half means - // embedding a query, which means loading the real model. The lexical half - // needs no model at all, so a fake-embedded vault still gets that much. + // The search bench (GH #201) is opt-in because the cosine half loads the real model to + // embed queries. A fake-embedded vault still gets the lexical half. let read_search = || -> Result> { if model == b2_core::embed::FAKE_MODEL_ID { read_search_transfer(&vault, &conn, &model, limit) @@ -1111,16 +984,14 @@ fn run() -> Result<(), Box> { ); } - // The table comes first and the model loads after it, so the discovery - // reading is on screen while the embedder is read off disk. + // After the table, so the discovery reading shows while the model loads. if search { print_search_transfer(&read_search()?); } Ok(()) } -/// The same reading as one JSON object on stdout — for scripting a sweep (the A7 -/// population-size sweep, a Phase-2 bake-off harness) without scraping the table. +/// The same reading as one JSON object on stdout, for scripting a sweep. #[allow(clippy::too_many_arguments)] fn print_json( vault_root: &std::path::Path, @@ -1141,12 +1012,9 @@ fn print_json( "dim": dim, "limit": limit, "gate": { "leader_z": gate.leader_z, "member_z": gate.member_z, "min_population": MIN_POPULATION }, - // The replayed fold's depth (GH #200): candidate B is reciprocal iff - // the anchor sits in B's own top `mutual_k`; `fold_serves` below is the - // ranked list's longest reciprocal prefix, capped at `limit`. + // The replayed fold's depth (GH #200). "mutual_k": mutual_k, - // Candidate 2's population and the bar read off it (GH #200) — `null` - // on a link-free vault, which is the rule's own reading there. + // `null` on a link-free vault (GH #200). "edge_bar": edge_bar.as_ref().map(|b| serde_json::json!({ "n": b.n, "bar": r4(b.bar()), @@ -1172,39 +1040,24 @@ fn print_json( })).collect::>(), })).collect::>(), "no_pool": poolless, - // Additive beside the discovery object, never redefining a key of it - // (GH #219): `null` means `--search` was not asked for, which is a fact - // about the invocation, not about the vault. + // Additive (GH #219); `null` means `--search` was not asked for. "search": search.as_ref().map(search_json), }); println!("{row}"); } -/// The search-side bench as JSON (GH #219) — the same reading -/// [`print_search_transfer`] renders as text. -/// -/// It follows the discovery object's **re-derivability** convention: every -/// summary the text block prints is left to the consumer, and what is emitted is -/// what the summary was computed *from* — each probe's own query, coverage, -/// cosine and the engine's own `vouched`, plus every served row's provenance. A -/// sweep that wants "how many titles did the bar cut" counts them; a sweep that -/// wants something the table never printed can have it too. -/// -/// The one exception is the tail families, which are emitted **derived**: a -/// family's edge is an extremum over the whole positive pile, and a consumer -/// re-deriving it would be re-implementing the fold's admissibility rule rather -/// than reading a number off it. They are re-derivable from `positives` all the -/// same. +/// The search-side bench as JSON (GH #219). Emits the per-probe inputs rather than the text +/// block's summaries, except the tail families' edges, which are emitted derived so a +/// consumer need not re-implement the fold. fn search_json(reading: &SearchReading) -> serde_json::Value { let probe = |p: &SearchProbe| { serde_json::json!({ "query": p.query, "coverage": p.coverage.map(r4), "best_cos": p.best_cos.map(r4), - // The engine's verdict, not a restatement of it. "vouched": p.vouched, "own_path": p.own_path, - // 0-based, like `bm25_rank`; `null` = never served within `limit`. + // 0-based; `null` when not served within `limit`. "own_rank": p.own_rank(), "rows": p.rows.iter().map(|r| serde_json::json!({ "path": r.path, @@ -1213,9 +1066,7 @@ fn search_json(reading: &SearchReading) -> serde_json::Value { })).collect::>(), }) }; - // A family reads one of three ways, exactly as the text block prints it: - // `needs` (this row pins the constant), `unconstrained` (no row engages the - // test), `dead` (a row no constant can admit). + // A family is `needs`, `unconstrained` or `dead`, as the text block prints it. let edge = |e: &Option| match e { None => serde_json::json!({ "status": "unconstrained" }), Some(e) => serde_json::json!({ @@ -1234,9 +1085,6 @@ fn search_json(reading: &SearchReading) -> serde_json::Value { }; serde_json::json!({ "chunk_total": reading.chunk_total, - // The lexical anchor's premise, model-free and read per vault: what a - // word this vault has never seen weighs, and what each function word - // weighs beside it. "function_words": { "absent_idf": r4(reading.absent_idf), "heaviest_share_pct": r4(reading.heaviest_share()), @@ -1245,12 +1093,7 @@ fn search_json(reading: &SearchReading) -> serde_json::Value { "idf": r4(*idf), })).collect::>(), }, - // One key, not four: the bar and the three piles read against it exist - // together or not at all, so they nest under the `Option` that decides - // it rather than each carrying its own `null` for a consumer to test. - // `null` = no calibrated bar for this model, so nothing under it was - // read: the piles need one to be judged (M2). The function-word reading - // above is model-free and is there either way. + // `null` when the model has no calibrated bar (M2). "judged": reading.judged.as_ref().map(|j| serde_json::json!({ "bar": { "min_term_coverage": j.bar.min_term_coverage, @@ -1277,8 +1120,7 @@ fn search_json(reading: &SearchReading) -> serde_json::Value { Some(d) => dead(d), None => edge(&j.tail.cos_edge), }, - // A dead row that is also dense-only kills the two-signal - // family too; one that is not leaves it constrained. + // Dead only when the dead row is also dense-only. "lex_or_cos": match &j.tail.cos_dead { Some(d) if d.dense_only => dead(d), _ => edge(&j.tail.lexcos_edge), @@ -1293,9 +1135,7 @@ fn search_json(reading: &SearchReading) -> serde_json::Value { }) } -/// Four decimals, the one rounding this object applies — enough to reproduce -/// every printed reading, short of dumping a float's full noise into a dataset -/// meant to be diffed. +/// Four decimals: enough to reproduce every printed reading. fn r4(x: f64) -> f64 { (x * 1e4).round() / 1e4 } diff --git a/crates/b2-embed/examples/common/mod.rs b/crates/b2-embed/examples/common/mod.rs index d3e82a0..f3082b2 100644 --- a/crates/b2-embed/examples/common/mod.rs +++ b/crates/b2-embed/examples/common/mod.rs @@ -1,17 +1,11 @@ //! Helpers the eval-harness examples share (`eval`, `calibrate`, `stability`, and //! `b2-llm`'s `groundedness`, which pulls this file in by `#[path]`). //! -//! Cargo discovers examples as `examples/*.rs` and `examples/*/main.rs`, so this -//! directory is a module, never an example of its own. Every example that says -//! `mod common;` compiles the whole file and uses a different subset of it, hence -//! the `dead_code` allowance — the same posture as `b2-core`'s `tests/common`. +//! Each example uses a different subset, hence the `dead_code` allowance. //! -//! What lives here is plumbing and the harness's **own** restatements of engine -//! maths. The restatements stay restatements on purpose: they exist so an -//! instrument can cross-check the engine, and a call into the engine would turn -//! each check into a comparison of a number with itself. Nothing here may import -//! beyond what `b2-llm`'s dev-dependencies also carry (b2-core, b2-embed, -//! serde_json, tempfile). +//! Engine maths restated here stays restated: calling the engine would make each +//! cross-check compare a number with itself. Import nothing beyond `b2-llm`'s +//! dev-dependencies (b2-core, b2-embed, serde_json, tempfile). #![allow(dead_code)] use b2_core::vault::{NoteSummary, SearchEvidenceView}; @@ -19,12 +13,8 @@ use b2_embed::{provision, EmbedConfig, LocalEmbedder}; use std::error::Error; use std::path::{Path, PathBuf}; -/// The strength-band landmarks the desktop paints (`ui/src/strength.ts`, GH #182): -/// `●●●` at or above the labelled-mate population's upper quartile, `●●○` at or -/// above the retired leader bar. Restated constants, not imports — the bands are -/// UI copy, and `make eval`'s calibration block is the instrument their values are -/// re-measured by (and what a loner anchor's always-served cards *claim*, GH #197's -/// A2 readout). +/// The strength-band z landmarks the desktop paints (`ui/src/strength.ts`, GH #182), +/// restated because they are UI copy; `make eval`'s calibration block re-measures them. pub const BAND_STRONG_Z: f64 = 2.52; pub const BAND_CLEAR_Z: f64 = 1.96; @@ -58,13 +48,9 @@ impl Band { } } -/// Z-score a population of squared distances, oriented nearer = higher — the -/// harness's own restatement of the arithmetic `discover::candidates` applies to -/// the stage-2 best-pair distances (GH #192), kept so the engine's z can be -/// cross-checked rather than merely trusted. `None` when no meaningful statistic -/// exists (under two values, or zero variance), mirroring the engine's own -/// inertness guard; a caller replaying a larger minimum population applies it -/// before calling. +/// Z-score squared distances, nearer = higher: a restatement of `discover::candidates` +/// (GH #192) to cross-check the engine. `None` under two values or at zero variance; a +/// caller with a larger minimum population applies it first. pub fn passage_z(d2: &[f64]) -> Option> { if d2.len() < 2 { return None; @@ -76,10 +62,8 @@ pub fn passage_z(d2: &[f64]) -> Option> { (sd > 0.0).then(|| d2.iter().map(|d| (mean - d) / sd).collect()) } -/// A `similar` score is negated L2 distance between L2-normalized vectors (the -/// real embedder normalizes every row), so it converts exactly: `cos = 1 − d²/2`. -/// Cosine is the unit the floor ruling is stated in and the unit that survives a -/// model swap comparison, so the piles are recorded in it. +/// A `similar` score is negated L2 distance between normalized vectors, so +/// `cos = 1 − d²/2` exactly. Piles are recorded in cosine, which compares across models. pub fn cosine_of(score: f64) -> f64 { 1.0 - (score * score) / 2.0 } @@ -100,12 +84,9 @@ pub fn pile_stats(pile: &[f64]) -> Option<(f64, f64, f64)> { Some((sorted[0], median, sorted[sorted.len() - 1])) } -/// A note's title as a search query: its frontmatter title, else its file slug with -/// `-`/`_` read as spaces (`drone-comb` → "drone comb"). `None` when that is blank. -/// -/// Titles are the label-free positives both search benches share (GH #201): a -/// note's own title names material the vault demonstrably holds, so nothing read -/// off one can be relabelled to clear it. +/// A note's title as a search query: frontmatter title, else the file slug with `-`/`_` as +/// spaces. `None` when blank. Titles are the label-free positives the search benches share +/// (GH #201). pub fn title_query(note: &NoteSummary) -> Option { let title = note.title.clone().unwrap_or_else(|| { Path::new(¬e.path) @@ -116,14 +97,9 @@ pub fn title_query(note: &NoteSummary) -> Option { (!title.trim().is_empty()).then_some(title) } -/// The share of a query's term IDF the vault carries, read off the view's own -/// per-term weights ([`b2_core::vault::QueryTermView::idf`]) rather than a second -/// copy of the formula. `None` when no term carries any weight (the lexical half -/// abstaining, not scoring zero). -/// -/// `make eval`'s labelled bake-off keeps its own restatement of this arithmetic -/// on purpose, as a drift check against the engine; one such check is the check, -/// so every other bench reads the engine's weights through here. +/// The share of a query's term IDF the vault carries, from the engine's own per-term +/// weights. `None` when no term has weight (the lexical half abstaining, not scoring zero). +/// `make eval`'s bake-off keeps the one independent restatement, as a drift check. pub fn term_coverage(view: &SearchEvidenceView) -> Option { let total: f64 = view.terms.iter().map(|t| t.idf).sum(); (total > f64::EPSILON).then(|| { @@ -136,11 +112,8 @@ pub fn term_coverage(view: &SearchEvidenceView) -> Option { }) } -/// A throwaway vault holding a copy of one **flat** corpus directory — the eval -/// corpora are single-level, so only regular files are copied: `fs::copy` errors -/// on a directory, and a future subfolder (or any stray non-file) must not abort -/// the run. The temp dir lives exactly as long as this value, so declare the -/// `Vault` opened on [`Self::root`] after it (locals drop in reverse order). +/// A throwaway vault copying the regular files of one flat corpus directory. The temp dir +/// lives as long as this value, so declare the `Vault` opened on [`Self::root`] after it. pub struct ScratchVault { _tmp: tempfile::TempDir, root: PathBuf, @@ -165,14 +138,8 @@ impl ScratchVault { } } -/// Load the configured embedding model, provisioning it first only when it is -/// not already loadable. -/// -/// `provision`'s idempotent fast path *is* a full model load, so calling it and -/// then loading again pays for the model twice on every warm run. Loading first -/// is the same decision in the other order: a model that loads is exactly what -/// the fast path would have accepted, and one that doesn't goes through -/// `provision` (fetch + verify) before the load that follows. +/// Load the configured model, provisioning only when it won't load. Loading first avoids +/// paying for two loads per warm run, since `provision`'s fast path is itself a full load. pub fn load_or_provision(config: &EmbedConfig) -> Result> { if let Ok(embedder) = LocalEmbedder::load(config) { return Ok(embedder); @@ -186,12 +153,8 @@ pub fn has_flag(args: &[String], name: &str) -> bool { args.iter().any(|a| a == name) } -/// A typo'd flag must not run a *different* measurement than the one asked for. -/// -/// `known` lists every accepted flag; `valued` the ones among them that take a -/// value as the next argument (`--vault path`; the `--vault=path` form needs no -/// entry). Positional arguments are refused too — every flag-driven example here -/// takes none. +/// Refuse unknown flags and positionals, so a typo can't run a different measurement. +/// `valued` lists the flags that take the next argument as a value. pub fn reject_unknown_flags( args: &[String], known: &[&str], @@ -213,8 +176,7 @@ pub fn reject_unknown_flags( if !known.contains(&name) { return Err(format!("unknown flag {arg:?}; known: {}", known.join(" ")).into()); } - // `--sweep=1` would pass the name check and then match no `has_flag` — - // the same silent wrong measurement as a typo, so a switch takes no value. + // `--sweep=1` would match no `has_flag`, so a switch takes no value. if inline_value && !valued.contains(&name) { return Err(format!("flag {name} takes no value (got {arg:?})").into()); } @@ -223,8 +185,7 @@ pub fn reject_unknown_flags( Ok(()) } -/// Append one row to a results log (creating it on first run). Append-only, so -/// runs accumulate into one dataset — the same convention as `B2_LOG_FILE`. +/// Append one row to a results log, creating it on first run. pub fn append_result(path: &Path, row: &serde_json::Value) -> Result<(), Box> { use std::io::Write; let mut f = std::fs::OpenOptions::new() @@ -248,10 +209,8 @@ pub fn git_short_sha() -> Option { Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) } -/// `s` cut to at most `max` chars, an ellipsis marking the cut. Counted in chars -/// and cut on a char boundary (byte-index arithmetic lands mid-codepoint on text -/// with em dashes or °C), and saturating, so `max == 0` yields a bare ellipsis -/// rather than an underflow panic. +/// `s` cut to at most `max` chars with an ellipsis. Counts chars, not bytes, so it never +/// splits a codepoint. pub fn truncate(s: &str, max: usize) -> String { if s.chars().count() <= max { s.to_string() diff --git a/crates/b2-embed/examples/eval/ablation.rs b/crates/b2-embed/examples/eval/ablation.rs index ec3aad0..9cc89ab 100644 --- a/crates/b2-embed/examples/eval/ablation.rs +++ b/crates/b2-embed/examples/eval/ablation.rs @@ -1,6 +1,5 @@ -//! The opt-in A/Bs, each judged against the default config's passes and logged as rows -//! of their own beside its row: `--stemmer` (the FTS tokenizer ablation, GH #157) and -//! `--sweep` (the in-process chunker sweep, GH #44). +//! The opt-in A/Bs, each judged against the default config's passes and logged as its own +//! row: `--stemmer` (FTS tokenizer, GH #157) and `--sweep` (chunker, GH #44). use crate::common::append_result; use crate::discovery::score_similar; @@ -13,37 +12,26 @@ use b2_core::db::FtsTokenizer; use b2_core::vault::Vault; use std::path::Path; -/// The default config's scored run — what every ablation row is judged against -/// (the paired per-query moves) and logged beside. A short-lived, read-only view -/// over values `run` owns, built once and handed to each A/B. +/// The default config's scored run, which every ablation row is judged against. A +/// short-lived, read-only view over values `run` owns. #[derive(Clone, Copy)] pub struct Baseline<'a> { - /// The results log every row is appended to. pub log: &'a Path, pub run: RunId<'a>, pub notes: usize, pub chunks: usize, pub embed_secs: f64, - /// The labelled positives every pass below was scored over. pub queries: &'a [Labelled], pub bm25: &'a Pass, pub vector: &'a Pass, pub hybrid: &'a Pass, } -/// The FTS tokenizer ablation (the #157 instrument). +/// The FTS tokenizer ablation (GH #157): shipped `porter unicode61` against unstemmed +/// `unicode61` over identical chunks and vectors. Discovery never touches FTS, so it isn't +/// re-scored; the dense ablation is, as a check that the harness itself is sound. /// -/// One lever, isolated: `rebuild_fts` swaps the tokenizer over the identical -/// chunk rows and vectors — the shipped `porter unicode61` against the -/// unstemmed `unicode61` the A/B retired, kept measurable so the verdict can -/// be re-tried as the corpus grows. Discovery is deliberately not re-scored — -/// `similar` never touches FTS (centroid shortlist + chunk vectors), so its -/// numbers cannot move and the ablation row records no `similar` keys. The -/// dense ablation IS re-scored, as an instrument check: FTS cannot reach it -/// either, so a moved dense rank means the harness is broken, not the engine. -/// -/// `bm25_unstemmed` is the lexical arm, which `run` scores while the vault is still -/// projected-but-unembedded (so `search` is honestly BM25-only under both tokenizers). +/// `bm25_unstemmed` is scored by `run` before embedding, so it is BM25-only. pub fn stemmer( vault: &Vault, base: &Baseline, @@ -102,8 +90,7 @@ pub fn stemmer( }, ), )?; - // Hand the vault back untainted, so a `--sweep` in the same run (and the - // reference numbers above) stay under the shipped default tokenizer. + // Restore the shipped tokenizer for a `--sweep` in the same run. vault.rebuild_fts(FtsTokenizer::PorterUnicode61)?; Ok(()) } @@ -116,14 +103,9 @@ pub fn sweep( sim_set: &SimilarSet, ) -> Result<(), Box> { let positives = base.queries; - // The #44 grid: both directions on each swept knob, plus the one - // interaction worth a row. `target_tokens` brackets the 450 default - // (250 / 350 / 600); `overlap_frac` brackets 0.15 (0.0 / 0.30); - // `target-250+heading-path` exists because a smaller chunk carries less - // of its own context, which is exactly when the breadcrumb prefix (D3) - // is most plausibly worth its tokens. `chars_per_token` and - // `backscan_tokens` stay unswept: they are calibration constants of the - // token proxy and the boundary search, not retrieval-quality levers. + // The #44 grid brackets each knob's default. `target-250+heading-path` pairs the heading + // prefix (D3) with small chunks, where it most plausibly helps. `chars_per_token` and + // `backscan_tokens` are calibration constants, not quality levers, so stay unswept. let variants: Vec<(&str, ChunkConfig)> = vec![ ( "target-250", @@ -178,12 +160,7 @@ pub fn sweep( ]; println!("\n{}", "=".repeat(78)); println!("chunker sweep (same model, same corpus; default row above for reference)"); - // `mate MRR` rather than the per-anchor `similar h@3` this column used - // to carry: that number saturates (GH #183), so as a *comparison* - // column across variants it could only ever print 1.00 (GH #188). - // `strangers` replaced `neg clean` when GH #197 retired the existence - // gate: under always-serve a negative anchor always serves, so that - // column could only ever print 0/5 — the same cannot-move failure. + // Per-mate MRR and strangers, because per-anchor hit@3 saturates (GH #183, #188). println!( "{:<24} {:>7} {:>8} note h@1/MRR vec h@1/MRR chunk h@1/MRR mate MRR strangers", "config", "chunks", "embed_s" @@ -209,10 +186,8 @@ pub fn sweep( sim.mate.mrr(), sim.strangers.len(), ); - // The readout the A/B is actually judged on: at this n every aggregate - // delta above is 1–2 queries, so the aggregate is a smoke alarm and the - // per-query win/loss list is the data (docs/evals.md, the - // process rules). + // What the A/B is judged on: at this n an aggregate delta is 1–2 queries, so the + // per-query moves are the data (docs/evals.md). print_rank_moves(positives, base.hybrid, &pass); append_result( base.log, diff --git a/crates/b2-embed/examples/eval/dense.rs b/crates/b2-embed/examples/eval/dense.rs index 32823cc..9ebece6 100644 --- a/crates/b2-embed/examples/eval/dense.rs +++ b/crates/b2-embed/examples/eval/dense.rs @@ -17,20 +17,15 @@ use b2_core::embed::Embedder; use b2_core::vault::Vault; use std::path::Path; -/// Score the dense single-domain fixture (GH #196/#197, Phase 0b) in a throwaway -/// vault of its own: per-mate discovery ranks against `similar-dense.json`, and -/// the empty-pane sweep across **every** note in the fixture — not only the -/// labelled anchors, because the assertion is about the surface ("no pane in a -/// dense vault is dark"), not about the labels. +/// Score the dense fixture in its own vault: per-mate ranks against `similar-dense.json`, +/// and the empty-pane sweep over every note, since that assertion is about the surface, not +/// the labels. pub fn score_dense( evals_dir: &Path, set: &SimilarSet, embedder: SharedEmbedder, ) -> Result> { let corpus_dir = evals_dir.join("corpus-dense"); - // The run's one loaded model, shared rather than reloaded: the fixture's whole - // point is an isolated run, and that isolation is the vault's — its own - // throwaway copy, its own index — not a second copy of the same weights. let model_id = embedder.model_id().to_string(); let scratch = ScratchVault::copy_flat(&corpus_dir)?; let vault = Vault::open_with_embedder(scratch.root(), Box::new(embedder))?; @@ -45,24 +40,19 @@ pub fn score_dense( mate_ranks: Vec::new(), empty_panes: Vec::new(), detail: Vec::new(), - // The bake-off's absolute bench (GH #200): every note is an anchor here, - // because the ruling being tested is about the *surface* — a vault where - // everything relates may never default to "nothing relates" — and the - // labels cover only a few of these notes. + // Every note is an anchor (GH #200): the ruling is about the surface. fold: score_fold(&vault, set, "dense", true)?, - // The bar's hardest bench, for the same reason the fold's is: this is the - // geometry that disqualified the rule that lost (GH #201). + // The geometry that disqualified the losing rule (GH #201). search: score_dense_search(&vault, &model_id)?, }; - // The pane sweep: every note is an anchor, labelled or not. + // Every note is an anchor, labelled or not. for note in vault.list_notes()? { pass.notes += 1; if vault.similar(¬e.path, SIM_K)?.is_empty() { pass.empty_panes.push(note.path); } } - // Per-mate ranks on the labelled anchors, the orthogonal corpus's metric - // re-used verbatim (GH #183's non-saturating readout). + // Per-mate ranks on the labelled anchors (GH #183). for label in &set.anchors { let candidates = vault.similar(&label.anchor, SIM_K)?; for expected in &label.expected { @@ -94,82 +84,51 @@ pub struct DensePass { pub notes: usize, pub chunks: usize, pub embed_secs: f64, - /// Per-mate ranks at [`SIM_K`] — the fixture's rank metric. + /// Per-mate ranks at [`SIM_K`]. pub mate: Agg, /// (anchor, mate, rank) per labelled mate, for the printed lines and the row. pub mate_ranks: Vec<(String, String, Option)>, - /// Notes whose discovery pane served nothing — asserted empty (GH #196/#197). + /// Notes whose discovery pane served nothing; asserted empty (GH #196/#197). pub empty_panes: Vec, pub detail: Vec, - /// The fold bake-off on this fixture (GH #200) — swept over **every** note, - /// which is where the candidates' hardest bench is: a rule whose default - /// view goes dark on a single-domain vault is disqualified, not re-tuned. + /// The fold bake-off over every note (GH #200): a rule whose default view goes dark here + /// is disqualified, not re-tuned. pub fold: FoldBench, - /// D2's shipped bar replayed on this fixture (GH #201) — see - /// [`score_dense_search`]. + /// D2's shipped bar replayed on this fixture (GH #201). pub search: DenseSearch, } -/// The shipped search evidence bar's reading **on the single-domain fixture** (ADR-0015). -/// -/// This exists because the bar's first form died here and nowhere else. A hard `df <= 10%` -/// content ceiling read 0 cut / 0 served on the labelled orthogonal corpus — clean by every -/// number that bench can produce — and then classed `drone` (df 3) and `comb` (df 7) as -/// stopwords in a vault about beekeeping, cutting 3 of 15 answerable queries. The lexical -/// rule's hazard is **topical concentration**, which the orthogonal corpus cannot express, so -/// a run that judges the bar only there judges it on the geometry it survives. -/// -/// The reading was first taken once, by hand. Taking it *once* is the thing GH #187 named, -/// so it is re-derived every run, here, beside the fold bench that already sweeps this -/// fixture. **In the exit gate** since GH #202, as a row of its own rather than headroom on -/// the labelled corpus's, because it watches a different *geometry*. +/// The shipped search evidence bar's reading on the single-domain fixture (ADR-0015). The +/// lexical rule's hazard is topical concentration, which the orthogonal corpus cannot +/// express. Gated since GH #202. pub struct DenseSearch { - /// Every note's own title replayed as a query — the **tripwire direction** - /// (D2: a labelled-relevant query cut is zero with no headroom). Titles need - /// no labels and so nothing here can be relabelled to clear a reading. + /// Every note's title replayed as a query: the tripwire direction (D2). pub titles: Vec, /// Nonsense, the defect direction. See [`DENSE_NONSENSE`]. pub nonsense: Vec, - /// `None` when the active model has no calibrated bar (M2) — the coverage - /// readings still print, the verdicts do not exist to print. + /// `None` when the model has no calibrated bar (M2); coverage still prints. pub bar: Option, } -/// One query's reading on the dense fixture: the two absolute signals D2 judges, -/// and the engine's own verdict rather than a restatement of it. +/// One query's reading on the dense fixture: D2's two signals and the engine's verdict. pub struct SearchProbe { pub query: String, - /// IDF-weighted term coverage; `None` when no term carries any weight (the - /// lexical half abstaining, not scoring zero). + /// IDF-weighted term coverage; `None` when no term carries weight. pub coverage: Option, pub best_cos: Option, - /// `Vault::search_evidence`'s verdict — what would actually ship. `None` - /// mirrors [`DenseSearch::bar`]. + /// `Vault::search_evidence`'s verdict. `None` mirrors [`DenseSearch::bar`]. pub vouched: Option, - /// The served list with its per-hit provenance — the tail bake-off's dense - /// bench (GH #206). On a title query `keep` is true for every row **by - /// geometry**, not by label: a single-domain vault's lists are all real - /// matches, so a tail rule that truncates one is disqualified — the same - /// absolute GH #200 enforced for discovery, and the reason this fixture - /// needs no `tail_relevant` labels. + /// The served list, for the tail bake-off (GH #206). On a title query every row is kept + /// by geometry, not label, so a tail rule that truncates one is disqualified. pub rows: Vec, } -/// The negatives replayed on the dense fixture: **nonsense only**. The labelled negatives in -/// `queries.json` are the *orthogonal* corpus's, and process rule 2's token audit is what -/// makes them negatives — an audit that says nothing about a different corpus. Against -/// `corpus-dense` the phrase-shaped ones disqualify themselves on their merits ("why parrots -/// mimic speech" shares `mimic` with `robbing-behavior.md`, which is a thing the rule -/// deliberately serves). Nonsense needs no audit in any vault, which is why it transfers. +/// The negatives replayed on the dense fixture: nonsense only. `queries.json`'s negatives +/// are audited against the orthogonal corpus and don't transfer; nonsense needs no audit. pub const DENSE_NONSENSE: [&str; 2] = ["shjfasd", "vrelqip zonktar wembleforth"]; -/// Replay the shipped bar over the dense fixture (see [`DenseSearch`]). -/// -/// Coverage is read off [`b2_core::vault::QueryTermView::idf`] — the view's own -/// weights, not a second copy of the formula ([`term_coverage`]). The orthogonal -/// corpus's bake-off re-derives its arithmetic deliberately, as a drift check -/// against the engine; one such check is the check, and a second would only be two -/// places to fix. +/// Replay the shipped bar over the dense fixture, reading coverage off the engine's weights +/// ([`term_coverage`]). pub fn score_dense_search( vault: &Vault, model_id: &str, @@ -195,9 +154,7 @@ pub fn score_dense_search( }; let mut titles = Vec::new(); for note in vault.list_notes()? { - // The fixture's notes carry no frontmatter title, so the slug is the - // query — `drone-comb` → "drone comb", which is the pair of words the - // retired ceiling called stopwords. + // No frontmatter titles here, so the slug is the query (`drone-comb`). if let Some(title) = title_query(¬e) { titles.push(read(&title, true)?); } @@ -217,12 +174,7 @@ pub fn print_dense_search(search: &DenseSearch) { println!( " search bar D2's shipped bar replayed on this geometry (GH #201; GATED since GH #202)" ); - // The coverage reading comes FIRST, and above the bar, because it is - // **model-free**: a fact about this vault's vocabulary, and the lexical - // half's whole premise. Gating it behind a calibrated bar would print - // nothing at all on a vault the harness can still say something true about — - // the defect PR #205's review already fixed in `calibrate.rs` (c03f8cd), met - // again here (PR #207 review). + // Coverage is model-free, so it prints before, and without, a calibrated bar (PR #207). let covs: Vec = search.titles.iter().filter_map(|p| p.coverage).collect(); let cov_line = match pile_stats(&covs) { Some((min, med, max)) => format!("{min:.2}/{med:.2}/{max:.2}"), @@ -230,7 +182,6 @@ pub fn print_dense_search(search: &DenseSearch) { }; println!(" title-as-query coverage min/med/max {cov_line}"); - // Only the *verdicts* below need a bar, so only they stop here. let Some(bar) = search.bar else { println!(" no calibrated bar for this model — no verdict is offered (M2)"); return; @@ -274,9 +225,8 @@ pub fn print_dense_search(search: &DenseSearch) { ); } -/// The dense fixture's search reading as JSON (`search_transfer` in the dense -/// row) — every probe, so any bar is re-derivable from a row without re-running -/// the model, the `discovery_fold` convention. +/// The dense fixture's search reading as JSON (`search_transfer`), every probe included so +/// any bar is re-derivable without re-running the model. pub fn dense_search_json(search: &DenseSearch) -> serde_json::Value { let probes = |pile: &[SearchProbe]| { pile.iter() @@ -286,9 +236,7 @@ pub fn dense_search_json(search: &DenseSearch) -> serde_json::Value { "coverage": p.coverage.map(|c| (c * 1e4).round() / 1e4), "best_cos": p.best_cos.map(|c| (c * 1e4).round() / 1e4), "vouched": p.vouched, - // NEW subkey (absent before GH #206): the served list's - // per-hit provenance, so the tail constraints below are - // re-derivable from a row without re-running the model. + // Absent before GH #206. "rows": p.rows.iter().map(|row| serde_json::json!({ "path": row.path, "bm25_rank": row.bm25_rank, @@ -307,8 +255,7 @@ pub fn dense_search_json(search: &DenseSearch) -> serde_json::Value { "nonsense_served": search.nonsense.iter().filter(|p| p.vouched == Some(true)).count(), "titles": probes(&search.titles), "nonsense": probes(&search.nonsense), - // NEW subkey (absent before GH #206): the per-hit tail families' - // dense-bench constraints — the absolute this fixture supplies. + // Absent before GH #206. "tail": dense_tail_json(&search.titles), }) } @@ -347,15 +294,12 @@ pub fn print_dense_report(dense: &DensePass) { ); } print_dense_search(&dense.search); - // Model-geometry reading, not a verdict, so it prints with or without a - // calibrated bar — the same posture as the coverage line above it. + // A geometry reading, not a verdict, so it prints without a calibrated bar too. print_dense_tail(&dense.search.titles); } -/// The dense fixture's own JSONL row. Tagged `"corpus": "dense"` — the key that -/// keeps rows from ever averaging across corpora (the orthogonal rows carry -/// `"corpus": "orthogonal"`); a smaller shape than the main row on purpose, -/// since the fixture scores discovery alone. +/// The dense fixture's own JSONL row, tagged `"corpus": "dense"` so rows never average +/// across corpora. pub fn dense_row(run: RunId, dense: &DensePass) -> serde_json::Value { let RunId { git, model, dim } = run; let ts = unix_secs(); @@ -374,13 +318,8 @@ pub fn dense_row(run: RunId, dense: &DensePass) -> serde_json::Value { })).collect::>(), "empty_panes": { "n": dense.notes, "empty": dense.empty_panes.len(), "detail": dense.empty_panes }, "discovery_fold": fold_json(&dense.fold), - // NEW key (absent from rows before 2026-08-22): the shipped search bar - // replayed on this fixture (GH #201). Deliberately NOT the orthogonal - // row's `search_evidence`: that key holds the *labelled* bake-off, and - // this is the label-free transfer reading — a different measurement, so - // it takes a different name. Same convention as every key above: new, - // never a redefinition, so no reader has to branch on `corpus` to learn - // which shape it is holding (PR #207 review). + // Absent before 2026-08-22 (GH #201). Not named `search_evidence`: that key is the + // labelled bake-off, and a key is never redefined across corpora (PR #207). "search_transfer": dense_search_json(&dense.search), "similar_detail": dense.detail.iter().map(|d| serde_json::json!({ "anchor": d.anchor, diff --git a/crates/b2-embed/examples/eval/discovery.rs b/crates/b2-embed/examples/eval/discovery.rs index a46ea76..7d4c6c0 100644 --- a/crates/b2-embed/examples/eval/discovery.rs +++ b/crates/b2-embed/examples/eval/discovery.rs @@ -8,145 +8,80 @@ use crate::metrics::{paths_match, Agg, Window}; use crate::{SIM_K, Z_SCAN_LIMIT}; use b2_core::vault::Vault; -/// A full pass over the discovery labels: rank aggregates over the positive -/// anchors, the suppression tally over the negative ones, and every surfaced -/// candidate's cosine sorted into the two calibration piles the quality floor -/// is read from (index-engine.md §3's ruling, PR #145). +/// A full pass over the discovery labels: rank aggregates, the negative-anchor tally, and +/// the cosine calibration piles (index-engine.md §3, PR #145). #[derive(Default)] pub struct SimilarPass { - /// Rank of the first `expected` hit per positive anchor — the pre-existing - /// hit@1 / hit@3 / MRR discovery metrics, unchanged. + /// Rank of the first `expected` hit per positive anchor. pub rank: Agg, - /// **Per-mate** ranks: one entry per `(anchor, expected mate)` pair rather than one per - /// anchor (GH #183). [`Self::rank`] takes the *first* labelled mate it finds and stops, - /// so an anchor scores a hit on its easiest one and every harder mate is invisible — - /// which is how that metric sat pinned at 1.000 across the ordering change it was - /// supposed to judge. Worse, *adding* a hard mate makes `rank` strictly easier. - /// - /// Scoring each mate on its own is the non-saturating readout GH #183 asked for: a mate - /// sliding from rank 2 to rank 4 moves this number even while `rank` stays at ceiling. A - /// `None` is honest — that mate did not surface at all. - /// - /// **In the exit gate** since GH #188, at [`FLOOR_MATE_MRR`](crate::gate::FLOOR_MATE_MRR). It shipped reporting-only - /// for one issue's worth of time on the measure-then-calibrate precedent, and that is - /// where the baseline came from. - /// - /// (A pass-vs-pass suppression diff — `mate_raw` / `mate_suppressed`, asserted at zero — - /// stood beside this until GH #217: once ADR-0014 retired the existence gate, both - /// passes read the *identical* `similar` call, so the diff compared a call against - /// itself and could never fire. An existence gate returning to the path is what the - /// dense fixture's zero-empty-panes assertion and both per-mate MRR floors catch — - /// against ground truth, not against a second read of the same surface.) + /// Per-mate ranks, one per `(anchor, expected mate)` pair (GH #183). [`Self::rank`] + /// stops at the easiest mate and saturates; this moves when any mate slides. Gated at + /// [`FLOOR_MATE_MRR`](crate::gate::FLOOR_MATE_MRR) (GH #188). pub mate: Agg, - /// **Strangers**: unlabelled notes the shipped surface serves on a *positive* anchor, - /// within the same top-`SIM_K` the ranks are read at — `(anchor, path)` per card, so the - /// smoke alarm comes with the list you argue against it with (process rule 1). - /// - /// This is discovery's **precision** side, and the harness had none: the gated numbers - /// watch mate ranks and negative anchors, and the negatives gate is structurally blind to - /// a member bar — while `member_z <= leader_z`, a negative anchor is clean iff its leader - /// is cut, so a relaxed member bar spends its entire cost here (GH #187/#188). - /// - /// Deliberately **reported, not gated**, and the reason is the gaming direction: the - /// cheapest way to shrink this count is to *label the stranger*, which silently moves the - /// per-mate metric too. The labels are not exhaustive, so an unlabelled note served is - /// not proof of junk — it reads as a smoke alarm with names attached. + /// Strangers: `(anchor, path)` for unlabelled notes served on a positive anchor within + /// `SIM_K`, discovery's precision side (GH #187/#188). Reported, not gated: the cheapest + /// way to shrink it is to label the stranger, and labels aren't exhaustive. pub strangers: Vec<(String, String)>, - /// Positive anchors serving at least one stranger — the spread behind - /// [`Self::strangers`], since one anchor with a long tail and five anchors - /// with one card each are different failures at the same count. + /// Positive anchors serving at least one stranger: the spread behind the count. pub stranger_anchors: usize, - /// Negative anchors asked. pub neg_n: usize, - /// Negative anchors that surfaced zero candidates. Under always-serve - /// (GH #197) an anchor with scorable candidates always serves, so this - /// reads 0 on this corpus — recorded for row comparability (the key's - /// meaning, "anchors serving nothing", is unchanged), no longer asserted: - /// the labels still say "nothing relates", and what the served cards - /// *claim* is measured by their bands (A2's readout, calibration block). + /// Negative anchors that surfaced zero candidates. Reads 0 under always-serve (GH #197); + /// kept for row comparability, not asserted. pub neg_clean: usize, - /// Candidates surfaced across all negative anchors — cards whose labelled - /// answer was "nothing". + /// Candidates surfaced across all negative anchors. pub neg_cards: usize, /// Cosines of surfaced candidates a human labelled genuinely related. pub related: Vec, - /// Cosines of everything else surfaced — non-expected candidates of positive - /// anchors, and every candidate of a negative anchor. + /// Cosines of everything else surfaced. pub junk: Vec, - /// Every anchor's surfaced list in rank order. The flat piles above judge the - /// absolute floor; the *relative* drop-off cutoff is judged within one - /// anchor's list (how far did #2 fall from #1?), and naming the pair behind a - /// pile value needs the anchor too — so the order is recorded, not just the - /// distribution. + /// Every anchor's surfaced list in rank order, for relative drop-off readings and for + /// naming the pair behind a pile value. pub detail: Vec, } /// One anchor's surfaced candidates, in rank order, for the results log. pub struct AnchorDetail { pub anchor: String, - /// True for a negative anchor (empty `expected`). pub negative: bool, /// (candidate path, cosine, human-labelled related) per surfaced candidate. pub candidates: Vec<(String, f64, bool)>, } -/// One row of the z dump: a candidate in the band's unit, with the human label -/// attached. +/// One row of the z dump: a candidate's z with its human label. pub struct ZCand { pub path: String, - /// The stage-2 best-passage z (GH #192's unit; the strength band's input, - /// gating nothing since GH #197) — straight from the engine's own - /// statistics. + /// The engine's stage-2 best-passage z (GH #192): the strength band's input. pub z: f64, - /// The same z recomputed harness-side from the served scores (z over squared - /// best-pair distance, nearer = higher). An instrument check, not a reading: - /// if this drifts from `z`, the engine's statistic moved and the harness's - /// model of it is stale. `None` only when the population had zero variance - /// and no z exists. + /// The same z recomputed from the served scores; a drift from `z` means the engine's + /// statistic moved. `None` at zero variance. pub z_recheck: Option, - /// The stage-2 score as served (negated best chunk-pair L2) — kept so the - /// row also records the model-comparable cosine, not only the - /// anchor-relative z. + /// The stage-2 score as served (negated best chunk-pair L2), for the cosine. pub score: f64, - /// Labelled a mate of this anchor. pub mate: bool, } -/// One anchor's **complete** reading in the band's unit — every candidate note -/// in the corpus, in z order, with the human label attached (GH #187; stage-2 -/// best-passage z since GH #192; the z travels ungated on the one shipped -/// surface since GH #197, so the dump *is* the served list read deep). -/// -/// The cosine piles above are the same numbers in a model-comparable unit; this -/// is the anchor-relative unit a z existence bar would judge in, which is why -/// the piles could never re-derive one's constants. +/// One anchor's complete reading in z order: every candidate note, labelled (GH #187). +/// The anchor-relative unit a z existence bar would judge in, unlike the cosine piles. pub struct AnchorZ { pub anchor: String, - /// True for a negative anchor (empty `expected`) — its whole list is - /// strangers by label, and its leader is what a leader gate would answer to. pub negative: bool, - /// Every candidate in served order, which *is* descending z. + /// Every candidate in served order, which is descending z. pub candidates: Vec, } impl AnchorZ { - /// The top candidate's z — what a leader gate would read. `None` only if - /// the anchor produced no scorable candidate at all. + /// The top candidate's z, what a leader gate would read. pub fn leader(&self) -> Option { self.candidates.first().map(|c| c.z) } - /// This anchor's labelled mates' z's (empty for a negative anchor). pub fn mates(&self) -> impl Iterator + '_ { self.candidates.iter().filter(|c| c.mate).map(|c| c.z) } - /// Everything on this anchor's list a human did *not* label — on a positive - /// anchor, exactly the population a member bar would have to cut. + /// Unlabelled candidates: on a positive anchor, what a member bar would have to cut. pub fn strangers(&self) -> impl Iterator + '_ { self.candidates.iter().filter(|c| !c.mate).map(|c| c.z) } - /// The worst disagreement between the engine's z and the harness's own - /// recomputation from the served scores, across this anchor's candidates — - /// the statistic-level drift check ([`ZCand::z_recheck`]). + /// The worst [`ZCand::z_recheck`] disagreement across this anchor's candidates. pub fn recheck_delta(&self) -> f64 { self.candidates .iter() @@ -155,36 +90,27 @@ impl AnchorZ { } } -/// The z dump across every discovery anchor, split into the populations an existence bar -/// would answer to (GH #187; the unit is the stage-2 best-passage z, and it gates nothing — -/// ADR-0014). +/// The z dump across every discovery anchor (GH #187; gates nothing, ADR-0014). /// -/// Three populations, because a two-bar rule's constants answer to different ones — the -/// conflation is what made "the negatives gate would catch a bad `member_z`" look true when -/// it was not. A leader gate is calibrated by negative-anchor leaders against positive-anchor -/// leaders; a member bar by strangers against labelled mates. Both windows are re-derived -/// every run — the standing record of why no such rule ships, and the first reading any -/// Phase-2 candidate answers to. +/// A leader gate is calibrated by negative- against positive-anchor leaders; a member bar by +/// strangers against mates. Keep the populations separate: conflating them made a bad +/// member bar look caught when it was not. #[derive(Default)] pub struct FloorZ { /// Every anchor with computed statistics, in label order. pub anchors: Vec, - /// Anchors whose candidate pool was under `STATS_MIN_POPULATION` or had zero - /// variance, so no z exists to calibrate from. Named rather than silently - /// dropped: on a corpus small enough for this to happen, every window below - /// is measured on fewer anchors than the labels suggest. + /// Anchors with no z (pool under `STATS_MIN_POPULATION`, or zero variance), named so the + /// windows aren't read as covering them. pub ungraded: Vec, } impl FloorZ { - /// (a) Labelled mates' z's — what a member bar would have to keep. + /// Labelled mates' z's: what a member bar would have to keep. pub fn mate_z(&self) -> Vec { self.anchors.iter().flat_map(|a| a.mates()).collect() } - /// (b) Strangers on **positive** anchors — what a member bar would have to - /// cut. The negative anchors' own candidates are deliberately excluded: - /// they are the leader pair's business, and folding them in here is what - /// would let a member window look wider than it is. + /// Strangers on positive anchors: what a member bar would have to cut. Negative anchors + /// are excluded; they would make the member window look wider than it is. pub fn stranger_z(&self) -> Vec { self.anchors .iter() @@ -192,7 +118,7 @@ impl FloorZ { .flat_map(|a| a.strangers()) .collect() } - /// (c) Negative anchors' leaders — what a leader gate would have to cut. + /// Negative anchors' leaders: what a leader gate would have to cut. pub fn neg_leader_z(&self) -> Vec { self.anchors .iter() @@ -200,8 +126,7 @@ impl FloorZ { .filter_map(|a| a.leader()) .collect() } - /// Positive anchors' leaders — what a leader gate would have to keep, or it - /// empties a list that has a real mate on it. + /// Positive anchors' leaders: what a leader gate would have to keep. pub fn pos_leader_z(&self) -> Vec { self.anchors .iter() @@ -210,11 +135,8 @@ impl FloorZ { .collect() } - /// The worst engine-vs-recomputed z disagreement across every anchor — the - /// statistic-level instrument check (see [`ZCand::z_recheck`]). Small fp - /// noise is expected (the engine z-scores f32 squared distances; the - /// harness recomputes them from the f32-sqrt'd scores), so the printed - /// check tolerates 1e-3 z and reports the observed maximum. + /// The worst [`ZCand::z_recheck`] disagreement across every anchor. f32 round-trips add + /// noise, so the printed check tolerates 1e-3. pub fn recheck_delta(&self) -> f64 { self.anchors .iter() @@ -223,15 +145,8 @@ impl FloorZ { } } -/// Score the discovery labels — **one pass, one surface** (GH #197). -/// -/// Everything reads `Vault::similar`, the always-served ranked list: the ranks, -/// the strangers, the negatives' card tally, and the cosine piles + per-anchor -/// detail the calibration blocks consume. (A second pass with a pass-vs-pass -/// suppression diff existed while the GH #150 existence gate shipped; under -/// always-serve it re-read the identical call and could never fire, so it -/// retired — GH #217, and see [`SimilarPass::mate`] for what catches a -/// returning gate instead.) +/// Score the discovery labels in one pass over `Vault::similar`, the always-served list +/// (GH #197). pub fn score_similar( vault: &Vault, set: &SimilarSet, @@ -252,8 +167,7 @@ pub fn score_similar( .position(|c| label.expected.iter().any(|e| paths_match(&c.path, e))) .map(|p| p + 1); pass.rank.add(rank); - // …and again per labelled mate, so a hard one can't hide behind an - // easy one (GH #183 — see `SimilarPass::mate`). + // Per labelled mate, so a hard one can't hide behind an easy one (GH #183). for expected in &label.expected { let mate_rank = candidates .iter() @@ -261,10 +175,7 @@ pub fn score_similar( .map(|p| p + 1); pass.mate.add(mate_rank); } - // The precision side of the same list (GH #188 — see - // `SimilarPass::strangers`): everything served here that no label - // claims. Scored on the shipped surface at the ranks' own depth, - // because that is the list a human is handed. + // Strangers (GH #188), at the ranks' own depth. let before = pass.strangers.len(); for c in &candidates { if !label.expected.iter().any(|e| paths_match(&c.path, e)) { @@ -275,7 +186,6 @@ pub fn score_similar( pass.stranger_anchors += 1; } } - // The calibration piles + per-anchor detail, off the same served list. let mut ordered = Vec::with_capacity(candidates.len()); for c in &candidates { let related = !negative && label.expected.iter().any(|e| paths_match(&c.path, e)); @@ -296,15 +206,9 @@ pub fn score_similar( Ok(pass) } -/// Dump every candidate's **band-unit z** (stage-2 best-passage z, GH #192) on -/// every discovery anchor — the deep discovery pass (GH #187). -/// -/// Since GH #197 the z travels ungated on the one shipped surface, so the dump -/// is simply `similar` read at [`Z_SCAN_LIMIT`] rather than `SIM_K`: a bar is -/// calibrated against the population it has to cut, and the strangers just -/// under a served prefix are precisely the ones a lower bar would admit. (This -/// used to need the engine's floor moved out of the way with `-∞` bars; the -/// machinery went with the gate.) +/// Dump every candidate's z on every discovery anchor (GH #187, #192): `similar` read at +/// [`Z_SCAN_LIMIT`], since the strangers just past a served prefix are what a lower bar +/// would admit. pub fn score_floor_z( vault: &Vault, set: &SimilarSet, @@ -312,18 +216,12 @@ pub fn score_floor_z( let mut dump = FloorZ::default(); for label in &set.anchors { let candidates = vault.similar(&label.anchor, Z_SCAN_LIMIT)?; - // z is uniform within one query by construction (`discover::candidates` - // gives every surviving candidate a z or none of them), so the leader's - // presence decides the whole list — an anchor with no statistics is - // named, never half-counted. + // `discover::candidates` gives every candidate a z or none, so the leader decides. let Some(true) = candidates.first().map(|c| c.z.is_some()) else { dump.ungraded.push(label.anchor.clone()); continue; }; - // The band z, recomputed harness-side over the same population the - // engine just served: z over squared best-pair distance (score is - // negated L2, so d² = score²), oriented nearer = higher. Should equal - // the engine's own z to fp noise — the statistic-level drift check. + // Score is negated L2, so d² = score². Should match the engine's z to fp noise. let d2: Vec = candidates.iter().map(|c| c.score * c.score).collect(); let recheck = passage_z(&d2); dump.anchors.push(AnchorZ { @@ -348,15 +246,8 @@ pub fn score_floor_z( Ok(dump) } -/// Re-derive the admissible windows a z existence rule *would* have on the -/// corpus as it stands today (GH #187) — the standing record of why none ships -/// (GH #197), and the first reading any Phase-2 bake-off candidate answers to. -/// -/// Printed every run, and recorded in the row, precisely so no number here ever -/// gets copied into a doc comment again: the #150 windows were measured once, -/// frozen into a rustdoc, and silently falsified the first time the corpus grew -/// a note shape they were never derived from. The instrument is the citable -/// thing; any constant stays in the code it governs. +/// Re-derive the windows a z existence rule would have on today's corpus (GH #187), the +/// record of why none ships (GH #197). Cite this output; don't copy its numbers into docs. pub fn print_floor_windows(z: &FloorZ) { let (mates, strangers) = (z.mate_z(), z.stranger_z()); let (neg_leaders, pos_leaders) = (z.neg_leader_z(), z.pos_leader_z()); @@ -364,9 +255,7 @@ pub fn print_floor_windows(z: &FloorZ) { " discovery z calibration (stage-2 best-passage z — the band's input; gates NOTHING \ since GH #197)" ); - // The statistic-level instrument check: the dump's z is recomputed from the - // served scores, so a drift in the engine's statistic can't pass silently - // (tolerance covers f32 sqrt/square round-trip noise). + // Tolerance covers f32 sqrt/square round-trip noise. let recheck = z.recheck_delta(); if recheck <= 1e-3 { println!(" [check] harness recomputation matches the engine z (max Δ {recheck:.1e})"); @@ -421,9 +310,7 @@ pub fn print_floor_windows(z: &FloorZ) { ), } } - // Every negative anchor's leader with the band it paints — A2's readout: - // under always-serve these cards ARE served, and the band is what they - // claim to the human whose labels say "nothing relates". + // Negative anchors' leaders are served under always-serve; the band is what they claim. println!( " negative anchors' leaders (served under always-serve; the band carries the honesty):" ); diff --git a/crates/b2-embed/examples/eval/evidence.rs b/crates/b2-embed/examples/eval/evidence.rs index 867da1f..6ed7abc 100644 --- a/crates/b2-embed/examples/eval/evidence.rs +++ b/crates/b2-embed/examples/eval/evidence.rs @@ -11,89 +11,60 @@ use b2_core::search::EvidenceBar; use b2_core::vault::Vault; use std::path::Path; -/// One served row of a labelled query's list (GH #206): the per-hit provenance -/// RRF discards, and the row's relevance **by label** — the two things a per-hit -/// tail rule is judged between. Rank is the row's position in `rows`, never a -/// stored field: the fused order is the identity D1's prefix requirement binds. +/// One served row of a labelled query's list (GH #206): per-hit provenance and relevance by +/// label. Rank is the position in `rows`, never stored: the fused order is what D1 binds. pub struct ServedRow { pub path: String, - /// 0-based rank in the BM25 list; `None` = the lexical half never ranked - /// this chunk — the "dense-only" row. + /// 0-based rank in the BM25 list; `None` is a dense-only row. pub bm25_rank: Option, - /// This row's own cosine to the query; `None` when the dense half never - /// ranked it (a row is in at least one list, so `bm25_rank` and `cos` are - /// never both absent). + /// The row's cosine to the query; `None` when the dense half never ranked it. Never + /// both `None`. pub cos: Option, - /// In the keep-set — `relevant` ∪ `tail_relevant`, exhaustive by label - /// since GH #206: a false here is a judgement ("filler"), not an omission. + /// In `relevant` ∪ `tail_relevant`. Exhaustive, so false is a judgement, not an omission. pub keep: bool, } -/// One labelled query's evidence reading (invariants.md D2; GH #201, Phase A of -/// the disclosure work): the signals a query-level evidence rule would judge, -/// dumped before any rule exists so GH #201 derives its rule from measurement -/// rather than assumption. Nothing here gates anything. +/// One labelled query's evidence reading (D2, GH #201): the signals a query-level evidence +/// rule judges. pub struct QueryEvidence { pub query: String, - /// Chunks the OR-sanitized FTS5 expression matches at all. **Not** a - /// lexical-anchor test on phrase queries — `fts5_query` ORs every - /// alphanumeric term, so stopwords saturate this count; recorded precisely - /// so that saturation stays measured instead of assumed away. + /// Chunks the OR-sanitized FTS5 expression matches. Not a lexical anchor: stopwords + /// saturate it. pub bm25_hits: usize, - /// Best BM25 score over those matches, sign-flipped so higher = better - /// (FTS5's `rank` is more-negative-is-better). `None` when nothing matches - /// — the honest zero the fused surface currently cannot say. + /// Best BM25 score, sign-flipped so higher = better. `None` when nothing matches. pub bm25_best: Option, - /// Best cosine between the embedded query and any stored chunk vector (the - /// dense top-1) — the strongest semantic evidence the vault holds for this - /// query, which RRF discards before the surface sees it. + /// The dense top-1 cosine, which RRF discards before the surface sees it. pub best_cos: Option, - /// The note that dense top-1 belongs to, naming what the number points at. + /// The note that dense top-1 belongs to. pub top: String, - /// The served list at [`K`], in fused order, one entry per row — path, - /// per-hit provenance, and the row's relevance **by label** (GH #206). For a - /// negative query every row's `keep` is false: the whole list is junk by - /// label. `served`/`dense_only` below read this, so the counts and the rows - /// they summarize cannot drift apart. + /// The served list at [`K`] in fused order (GH #206). pub rows: Vec, - /// Chunks in the index — the denominator every `df` below is judged against. + /// Chunks in the index, the denominator for every `df`. pub chunk_total: usize, - /// Every query term with its document frequency, in query order. The - /// population the lexical-anchor rule is swept over: raw hit count and raw - /// best-BM25 both failed to separate the piles (Phase A), so the anchor is - /// derived from *these* rather than from either of those. + /// Every query term with its document frequency, in query order: what the lexical + /// anchor is derived from, since raw hits and best-BM25 failed to separate the piles. pub terms: Vec<(String, usize)>, - /// **The engine's own verdict** for this query — `Vault::search_evidence`'s `vouched`, - /// i.e. exactly what the surfaces act on (GH #202). The exit gate counts on *this*, never - /// on the harness's restatement below: an assertion about what ships must read what - /// ships. `None` when the model has no calibrated bar. - /// - /// The restatement is kept beside it because the sweep needs it — [`bake_off`] evaluates - /// the rule at coverages the engine cannot be asked about — and because two independent - /// readings of one rule are a drift check when compared. So they are: [`read_shipped_bar`] - /// prints a `[FAULT]` on any query where they disagree. + /// The engine's own verdict (GH #202), which the exit gate counts on. `None` when the + /// model has no calibrated bar. [`coverage`](Self::coverage) restates the rule for the + /// sweep; [`read_shipped_bar`] flags any disagreement. pub vouched: Option, } impl QueryEvidence { - /// What the shipped surface serves at [`K`] today. For a negative query - /// this is the measured D2 defect: `limit` confident-looking results for a - /// query the vault holds nothing for. + /// What the shipped surface serves at [`K`]; on a negative query, the D2 defect. pub fn served(&self) -> usize { self.rows.len() } - /// Of the served rows, how many the **lexical half never ranked at all** - /// (`bm25_rank: None`) — the per-hit shape of the same defect, and the - /// signal the `lexical` tail rule folds on (GH #206). + /// Served rows the lexical half never ranked, which the `lexical` tail rule folds on + /// (GH #206). pub fn dense_only(&self) -> usize { self.rows.iter().filter(|r| r.bm25_rank.is_none()).count() } - /// The share of this query's term IDF the vault carries — the harness's own - /// restatement of [`b2_core::search::LexicalEvidence::term_coverage`], kept - /// separate so a drift in the engine's rule shows up as the two disagreeing - /// rather than as silence. `None` when nothing in the query carries weight. + /// The share of this query's term IDF the vault carries: an independent restatement of + /// [`b2_core::search::LexicalEvidence::term_coverage`], as a drift check. `None` when no + /// term carries weight. pub fn coverage(&self) -> Option { let idf = |df: usize| ((self.chunk_total as f64 + 1.0) / (df as f64 + 1.0)).ln(); let total: f64 = self.terms.iter().map(|(_, df)| idf(*df)).sum(); @@ -115,47 +86,33 @@ impl QueryEvidence { } } -/// The `min_term_coverage` grid the bake-off sweeps: how much of a query's own -/// weight the vault must carry for the lexical half to vouch for it. Spans "a -/// twentieth" to "all of it", so the printed rows show the rule's whole -/// behaviour rather than a neighbourhood of the shipped constant. +/// The `min_term_coverage` grid the bake-off sweeps, spanning the rule's whole range rather +/// than a neighbourhood of the shipped constant. pub const COVERAGES: [f64; 10] = [0.05, 0.10, 0.15, 0.20, 0.25, 0.34, 0.50, 0.67, 0.85, 1.00]; -/// One coverage-bar cell of the bake-off: how the lexical half alone splits the -/// labelled piles there, and what the cosine half would then have to do for the -/// queries it leaves undecided. +/// One coverage cell of the bake-off: how the lexical half splits the labelled piles, and +/// what the cosine half must do with the rest. pub struct EvidenceCell { pub coverage: f64, - /// Positives the lexical half already vouches for. pub pos_anchored: usize, - /// Negatives the lexical half wrongly vouches for. **Any nonzero value - /// disqualifies the cell**: an anchored negative is served whatever the - /// cosine bar says, so no `min_cos` can rescue it. + /// Negatives the lexical half vouches for. Nonzero disqualifies the cell: no `min_cos` + /// can cut an anchored query. pub neg_anchored: usize, - /// Cosines of the positives the lexical half left undecided — the pile a - /// `min_cos` must KEEP. + /// Cosines of undecided positives, which a `min_cos` must keep. pub undecided_pos: Vec, - /// Cosines of the negatives left undecided — the pile it must CUT. + /// Cosines of undecided negatives, which it must cut. pub undecided_neg: Vec, } impl EvidenceCell { - /// The cosine window this cell leaves for `min_cos`, over **only** the - /// queries the lexical half did not already decide — which is the whole - /// point of reading it here rather than over every query: D2's rule is - /// lexical OR semantic, so a positive with an anchor never needs its cosine - /// kept, and a pure-cosine window (Phase A's) overstates the keep set. + /// The cosine window for `min_cos` over only the undecided queries: D2 is lexical OR + /// semantic, so an anchored positive never needs its cosine kept. pub fn window(&self) -> Option { Window::read(&self.undecided_neg, &self.undecided_pos) } - /// Whether some `min_cos` completes this cell into a rule that keeps every - /// positive and cuts every negative. - /// - /// Three ways to be admissible, and the two degenerate ones are real - /// readings rather than edge-case bookkeeping: with no undecided negatives - /// the cosine half is inert (the lexical rule did the whole job), and with - /// no undecided positives any bar above the negatives' best cosine works. + /// Whether some `min_cos` completes this cell into a rule that keeps every positive and + /// cuts every negative. An empty undecided pile on either side is admissible. pub fn admissible(&self) -> bool { if self.neg_anchored > 0 { return false; @@ -166,8 +123,7 @@ impl EvidenceCell { } } - /// The lowest `min_cos` that cuts every undecided negative — the bar's - /// measured floor. `None` when nothing is left to cut. + /// The bar's measured floor. `None` when nothing is left to cut. pub fn cut_floor(&self) -> Option { let max = self .undecided_neg @@ -177,8 +133,7 @@ impl EvidenceCell { max.is_finite().then_some(max) } - /// The highest `min_cos` that still keeps every undecided positive — the - /// bar's measured ceiling. `None` when nothing is left to keep. + /// The bar's measured ceiling. `None` when nothing is left to keep. pub fn keep_ceiling(&self) -> Option { let min = self .undecided_pos @@ -216,35 +171,25 @@ pub fn bake_off(ev: &SearchEvidence) -> Vec { cells } -/// The search evidence dump (GH #201): every labelled query's reading, split -/// into the piles a query-level evidence bar answers to — positives it must -/// keep reachable, negatives it must answer "no matches". +/// The search evidence dump (GH #201): positives the bar must keep, negatives it must cut. pub struct SearchEvidence { pub positives: Vec, pub negatives: Vec, } -/// One side's best-cos pile — the population the query-window derivation and -/// the printed pile lines both read (queries with no dense reading drop out). +/// One side's best-cos pile; queries with no dense reading drop out. pub fn best_cos_pile(rows: &[QueryEvidence]) -> Vec { rows.iter().filter_map(|r| r.best_cos).collect() } -/// Score the search evidence calibration (invariants.md D2; GH #201) — a pure -/// read over the already-built vault. Per labelled query: the lexical half's -/// reading via a direct `chunks_fts` probe (the engine's own sanitized -/// expression — the absolute signals RRF discards), the dense top-1 via the -/// vector-only ablation path, and the shipped surface's served count. +/// Score the search evidence calibration (D2, GH #201), a pure read over the built vault. pub fn score_search_evidence( vault_root: &Path, vault: &Vault, positives: &[Labelled], negatives: &[Labelled], ) -> Result> { - // A second read connection beside the Vault's own — C1: readers are - // unrestricted, and the probe wants FTS5's `rank`, which no façade read - // exposes (deliberately: bm25 units are engine internals everywhere but - // this instrument). + // A second reader (C1): the probe wants FTS5's `rank`, which no façade read exposes. let conn = b2_core::open(&vault_root.join(".b2").join("b2.sqlite"))?; let rows = |queries: &[Labelled]| -> Result, Box> { queries @@ -257,9 +202,7 @@ pub fn score_search_evidence( .first() .map(|h| h.path.clone()) .unwrap_or_else(|| "—".to_string()); - // The façade's own evidence read (GH #201) — the term dfs, the - // per-hit provenance, and the served list in one call, so the - // sweep judges exactly what the engine would. + // The façade's own evidence read, so the sweep judges what the engine would. let view = vault.search_evidence(&q.query, K)?; Ok(QueryEvidence { query: q.query.clone(), @@ -294,12 +237,8 @@ pub fn score_search_evidence( }) } -/// The lexical half's reading for one raw query, probed directly over -/// `chunks_fts` with the engine's own sanitized expression -/// ([`b2_core::search::fts5_query`]): how many chunks match at all, and the -/// best BM25 score among them (FTS5 `rank`, sign-flipped so higher = better). -/// These are exactly the absolute signals the fused path computes and then -/// discards at RRF — which is why the instrument reads them raw. +/// Match count and best BM25 (sign-flipped) for one query, probed over `chunks_fts` with the +/// engine's [`b2_core::search::fts5_query`]: the absolute signals RRF discards. pub fn bm25_probe( conn: &rusqlite::Connection, query: &str, @@ -325,13 +264,9 @@ pub fn bm25_probe( Ok((hits, Some(-best))) } -/// Print the search evidence calibration (invariants.md D2; GH #201) — the -/// query-side sibling of [`print_floor_windows`](crate::discovery::print_floor_windows). Reported, nothing gates: the -/// piles are what GH #201's query-level bar is argued from, and any constant -/// read off them owes process rule 5's real-vault transfer check before it -/// ships. The window's caveat is structural: D2's rule is lexical OR semantic -/// evidence, so a *pure-cosine* window overstates what a real bar must keep — -/// a positive query with a lexical anchor never needs its cosine kept. +/// Print the search evidence calibration (D2, GH #201). Reported, not gated. The +/// pure-cosine window overstates what a real bar must keep, since lexical evidence keeps +/// its own. pub fn print_search_evidence(ev: &SearchEvidence) { println!( " search evidence calibration (D2 — reported, nothing gates; the query bar is GH #201's \ @@ -408,11 +343,8 @@ pub fn print_search_evidence(ev: &SearchEvidence) { } } -/// Print the **search evidence bake-off** (ADR-0015, GH #201) — the query-level rule's -/// window, re-derived from the labelled piles on every run rather than quoted from the day it -/// was read. The GH #187 idiom on search's side: the constant lives in -/// [`b2_core::search::BGE_BASE_EVIDENCE_BAR`] and its *justification* is recomputed here, -/// including whether the shipped bar still sits inside the window it was read from. +/// Print the search evidence bake-off (ADR-0015, GH #201). The constant lives in +/// [`b2_core::search::BGE_BASE_EVIDENCE_BAR`]; its justification is recomputed here each run. pub fn print_search_bakeoff(ev: &SearchEvidence, cells: &[EvidenceCell], model_id: &str) { println!( " search evidence bake-off (D2 — the query-level rule, re-derived every run; GH #201)" @@ -499,15 +431,8 @@ pub fn print_search_bakeoff(ev: &SearchEvidence, cells: &[EvidenceCell], model_i println!(" the tail bake-off's, below (GH #206)."); } -/// Name the admissible cells with the most cosine headroom, and within that band -/// the **most conservative corner** — the tightest `df` fraction and the -/// strictest coverage that still buy the widest window. -/// -/// The tie matters more than the maximum does, and printing only a winner would -/// hide it: the grid's cells are not distinct rules but a plateau, and the -/// reading to carry into process rule 5's transfer check is "anywhere in this -/// band", not "at this point". A constant placed at a lone maximum would be -/// fitted to the grid's resolution. +/// Name the plateau of admissible cells with the most cosine headroom and its strictest +/// coverage. Report the plateau, not a lone maximum, which would fit the grid's resolution. pub fn widest(cells: &[&EvidenceCell]) -> String { let best = cells .iter() @@ -538,10 +463,8 @@ pub fn widest(cells: &[&EvidenceCell]) -> String { } } -/// A cell's cosine headroom: how much room a `min_cos` has between the -/// negatives it must cut and the positives it must keep. An inert cosine half -/// (nothing left undecided on one side) has unbounded room and is reported as -/// such rather than scored against bounded cells. +/// A cell's cosine headroom between the negatives to cut and positives to keep; infinite +/// when the cosine half is inert. pub fn headroom(cell: &EvidenceCell) -> f64 { match (cell.cut_floor(), cell.keep_ceiling()) { (Some(cut), Some(keep)) => keep - cut, @@ -549,33 +472,20 @@ pub fn headroom(cell: &EvidenceCell) -> f64 { } } -/// Where the **shipped** constant stands against this run's piles: the tripwire (a labelled -/// positive the bar would cut — ADR-0015 asserts zero, no headroom) and the defect it exists -/// to fix (a labelled negative it still serves). -/// -/// **In the exit gate** since GH #202, at [`MAX_POSITIVES_CUT`] and [`MAX_NEGATIVES_SERVED`]. -/// Read here rather than in the printer, so the number asserted and the number explained are -/// one number. `None` when the active model has no calibrated bar. +/// Where the shipped bar stands against this run's piles. Gated at [`MAX_POSITIVES_CUT`] and +/// [`MAX_NEGATIVES_SERVED`] (GH #202); read here so the gate and the printer share one number. pub struct ShippedBar { pub bar: EvidenceBar, - /// Labelled positives the bar cuts — the tripwire's direction. pub pos_cut: usize, - /// Labelled negatives it still serves — the defect's direction. pub neg_served: usize, - /// Queries where the engine's verdict and the harness's independent - /// restatement of the same rule **disagree** — a drift between the two, which - /// is silence unless something looks. Expected empty; printed, not gated, - /// because it accuses the instrument as readily as the engine and the reader - /// has to say which. + /// Queries where the engine's verdict and the harness's restatement disagree. Printed, + /// not gated: it may be the instrument that drifted. pub faults: Vec, } pub fn read_shipped_bar(ev: &SearchEvidence, model_id: &str) -> Option { let bar = EvidenceBar::for_model(model_id)?; - // The ENGINE's verdict, not a restatement of it: this is what the gate - // asserts and what the surfaces act on. `None` cannot occur here — the model - // has a bar or this function returned already — so it is counted as served, - // the same direction the surfaces take it (M2: no verdict is not "no match"). + // The engine's verdict. A `None` counts as served, as on the surfaces (M2). Some(ShippedBar { bar, pos_cut: ev @@ -613,8 +523,7 @@ pub fn print_shipped_bar(ev: &SearchEvidence, model_id: &str) { println!(" shipped bar: none for this model — no verdict is offered (M2)"); return; }; - // Every line below reads the ENGINE's verdict, so the numbers printed are the - // numbers gated — one reading, not two that can drift apart. + // The engine's verdict, so the numbers printed are the numbers gated. let vouches = |r: &QueryEvidence| r.vouched != Some(false); println!( " shipped bar: coverage ≥ {:.2}, cos ≥ {:.3}", diff --git a/crates/b2-embed/examples/eval/fold.rs b/crates/b2-embed/examples/eval/fold.rs index 0c35dad..d0621eb 100644 --- a/crates/b2-embed/examples/eval/fold.rs +++ b/crates/b2-embed/examples/eval/fold.rs @@ -1,6 +1,6 @@ -//! The discovery fold bake-off (GH #200, Phase B) — the *default disclosure boundary*, -//! priced on every bench this harness carries. The ruling of record is **no fold ships** -//! (docs/evals.md); the bake-off keeps re-deriving the window every run. +//! The discovery fold bake-off (GH #200): the default disclosure boundary, priced on every +//! bench. The ruling of record is that no fold ships (docs/evals.md); the window is +//! re-derived every run. use crate::common::{cosine_of, truncate}; use crate::labels::{SimilarLabel, SimilarSet}; @@ -9,46 +9,27 @@ use crate::{SIM_K, Z_SCAN_LIMIT}; use b2_core::vault::Vault; use std::collections::HashMap; -/// The reciprocity depths GH #200's candidate 1 is shown at in the headline -/// table and the per-anchor lists — [`SIM_K`] (the pane itself) with one depth -/// either side of it. They are a *display* choice only: the window below is -/// derived from the whole sweep, so no verdict turns on these three. +/// The reciprocity depths shown in the headline table: [`SIM_K`] and one either side. +/// Display only; the window is derived from the whole sweep. pub const FOLD_MUTUAL_K: [usize; 3] = [3, 5, 10]; -/// How far the reciprocity depth is swept when re-deriving candidate 1's -/// admissible window each run (GH #187's idiom on the disclosure axis — a -/// measurement in the harness, never a constant frozen into a docstring). Half -/// the orthogonal corpus's note count: a `k` past that admits most of the vault -/// as "mutually near", which is the rule going vacuous rather than a depth -/// anyone would ship. +/// How far the reciprocity depth is swept. Half the orthogonal corpus's note count: past +/// that, most of the vault is "mutually near" and the rule is vacuous. pub const FOLD_K_SWEEP: usize = 15; -/// A **default disclosure rule**: a candidate answer to "how much of the ranked -/// list does the default view vouch for?" (invariants.md D1 as redrafted; -/// GH #200). Every variant is a **prefix** rule *by construction* — it returns a -/// depth, never a set — so a rule that admits rank 5 while folding rank 2 is not -/// expressible here. D1 states that admissibility requirement; this type makes -/// violating it unrepresentable, which is the difference between a check and a -/// guarantee. +/// A default disclosure rule: how much of the ranked list the default view vouches for +/// (D1, GH #200). Returns a depth, never a set, so a non-prefix rule is unrepresentable. #[derive(Clone, Copy, PartialEq)] pub enum FoldRule { - /// **Candidate 3, the incumbent**: no fold. The default view is the whole - /// served prefix — GH #197's always-serve ruling, admissible per D1 and the - /// thing the other candidates have to beat. + /// Candidate 3, the incumbent: no fold, GH #197's always-serve. NoFold, - /// **Candidate 1**: mutual-kNN reciprocity in prefix form. Candidate B is - /// *reciprocal* for anchor A iff A sits in B's own top `k` candidates, and - /// the default view is the ranked list's longest reciprocal prefix. Rank- - /// based (the hubness-correction family: mutual proximity, CSLS), so it - /// carries no cosine or z constant — though `k` itself is measured below to - /// be scale-dependent, which is a different objection and the decisive one. + /// Candidate 1: the longest prefix whose candidates each have the anchor in their own + /// top `k`. Rank-based, so no cosine or z constant, but `k` measures scale-dependent. Mutual(usize), } impl FoldRule { - /// The rule's name in every table, row and window line — one spelling, so a - /// printed verdict and a recorded row can never name the same rule - /// differently. + /// The rule's one spelling in every table, row and window line. pub fn label(self) -> String { match self { FoldRule::NoFold => "no fold".to_string(), @@ -65,53 +46,37 @@ impl FoldRule { } } -/// One served candidate on one anchor's list, carrying everything a candidate -/// rule judges: the label, the score, and — the reciprocity signal — **where the -/// anchor sits in this candidate's own ranked list**. +/// One served candidate on one anchor's list, with what a rule judges. pub struct FoldRow { pub path: String, pub cos: f64, - /// A labelled mate of this anchor (always `false` on a negative or - /// unlabelled anchor). pub mate: bool, - /// The **anchor's** rank in *this candidate's* full-depth candidate list, or - /// `None` if the anchor is not in it at all. Storing the rank rather than a - /// per-`k` boolean is what makes the whole sweep below free: reciprocity at - /// depth `k` is `rank ≤ k`, so every `k` is priced off one discovery pass - /// per note instead of one per note per depth. + /// The anchor's rank in this candidate's full-depth list. A rank rather than a per-`k` + /// flag, so one pass prices every `k`. pub recip_rank: Option, } impl FoldRow { - /// Whether this candidate is *reciprocal* at depth `k` — i.e. the anchor - /// sits in its own top `k`. Reciprocity is necessary but not sufficient for - /// being above the fold: the fold is the longest reciprocal **prefix**, so a - /// reciprocal candidate ranked after a non-reciprocal one is below it too. + /// Whether the anchor sits in this candidate's top `k`. Not sufficient for being above + /// the fold, which is the longest reciprocal prefix. pub fn reciprocal_at(&self, k: usize) -> bool { self.recip_rank.is_some_and(|r| r <= k) } } -/// One anchor's served list, read once and judged by every candidate rule — so -/// the rules are compared on identical rows rather than on separate passes that -/// could drift. +/// One anchor's served list, read once so every rule is judged on identical rows. pub struct FoldAnchor { pub anchor: String, - /// A labelled **loner** (empty `expected`): its correct default view is - /// *empty above the fold* — the assertion GH #197 retired, returning on the - /// disclosure axis. + /// A labelled loner: its correct default view is empty above the fold. pub negative: bool, - /// Labelled mates in label order. pub expected: Vec, /// The served prefix (`limit` = [`SIM_K`]) in rank order. pub rows: Vec, } impl FoldAnchor { - /// This anchor's labelled mates that the *served prefix* reaches, as - /// `(mate, rank)`. A mate ranked past `limit` is not among them — it is - /// always-serve's own miss at this depth, and counting it against a fold - /// would charge every rule for a cost none of them caused. + /// Labelled mates the served prefix reaches, as `(mate, rank)`. A mate past `limit` is + /// always-serve's miss, not a fold's cost. pub fn served_mates(&self) -> Vec<(String, usize)> { self.expected .iter() @@ -125,71 +90,41 @@ impl FoldAnchor { } } -/// What one rule reads on one bench — GH #200's judged quantities, each carrying -/// the per-anchor list it is argued from (process rule 1: the aggregate is a -/// smoke alarm, the named list is the data). +/// What one rule reads on one bench, each quantity with its named list (process rule 1). pub struct FoldReading { pub rule: FoldRule, - /// Labelled mates the served prefix reaches but the default view does - /// **not** vouch for: `(anchor, mate, rank)`. **The fold's own cost**, and - /// the quantity GH #200 judges a candidate at zero on — a human-labelled - /// relation hidden by default. Reported rather than gated while nothing - /// folds (it reads a structural 0 under always-serve) — and GH #202 shipped - /// no fold to charge, so it stays reporting-only; it becomes an exit-gate - /// row with the first fold that does ship. + /// Served labelled mates hidden by default, `(anchor, mate, rank)`: the fold's own cost, + /// judged at zero (GH #200). Reported only until a fold ships. pub mates_folded: Vec<(String, String, usize)>, - /// Labelled mates above the fold — the other half of the same count. pub mates_above: usize, - /// Unlabelled notes the default view vouches for on a *positive* anchor — - /// the strangers count, read above the fold instead of over the whole served - /// prefix. Expected to shrink; deliberately ungated for the standing reason - /// (the cheapest way to shrink it is to label the stranger). + /// Strangers above the fold on positive anchors. Ungated, like the strangers count. pub strangers_above: Vec<(String, String)>, - /// Negative (loner) anchors whose default view is empty — the honest answer - /// their label always claimed, and what always-serve cannot assert. + /// Loner anchors whose default view is empty. pub neg_empty: usize, - /// Non-negative anchors whose default view is empty. On the dense fixture - /// **any** entry here is disqualifying (a vault where everything relates may - /// never *default* to "nothing relates"); on the orthogonal corpus it is the - /// same event read through the labels. + /// Non-negative anchors whose default view is empty. Any entry disqualifies on the dense + /// fixture. pub dark_panes: Vec, - /// Cards above the fold, summed — the default view's size against - /// always-serve's. + /// Cards above the fold, summed. pub cards_above: usize, } -/// The **admissible-`k` window** for candidate 1, re-derived from this run -/// (GH #187's idiom, moved onto the disclosure axis: print the window a rule -/// *would* have rather than freezing a constant into a docstring). Two bounds -/// that must overlap for any `k` to be shippable on this bench: -/// -/// - `keep_min` — the smallest `k` that folds **no** labelled mate and darkens -/// **no** pane. Below it the rule hides relations a human labelled. -/// - `loner_max` — the largest `k` at which **every** labelled loner's default -/// view is empty. Above it the fold stops making the claim it exists to make, -/// and a loner's pane fills up again. +/// The admissible-`k` window for candidate 1, re-derived each run. Shippable only if +/// `keep_min` (smallest `k` folding no mate and darkening no pane) ≤ `loner_max` (largest +/// `k` emptying every loner's view). pub struct FoldWindow { pub keep_min: Option, pub loner_max: Option, - /// The smallest swept `k` whose fold equals always-serve on every anchor — - /// where the rule becomes vacuous (it vouches for everything). + /// The smallest `k` whose fold equals always-serve everywhere: the rule goes vacuous. pub vacuous_at: Option, - /// The best the loner claim ever gets on this bench, as `(empty, largest k - /// still achieving it)` — printed because "never all of them" and "four of - /// five up to k = 11" are different findings, and only the second says how - /// close the rule came. + /// The best loner claim, as `(empty, largest k still achieving it)`. pub loner_best: Option<(usize, usize)>, - /// The largest swept `k` that still darkens a pane. Every `k` at or below it - /// is disqualified outright on a corpus where everything relates (D1's - /// absolute), so this is the hard floor under any window. + /// The largest `k` that still darkens a pane: the hard floor under any window (D1). pub dark_below: Option, } impl FoldWindow { - /// Whether some swept `k` satisfies both bounds at once. A bench with no - /// labelled loner has no upper bound to satisfy and so can never report an - /// open window on its own — it contributes the lower bound to the - /// bake-off's joint window, which is what the printed verdict says. + /// Whether some swept `k` satisfies both bounds. Never true on a bench with no loner, + /// which supplies only the lower bound. pub fn open(&self) -> bool { matches!((self.keep_min, self.loner_max), (Some(a), Some(b)) if a <= b) } @@ -201,36 +136,22 @@ pub struct FoldBench { pub anchors: Vec, /// The headline rules: always-serve plus [`FOLD_MUTUAL_K`]'s detail depths. pub readings: Vec, - /// `(k, reading)` across the whole swept range — what the window is derived - /// from, and printed as a compact table so the shape of the trade is visible - /// rather than asserted. + /// `(k, reading)` across the swept range, which the window is derived from. pub sweep: Vec<(usize, FoldReading)>, pub window: FoldWindow, - /// Negative anchors on this bench (0 on the dense fixture, which has no - /// loner by construction). pub neg_n: usize, - /// Labelled mates the served prefix never reaches at `limit` — always-serve's - /// own miss, identical under every rule. Printed beside the folded count so - /// no rule is charged for it. + /// Labelled mates never served at `limit`: always-serve's miss, charged to no rule. pub mates_unserved: usize, - /// The median *full-depth* candidate pool an anchor here has — what a - /// reciprocity depth is a fraction OF. A `k` is only meaningful against it: - /// "top 7" of a 14-candidate pool and "top 7" of a 30-candidate one are - /// different claims, and the two corpora's windows below are only - /// comparable in this unit. + /// The median full-depth candidate pool. A `k` only compares across corpora as a + /// fraction of it. pub pool_median: usize, - /// Authored edges in the built vault — **candidate 2's** entire calibration - /// population ("what related looks like in this vault", read off the human's - /// own committed links). Both eval corpora are link-free by construction, so - /// this reads 0 and candidate 2 is *unpriceable here*: it is measured where - /// its population exists, by `make calibrate` on a real vault. + /// Authored edges: candidate 2's calibration population. The eval corpora are link-free, + /// so candidate 2 is only priced by `make calibrate` on a real vault. pub authored_edges: usize, } -/// Every note's own ranked candidate list as `path → rank`, read at full depth — -/// the reciprocity lookup candidate 1 is judged on. Read over **every note in -/// the vault**, not only the labelled anchors: reciprocity is decided by the -/// *candidate's* list, and most candidates are not anchors. +/// Every note's full-depth candidate list as `path → rank`, the reciprocity lookup. Over +/// every note, since most candidates are not anchors. pub fn reciprocity_ranks( vault: &Vault, ) -> Result>, Box> { @@ -247,10 +168,9 @@ pub fn reciprocity_ranks( Ok(out) } -/// Score the fold bake-off on one built vault. `sweep_all` reads **every note** -/// as an anchor (the dense fixture's pane sweep, where the assertion is about -/// the surface rather than the labels); otherwise only the labelled anchors are -/// read, matching [`score_similar`](crate::discovery::score_similar)'s depth and population exactly. +/// Score the fold bake-off on one built vault. `sweep_all` reads every note as an anchor; +/// otherwise only the labelled anchors, matching +/// [`score_similar`](crate::discovery::score_similar). pub fn score_fold( vault: &Vault, set: &SimilarSet, @@ -274,11 +194,8 @@ pub fn score_fold( } } - // Outbound only. `neighbors` answers "what is 1 hop from here" and so returns - // an edge from *both* of its endpoints; summing the raw counts over every - // note would census edge **endpoints** and report candidate 2's population at - // twice its size (PR #204 review). Counting each edge once from its source is - // the number the rule would calibrate from. + // Outbound only: `neighbors` returns an edge from both endpoints, so counting all would + // double it (PR #204). let mut authored_edges = 0; for note in vault.list_notes()? { authored_edges += vault @@ -292,10 +209,7 @@ pub fn score_fold( let mut neg_n = 0; for (anchor, label) in anchors { let expected: Vec = label.map(|l| l.expected.clone()).unwrap_or_default(); - // A *labelled* loner. An unlabelled anchor on the dense sweep is not a - // negative — that fixture carries no loner at all, and reading "no - // label" as "nothing relates" would invent the claim the bench exists - // to test. + // An unlabelled anchor on the dense sweep is not a negative. let negative = label.is_some() && expected.is_empty(); if negative { neg_n += 1; @@ -421,8 +335,6 @@ pub fn read_fold(anchors: &[FoldAnchor], rule: FoldRule) -> FoldReading { reading.mates_folded.push((a.anchor.clone(), mate, rank)); } } - // The precision side, read above the fold: unlabelled cards the default - // view vouches for on a positive anchor. if !a.negative && !a.expected.is_empty() { for row in a.rows.iter().take(fold) { if !row.mate { @@ -436,9 +348,7 @@ pub fn read_fold(anchors: &[FoldAnchor], rule: FoldRule) -> FoldReading { reading } -/// The bake-off's printed readout: the headline rules side by side, the swept -/// `k` the window is derived from, and the per-anchor folds — which is the list -/// the verdict is argued from (process rule 1). +/// The bake-off's printed readout: headline rules, the `k` sweep, and per-anchor folds. pub fn print_fold_bench(bench: &FoldBench) { println!("\n{}", "=".repeat(78)); println!( @@ -477,7 +387,7 @@ pub fn print_fold_bench(bench: &FoldBench) { bench.mates_unserved ); - // The named cost of every rule that has one — the list, not the count. + // The named cost of every rule that has one. for r in &bench.readings { if r.mates_folded.is_empty() { continue; @@ -488,8 +398,7 @@ pub fn print_fold_bench(bench: &FoldBench) { } } - // The swept window — the #187 idiom on the disclosure axis: derive the - // rule's admissible range from THIS run rather than quote a constant. + // The swept window, derived from this run. println!( "\n mutual-k sweep (k = 1..{FOLD_K_SWEEP}; the window a shippable k would have to sit in)" ); @@ -555,9 +464,7 @@ pub fn print_fold_bench(bench: &FoldBench) { bench.neg_n ); } - // The verdict names *which* bound failed. "No k works" and "this corpus - // cannot decide alone" are different sentences, and a bench with no loner - // can only ever supply the lower bound. + // Name which bound failed; a bench with no loner can only supply the lower bound. println!( " → window {}", match (bench.neg_n, bench.window.keep_min, bench.window.loner_max) { @@ -592,7 +499,6 @@ pub fn print_fold_bench(bench: &FoldBench) { } ); - // The per-anchor lists at the headline depths. println!("\n per anchor — served / above the fold, by rule:"); print!(" {:<40} {:>7}", "anchor", "served"); for r in &bench.readings { @@ -632,9 +538,7 @@ pub fn print_fold_bench(bench: &FoldBench) { println!(); } - // Candidate 2's census — printed on every bench, because "we did not measure - // it" and "there was nothing to measure it on" are different sentences and - // only the second is an argument. + // Printed on every bench: "nothing to measure it on" is a finding, not an omission. println!( "\n candidate 2 (authored-edge reference bar): {} authored edges in this corpus — {}", bench.authored_edges, @@ -647,10 +551,8 @@ pub fn print_fold_bench(bench: &FoldBench) { ); } -/// The bake-off as one JSON subtree (`discovery_fold`): every rule's reading, -/// the swept window it was judged against, and the per-anchor rows both were -/// computed from — so a verdict cited from a row can be re-derived without -/// re-running the model, which is what `results.jsonl` is for. +/// The bake-off as one JSON subtree (`discovery_fold`), with the per-anchor rows so a +/// verdict can be re-derived without re-running the model. pub fn fold_json(bench: &FoldBench) -> serde_json::Value { let reading = |r: &FoldReading| { serde_json::json!({ @@ -672,11 +574,8 @@ pub fn fold_json(bench: &FoldBench) -> serde_json::Value { "limit": SIM_K, "pool_median": bench.pool_median, "neg_n": bench.neg_n, - // Always-serve's own miss at this limit, recorded apart from every - // rule's cost so the two can never be added together by a later reader. "mates_unserved": bench.mates_unserved, - // Candidate 2's population, recorded even at zero: "unpriceable here" - // is a measurement, and a row that omitted it would read as untried. + // Recorded even at zero, so the row doesn't read as untried. "authored_edges": bench.authored_edges, "rules": bench.readings.iter().map(reading).collect::>(), "sweep": bench.sweep.iter().map(|(k, r)| serde_json::json!({ @@ -706,8 +605,6 @@ pub fn fold_json(bench: &FoldBench) -> serde_json::Value { "path": row.path, "cos": (row.cos * 1e4).round() / 1e4, "mate": row.mate, - // The anchor's rank in THIS candidate's own list — the whole - // reciprocity signal, from which any k is re-derivable. "recip_rank": row.recip_rank, })).collect::>(), })).collect::>(), diff --git a/crates/b2-embed/examples/eval/gate.rs b/crates/b2-embed/examples/eval/gate.rs index 9f23312..f61515b 100644 --- a/crates/b2-embed/examples/eval/gate.rs +++ b/crates/b2-embed/examples/eval/gate.rs @@ -1,10 +1,9 @@ -//! The exit gate: the assertions `make eval` exits 2 on, read off the **default** config's +//! The exit gate: the assertions `make eval` exits 2 on, read off the default config's //! passes (docs/evals.md, "The exit gate"). A manual quality gate, never a CI test. //! -//! How each row is placed is the point, and the constants' own docs say why: a rank floor -//! sits *below* its reading (corpus-drift headroom, run noise being zero), while the -//! search-evidence rows sit *at* their structural zeros, because headroom there would read -//! as permission to serve a nonsense query or cut a real one. +//! A rank floor sits below its reading (corpus-drift headroom; run noise is zero). The +//! search-evidence rows sit at zero, since headroom there would permit serving a nonsense +//! query or cutting a real one. use crate::dense::DensePass; use crate::discovery::SimilarPass; @@ -13,62 +12,35 @@ use crate::retrieval::Pass; use crate::SIM_K; /// The soft reference floor on the default config's hybrid note hit@1. -/// Untouched by the GH #197 re-derivation: retrieval never had a gate to -/// retire. pub const FLOOR_HIT1: f64 = 0.75; -/// The floor on **per-mate** discovery MRR@[`SIM_K`] (GH #188) — the non-saturating rank -/// metric, gated once a baseline existed to price it. Re-derived for always-serve -/// (ADR-0014): the reading moved 0.633 -> 0.650 when the existence gate retired. +/// The floor on per-mate discovery MRR@[`SIM_K`] (GH #188, ADR-0014); reading 0.650. /// -/// Placed **below** the reading, never at it: a gate pinned to today's number fails on the -/// first legitimate corpus edit, which trains the one habit this harness must never train -/// (process rule 2 — editing a *label* to get green). Sizing is measured, not guessed: five -/// consecutive runs on an unchanged corpus/model/build reproduce every rank exactly, so the -/// noise floor is 0 and the headroom exists for *corpus* drift. At n = 15 mates one mate -/// lost from rank 1 costs 1/15, and this floor sits ~2 such losses under the reading. +/// Below the reading so a legitimate corpus edit doesn't tempt a relabel (process rule 2). +/// Runs are bit-reproducible, so the headroom is for corpus drift: ~2 mates lost from rank +/// 1 at n = 15. pub const FLOOR_MATE_MRR: f64 = 0.52; -/// The floor on the **dense fixture's** per-mate MRR@[`SIM_K`] (ADR-0014, Phase 0b), gated -/// once its baseline existed — the same measure-then-calibrate order as [`FLOOR_MATE_MRR`]. -/// -/// The reading is 0.467 (n = 14): the model recovers the within-cluster labels and ranks the -/// three cross-cluster claims lower, which is headroom in both directions. (Its first -/// reading was 0.502; a one-word grammar fix moved one mate a rank — the worked example of -/// why these floors carry corpus-drift margin.) At n = 14 one mate lost from rank 1 costs -/// 1/14, and this sits ~2 such losses under. Process rule 2 binds hard here: in a corpus -/// where everything relates, relabelling toward the model's order would always look -/// plausible — a red reading argues about the notes. +/// The floor on the dense fixture's per-mate MRR@[`SIM_K`] (ADR-0014); reading 0.467, sized +/// like [`FLOOR_MATE_MRR`] (~2 losses at n = 14). Where everything relates, relabelling +/// toward the model always looks plausible: a red reading argues about the notes. pub const FLOOR_DENSE_MATE_MRR: f64 = 0.32; -/// How many **labelled negative queries** the shipped evidence bar may still serve -/// (ADR-0015, GH #202). Zero: this is the defect the bar exists to fix. -/// -/// A floor at its measured value rather than below it, which is the deliberate exception to -/// the house sizing method — the "headroom" a floor normally carries would be *permission to -/// serve a nonsense query*. A new negative the bar serves is either a real regression or a -/// mislabelled query; both want a red reading. +/// How many labelled negative queries the shipped evidence bar may serve (ADR-0015, +/// GH #202). Zero: this is the defect the bar exists to fix, so no headroom. pub const MAX_NEGATIVES_SERVED: usize = 0; -/// How many **labelled relevant queries** the bar may cut (GH #202) — the search-side -/// tripwire ADR-0015 asserts at zero with no headroom, and the direction that costs a user -/// something real: a served nonsense row costs a little trust, a cut positive costs the -/// answer. Its precondition was met by GH #208, which labelled the date-shaped query pile. -/// The reading is 0 of 44. A nonzero value is never a calibration nudge: it means the *rule* -/// is wrong for a shape the corpus now carries. +/// How many labelled relevant queries the bar may cut (ADR-0015, GH #202, #208). A cut +/// positive costs the user the answer. A nonzero reading means the rule is wrong, not the +/// constant. pub const MAX_POSITIVES_CUT: usize = 0; -/// How many of the **dense fixture's title-as-query probes** the bar may cut (GH #202). -/// Zero, and this is the assertion that would have caught the losing rule: a note's own -/// title names a note the vault demonstrably holds, so cutting one is indefensible whatever -/// a labelled corpus says. A third row rather than headroom on [`MAX_POSITIVES_CUT`] because -/// it gates a different *geometry*: the labelled corpus minimizes shared vocabulary by -/// construction, so topical concentration is only expressible here. Titles need no labels, -/// so nothing in this reading can be relabelled to clear it. +/// How many of the dense fixture's title-as-query probes the bar may cut (GH #202). A note's +/// own title names a note the vault holds, and topical concentration is only expressible in +/// this fixture. Titles carry no labels, so nothing here can be relabelled to clear it. pub const MAX_DENSE_TITLES_CUT: usize = 0; -/// Whether the default config clears every exit-gate row, printing a `[warn]` naming the -/// first row that fails (so a red run says which reading to argue with). +/// Whether the default config clears every exit-gate row; warns naming the first failure. pub fn passes( hybrid: &Pass, similar: &SimilarPass, @@ -76,8 +48,6 @@ pub fn passes( evidence: &SearchEvidence, model_id: &str, ) -> bool { - // The soft floors, on the DEFAULT config's passes — so this can double as - // a manual quality gate. Not a CI test. if hybrid.note.hit1() < FLOOR_HIT1 { eprintln!( "\n[warn] hybrid hit@1 {:.2} is below the {FLOOR_HIT1} reference floor — inspect the misses above.", @@ -85,16 +55,9 @@ pub fn passes( ); return false; } - // The negatives' suppression assertion RETIRED with the gate it watched (ADR-0014): - // under always-serve a loner anchor serves its ranked nearest — that is the ruling, not - // a regression. The anchors stay labelled, the strangers instrument keeps counting, and - // what the served cards *claim* is the calibration block's band readout. (The - // pass-vs-pass suppression assertion retired later, for a different reason: it compared - // a call against itself and could never fire — GH #217. A returning existence gate is - // the dense zero-empty-panes assertion's and the per-mate floors' to catch.) - // - // Discovery **rank** (GH #188). The rank floor sits below its measured reading (corpus - // drift headroom, run noise being zero). + // No suppression assertion on negative anchors: under always-serve (ADR-0014) a loner + // serves its nearest by ruling. A returning existence gate is caught by the dense + // empty-pane check and the per-mate floors (GH #217). if similar.mate.mrr() < FLOOR_MATE_MRR { eprintln!( "\n[warn] per-mate MRR@{SIM_K} {:.3} is below the {FLOOR_MATE_MRR:.2} floor — discovery ranking regressed \ @@ -103,10 +66,7 @@ pub fn passes( ); return false; } - // The dense fixture's existence assertion (GH #196/#197): every note in a - // corpus where everything relates must serve candidates. An empty pane here - // can only come from an anchor-local statistic claiming "nothing relates" - // on a vault where that is false — the exact failure GH #196 measured. + // Every note in a corpus where everything relates must serve candidates (GH #196/#197). if !dense.empty_panes.is_empty() { eprintln!( "\n[warn] {} of {} dense-fixture notes serve an EMPTY pane ({}) — an existence \ @@ -117,7 +77,6 @@ pub fn passes( ); return false; } - // …and its rank floor, the dense sibling of FLOOR_MATE_MRR. if dense.mate.mrr() < FLOOR_DENSE_MATE_MRR { eprintln!( "\n[warn] dense per-mate MRR@{SIM_K} {:.3} is below the {FLOOR_DENSE_MATE_MRR:.2} floor — \ @@ -126,15 +85,8 @@ pub fn passes( ); return false; } - // The search-evidence rows (ADR-0015, GH #202), landed with the surfaces that consume - // the verdict. Everything above is discovery's and is deliberately UNCHANGED: search's - // bar moves no discovery rank and no reachability, so movement up there is a bug. - // - // All three sit at their structural zeros with no headroom — the exception to the house - // sizing method rather than an oversight, since headroom here would read as permission - // to serve a nonsense query or cut a real one. Skipped entirely when the model has no - // calibrated bar (ADR-0007): asserting the absence of a verdict would fail every run on - // a model the harness has simply not measured yet. + // The search-evidence rows (ADR-0015, GH #202). Search's bar moves no discovery rank, so + // movement above is a bug. Skipped when the model has no calibrated bar (ADR-0007). match read_shipped_bar(evidence, model_id) { None => eprintln!( "\n[note] no calibrated evidence bar for {model_id} — D2's exit-gate rows are not \ @@ -152,9 +104,6 @@ pub fn passes( ); return false; } - // The tripwire, and the direction that costs a user the answer rather - // than a little trust. Its precondition is GH #208's date-shaped pile: - // the assertion is only worth what the query shapes behind it are. if reading.pos_cut > MAX_POSITIVES_CUT { eprintln!( "\n[warn] the shipped evidence bar CUTS {} of {} labelled relevant queries where D2 \ @@ -167,11 +116,8 @@ pub fn passes( } } } - // The same two directions on the dense fixture — a different geometry, not a - // different threshold. Topical concentration is what killed the losing rule - // and is structurally inexpressible on the orthogonal corpus (process rule 2's - // token audit minimizes shared vocabulary), so these are their own rows. - // Titles carry no labels, so nothing here can be relabelled to clear it. + // The same two directions on the dense fixture: a different geometry, not a different + // threshold. let titles_cut = dense .search .titles diff --git a/crates/b2-embed/examples/eval/instrument.rs b/crates/b2-embed/examples/eval/instrument.rs index c5fe193..d961ef8 100644 --- a/crates/b2-embed/examples/eval/instrument.rs +++ b/crates/b2-embed/examples/eval/instrument.rs @@ -9,16 +9,10 @@ use std::ops::ControlFlow; use std::rc::Rc; use std::time::Instant; -/// One loaded model serving every throwaway vault of the run. +/// One loaded model serving every throwaway vault of the run, instead of a load per vault. /// -/// `Vault::open_with_embedder` takes its embedder by value, so without this each vault -/// (the orthogonal corpus's, the dense fixture's) loaded its own copy of the same -/// weights — a full model load apiece. The model is read-only and deterministic -/// (`&self` throughout), so sharing it costs the fixture none of its isolation: that -/// is a property of the *vault*, which stays its own. Every [`Embedder`] method -/// forwards, including the ones with trait defaults `LocalEmbedder` overrides (the -/// query prefix, the batched forward pass) — a method added to the trait later must -/// be forwarded here too, or the shared model silently runs the default instead. +/// Forwards every [`Embedder`] method, including the defaults `LocalEmbedder` overrides: a +/// method added to the trait must be forwarded here too, or the default silently runs. #[derive(Clone)] pub struct SharedEmbedder(Rc); @@ -46,16 +40,11 @@ impl Embedder for SharedEmbedder { } } -/// `LocalEmbedder::embed_batch` must be a faithful map of `embed`: right-padding short rows -/// to the batch's longest and masking them out has to leave each row's CLS vector unchanged. -/// The reindex path batches freely, so a regression here would silently corrupt every stored -/// vector — and every score this eval prints. -/// -/// It lives in the eval rather than `cargo test` because it needs the provisioned model, -/// which the fast suite deliberately never touches (ADR-0013). Running it here means it -/// actually runs, instead of sitting behind an `#[ignore]` nobody passes `--ignored` to. +/// `LocalEmbedder::embed_batch` must equal `embed` row by row: padding must not move a CLS +/// vector, or every stored vector and every score here is silently wrong. Lives here, not in +/// `cargo test`, because it needs the real model (ADR-0013). pub fn check_batch_matches_single(model: &dyn Embedder) -> Result<(), Box> { - // Deliberately varied lengths, so batching pads the short rows to the longest. + // Varied lengths, so batching pads. let texts = [ "Spaced repetition schedules reviews at increasing intervals.", "Sleep consolidates memory.", @@ -78,11 +67,8 @@ pub fn check_batch_matches_single(model: &dyn Embedder) -> Result<(), Box Result<(usize, f64), Box, - /// A short verbatim phrase from the target passage; when present the query is - /// also scored at chunk level (does a top-K chunk of a relevant note contain - /// it?). See queries.json's description for the labelling rules. + /// A verbatim phrase from the target passage; when present the query is also scored at + /// chunk level. Labelling rules are in queries.json. #[serde(default)] pub passage: Option, - /// Notes beyond `relevant` that are honest evidence for the query (GH #206) — - /// the per-hit tail depth. The judgement is **exhaustive** for every positive - /// query: a served note in neither `relevant` nor here is irrelevant *by - /// label*, which is the statement the tail bake-off is judged on. Never enters - /// a rank aggregate — `relevant` alone says what should rank first. + /// Other notes that are honest evidence for the query (GH #206). Exhaustive: a served + /// note in neither list is irrelevant by label. Never enters a rank aggregate. #[serde(default)] pub tail_relevant: Vec, } @@ -43,15 +35,13 @@ pub struct SimilarSet { #[derive(Deserialize)] pub struct SimilarLabel { pub anchor: String, - /// Corpus notes a human says belong next to `anchor`. **Empty = a negative - /// anchor**: the labelled answer is "nothing relates", so the right result is - /// zero candidates and everything surfaced is junk by label (similar.json). + /// Corpus notes a human says belong next to `anchor`. Empty means a negative anchor: + /// everything surfaced is junk by label. pub expected: Vec, } -/// Every note in one corpus dir, as `file name → lowercased content` — the -/// ground the label lint checks against. Lowercased once, so the passage check -/// matches the way [`score_pass`](crate::retrieval::score_pass) will (case-insensitive containment). +/// Every note in one corpus dir as `file name → lowercased content`, lowercased to match +/// [`score_pass`](crate::retrieval::score_pass)'s case-insensitive containment. pub fn corpus_texts(dir: &Path) -> Result, Box> { let mut out = HashMap::new(); for entry in std::fs::read_dir(dir)? { @@ -66,15 +56,9 @@ pub fn corpus_texts(dir: &Path) -> Result, Box Result> { - // A bare `--` is dropped rather than refused (`make stability`'s posture: a pasted - // `cargo run … -- --sweep` must not fail on its own separator); anything else - // unrecognised is refused, since a typo'd flag would otherwise run the default - // measurement while the reader believes an A/B ran. + // A bare `--` is dropped so a pasted `cargo run … -- --sweep` works. let args: Vec = std::env::args().skip(1).filter(|a| a != "--").collect(); reject_unknown_flags(&args, &["--sweep", "--stemmer"], &[])?; let sweep = has_flag(&args, "--sweep"); @@ -144,10 +98,8 @@ fn run() -> Result> { let corpus_dir = evals_dir.join("corpus"); let results_path = evals_dir.join("results.jsonl"); - // Load the labelled sets. Queries split on their labels: positives carry - // rank labels; an empty `relevant` is a negative query (invariants.md D2, - // GH #201), scored only by the search evidence calibration so its presence - // moves no rank aggregate. + // An empty `relevant` is a negative query (D2, GH #201), scored only by the search + // evidence calibration so it moves no rank aggregate. let set: QuerySet = serde_json::from_str(&std::fs::read_to_string(evals_dir.join("queries.json"))?)?; let (negatives, positives): (Vec, Vec) = @@ -158,12 +110,7 @@ fn run() -> Result> { evals_dir.join("similar-dense.json"), )?)?; - // Lint the labels against the corpora they claim to describe, before - // anything expensive runs. A typo'd label path fails nothing on its own — - // it just reads as a permanent miss (a rank of `None`, a served row - // downgraded to filler by label) and gets chased as an engine regression — - // so the run refuses to score against labels the corpus cannot honour - // (exit 1: the run broke, not the gate). + // A typo'd label path reads as a permanent miss, not an error, so lint first (exit 1). lint_labels( &corpus_dir, &evals_dir.join("corpus-dense"), @@ -173,27 +120,22 @@ fn run() -> Result> { &dense_set, )?; - // Ensure the model is available and load it — once for the whole run: both - // corpora's vaults share this one copy (a missing model is fetched here first). + // Loaded once; both corpora's vaults share it. let config = EmbedConfig::load()?; let embedder = SharedEmbedder::new(load_or_provision(&config)?); let model_id = embedder.model_id().to_string(); let dim = embedder.dim(); eprintln!("[eval] model = {model_id} (dim {dim})\n"); - // A correctness gate, not a score: every number below is computed from batched - // embeddings, so they only mean anything if batching is faithful. + // Every number below comes from batched embeddings, so check batching first. check_batch_matches_single(&embedder)?; - // Build a throwaway vault from the corpus. let scratch = ScratchVault::copy_flat(&corpus_dir)?; let vault_root = scratch.root(); let mut vault = Vault::open_with_embedder(vault_root, Box::new(embedder.clone()))?; // ---- Phase 1: projection only → the BM25-only baseline. ------------------ - // The vector space does not exist yet, so `search`/`search_chunks` run - // keyword-only (index-engine.md) — the ablation costs nothing - // extra: it is the same vault, paused between the two passes. + // No vectors yet, so search runs keyword-only (index-engine.md). let report = vault.project(false)?; let bm25 = score_pass(&vault, &positives, Retrieval::Fused)?; eprintln!( @@ -201,10 +143,7 @@ fn run() -> Result> { report.indexed ); - // The stemmer instrument's lexical arm is scored HERE, while the vault is - // still projected-but-unembedded, so `search` is honestly BM25-only under - // both tokenizers; the vault is handed back to the shipped default before - // anything embeds. + // Scored here, before embedding, so both tokenizers are BM25-only; restored after. let bm25_unstemmed = if stemmer { vault.rebuild_fts(FtsTokenizer::Unicode61)?; let pass = score_pass(&vault, &positives, Retrieval::Fused)?; @@ -219,15 +158,11 @@ fn run() -> Result> { let vector = score_pass(&vault, &positives, Retrieval::VectorOnly)?; let hybrid = score_pass(&vault, &positives, Retrieval::Fused)?; let similar = score_similar(&vault, &sim_set)?; - // The fold bake-off (GH #200, Phase B) — the candidate default-disclosure - // rules judged on the same served lists `similar` above was scored from, so - // no rule is compared against a surface the others did not see. + // The fold bake-off (GH #200), on the same served lists as `similar`. let fold = score_fold(&vault, &sim_set, "orthogonal", false)?; - // The z calibration dump (GH #187) — the same shipped surface read deep, - // since the z travels ungated on it (GH #197). + // The z calibration dump (GH #187, #197). let floor_z = score_floor_z(&vault, &sim_set)?; - // The search evidence dump (invariants.md D2, GH #201) — the query-side - // sibling of the z calibration, read over the same built vault. + // The search evidence dump (D2, GH #201). let evidence = score_search_evidence(vault_root, &vault, &positives, &negatives)?; eprintln!( "[eval] embedded {chunks} chunks in {embed_secs:.1}s ({} candidates per signal at K={K}, \ @@ -241,8 +176,7 @@ fn run() -> Result> { print_fold_bench(&fold); print_search_evidence(&evidence); print_search_bakeoff(&evidence, &bake_off(&evidence), &model_id); - // The per-hit tail bake-off (GH #206) — judged from the same served lists - // the evidence dump above recorded, against the tail_relevant keep-set. + // The per-hit tail bake-off (GH #206), from the evidence dump's served lists. let tail = score_search_tail(&evidence); print_search_tail(&tail); @@ -279,15 +213,13 @@ fn run() -> Result> { )?; // ---- Phase 3: the dense single-domain fixture (GH #196/#197, Phase 0b). -- - // Its own throwaway vault (sharing the loaded model), its own results row (corpus - // id `dense`) — the fixture measures a *vault-level* geometry, so nothing - // about it may share state with the orthogonal corpus's run above. + // Its own vault and results row: it measures vault-level geometry, so it shares no + // state with the run above. let dense = score_dense(&evals_dir, &dense_set, embedder)?; print_dense_report(&dense); print_fold_bench(&dense.fold); append_result(&results_path, &dense_row(run_id, &dense))?; - // The tail bake-off's cross-bench join (GH #206) — printable only here, - // where both corpora's readings exist in one run. + // Needs both corpora's readings (GH #206). print_tail_join(&evidence, &tail, &dense.search.titles); // ---- Optional: the A/Bs, each against the default passes above. ---------- diff --git a/crates/b2-embed/examples/eval/metrics.rs b/crates/b2-embed/examples/eval/metrics.rs index bcfe107..5bcbec5 100644 --- a/crates/b2-embed/examples/eval/metrics.rs +++ b/crates/b2-embed/examples/eval/metrics.rs @@ -3,9 +3,8 @@ use crate::K; -/// Running hit@1 / hit@3 / MRR@K over a set of 1-based ranks. ("hit@k" — each -/// query has essentially one relevant target, so precision@k and recall@k -/// coincide with it.) +/// Running hit@1 / hit@3 / MRR@K over 1-based ranks. With one target per query, +/// precision@k and recall@k coincide with hit@k. #[derive(Default)] pub struct Agg { pub n: usize, @@ -38,20 +37,15 @@ impl Agg { } } -/// The admissible interval for one threshold, read straight off two labelled -/// populations: a bar `t` keeps everything in `keep` iff `t ≤ min(keep)`, and -/// cuts everything in `cut` iff `t > max(cut)`, so every workable constant lies -/// in `(cut_max, keep_min]` — and **no constant works at all** when the two -/// populations overlap, which is a measured result rather than a failure to -/// search harder. +/// The admissible interval `(cut_max, keep_min]` for one threshold, read off two labelled +/// populations. When they overlap no constant works, which is a result, not a failure. pub struct Window { pub cut_max: f64, pub keep_min: f64, } impl Window { - /// Both edges, or `None` while either population is empty (nothing to - /// separate — an honest no-reading, not a wide-open window). + /// `None` while either population is empty: no reading, not a wide-open window. pub fn read(cut: &[f64], keep: &[f64]) -> Option { let cut_max = cut.iter().copied().fold(f64::NEG_INFINITY, f64::max); let keep_min = keep.iter().copied().fold(f64::INFINITY, f64::min); diff --git a/crates/b2-embed/examples/eval/report.rs b/crates/b2-embed/examples/eval/report.rs index e604b4c..73d3b0d 100644 --- a/crates/b2-embed/examples/eval/report.rs +++ b/crates/b2-embed/examples/eval/report.rs @@ -64,11 +64,7 @@ pub fn print_default_report( hybrid.note.mrr(), hybrid.note.hit1() - bm25.note.hit1(), ); - // The standing form of the fusion finding (GH #158): every query - // where fusing the two signals ranked the labelled answer WORSE than the - // dense signal alone would have. RRF's consensus bias makes some of this - // inevitable; the point is that it is counted and named on every run instead - // of rediscovered by hand-decomposing scores. + // Every query that fusion ranks worse than the dense signal alone (GH #158). let demoted: Vec = (0..queries.len()) .filter(|&i| { let Some(v) = vector.scores[i].note else { @@ -116,12 +112,7 @@ pub fn print_default_report( "similar (n={} positive + {} negative, K={SIM_K}):", similar.rank.n, similar.neg_n ); - // The per-anchor metric (first mate found, then stop) is **no longer - // printed** (GH #188): it read 1.000 across every change it was meant to - // judge, and a line that cannot move is a line that trains skimming. It is - // still recorded — `results.jsonl`'s `"similar"` key is unchanged, so rows - // stay comparable back to the first run — so retiring the line costs the - // dataset nothing. + // The saturated per-anchor metric is recorded in the row but not printed (GH #188). println!( " per-mate hit@1={:.2} hit@3={:.2} MRR@{SIM_K}={:.3} (n={} mates, GATED at MRR@{SIM_K} ≥ {FLOOR_MATE_MRR:.2})", similar.mate.hit1(), @@ -129,8 +120,7 @@ pub fn print_default_report( similar.mate.mrr(), similar.mate.n ); - // Discovery's precision side (GH #188) — reported with names, never gated; - // see `SimilarPass::strangers` for why gating it would reward labelling. + // Never gated; see `SimilarPass::strangers`. println!( " strangers {} card{} on {}/{} positive anchors (unlabelled notes served in the top-{SIM_K} — \ a smoke alarm, not a gate: labels aren't exhaustive)", @@ -143,19 +133,14 @@ pub fn print_default_report( println!(" {anchor} → {path}"); } if similar.neg_n > 0 { - // Under always-serve (GH #197) a negative anchor serves its ranked - // nearest like any other — that is the ruling, not a regression. What - // these anchors measure now is what the served cards *claim*: their - // bands (A2's readout, printed per leader in the calibration block). + // Under always-serve (GH #197) a negative anchor serves; its bands say what it claims. println!( " negatives {} loner anchors serve {} cards under always-serve (labels still say \ \"nothing relates\"; the bands carry the honesty — leaders below)", similar.neg_n, similar.neg_cards ); } - // The two calibration piles. If they separate, the gap IS the floor, read - // off measured data; if they overlap, no simple floor can hold and the - // escalation path (a discovery-side pair-scorer) is justified by data. + // If the piles separate, the gap is the floor; if they overlap, no simple floor holds. if let (Some((r_min, r_med, r_max)), Some((j_min, j_med, j_max))) = (pile_stats(&similar.related), pile_stats(&similar.junk)) { @@ -184,20 +169,13 @@ pub fn print_default_report( } ); } - // The same question in the floor's own anchor-relative unit, where an - // answer is actionable: the piles are absolute cosines, and no constant in - // the code is ever compared against one. + // The same question in z, the unit a constant would be stated in. print_floor_windows(floor_z); } -/// The paired per-query diff between two fused passes — what an A/B is actually -/// judged on. At this corpus's n, every aggregate delta is worth 1–2 queries, so -/// "hit@1 +0.05" and "these two queries flipped, this one broke" are the same -/// fact — but only the second form can be argued with, per-query, against the -/// labels (docs/evals.md, the process rules). Prints nothing but a -/// no-moves line when the variant reproduced the reference ranking exactly — -/// which, per the same rules, is itself a claim to verify against a -/// continuous quantity (the piles), never bare proof of "no effect". +/// The paired per-query diff between two fused passes, which an A/B is judged on +/// (docs/evals.md). No moves is a claim to verify against the piles, not proof of no +/// effect. pub fn print_rank_moves(queries: &[Labelled], reference: &Pass, variant: &Pass) { let improved = |a: Option, b: Option| match (a, b) { (None, Some(_)) => true, @@ -244,8 +222,7 @@ pub fn print_rank_moves(queries: &[Labelled], reference: &Pass, variant: &Pass) } } -/// Who scored a run and in which embedding space — the identity every row the run -/// appends carries (the orthogonal corpus's rows and the dense fixture's alike). +/// Who scored a run and in which embedding space; carried by every row the run appends. #[derive(Clone, Copy)] pub struct RunId<'a> { /// The repo's short commit, `None` outside a git checkout. @@ -254,8 +231,7 @@ pub struct RunId<'a> { pub dim: usize, } -/// One scored configuration's inputs to [`result_row`]: a short-lived, read-only view -/// over passes `run` (or an A/B) owns, assembled at the call and consumed there. +/// One scored configuration's inputs to [`result_row`]: a short-lived, read-only view. #[derive(Clone, Copy)] pub struct RowInputs<'a> { /// The config's name in the row (`"default"`, `"unicode61"`, a sweep variant). @@ -266,14 +242,12 @@ pub struct RowInputs<'a> { pub notes: usize, pub chunks: usize, pub embed_secs: f64, - /// The labelled positives every pass here was scored over. pub queries: &'a [Labelled], pub bm25: Option<&'a Pass>, pub vector: Option<&'a Pass>, pub hybrid: &'a Pass, pub similar: Option<&'a SimilarPass>, - /// The calibration blocks, which only the default row re-derives; `None` on the - /// ablation and sweep rows, whose keys then record `null`. + /// Only the default row re-derives these; other rows record `null`. pub calibration: Option>, } @@ -295,6 +269,9 @@ pub fn unix_secs() -> u64 { } /// One appendable JSONL row for a scored configuration. +/// +/// Row convention: a key is added, never redefined, so a mixed file shows a missing field +/// rather than a number whose meaning changed. Dates note when a key first appeared. pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { let RunId { git, model, dim } = run; let RowInputs { @@ -322,10 +299,7 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "git": git, "model": model, "dim": dim, - // NEW key (absent from rows before 2026-08-18): which corpus this row - // scored — the dense single-domain fixture appends its own `"dense"` - // rows (GH #196/#197), and rows must never average across corpora. - // Absent = this corpus, so every older row stays comparable unchanged. + // Since 2026-08-18; absent means orthogonal. Never average across corpora. "corpus": "orthogonal", "config": { "label": label, @@ -335,31 +309,19 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "backscan_tokens": cfg.backscan_tokens, "prepend_heading_path": cfg.prepend_heading_path, }, - // NEW key (absent from rows before 2026-08-11, which were all scored - // under the then-default `unicode61`): the chunks_fts tokenizer this row - // was scored under (GH #157). Top-level rather than inside `config`, - // which stays the ChunkConfig alone. + // Since 2026-08-11; earlier rows are `unicode61` (GH #157). "tokenizer": tokenizer, "notes": notes, "chunks": chunks, - // The caveat travels with the numbers: a row whose corpus fit inside the - // retrieval pool had both candidate lists complete, so no candidate-width - // change could have affected it and comparing it across one proves nothing - // (GH #141). Equality is blind too — a pool exactly the size of the corpus - // truncates nothing either. Both depths are recorded because since #142 the - // two views differ; the flat `pool` key of earlier rows is deliberately - // *not* reused, so a reader of a mixed file sees a missing field rather than - // one number silently meaning something narrower than it used to. + // A `pool_blind` row can't show a candidate-width change (GH #141). Earlier rows' + // flat `pool` key is not reused. "pool_note": note_candidate_pool(K), "pool_chunk": chunk_candidate_pool(K), "pool_blind": chunks <= chunk_candidate_pool(K).min(note_candidate_pool(K)), "embed_secs": embed_secs, "note": { "bm25": bm25.map(|p| agg(&p.note)), - // NEW key (absent from rows before 2026-08-10): the dense ablation — - // `Vault::search_vector_only`, the single-signal baseline fusion is - // judged against (GH #158). Same convention as pool_note/pool_chunk: - // a new key, never a redefined one. + // Since 2026-08-10 (GH #158). "vector": vector.map(|p| agg(&p.note)), "hybrid": agg(&hybrid.note), }, @@ -367,36 +329,17 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "bm25": bm25.map(|p| agg(&p.chunk)), "hybrid": agg(&hybrid.chunk), }, - // "similar" keeps its pre-negative shape (the positive anchors' rank agg) - // so rows stay comparable across the change; the negative-anchor tally and - // the calibration piles are NEW keys, absent from older rows rather than - // redefining an existing one — the same convention as pool_note/pool_chunk. "similar": similar.map(|s| agg(&s.rank)), - // NEW key (absent from rows before 2026-08-17): the per-mate, - // non-saturating companion to "similar" (GH #183). Same convention - // again — a new key, never a redefined one, so every older row stays - // comparable on "similar" itself. + // Since 2026-08-17 (GH #183). "similar_per_mate": similar.map(|s| agg(&s.mate)), - // `similar_per_mate_raw` / `similar_mates_suppressed` retired with the - // pass-vs-pass tripwire (GH #217): under always-serve both passes read - // the one surface through the same call, so the diff was a tautology. - // Absent from newer rows, never redefined — the row conventions. - // NEW key (absent from rows before 2026-08-17): discovery's precision - // side — unlabelled notes served on positive anchors at the ranks' own - // depth (GH #188). Same convention as every key above: new, never a - // redefinition. Recorded with the pairs, because the count alone is - // unarguable and the pairs are what a reader checks against the notes. + // `similar_per_mate_raw` / `similar_mates_suppressed` retired (GH #217). + // Since 2026-08-17 (GH #188), with the pairs so a reader can check them. "similar_strangers": similar.map(|s| serde_json::json!({ "cards": s.strangers.len(), "anchors": s.stranger_anchors, "positives": s.rank.n, - // The depth the count is read at — the ranks' own top-K, so a - // future SIM_K change shows up in the row instead of silently - // redefining the number. + // So a SIM_K change shows in the row rather than redefining the count. "depth": SIM_K, - // The pairs, because a bare count is unarguable: this metric is a - // smoke alarm whose correct answer is sometimes "that label is - // missing", and that argument needs the notes named. "detail": s.strangers.iter().map(|(anchor, path)| serde_json::json!({ "anchor": anchor, "path": path, @@ -405,18 +348,12 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "similar_negatives": similar.map(|s| serde_json::json!({ "n": s.neg_n, "clean": s.neg_clean, "cards": s.neg_cards, })), - // Cosine, 4 decimals: enough to place a floor, short enough to keep rows - // readable. Related = human-labelled matches; junk = everything else - // surfaced (see score_similar). + // Cosine, 4 decimals. "similar_piles": similar.map(|s| serde_json::json!({ "related": s.related.iter().map(|c| (c * 1e4).round() / 1e4).collect::>(), "junk": s.junk.iter().map(|c| (c * 1e4).round() / 1e4).collect::>(), })), - // The same scores with their per-anchor rank order kept: the relative - // drop-off cutoff is judged within one anchor's list, and tracing a pile - // value back to its pair needs the anchor. The piles above are this, - // flattened — kept anyway, because the flat distributions are what a - // quick jq/pandas histogram wants. + // The piles above with per-anchor rank order kept. "similar_detail": similar.map(|s| s.detail.iter().map(|d| serde_json::json!({ "anchor": d.anchor, "negative": d.negative, @@ -426,15 +363,7 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "related": related, })).collect::>(), })).collect::>()), - // NEW key (absent from rows before 2026-08-17): the floor's own - // calibration data in the floor's own unit — every candidate's ungated - // judge z, the populations the two constants answer to, and both - // re-derived windows (GH #187; the unit changed from stage-1 centroid z - // to stage-2 best-passage z with GH #192 — the row's "unit" field is - // what tells the two apart). Same convention as every key above: new, - // never a redefinition. This is the row the next recalibration reads, and - // the reason no window belongs in a doc comment — `null` on the ablation - // rows, which do not re-derive it. + // Since 2026-08-17 (GH #187): the z populations and both re-derived windows. "discovery_z": floor_z.map(|z| { let (mates, strangers) = (z.mate_z(), z.stranger_z()); let (neg_leaders, pos_leaders) = (z.neg_leader_z(), z.pos_leader_z()); @@ -445,12 +374,8 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { })); let round = |v: &[f64]| v.iter().map(|z| (z * 1e4).round() / 1e4).collect::>(); serde_json::json!({ - // The unit these piles/windows are measured in. Rows before - // GH #192 carried no key here and are stage-1 centroid z — a - // different unit; never compare across the flip. Since GH #197 - // the z gates nothing (the "shipped" and "replay_faults" keys - // of earlier rows retired with the gate — absent, per the - // convention, rather than redefined). + // Rows before GH #192 lack this and are stage-1 centroid z: never compare + // across the flip. "unit": "stage2-best-passage", "piles": { "mates": round(&mates), @@ -461,12 +386,9 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "window_leader": win(&neg_leaders, &pos_leaders), "window_member": win(&strangers, &mates), "ungraded_anchors": z.ungraded, - // ~0 is the only trustworthy reading — see FloorZ::recheck_delta. + // ~0 is the only trustworthy reading (FloorZ::recheck_delta). "z_recheck_max_delta": z.recheck_delta(), - // Per-anchor and per-candidate, because a window edge is only - // arguable once you can name the pair that set it. `z` is the - // stage-2 best-passage z, `cos` the same pair's cosine (the - // model-comparable unit). + // Per candidate, so the pair behind a window edge can be named. "detail": z.anchors.iter().map(|a| serde_json::json!({ "anchor": a.anchor, "negative": a.negative, @@ -479,13 +401,7 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { })).collect::>(), }) }), - // NEW key (absent from rows before 2026-08-22): the search evidence - // dump (invariants.md D2, GH #201) — per labelled query, the absolute - // signals RRF discards (BM25 hit count + best score, dense top-1 - // cosine) and the shipped surface's served count, positives and - // negatives apart, with the would-be pure-cosine query window. Same - // convention as every key above: new, never a redefinition. `null` on - // ablation/sweep rows, which do not re-derive it. + // Since 2026-08-22 (D2, GH #201). "search_evidence": evidence.map(|e| { let bar = EvidenceBar::for_model(model); let row = |r: &QueryEvidence| serde_json::json!({ @@ -495,16 +411,9 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "best_cos": r.best_cos.map(|c| (c * 1e4).round() / 1e4), "top": r.top, "served": r.served(), - // NEW keys (absent from rows before 2026-08-22, GH #201 Phase C): - // the per-query evidence the bake-off is swept over, recorded raw - // so any (fraction, coverage, cos) cell is re-derivable from a row - // without re-running the model — the `discovery_fold` convention. + // Raw, so any bake-off cell is re-derivable from the row. "dense_only": r.dense_only(), - // NEW key (absent from rows before GH #206): the served list - // itself, per row — path, per-hit provenance, and relevance by - // label — so any per-hit tail rule is re-derivable from a row - // without re-running the model, the `discovery_fold` convention - // at hit granularity. `keep` reads `relevant` ∪ `tail_relevant`. + // Since GH #206. `keep` reads `relevant` ∪ `tail_relevant`. "rows": r.rows.iter().map(|row| serde_json::json!({ "path": row.path, "bm25_rank": row.bm25_rank, @@ -514,17 +423,12 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "chunk_total": r.chunk_total, "terms": r.terms.iter().map(|(t, df)| serde_json::json!([t, df])) .collect::>(), - // The ENGINE's verdict, matching the dense row's convention and - // what the exit gate asserts — recording the harness's - // restatement here instead would mask exactly the drift the - // [FAULT] check exists to surface (the restatement is - // re-derivable from `terms` + `best_cos` + `bar` regardless). + // The engine's verdict, not the restatement, which would mask drift. "vouched": r.vouched, }); let (pos_cos, neg_cos) = (best_cos_pile(&e.positives), best_cos_pile(&e.negatives)); serde_json::json!({ - // The depth `served` is read at, so a future K change shows up - // in the row instead of silently redefining the number. + // So a K change shows in the row rather than redefining `served`. "k": K, "positives": e.positives.iter().map(row).collect::>(), "negatives": e.negatives.iter().map(row).collect::>(), @@ -533,11 +437,7 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { "keep_min": (w.keep_min * 1e4).round() / 1e4, "open": w.open(), })), - // NEW key (GH #201, Phase C): the whole bake-off grid, so the - // admissible window is re-derivable from the row — including the - // shipped bar's own three constants, which is what makes a later - // reader able to see that a bar has drifted out of the window it - // was read from. + // The whole grid and the shipped bar, so drift out of the window is visible. "bar": bar.map(|b| serde_json::json!({ "min_term_coverage": b.min_term_coverage, "min_cos": b.min_cos, @@ -552,17 +452,9 @@ pub fn result_row(run: RunId, row: RowInputs) -> serde_json::Value { })).collect::>(), }) }), - // The fold bake-off (GH #200, Phase B) — every candidate rule's reading - // on this run, with the per-anchor folds it was read from. Recorded on - // the default row only: the sweep's variants re-chunk the corpus, and a - // disclosure rule judged on a non-shipped chunker is a number about the - // chunker. + // Default row only (GH #200): on a sweep variant it would measure the chunker. "discovery_fold": fold.map(fold_json), - // NEW key (absent from rows before GH #206): the per-hit tail bake-off - // — each family's re-derived constraint and edge payoff. Default row - // only, for the same reason as `discovery_fold`; the served rows it was - // read from are under `search_evidence`, so any other constant is - // re-derivable from the row. + // Since GH #206. Default row only, like `discovery_fold`. "search_tail": tail.map(tail_json), "queries": queries.iter().enumerate().map(|(i, q)| serde_json::json!({ "q": q.query, diff --git a/crates/b2-embed/examples/eval/retrieval.rs b/crates/b2-embed/examples/eval/retrieval.rs index e3fb0da..b988fef 100644 --- a/crates/b2-embed/examples/eval/retrieval.rs +++ b/crates/b2-embed/examples/eval/retrieval.rs @@ -6,40 +6,33 @@ use crate::metrics::{paths_match, Agg}; use crate::K; use b2_core::vault::Vault; -/// One query's ranks in one retrieval mode: 1-based note rank, 1-based chunk rank -/// (only for passage-labelled queries), and the top note hit for display. +/// One query's 1-based ranks in one retrieval mode; `chunk` only for passage-labelled +/// queries. pub struct QueryScore { pub note: Option, pub chunk: Option, pub top: String, } -/// A full pass over the query set in the vault's current state (keyword-only -/// before `embed`, hybrid after): per-query scores plus note- and chunk-level -/// aggregates. +/// A pass over the query set in the vault's current state (keyword-only before `embed`, +/// hybrid after). pub struct Pass { pub scores: Vec, pub note: Agg, pub chunk: Agg, } -/// Which retrieval a pass scores. `Fused` is the shipped path (`Vault::search` — -/// BM25-only before embed, hybrid after) plus chunk-level scoring for -/// passage-labelled queries; `VectorOnly` is the dense ablation -/// (`Vault::search_vector_only`, GH #158), note-level only — its chunk aggregate -/// stays empty. The ablation column is what lets a run say whether fusion paid -/// rent: the finding that RRF demotes dense rank-1 hits was established -/// by hand-decomposing fused scores once; this makes it a standing measurement. +/// Which retrieval a pass scores. `Fused` is the shipped `Vault::search` plus chunk-level +/// scoring; `VectorOnly` is the dense ablation (GH #158), note-level only, which shows +/// whether fusion pays its way. #[derive(Clone, Copy, PartialEq)] pub enum Retrieval { Fused, VectorOnly, } -/// Score every labelled query against the vault's current state: note rank via -/// the selected retrieval, and — for passage-labelled queries on the fused path — -/// chunk rank via `search_chunks` (the first top-K chunk that belongs to a -/// relevant note AND contains the labelled phrase, case-insensitively). +/// Score every labelled query. A chunk hit is the first top-K chunk of a relevant note that +/// contains the labelled phrase, case-insensitively. pub fn score_pass( vault: &Vault, queries: &[Labelled], diff --git a/crates/b2-embed/examples/eval/tail.rs b/crates/b2-embed/examples/eval/tail.rs index d05ea4d..2f57931 100644 --- a/crates/b2-embed/examples/eval/tail.rs +++ b/crates/b2-embed/examples/eval/tail.rs @@ -1,46 +1,27 @@ -//! The per-hit search tail bake-off (GH #206): the candidate prefix-cut families, their -//! constraints re-derived on the labelled corpus and on the dense fixture's title lists, -//! and the cross-bench join. The ruling of record is **no tail fold ships** -//! (docs/evals.md); every constraint is still re-derived each run. +//! The per-hit search tail bake-off (GH #206): candidate prefix-cut families, their +//! constraints on the labelled corpus and the dense fixture, and the cross-bench join. The +//! ruling of record is that no tail fold ships (docs/evals.md). use crate::common::truncate; use crate::dense::SearchProbe; use crate::evidence::{QueryEvidence, SearchEvidence, ServedRow}; -/// A candidate **per-hit tail rule** family (GH #206) — the fold where a real -/// query's evidence runs out, D2's per-hit half. Every family folds as a -/// **prefix cut** (invariants.md D1): the default view ends at the first served -/// row failing the family's test, and every row below folds with it — a passing -/// row under a failing one folds too, because a fold that punched holes in the -/// fused order would let row order and fold visibly disagree. -/// -/// The consequence that does all the work below: a filler row served *above* a -/// keep-labelled row must pass the test too, or the keep row under it folds. So -/// each family's constraint is read over the **keep-prefix** — every row at or -/// above a list's deepest keep row — not over the keep rows alone. +/// A candidate per-hit tail rule family (D2, GH #206). Every family is a prefix cut (D1): +/// the view ends at the first failing row. So a filler row above a keep row must pass too, +/// and constraints are read over the keep-prefix, not the keep rows alone. #[derive(Clone, Copy, PartialEq)] pub enum TailRule { - /// Fold at the first row the lexical half never ranked (`bm25_rank: None`) - /// — the dense-only fold, the signal GH #201 measured (0 of 410 positive - /// rows vs 20 of 50 negative ones). Parameterless, so nothing - /// distributional to place — but **not** scale-free: `bm25_rank` is a rank - /// in a pool-truncated list (`search::pool_size`), so "never ranked" is - /// partly a fact about pool depth against vault size. `make calibrate - /// VAULT= ARGS=--search` measures that rather than assuming it - /// (process rule 5's posture, owed even without a constant). + /// Fold at the first dense-only row (GH #201). Parameterless but not scale-free: "never + /// ranked" depends on `search::pool_size` against vault size, which `make calibrate + /// ARGS=--search` measures. Lexical, - /// Fold at the first row that is dense-only **and** under a per-hit cosine - /// bar — the shipped query rule's shape (lexical OR semantic) read per hit. + /// Fold at the first row that is dense-only and under a cosine bar: the query rule's + /// shape, per hit. LexOrCos, - /// Fold at the first row under a per-hit cosine bar, lexical rank ignored - /// (a row the dense half never ranked fails at any bar). The single-signal - /// baseline the two-signal family must beat — the pure-cosine window at hit - /// granularity, included to be retired the way GH #201 retired its - /// query-level twin. + /// Fold at the first row under a cosine bar, lexical rank ignored: the single-signal + /// baseline to beat. Cos, - /// Fold at the first row whose cosine sits more than δ under the list's own - /// best — the "drop-off" shape, scale-adaptive in the query and still - /// distributional in δ. + /// Fold at the first row more than δ under the list's best cosine. CosDrop, } @@ -61,9 +42,8 @@ impl TailRule { } } - /// Whether `row` passes this family's per-hit test at constant `p` - /// (ignored by `Lexical`; `best` is the row's list's best served cosine, - /// read only by `CosDrop`). + /// Whether `row` passes at constant `p` (ignored by `Lexical`). `best` is read only by + /// `CosDrop`. pub fn passes(self, row: &ServedRow, p: f64, best: f64) -> bool { match self { TailRule::Lexical => row.bm25_rank.is_some(), @@ -73,9 +53,7 @@ impl TailRule { } } - /// Where this family folds `rows` at constant `p`: the index of the first - /// failing row, `rows.len()` when nothing folds. The prefix-cut definition - /// lives here and nowhere else. + /// The index of the first failing row, or `rows.len()`. The one prefix-cut definition. pub fn fold(self, rows: &[ServedRow], p: f64) -> usize { let best = best_served_cos(rows); rows.iter() @@ -84,18 +62,14 @@ impl TailRule { } } -/// The best served cosine of one list — [`TailRule::CosDrop`]'s reference -/// point. Read off the *served* rows, not the vault-wide dense top-1: the drop -/// rule is a claim about a list's own shape. +/// The best served cosine of one list, [`TailRule::CosDrop`]'s reference point. pub fn best_served_cos(rows: &[ServedRow]) -> f64 { rows.iter() .filter_map(|r| r.cos) .fold(f64::NEG_INFINITY, f64::max) } -/// The keep-prefix of one served list: every row at or above its deepest -/// keep-labelled row. Empty when nothing served is keep-labelled — such a list -/// constrains no rule (there is nothing a fold could wrongly hide). +/// Every row at or above the deepest keep row; empty when none is kept. pub fn keep_prefix(rows: &[ServedRow]) -> &[ServedRow] { match rows.iter().rposition(|r| r.keep) { Some(last) => &rows[..=last], @@ -103,35 +77,23 @@ pub fn keep_prefix(rows: &[ServedRow]) -> &[ServedRow] { } } -/// One family's constraint, re-derived from a set of served lists (GH #187's -/// idiom on the per-hit axis): the range of its constant, if any, at which no -/// keep-prefix row anywhere fails. Both edges carry the row that set them, -/// because a window edge is only arguable once the pair is named. +/// One family's constraint: the range of its constant at which no keep-prefix row fails, +/// with the row that set each edge. pub struct TailConstraint { - /// A keep-prefix row that cannot pass at **any** constant — the family is - /// dead on this bench (e.g. a row with no cosine under a cosine family). + /// A keep-prefix row that passes at no constant: the family is dead on this bench. pub dead: Option<(String, String)>, - /// `Cos`/`LexOrCos`: the highest admissible bar (the binding keep-prefix - /// row's own cosine). `CosDrop`: the lowest admissible δ (the binding - /// row's drop from its list's best). `None` = unconstrained — no - /// keep-prefix row engages the family's test at all. + /// `Cos`/`LexOrCos`: the highest admissible bar. `CosDrop`: the lowest admissible δ. + /// `None` when no keep-prefix row engages the test. pub edge: Option, /// The (query-or-title, path) that set `edge`. pub edge_row: Option<(String, String)>, - /// `Lexical` only: keep-prefix rows failing its fixed test. Nonzero = - /// inadmissible, and each failure is a keep row the fold would hide. + /// `Lexical` only: keep-prefix rows failing its test. Nonzero is inadmissible. pub lexical_violations: usize, } -/// Read one family's [`TailConstraint`] over `(list name, full rows, keep-prefix -/// length)` triples, where every keep-prefix row must pass. The dense fixture -/// reuses this with whole lists as keep-prefixes: on a single-domain vault every -/// served row is a real match, so the absolute "truncate nothing" is the same -/// constraint with the keep-set saturated. The full list rides along because -/// [`TailRule::CosDrop`]'s reference is the **list's** best served cosine — -/// reading it off the prefix alone could understate a required δ when the best -/// row sits below the prefix, and the constraint must read exactly what the -/// fold would. +/// Read one family's [`TailConstraint`] over `(list name, full rows, keep-prefix length)`. +/// The dense fixture passes whole lists as keep-prefixes. The full list is needed because +/// [`TailRule::CosDrop`]'s reference is the whole list's best cosine. pub fn tail_constraint<'a>( rule: TailRule, lists: impl Iterator, @@ -154,20 +116,11 @@ pub fn tail_constraint<'a>( } } } - // The cosine arms read the row's cosine through a finiteness - // filter (PR #212 review): a NaN would sail through `c < e` - // comparisons as silently-true-nowhere — worse, `is_none_or` - // admits the FIRST row unconditionally, so a NaN first row - // would seed the edge. A non-finite reading is *no reading*, - // and the honest arm for that is `dead`, with the row named — - // never a silent skip that reports the family unconstrained. + // Filter non-finite cosines (PR #212): `is_none_or` admits the first row + // unconditionally, so a NaN would seed the edge. No reading means `dead`. TailRule::LexOrCos => { if row.bm25_rank.is_none() { match row.cos.filter(|c| c.is_finite()) { - // A served row is in at least one list, so a - // dense-only row carries a cosine unless it is - // non-finite — either way, no finite reading means - // this row can pass at no bar. None => { if out.dead.is_none() { out.dead = Some((name.to_string(), row.path.clone())); @@ -215,11 +168,8 @@ pub fn tail_constraint<'a>( out } -/// What a family buys at constant `p` on the labelled lists: rows cut, split by -/// label. At an admissible constant `kept_cut` is zero **by construction** — -/// the constraint above is exactly "every keep-prefix row passes" — so a -/// nonzero here is an arithmetic fault, printed as such rather than assumed -/// away. +/// Rows a family cuts at constant `p`, by label. At an admissible constant `kept_cut` is +/// zero by construction, so nonzero is printed as a fault. pub struct TailPayoff { pub filler_cut: usize, pub kept_cut: usize, @@ -243,28 +193,19 @@ pub fn tail_payoff(rule: TailRule, lists: &[&QueryEvidence], p: f64) -> TailPayo out } -/// The per-hit tail bake-off's labelled-corpus reading (GH #206): every -/// family's re-derived constraint plus its payoff at the constraint's own edge -/// — the most aggressive admissible point, since payoff is monotone in the -/// constant. Judged over the **positives**: the query-level bar already -/// answers the negatives whole (GH #201), so a tail rule ships riding on a -/// vouched verdict; the negatives' would-be reading is kept as context for -/// what the rule would buy where a query bar had missed. +/// The tail bake-off's labelled-corpus reading (GH #206): each family's constraint and its +/// payoff at the edge. Judged over the positives, since the query bar already cuts the +/// negatives (GH #201). pub struct TailBench { pub keep_rows: usize, pub filler_rows: usize, - /// Positives whose keep-prefix is the whole served list — a fold has - /// nothing to cut there under any admissible rule. + /// Positives whose keep-prefix is the whole served list. pub saturated: usize, - /// The **oracle** ceiling: rows below each list's last keep row — what a - /// perfect prefix fold (one placed by the labels themselves) would cut. - /// Every family's payoff is read against this, because "cuts N rows" means - /// nothing until the reachable maximum is beside it. + /// The oracle ceiling: rows below each list's last keep row, what a label-placed fold + /// would cut. pub oracle: usize, pub families: Vec, - /// Junk rows the parameterless lexical fold would cut on the negatives' - /// served lists, of their total — the GH #201 `dense_only` reading, priced - /// as a fold. + /// Rows the lexical fold would cut on the negatives' lists. pub neg_lexical_cut: usize, pub neg_rows: usize, } @@ -272,9 +213,7 @@ pub struct TailBench { pub struct TailFamilyReading { pub rule: TailRule, pub constraint: TailConstraint, - /// Payoff at the constraint's edge; `None` when the family is dead here or - /// (for the constant families) unconstrained-and-therefore-identical to a - /// simpler family. + /// Payoff at the constraint's edge; `None` when dead or unconstrained. pub payoff: Option, } @@ -421,10 +360,8 @@ pub fn print_search_tail(bench: &TailBench) { ); } -/// The tail bake-off's JSON (`search_tail` in the orthogonal row): each -/// family's re-derived constraint and edge payoff. The served rows it was read -/// from are already in `search_evidence` per query, so any other constant is -/// re-derivable from the row — the `discovery_fold` convention. +/// The tail bake-off's JSON (`search_tail`). The served rows are in `search_evidence`, so +/// any other constant is re-derivable from the row. pub fn tail_json(bench: &TailBench) -> serde_json::Value { let round = |v: f64| (v * 1e4).round() / 1e4; serde_json::json!({ @@ -446,11 +383,8 @@ pub fn tail_json(bench: &TailBench) -> serde_json::Value { }) } -/// One family's dense-fixture reading (GH #206): the same [`TailConstraint`] -/// read over **whole** title lists — every served row is a real match by -/// geometry, so the keep-prefix is the entire list and the constraint *is* the -/// absolute ("truncate nothing"). `lexical_cut` is the parameterless family's -/// visible cost here: rows its fold would remove from title queries' views. +/// One family's dense-fixture reading (GH #206), over whole title lists: every row is a real +/// match, so the constraint is "truncate nothing". `lexical_cut` is the lexical fold's cost. pub struct TailFamilyTransfer { pub rule: TailRule, pub constraint: TailConstraint, @@ -479,9 +413,7 @@ pub fn dense_tail_families(titles: &[SearchProbe]) -> Vec { .collect() } -/// Print the dense fixture's tail-transfer reading (GH #206) — the per-hit -/// sibling of the query bar's `search_transfer` block, and the bench that -/// carries D1's absolute for this bake-off. +/// Print the dense fixture's tail-transfer reading (GH #206). pub fn print_dense_tail(titles: &[SearchProbe]) { println!( " tail the per-hit tail bench on this geometry (GH #206): every served row of a" @@ -530,8 +462,7 @@ pub fn print_dense_tail(titles: &[SearchProbe]) { } } -/// The dense tail-transfer reading as JSON, nested under the dense row's -/// `search_transfer` key (additive subkey, per the row conventions). +/// The dense tail-transfer reading as JSON, under the dense row's `search_transfer`. pub fn dense_tail_json(titles: &[SearchProbe]) -> serde_json::Value { let round = |v: f64| (v * 1e4).round() / 1e4; serde_json::json!(dense_tail_families(titles) @@ -546,11 +477,8 @@ pub fn dense_tail_json(titles: &[SearchProbe]) -> serde_json::Value { .collect::>()) } -/// The tail bake-off's **cross-bench join** (GH #206), printed once both -/// corpora have been read in the same run: a family ships only if some -/// constant is admissible on the labelled corpus *and* folds nothing on the -/// dense fixture *and* still cuts labelled filler there — read at the joint -/// edge, the most aggressive point both benches allow. +/// The tail bake-off's cross-bench join (GH #206): a family survives only if some constant +/// is admissible on both benches and still cuts labelled filler at the joint edge. pub fn print_tail_join(ev: &SearchEvidence, orth: &TailBench, titles: &[SearchProbe]) { println!("\n{}", "=".repeat(78)); println!("search tail — the cross-bench join (GH #206; both corpora, one run)"); @@ -597,8 +525,7 @@ pub fn print_tail_join(ev: &SearchEvidence, orth: &TailBench, titles: &[SearchPr if orth_dead { "labelled" } else { "dense" } ) } else { - // Joint edge: the tighter bench binds. `None` = that bench - // leaves the constant free. + // The tighter bench binds; `None` leaves the constant free. let joint = match rule { TailRule::CosDrop => { match (orth_f.constraint.edge, dense_f.constraint.edge) { diff --git a/crates/b2-embed/examples/stability.rs b/crates/b2-embed/examples/stability.rs index 923f628..76b8096 100644 --- a/crates/b2-embed/examples/stability.rs +++ b/crates/b2-embed/examples/stability.rs @@ -1,29 +1,16 @@ -//! Rank-stability probe — the **large-corpus** half of the eval harness (GH #141). +//! Rank-stability probe: the large-corpus half of the eval harness (GH #141). //! -//! The scored eval's corpus is barely bigger than the candidate pools each signal retrieves -//! (since GH #183 it crosses the narrower passage-view pool by a few chunks; the note view's -//! not at all) — so at that scale a pool is scarcely ever truncated, widening it can add -//! almost nothing, and candidate-width changes are mostly **invisible** there. A change to a -//! hit pool or to `search::pool_size` reads as "no change" on the eval while genuinely -//! reordering a real vault: GH #140's 3x widening moved 5 of 7 probe top-10s on -//! `fixtures/test-vault` and printed bit-identical eval numbers. This probe priced that, and -//! GH #142 reverted on the strength of it. Width is the whole of the gap — `RRF_K` -//! re-weights the *same* lists, which the scored eval already sees. +//! The scored eval's corpus is too small for candidate pools to truncate, so candidate-width +//! changes are invisible there. This runs on a vault big enough for the pool to bind and +//! reports: //! -//! It runs on a vault big enough for the pool to **bind** (~200 notes / ~790 chunks) and -//! reports two things: +//! 1. **Pool sensitivity**: the same query asked at several depths, hence pool widths. RRF +//! over a wider pool need not keep the shallow answer as a prefix; how often it breaks is +//! what candidate width is worth here. +//! 2. **Baseline drift**: the shipped top-K against the committed baseline. `--bless` +//! accepts it. //! -//! 1. **Pool sensitivity** — retrieval width is a function of the ask, so asking the same -//! query at several depths asks it at several pool widths. A pool-invariant retriever -//! would answer a shallow ask with an exact prefix of the deep one; RRF over a widened -//! pool does not, because a candidate ranked in *both* lists can outscore one ranked top -//! of a single list (`2/121 > 1/61` at k = 60). How often the prefix breaks is how much -//! candidate width is worth on this vault. -//! 2. **Baseline drift** — the shipped top-K against the committed baseline, so any future -//! ranking change is visible as movement rather than inferred. `--bless` accepts it. -//! -//! The corpus is **unlabelled**: this scores no relevance and never says *better*, only -//! *different, and by how much*. +//! The corpus is unlabelled: this says different, never better. //! //! ```console //! cargo run -p b2-embed --example stability # the probe (fake embedder) @@ -33,19 +20,11 @@ //! cargo run -p b2-embed --example stability -- --vault path/to/vault //! ``` //! -//! **Why the fake embedder by default.** It is deterministic, so the committed baseline means -//! the same thing on every machine — a real-model baseline would be device-specific -//! (ADR-0007) and could not be committed. The cost is that fake vector ranking is -//! uncorrelated with BM25, which *exaggerates* how much a pool change moves results; use -//! `--model` when the real magnitude is what the decision needs. -//! -//! **The control experiment:** `--vault crates/b2-embed/evals/corpus` runs the same probe on -//! the eval corpus, where every prefix holds at every depth — that *is* #141: not stability, -//! blindness. +//! The fake embedder is the default because it is deterministic, so the baseline can be +//! committed (a real-model one is device-specific, ADR-0007). It exaggerates pool effects; +//! use `--model` for real magnitudes. //! -//! Never a gate, and drift is not failure: a knob change is *supposed* to move ranking, and -//! an unlabelled corpus cannot say whether it improved. Exit status is 0 for any completed -//! measurement. +//! Never a gate: exit status is 0 for any completed measurement. mod common; @@ -61,24 +40,16 @@ use std::time::Instant; /// The vault the committed baseline is defined over, relative to the repo root. const DEFAULT_VAULT: &str = "fixtures/test-vault"; -/// The depths each probe is asked at; each widens the pool it retrieves from, by view since -/// #142, and the table prints each column's own widths. -/// -/// The first pair is what settled #142: at a 10-result ask the passage view's shipped 3x -/// headroom retrieved 150 candidates per signal against the conservative `limit + 2`'s 60 — -/// the same step measured here between depths 4 and 10 under the note view's 3x. It moved 10 -/// of 10 probes' top-4 passages: real movement, unpriced by any relevance eval. +/// The depths each probe is asked at; each widens the pool it retrieves from (GH #142). const DEPTHS: [usize; 3] = [4, 10, 30]; -/// How deep a prefix the depths are compared over — capped by the shallowest ask. +/// How deep a prefix the depths are compared over, capped by the shallowest ask. const PREFIX: usize = DEPTHS[0]; /// The depth the committed baseline records: what an adapter's default search shows. const BASELINE_K: usize = DEPTHS[1]; /// How much of a chunk's text goes into its baseline key (see [`chunk_key`]). const KEY_CHARS: usize = 48; -/// Report column widths: the probe text, then one column per compared view. const PROBE_COL: usize = 44; const CELL_COL: usize = 26; -/// Every flag this probe takes; anything else is refused rather than ignored. const KNOWN_FLAGS: [&str; 4] = ["--bless", "--model", "--verbose", "--vault"]; /// The hand-authored probe set (`evals/stability.json`) — plain queries, no labels. @@ -90,13 +61,12 @@ struct ProbeSet { /// The committed ranking snapshot (`evals/stability-baseline.json`). #[derive(Serialize, Deserialize)] struct Baseline { - /// Kept in the file so the artifact explains itself where it is read. #[serde(rename = "_note")] note: String, vault: String, embedder: String, k: usize, - /// The commit the snapshot was blessed at, best-effort (`None` outside a checkout). + /// The commit the snapshot was blessed at, best-effort. blessed_at: Option, probes: Vec, } @@ -108,8 +78,7 @@ struct BaselineProbe { chunks: Vec, } -/// One probe's answer at one depth: the ranked note paths (`Vault::search`) and the -/// ranked chunk keys (`Vault::search_chunks`). +/// One probe's answer at one depth: ranked note paths and chunk keys. struct Answer { notes: Vec, chunks: Vec, @@ -123,11 +92,7 @@ fn main() { } fn run() -> Result<(), Box> { - // A bare `--` is dropped, not parsed, defensively. `make stability ARGS=--verbose` - // needs no `--` of its own (`cargo run ... -- $(ARGS)` already supplies the one - // separator), but the habit of typing one anyway — e.g. pasting a raw - // `cargo run ... -- --verbose` invocation into ARGS — is common enough that the - // example should not reject its own documented usage on a literal `--`. + // A bare `--` is dropped so a pasted `cargo run ... -- --verbose` in ARGS still works. let args: Vec = std::env::args().skip(1).filter(|a| a != "--").collect(); let bless = has_flag(&args, "--bless"); let real_model = has_flag(&args, "--model"); @@ -135,9 +100,7 @@ fn run() -> Result<(), Box> { let vault_arg = flag_value(&args, "--vault")?; reject_unknown_flags(&args, &KNOWN_FLAGS, &["--vault"])?; - // A baseline is a *deterministic* artifact over a *committed* vault; blessing - // one from a real-model run or a private vault would commit a number nobody - // else can reproduce. + // A baseline must be reproducible: fake embedder, committed vault. if bless && real_model { return Err("--bless needs the deterministic fake embedder; drop --model".into()); } @@ -163,10 +126,7 @@ fn run() -> Result<(), Box> { return Err("stability.json lists no probes".into()); } - // Work on a throwaway copy: indexing writes `.b2/`, and the committed fixtures - // are never ours to mutate — the same isolation the integration tests and - // `make compare-device` use. (Indexing no longer writes to the vault itself at - // all — W1, GH #170 — so the copy is now about `.b2/` alone.) + // A throwaway copy, since indexing writes `.b2/` into the committed fixture. let tmp = tempfile::TempDir::new()?; let root = tmp.path().join("vault"); copy_dir_all(&source, &root)?; @@ -192,11 +152,7 @@ fn run() -> Result<(), Box> { report.indexed, t0.elapsed().as_secs_f64(), ); - // Checked against the *widest* pool a probe reaches (the note view's): a vault between - // the two widths leaves the note half stable by construction while the passage half - // still measures something, which is a partly-blind run worth saying so about. - // Inclusive, like the eval's own blindness check — a pool exactly the size of the corpus - // truncates nothing either. + // Checked against the widest pool (the note view's), so a partly-blind run warns too. if chunks <= note_candidate_pool(BASELINE_K) { eprintln!( "[warn] {chunks} chunks ≤ the {}-candidate pool the note view reaches (the passage view\n\ @@ -209,7 +165,7 @@ fn run() -> Result<(), Box> { } eprintln!(); - // One retrieval per (probe, depth); everything below is computed from these. + // One retrieval per (probe, depth). let mut answers: Vec> = Vec::with_capacity(probes.probes.len()); for query in &probes.probes { let mut per_depth = Vec::with_capacity(DEPTHS.len()); @@ -221,8 +177,6 @@ fn run() -> Result<(), Box> { report_pool_sensitivity(&probes.probes, &answers, verbose); - // The baseline is the deterministic mode's artifact over the committed vault; - // any other run still gets the sensitivity report above, which needs no baseline. let baseline_path = evals_dir.join("stability-baseline.json"); if bless { write_baseline(&baseline_path, &probes.probes, &answers)?; @@ -242,10 +196,8 @@ fn run() -> Result<(), Box> { Ok(()) } -/// Ask one probe at one depth, keeping only what identifies a result: the note path -/// and the chunk key. Scores are deliberately not compared — RRF scores move -/// whenever the pool does, so comparing them would report "different" for changes -/// that reorder nothing, and the question here is what the human *sees*. +/// Ask one probe at one depth, keeping only result identities. Scores aren't compared: they +/// move with the pool even when nothing reorders. fn answer(vault: &Vault, query: &str, depth: usize) -> Result> { Ok(Answer { notes: vault @@ -261,11 +213,8 @@ fn answer(vault: &Vault, query: &str, depth: usize) -> Result String { let head: String = hit .text @@ -282,9 +231,7 @@ fn chunk_key(hit: &ChunkSearchResult) -> String { ) } -/// One label per adjacent depth pair: the per-signal candidate widths that pair -/// spans, for the view whose pool function is passed. A `fn` pointer rather than a -/// generic — two call sites, one line each, and no reason for a type parameter. +/// One label per adjacent depth pair: the per-signal candidate widths it spans for `pool`. fn width_steps(pool: fn(usize) -> usize) -> Vec { DEPTHS .windows(2) @@ -300,8 +247,7 @@ fn report_pool_sensitivity(probes: &[String], answers: &[Vec], verbose: " a cell is how many of the top-{PREFIX} results the two asks agree on, position by position;" ); println!(" `=` means the shallow answer is an exact prefix of the deep one (pool-invariant)."); - // Each view states its *own* widths: the two headroom rules differ (GH #142), so - // one shared pair of numbers would mislabel a column. + // Each view has its own widths (GH #142). let chunk_steps = width_steps(chunk_candidate_pool); let note_steps = width_steps(note_candidate_pool); println!(); @@ -348,11 +294,7 @@ fn report_pool_sensitivity(probes: &[String], answers: &[Vec], verbose: } println!(); - // Denominators are the probes that actually produced something to compare, not - // the probe count: a probe that returned nothing measured nothing, and folding - // it in either direction would misreport (see [`prefix_cell`]). - // Keyed by the *ask*, since the two views no longer span the same widths across - // one depth pair — each view's own candidates/signal step is named beside it. + // Denominators count only measured probes (see [`prefix_cell`]). for s in 0..DEPTHS.len() - 1 { println!( " ask {:>2}→{:<2}: {}/{} probes changed their top-{PREFIX} chunks ({} candidates/signal), \ @@ -391,12 +333,9 @@ fn report_pool_sensitivity(probes: &[String], answers: &[Vec], verbose: } } -/// One depth-pair cell: how much of the shallow answer the deeper ask preserved. -/// -/// Three outcomes, not two. A comparison with **no overlapping prefix** — a probe -/// that matched nothing, so both asks returned empty — is `measured = false`: it is -/// neither stable nor moved, and counting it as "stable" would let a report that -/// measured nothing announce that everything is pool-invariant. +/// One depth-pair cell: how much of the shallow answer the deeper ask preserved. An empty +/// comparison is unmeasured, not stable, so a report that measured nothing can't claim +/// invariance. struct Cell { label: String, moved: bool, @@ -424,16 +363,13 @@ fn prefix_cell(shallow: &[String], deep: &[String]) -> Cell { } } -/// `--verbose`: the two rankings that disagreed, so the movement is inspectable -/// rather than a bare fraction. Only the chunk lists — they are the un-deduped view -/// the fusion actually produces. +/// `--verbose`: the two chunk rankings that disagreed (the un-deduped fusion output). fn print_divergence(per_depth: &[Answer]) { for w in 0..DEPTHS.len() - 1 { let (lo, hi) = (&per_depth[w], &per_depth[w + 1]); if !prefix_cell(&lo.chunks, &hi.chunks).moved { continue; } - // The chunk lists, so the passage view's own widths label them. println!( " pool {} vs {}:", chunk_candidate_pool(DEPTHS[w]), @@ -465,10 +401,7 @@ fn report_baseline_drift( return Ok(()); }; let baseline: Baseline = serde_json::from_str(&raw)?; - // The snapshot records what produced it, and that provenance is checked rather - // than decorative: comparing today's ranking against one blessed from a - // different vault or a different embedder would report drift that is really a - // mismatch, and the reader would have no way to tell the two apart. + // Otherwise drift would really be an instrument mismatch. if baseline.vault != DEFAULT_VAULT || baseline.embedder != "fake" { return Err(format!( "baseline was blessed from {} under the {} embedder, not {DEFAULT_VAULT} under fake — \ @@ -615,8 +548,7 @@ fn write_baseline( Ok(()) } -/// Recursive copy — the fixture has topic subfolders, so the flat copy the scored -/// eval uses for its single-level corpus would silently drop 9 of 10 topics. +/// Recursive copy: the fixture has topic subfolders, which the eval's flat copy would drop. fn copy_dir_all(from: &Path, to: &Path) -> std::io::Result<()> { std::fs::create_dir_all(to)?; for entry in std::fs::read_dir(from)? { @@ -632,8 +564,7 @@ fn copy_dir_all(from: &Path, to: &Path) -> std::io::Result<()> { Ok(()) } -/// `--vault ` / `--vault=`, erroring on a flag given without a value -/// rather than silently probing the default vault under a name the user did not mean. +/// `--vault ` / `--vault=`, erroring when the value is missing. fn flag_value(args: &[String], name: &str) -> Result, Box> { for (i, arg) in args.iter().enumerate() { if let Some(rest) = arg.strip_prefix(&format!("{name}=")) { @@ -649,8 +580,7 @@ fn flag_value(args: &[String], name: &str) -> Result, Box bool { match (a.canonicalize(), b.canonicalize()) { (Ok(a), Ok(b)) => a == b, @@ -658,12 +588,8 @@ fn same_dir(a: &Path, b: &Path) -> bool { } } -/// Paths are printed relative to the repo root where possible — an absolute -/// `/Users/…/B2/crates/…` line is noise in a report meant to be pasted into an issue. -/// -/// A path that does not exist yet still resolves: the baseline is *named* before it -/// is written (`--bless` writes crates/…/stability-baseline.json), and canonicalize -/// only works on what exists — so a missing leaf falls back to its parent. +/// `path` relative to the repo root where possible. A missing leaf (a baseline not yet +/// written) resolves through its parent, since canonicalize needs an existing path. fn display_from_repo(path: &Path) -> String { let Ok(root) = Path::new(env!("CARGO_MANIFEST_DIR")) .join("../..") diff --git a/crates/b2-llm/examples/groundedness.rs b/crates/b2-llm/examples/groundedness.rs index 2ec11cb..b243042 100644 --- a/crates/b2-llm/examples/groundedness.rs +++ b/crates/b2-llm/examples/groundedness.rs @@ -1,39 +1,21 @@ -//! Groundedness + citation-accuracy smoke for the chat seam — the embedder eval's posture -//! (ADR-0013) applied to the second seam. -//! -//! It lives as an **example**, not a test, for the reason `b2-embed`'s does: it needs two -//! real models, it is non-deterministic, and it talks to the network. `cargo test` must never -//! do any of those. +//! Groundedness + citation-accuracy smoke for the chat seam (ADR-0013). An example, not a +//! test: it needs real models and the network, and is non-deterministic. //! //! ```console //! ollama serve & # or any OpenAI-compatible server //! cargo run -p b2-llm --example groundedness # B2_LLM_URL / B2_LLM_MODEL apply //! ``` //! -//! One run builds a throwaway vault from the retrieval eval's corpus, embeds it, and asks -//! each labelled question through the real `Vault::ask`. Four things are scored, deliberately -//! separable because a bad answer has more than one possible author: -//! -//! 1. **Retrieval reach** — did the labelled note make it into the passages at all? The -//! ceiling: a question that fails here is a *retrieval* result, not a chat one. -//! 2. **Grounding** — did the answer cite anything? An uncited answer is a refusal or general -//! knowledge, and the grounded prompt forbids the second. -//! 3. **Citation accuracy** — did it cite the *labelled* note? The headline number, scored -//! only over questions retrieval actually reached. -//! 4. **Refusal** — on the negative questions, did it say "I don't find that in your notes" -//! instead of confabulating? Refusing when the corpus *does* answer is scored too. -//! -//! Hallucinated `[n]` markers are counted alongside: a marker naming no passage resolves to -//! no citation, so it would otherwise go unmeasured. +//! Each labelled question goes through the real `Vault::ask` over the retrieval eval's +//! corpus. Scored separately, since a bad answer has more than one possible author: +//! retrieval reach (the ceiling), grounding (cited anything), citation accuracy (cited the +//! labelled note, over reached questions), refusal on negatives, and hallucinated `[n]` +//! markers. //! -//! Every run appends one JSON line to `evals/results.jsonl` (gitignored, since the numbers -//! depend on the machine's models). **Exit code**: 0 normally, 2 when retrieval reached -//! labelled notes and yet *no* answer cited one — a broken pipeline rather than a weak model. -//! Model quality is read off the numbers, not the exit code: a gate that fails on every small -//! local model would be a gate nobody runs. +//! Appends one line to `evals/results.jsonl`. Exits 2 only when retrieval reached labelled +//! notes and no answer cited one (a broken pipeline); model quality is read off the numbers. -// The retrieval harness's shared helpers, reused rather than copied: everything the -// module imports is in this crate's (dev-)dependencies too. +// The retrieval harness's shared helpers; everything they import is in our dev-deps too. #[path = "../../b2-embed/examples/common/mod.rs"] mod common; @@ -59,8 +41,8 @@ struct QuestionSet { #[derive(Debug, Deserialize)] struct Question { question: String, - /// The note(s) a correct answer cites. **Empty means unanswerable** — the - /// only correct answer is the refusal. + /// The note(s) a correct answer cites. Empty means unanswerable: only a refusal is + /// correct. #[serde(default)] expect: Vec, } @@ -76,11 +58,9 @@ impl Question { struct Scored { question: String, unanswerable: bool, - /// Did retrieval hand the model a passage from a labelled note? retrieval_hit: bool, passages: usize, citations: usize, - /// Did the answer cite a labelled note? cited_expected: bool, /// `[n]` markers naming no passage — invisible in the citation list. hallucinated_markers: usize, @@ -108,17 +88,14 @@ fn main() { fn run() -> Result> { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); let evals_dir = manifest.join("evals"); - // Deliberately the *retrieval* eval's corpus: chat is scored over the notes - // whose retrieval behaviour is already characterised, so a surprise here can - // be attributed rather than guessed at. + // The retrieval eval's corpus, whose retrieval behaviour is already characterised. let corpus_dir = manifest.join("../b2-embed/evals/corpus"); let results_path = evals_dir.join("results.jsonl"); let set: QuestionSet = serde_json::from_str(&std::fs::read_to_string(evals_dir.join("questions.json"))?)?; - // The chat provider first: it is the one that needs a server running, and - // finding that out after a model download and an embed pass would be rude. + // Probe the chat server first, before a model download and an embed pass. let llm_config = LlmConfig::from_env(); let llm = OpenAiCompatProvider::new(llm_config.clone()); llm.probe()?; @@ -132,7 +109,6 @@ fn run() -> Result> { let embed_model = embedder.model_id().to_string(); eprintln!("[eval] embedder = {embed_model}"); - // A throwaway vault from the corpus — nothing here touches a real one. let scratch = ScratchVault::copy_flat(&corpus_dir)?; let vault = Vault::open_with_embedder(scratch.root(), Box::new(embedder))?; let report = vault.reindex()?; @@ -153,11 +129,8 @@ fn run() -> Result> { append_result(&results_path, &row)?; eprintln!("\n[eval] appended one row to {}", results_path.display()); - // The floor is a liveness check, not a quality bar (see the module doc): fail - // only when retrieval *did* hand the model labelled notes and nothing cited - // one — a broken pipeline. When retrieval reached nothing there is no chat - // result to judge at all; that is `--example eval`'s finding, printed above as - // a retrieval reach of 0, and failing here would blame the wrong half. + // A liveness check, not a quality bar. Reaching nothing is retrieval's finding, not + // chat's, so it doesn't fail here. let reached = scored .iter() .filter(|s| !s.unanswerable && s.retrieval_hit) @@ -171,10 +144,7 @@ fn run() -> Result> { /// Ask one question through the real flow ④ and score what came back. fn ask_one(vault: &Vault, llm: &dyn LlmProvider, q: &Question) -> Result> { - // Retrieval, scored separately *before* the ask: `Vault::ask` retrieves the - // same way for a single-turn question (no condensation), so this is the same - // passage set the model is about to see — and it is the ceiling the citation - // score is judged against. + // A single-turn `Vault::ask` retrieves exactly this passage set (no condensation). let passages = vault.search_chunks(&q.question, ASK_PASSAGES)?; let retrieval_hit = passages.iter().any(|p| q.expect.contains(&p.path)); @@ -188,9 +158,7 @@ fn ask_one(vault: &Vault, llm: &dyn LlmProvider, q: &Question) -> Result Result Result serde_json::Value { let answerable: Vec<&Scored> = scored.iter().filter(|s| !s.unanswerable).collect(); let negatives: Vec<&Scored> = scored.iter().filter(|s| s.unanswerable).collect(); @@ -277,7 +239,6 @@ fn summarize(scored: &[Scored], llm: &LlmConfig, embed_model: &str) -> serde_jso "chat_endpoint": llm.base_url, "embed_model": embed_model, "questions": scored.len(), - // The ceiling: chat can only be scored where retrieval delivered. "retrieval_reach": share(reached.len(), answerable.len()), "citation_accuracy": share( reached.iter().filter(|s| s.cited_expected).count(), diff --git a/ui/scripts/gen-icons.ts b/ui/scripts/gen-icons.ts index 8f87d1d..407d3af 100644 --- a/ui/scripts/gen-icons.ts +++ b/ui/scripts/gen-icons.ts @@ -1,29 +1,15 @@ -// The icon vendoring step: Bootstrap Icons (twbs/icons, MIT) → `src/icons.gen.ts`. +// Vendors a named subset of Bootstrap Icons (twbs/icons, MIT) into `src/icons.gen.ts`. // -// Why generate rather than depend at runtime. Three constraints meet here and only one -// shape satisfies all of them: -// -// 1. **The webview CSP is `default-src 'self'`** (crates/b2-desktop/tauri.conf.json), so -// there is no CDN option — whatever we use has to be in the bundle. -// 2. **`npm test` runs off the source through node**, not Vite (`node -// --experimental-strip-types --test src/*.test.ts`). So the obvious Vite-native move — -// `import chevron from "bootstrap-icons/icons/chevron-right.svg?raw"` — would resolve -// under `vite build` and fail under the test runner, taking every test that transitively -// imports render.ts with it. Plain TypeScript is the only form both hosts can read. -// 3. **2078 icons ship in the package; we use ~24.** Vendoring the ones we name keeps the -// bundle honest and the diff reviewable. -// -// So the package is a **devDependency** — the provenance and the upgrade path — and this -// script lifts the path data of the named subset into a checked-in `.ts` file. +// Generated rather than imported at runtime: the webview CSP is `default-src 'self'` (no +// CDN); `npm test` runs the source through node, not Vite, so `?raw` SVG imports would +// break the tests; and we use ~24 of 2078 icons. The package is a devDependency for +// provenance and upgrades. // // node --experimental-strip-types scripts/gen-icons.ts # regenerate (make icons) // node --experimental-strip-types scripts/gen-icons.ts --check # fail if stale // -// `--check` is what makes the generated file un-driftable: it runs first in `npm test` -// (so `make check` and `make ci` both carry it), and it exits non-zero — a check that -// reports a problem and exits 0 is a hole in the gate (CLAUDE.md → "Commands"). Bump the -// dependency and the check fails until you re-run the generator, which is the only -// moment upstream path data can change under us. +// `--check` runs first in `npm test`, so bumping the dependency fails the gate until the +// file is regenerated. import { readFileSync, writeFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; @@ -34,19 +20,15 @@ const PKG = join(HERE, "..", "node_modules", "bootstrap-icons"); const OUT = join(HERE, "..", "src", "icons.gen.ts"); /** - * The subset, grouped by what asks for it. Adding an icon is: a name here, `make icons`, - * then use it — `IconName` is `keyof typeof ICON_BODIES`, so a typo is a type error and a - * name nobody imports is dead weight the reviewer can see. + * The subset, grouped by what asks for it. To add an icon: name it here, `make icons`, use + * it (`IconName` is `keyof typeof ICON_BODIES`, so a typo is a type error). */ const MANIFEST: Record = { - // Fold state + directional chrome. `chevron-right`/`-down` are the app's one fold - // affordance (file tree, frontmatter drawer, discovery sections and cards); `-left`/`-up` - // are history back and find-previous. + // `chevron-right`/`-down` are the app's one fold affordance; `-left`/`-up` are history + // back and find-previous. chevrons: ["chevron-down", "chevron-left", "chevron-right", "chevron-up"], - // The file tree's two slots: a folder's open/closed state, and one icon per thing a row - // can be. The per-class file icons are the point of the whole change — `▣ ▶ ▤ ◇ ≡ ◆` said - // "not a note" and nothing else. + // The file tree: a folder's open/closed state, and one icon per file class. tree: [ "file-earmark-binary", "file-earmark-code", @@ -59,22 +41,17 @@ const MANIFEST: Record = { "folder2-open", ], - // Create actions (tree head) and the top bar — `chat-dots` is the chat pane's toggle - // (GH #155), which is a top-bar control beside Settings and the vault switcher. + // Create actions (tree head) and the top bar; `chat-dots` is the chat toggle (GH #155). actions: ["chat-dots", "file-earmark-plus", "folder-plus", "gear", "search", "x-lg"], - // Note-bar toggles: the connection graph and the Markdown-source view. They sit side by - // side, so they have to come from the same family. + // Note-bar toggles: the connection graph and the Markdown-source view. toggles: ["code-slash", "diagram-3"], - // Status: the anomaly badge, the graph's dangling node, a broken link's card marker, the - // "no model provisioned yet" banner, and the side pane's caveats about a list (ungraded - // discovery, raw mode) — `info-circle`, not the triangle, because those are notices - // rather than faults. + // Status: faults take the triangle; notices (ungraded discovery, raw mode) take + // `info-circle`. status: ["exclamation-triangle", "info-circle", "stars"], - // The Explain view's "?" (GH #236): the longer account of a figure, one click away - // rather than printed under it. + // The Explain view's "?" (GH #236). help: ["question-circle"], // A connection's direction, on the discovery card that names it. @@ -84,11 +61,8 @@ const MANIFEST: Record = { const NAMES = Object.values(MANIFEST).flat().sort(); /** - * An icon file's *children* — everything between `` and ``, with the - * formatting whitespace collapsed. Children rather than the whole element because the - * caller owns the wrapper: `icon()` sets the size, the class, and `aria-hidden`, and an - * icon that carried its own `` would carry upstream's `width="16"` and `class="bi"` - * into every call site. + * An icon file's children (between `` and ``), whitespace collapsed. The + * wrapper is `icon()`'s, so upstream's `width` and `class` stay out of call sites. */ export function iconBody(svg: string): string { const open = svg.indexOf(">"); @@ -109,23 +83,10 @@ function read(name: string): string { } /** - * The upstream license, reproduced verbatim. - * - * Not a formality, and not satisfied by a pointer: the path data we vendor *is* a portion of - * Bootstrap Icons, and the MIT License requires the copyright notice **and the permission - * notice** to travel with it. An earlier version of this generator cited - * `ui/node_modules/bootstrap-icons/LICENSE`, which is neither committed nor shipped — so - * anyone receiving the generated file received the icons without their terms. - * - * It is emitted **twice**, from this one source, because source and bundle are two different - * distributions and a comment only covers the first: - * - * - as a `/*!` **legal comment**, the marker esbuild (Vite's minifier) keeps when it strips - * every other comment — so the notice survives into `ui/dist` and therefore into the app; - * - as an exported **string**, which is what `icons.test.ts` asserts against. A comment is - * invisible to a test that can't read the filesystem, and `--check` alone can't cover this - * (it compares the file to what the generator emits, so a generator edited to drop the - * notice would pass). The obligation deserves a check that fails. + * The upstream license, verbatim. MIT requires the copyright and permission notice to travel + * with the vendored path data, so it is emitted twice: as a `/*!` legal comment (which + * esbuild keeps, so it reaches `ui/dist`) and as an exported string that `icons.test.ts` + * asserts (`--check` alone would pass a generator edited to drop it). */ const LICENSE_TEXT = `The MIT License (MIT) diff --git a/ui/src/api.ts b/ui/src/api.ts index 599d661..3eabd67 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -1,8 +1,6 @@ -// THE ONE IPC SEAM (crates/b2-desktop/CLAUDE.md). Every `invoke()` in the frontend -// lives here — the presentation-side mirror of the `Vault` façade. Keeping it in one -// module means the rest of the UI never imports Tauri directly: it can be unit-tested -// by mocking this module, and a future `serve`/HTTP transport swap touches ~this file -// only. Do not call `invoke` anywhere else. +// The one IPC seam (crates/b2-desktop/CLAUDE.md): every `invoke()` lives here, so the rest +// of the UI never imports Tauri and can be tested by mocking this module. Do not call +// `invoke` anywhere else. import { Channel, invoke } from "@tauri-apps/api/core"; import { listen, type UnlistenFn } from "@tauri-apps/api/event"; @@ -38,48 +36,34 @@ import type { WriteReport, } from "./types"; -// A rejected `invoke` resolves to the host's user-facing string (CmdError serializes -// to `user_message`), so surface it directly — it's already generic and actionable. +// A rejected `invoke` carries the host's user-facing string (CmdError's `user_message`). export function errText(e: unknown): string { return typeof e === "string" ? e : e instanceof Error ? e.message : String(e); } /** - * The host's exact `WriteConflict` message — part of the IPC contract - * (crates/b2-desktop/CLAUDE.md): the frontend recognizes a save conflict by matching this - * stable constant. Pinned host-side by the `write_conflict_is_generic_and_recognizable` - * test in `b2-desktop/src/commands.rs` — change them together. + * The host's exact `WriteConflict` message, part of the IPC contract. Pinned host-side by + * `write_conflict_is_generic_and_recognizable` (`b2-desktop/src/commands.rs`); change both. */ export const WRITE_CONFLICT_MESSAGE = "This note changed on disk since it was opened. Reload the note, then reapply your edit."; -/** - * Whether an IPC rejection is the save guard refusing a stale revision. - * `startsWith`, not equality: `B2_DEBUG` appends `\n(debug: …)` to every message. - */ +/** Whether an IPC rejection is a stale-revision save. `startsWith`: `B2_DEBUG` appends detail. */ export function isWriteConflict(e: unknown): boolean { return errText(e).startsWith(WRITE_CONFLICT_MESSAGE); } /** - * The host's filesystem-watch pulse (crates/b2-desktop/CLAUDE.md / #14): the Rust watcher emits - * this event, debounced, whenever the vault's Markdown changes on disk from outside the app - * (an external editor, a `git pull`). Must equal the host's `VAULT_CHANGED_EVENT` - * (`b2-desktop/src/watch.rs`) — pinned by the `vault_changed_event_matches_the_frontend` - * test there; change both together. The pulse carries no payload: it's a bare "reconcile - * now" signal, and the frontend re-reads through the façade to see *what* changed. + * The host's debounced filesystem-watch pulse (#14): the vault's Markdown changed outside the + * app. No payload; the frontend re-reads to see what changed. Must equal the host's + * `VAULT_CHANGED_EVENT` (`b2-desktop/src/watch.rs`, pinned by a test there). */ export const VAULT_CHANGED_EVENT = "vault-changed"; /** - * A menu line of **B2's own** was chosen — the View menu's three sizes, today. The payload - * is the item's id as `crates/b2-desktop/src/menu.rs` spells it (`view.zoom-in`), which is - * the same id `menukeys.ts` mirrors. Must equal the host's `MENU_COMMAND_EVENT`; change - * both together. - * - * The host forwards rather than acts because AppKit is only where the *keystroke* lands: - * a menu accelerator never reaches the webview's keydown handler, so this event is how a - * chord the menu owns reaches the code that owns what it means (`zoom.ts`). + * One of B2's own menu lines was chosen; the payload is its id from `menu.rs` + * (`view.zoom-in`). A menu accelerator never reaches the webview's keydown, so the host + * forwards it. Must equal the host's `MENU_COMMAND_EVENT`. */ export const MENU_COMMAND_EVENT = "menu-command"; @@ -87,8 +71,7 @@ export const api = { /** The active vault root + whether semantic ranking is live (real model). */ vaultInfo: (): Promise => invoke("vault_info"), - /** Open a native folder picker to switch the active vault. Resolves to the new - * `VaultInfo`, or `null` if the user cancelled (the current vault stays put). */ + /** Native folder picker to switch vault; `null` if the user cancelled. */ chooseVault: (): Promise => invoke("choose_vault"), /** A note's body + metadata for the left pane, by vault-relative path. */ @@ -97,12 +80,10 @@ export const api = { /** Every indexed note (path, title; no body) — the file tree's source. */ listNotes: (): Promise => invoke("list_notes"), - /** Every inventoried non-`.md` file — the tree's resource half (slice 1). */ + /** Every inventoried non-`.md` file — the tree's resource half. */ listResources: (): Promise => invoke("list_resources"), - /** Every folder in the vault, empty ones included — the tree's structure half, - * read live off the filesystem (never the index) so the tree is one-to-one - * with disk in both directions. */ + /** Every folder, empty ones included, read live off disk (never the index). */ listDirs: (): Promise => invoke("list_dirs"), /** The fallback card's data: a resource's metadata + backlinks. */ @@ -110,74 +91,49 @@ export const api = { invoke("explain_resource", { path }), /** - * A resource's bytes, base64 — what the in-app viewer shows in place of the - * *No viewer available* card. This carries a copy of the whole file across the IPC, - * so ask only for what will actually be displayed; the size is on - * `explainResource`'s view before this is called. The host re-validates the path - * against the inventory. + * A resource's bytes, base64, for the in-app viewer. Copies the whole file across the IPC, + * so ask only for what will be displayed (the size is on `explainResource`'s view). */ readResource: (path: string): Promise => invoke("read_resource", { path }), - /** - * *Open in system default* — an OS handoff performed host-side (the webview holds - * no opener permission); the host validates the path against the inventory first. - */ + /** *Open in system default*, host-side (the webview holds no opener permission). */ openResource: (path: string): Promise => invoke("open_resource", { path }), /** - * *Open a web link in the system default browser* — `openResource`'s sibling for the - * links **inside** a note. The same OS handoff, and host-side for the same reason: the - * webview holds no opener permission, and following the link in place would replace - * the whole app with a web page. The host re-checks the URL's scheme against its own - * allow-list (http / https / mailto), so `ui/src/links.ts` decides what gets *routed* - * here and the host decides what actually opens. + * Open a web link from inside a note in the default browser. `links.ts` decides what is + * routed here; the host re-checks the scheme (http / https / mailto) and decides what opens. */ openExternal: (url: string): Promise => invoke("open_external", { url }), /** - * The clipboard's plain-text flavor, for the editor's ⌘⇧V (paste as plain text). - * Host-side of necessity: WebKit runs no editing command for a raw ⌘⇧V and gates a - * programmatic `navigator.clipboard` read behind a native confirmation. + * The clipboard's plain text, for ⌘⇧V. Host-side: WebKit gates a programmatic + * `navigator.clipboard` read behind a native confirmation. */ clipboardText: (): Promise => invoke("clipboard_text"), - /** Semantically-near, not-yet-linked candidates for a note — the ranked - * nearest, always served (GH #197). */ + /** Semantically near, not-yet-linked candidates for a note (GH #197). */ similar: (note: string, limit = 10): Promise => invoke("similar", { note, limit }), - /** **Explain** one Similar card (GH #236): where `candidate` stands in `anchor`'s list - * and the passage pairs behind it. Model-free; `limit` is the list length the pane - * asked `similar` for, so the rank is the card's. */ + /** Explain one Similar card (GH #236). `limit` is the pane's `similar` limit, so the + * rank matches the card's. */ explainSimilar: (anchor: string, candidate: string, limit: number): Promise => invoke("explain_similar", { anchor, candidate, limit }), - /** Hybrid keyword+semantic search across the vault, **with its evidence - * reading** (invariants.md D2, GH #202): the same rows in the same order, plus - * the query-level verdict that says whether the vault holds evidence for the - * query at all. The host returns the view whole — what the pane *shows* on - * each of `vouched`'s three states is `doSearch`'s call, in `main.ts`. */ + /** Hybrid search plus the query-level evidence verdict (invariants.md D2, GH #202). */ search: (query: string, limit = 20): Promise => invoke("search", { query, limit }), /** - * Save a note's body — Markdown-first through `Vault::write` - * (crates/b2-desktop/CLAUDE.md): a byte-honest body splice guarded by the `revision` captured at read, then - * a model-free re-projection. Rejects with `WRITE_CONFLICT_MESSAGE` when the file - * changed on disk since. (Tauri v2 maps camelCase keys to the command's snake_case - * params — `baseRevision` → `base_revision` — so no hand-written snake_case here.) + * Save a note's body via `Vault::write`, guarded by the `revision` captured at read. + * Rejects with `WRITE_CONFLICT_MESSAGE` if the file changed on disk since. */ writeNote: (note: string, body: string, baseRevision: string): Promise => invoke("write_note", { note, body, baseRevision }), /** - * Save a note's frontmatter — `writeNote`'s frontmatter sibling (GH #79): - * the raw YAML is spliced verbatim between the fences (`Vault::write_frontmatter`), - * body untouched, model-free, guarded by the same `revision` contract (rejects - * with `WRITE_CONFLICT_MESSAGE` on an external change). The host's one refusal is - * a `---` line that would end the block early and shift bytes into the body; it - * comes back as an actionable message to show inline. Everything else — including - * YAML B2 can't read — saves fine: B2 owns no line inside the block. + * Save a note's raw frontmatter YAML verbatim (GH #79), under the same `revision` guard. + * The host's one refusal is a `---` line that would end the block early. */ writeFrontmatter: ( note: string, @@ -186,55 +142,36 @@ export const api = { ): Promise => invoke("write_frontmatter", { note, frontmatter, baseRevision }), /** - * Create a new, empty note at a vault-relative path (`.md` optional; missing - * parent folders are created, like `b2 add`). Model-free like `writeNote`: the - * note is projected immediately (tree/search/graph) and its vectors fill on - * the next embed pass. + * Create an empty note (`.md` optional; missing parents created, like `b2 add`). + * Model-free: vectors fill on the next embed pass. */ createNote: (path: string): Promise => invoke("create_note", { path }), - /** - * Create a folder — a real on-disk create (missing parents included, an - * occupied target refused). A folder is user-authored vault structure, - * immediately visible to Finder, the CLI, and any sync; no index rows are - * touched. - */ + /** Create a folder on disk (missing parents included, an occupied target refused). */ createDir: (dir: string): Promise => invoke("create_dir", { dir }), /** - * Import a file from outside the vault into the folder `dir` (`""` for the root), - * from its **bytes** — the file tree's drop target. `data` is base64 (see - * `importfiles.ts`): a file dropped on the webview arrives as content, not a path, - * and Tauri's JSON IPC carries no byte array cheaply. Model-free like `createNote`: - * the file is projected immediately and any vectors fill on the next embed pass. + * Import a dropped file's bytes into `dir` (`""` for the root). `data` is base64: a drop + * arrives as content, not a path, and JSON IPC carries no byte array cheaply. */ importFile: (dir: string, name: string, data: string): Promise => invoke("import_file", { dir, name, data }), - /** - * `importFile` from a path instead of bytes — what the Import files… picker's - * selections come back as. Same op, and the bytes never cross the IPC. - */ + /** `importFile` from a path (the Import files… picker); the bytes never cross the IPC. */ importPath: (dir: string, source: string): Promise => invoke("import_path", { dir, source }), - /** - * Open the native multi-select file picker behind Import files… — the keyboard path - * to the drop gesture (K1). Resolves to the chosen absolute paths for `importPath` - * to place, or an empty list if the user cancelled. Host-side like `chooseVault`: - * the webview holds no dialog permission. - */ + /** The native file picker behind Import files… (K1); empty if cancelled. */ pickImportFiles: (): Promise => invoke("pick_import_files"), /** - * Move/rename a note to a new vault-relative path — inbound - * links are rewritten and the index re-projects. Needs the real model (the - * rewritten files re-embed), so it can reject with the "run `b2 init`" state. + * Move/rename a note, rewriting inbound links. Needs the real model (rewritten files + * re-embed), so it can reject with the "run `b2 init`" state. */ moveNote: (note: string, to: string): Promise => invoke("move_note", { note, to }), - /** `moveNote`'s resource sibling — same posture, same report shape. */ + /** `moveNote` for a resource. */ moveResource: (path: string, to: string): Promise => invoke("move_resource", { path, to }), @@ -242,13 +179,10 @@ export const api = { moveDir: (from: string, to: string): Promise => invoke("move_dir", { from, to }), - /** - * Delete a note from the vault *and* the disk. Model-free — - * inbound links dangle (surfacing as unresolved), they are never rewritten. - */ + /** Delete a note from disk. Inbound links dangle; they are never rewritten. */ deleteNote: (note: string): Promise => invoke("delete_note", { note }), - /** `deleteNote`'s resource sibling — same posture, same report shape. */ + /** `deleteNote` for a resource. */ deleteResource: (path: string): Promise => invoke("delete_resource", { path }), @@ -268,19 +202,14 @@ export const api = { ): Promise => invoke("link", { src, dst, relation, explanation }), /** - * Phase 1 of a reindex — the fast, **model-free** projection pass - * (docs/index-engine.md): notes + keyword index + graph, writing nothing - * to the vault. Once it resolves, the tree and keyword search are live; call - * `embed` to fill the vectors behind it. + * Reindex phase 1: the model-free projection (docs/index-engine.md). Once it resolves the + * tree and keyword search are live; `embed` fills the vectors. */ project: (): Promise => invoke("project"), /** - * Phase 2 of a reindex — fill the missing vectors (real model) as a cancellable - * background action. `onProgress` fires per embed batch over a typed Tauri - * `Channel` (docs/index-engine.md), determinate from the first batch; the returned - * Promise resolves with the final report (its `cancelled` flag set if - * `cancelReindex` was called mid-run). + * Reindex phase 2: fill missing vectors, cancellable. `onProgress` fires per batch; the + * report's `cancelled` is set if `cancelReindex` was called mid-run. */ embed: (onProgress: (p: ReindexProgress) => void): Promise => { const channel = new Channel(); @@ -292,13 +221,8 @@ export const api = { cancelReindex: (): Promise => invoke("cancel_reindex"), /** - * **Flow ④ — one grounded answer** (GH #151/#153/#155): condense → retrieve → assemble - * → stream → cite, behind `Vault::ask`. `onToken` fires per token over a typed Tauri - * `Channel` (the `embed` progress precedent) and the Promise resolves with the resolved - * [`AnswerView`] — the same two facts `b2 ask --json` emits as a JSONL event stream. - * - * `history` is the caller's and session-only (S4): the pane holds the conversation and - * hands it back each turn. Nothing about a chat is stored anywhere. + * One grounded answer (flow ④, `Vault::ask`), streamed per token. `history` is the + * caller's and session-only (S4): nothing about a chat is stored. */ ask: ( question: string, @@ -311,12 +235,8 @@ export const api = { }, /** - * **Why was this suggested?** — the chat turn behind a *Similar & unlinked* card's - * **Why?**. The host gathers B2's own discovery evidence for the pair (the matched - * passages, the card's rank and strength, the graph facts) and streams a grounded, - * cited explanation — delivered exactly as `ask` is, and stopped by the same - * `cancelAsk`. `limit` is the list length the pane showed, so the rank the explanation - * quotes is the card's own. + * The Similar card's **Why?**: a grounded, cited explanation streamed like `ask` and + * stopped by `cancelAsk`. `limit` is the pane's list length, so the quoted rank matches. */ whySimilar: ( anchor: string, @@ -330,35 +250,22 @@ export const api = { }, /** - * Stop the streaming answer at its next token — the pane's Esc. Cooperative: the - * in-flight `ask` resolves normally, with `cancelled` set and the partial text intact, - * so a stopped answer renders honestly rather than as a failure. + * Stop the streaming answer (Esc). The in-flight `ask` resolves normally with `cancelled` + * set and the partial text intact. */ cancelAsk: (): Promise => invoke("cancel_ask"), /** - * What the chat surface needs before a question is asked: the endpoint and model in - * force, whether that is the **Local** or the **Cloud models** configuration, and — for - * an Ollama endpoint — the native inventory behind the setup card (is the daemon up, - * what is installed, what to pull on a machine this size). - * - * Never rejects: "the daemon isn't running" is the answer the card is asking for. + * The chat configuration in force and, for Ollama, the daemon's inventory. Never rejects: + * "the daemon isn't running" is an answer the setup card needs. */ chatSetup: (): Promise => invoke("chat_setup"), /** - * Save the chat configuration and re-probe it. Adapter state, never vault or index - * state — a chat model swap costs no reindex (contrast M2). - * - * `apiKey` is held in memory for the run and, when the platform will take it, in the - * macOS Keychain — never in a plain file, and never in a way that comes back here: the - * returned setup carries `api_key_source`, not the key (`b2-desktop/src/keychain.rs` - * argues both halves). Passing `null` keeps whatever is already in force, so re-saving - * the endpoint can't silently sign you out; `""` is the explicit *clear*. - * - * `maxToolCalls` follows the key's three-state rule for the same reason — most callers - * of this save never mention it: `null` keeps the cap, `""` clears it back to the - * environment/default, a number sets it. chat.ts's `toolCapInput` decides which. + * Save the chat configuration and re-probe it (adapter state; no reindex, contrast M2). + * `apiKey` and `maxToolCalls` are three-state: `null` keeps what is in force, `""` clears, + * a value sets. The key never comes back; the setup carries `api_key_source` + * (`b2-desktop/src/keychain.rs`). */ setChatConfig: ( baseUrl: string | null, @@ -372,60 +279,38 @@ export const api = { listModels: (): Promise => invoke("list_models"), /** - * Persist the chosen embedding model into the shared config B2 reads. Returns the - * refreshed list. Selecting a *different* model is a model swap — it takes effect only - * after that model is downloaded (`b2 init`) and the vault is reindexed; this call - * just records the choice. + * Record the chosen embedding model. A different model takes effect only after it is + * downloaded (`b2 init`) and the vault is reindexed. */ setModel: (model: string): Promise => invoke("set_model", { model }), - /** - * Download + verify the currently-selected model into the shared cache — the in-app - * `b2 init`. Idempotent, network-bound (can take minutes). Resolves with the refreshed - * model list, the just-installed model now flagged `installed`. - */ + /** The in-app `b2 init`: download and verify the selected model. Can take minutes. */ provisionModel: (): Promise => invoke("provision_model"), - /** Per-model cumulative embedding time (Settings), accumulated across sessions. */ + /** Per-model cumulative embedding time across sessions (Settings). */ embedStats: (): Promise => invoke("embed_stats"), - /** The shared directory where downloaded model files are saved (shown in Settings). */ + /** Where downloaded model files are saved (Settings). */ modelsDir: (): Promise => invoke("models_dir"), - /** Compute device the embedder runs on for this build — "Metal" or "CPU" (Settings badge). */ + /** This build's embedder device, "Metal" or "CPU" (Settings badge). */ embedDevice: (): Promise => invoke("embed_device"), /** - * Every chord the app's **menu bar** takes (#119) — the host declares the menu - * (b2-desktop `menu.rs`), so this is the authority on chords the webview never sees: - * AppKit dispatches a menu key equivalent before the key window's responder chain. - * Read once, at boot, to check `menukeys.ts`'s mirror against (main.ts - * `checkMenuDrift`); everything else — the recorder's refusal of a menu chord included - * — reads that mirror, never this. + * Every chord the menu bar takes (#119), the authority on chords the webview never sees. + * Read once at boot to check `menukeys.ts`'s mirror (`checkMenuDrift`); all else reads + * the mirror. */ menuChords: (): Promise => invoke("menu_chords"), - /** - * Scale the whole window — WebKit page zoom, what ⌘= / ⌘- / ⌘0 drive (`zoom.ts`). - * The host is a pass-through here: `factor` is already a rung off the ladder, because - * the ladder and its walls are the frontend's rule, not the host's. - */ + /** WebKit page zoom (`zoom.ts`). `factor` is already a rung of the frontend's ladder. */ setZoom: (factor: number): Promise => invoke("set_zoom", { factor }), - /** - * Subscribe to the host's debounced filesystem-watch pulse (#14). `handler` fires once - * per burst of external Markdown changes; the returned promise resolves to an unlisten - * function (unused here — the subscription lives for the window's lifetime). Kept - * behind the seam like every `invoke`. - */ + /** Subscribe to the filesystem-watch pulse (#14), once per burst of external changes. */ onVaultChanged: (handler: () => void): Promise => listen(VAULT_CHANGED_EVENT, () => handler()), - /** - * Subscribe to B2's own menu lines (the View menu's sizes). `handler` gets the item's - * id; an id it doesn't recognize is not an error — the host declares the menu, so the - * honest response to a line this build doesn't know about is to ignore it. - */ + /** Subscribe to B2's own menu lines. `handler` gets the id; ignore unknown ids. */ onMenuCommand: (handler: (id: string) => void): Promise => listen(MENU_COMMAND_EVENT, (e) => handler(e.payload)), }; diff --git a/ui/src/bindings.test.ts b/ui/src/bindings.test.ts index 8512889..2af7c02 100644 --- a/ui/src/bindings.test.ts +++ b/ui/src/bindings.test.ts @@ -1,19 +1,5 @@ -// The keyboard registry (bindings.ts), pinned — and the conflict gate itself. Pure — -// no DOM — so node runs it straight off the source: `npm test`. Dependency-free like the -// others. -// -// Two jobs. The first is that B2's own chords don't conflict: `conflicts(DEFAULT_BINDINGS)` -// must be empty, and because `npm test` runs inside both `make check` and `make ci`, that -// assertion *is* the gate. The second is proving the gate can fail — a checker that has -// only ever seen a clean table is indistinguishable from one that returns `[]` -// unconditionally, so the interesting cases below are synthetic tables built to clash. -// -// The matcher gets the same treatment. What a keyboard layer actually gets wrong is -// dull and invisible: a chord that never fires because the table spells a key the way -// the docs write it rather than the way `KeyboardEvent.key` reports it, a `?` that -// demands ⇧ on top of the ⇧ the browser already applied, a ⌘-chord that also answers to -// ⌥⌘ because nobody checked `altKey`. None of that shows up until a user reports that a -// shortcut "sometimes" doesn't work. +// The keyboard registry (bindings.ts), and the conflict gate itself: `conflicts` on the +// shipped table must be empty. Synthetic tables built to clash prove the gate can fail. import { FORMATS } from "./format.ts"; import { type Binding, @@ -36,16 +22,12 @@ import { shiftDistinguishes, } from "./bindings.ts"; -/** A synthetic row. These tables exist to make a checker *fail*, so the label carries no - * meaning here — it is required by the type and named after the id to keep it honest. */ +/** A synthetic row, labelled after its id. */ function row(b: Omit): Binding { return { label: b.id, ...b }; } -/** The shipped table under its declared type. `DEFAULT_BINDINGS` is `as const` so that - * `BindingId` can be derived from it, which also means reading `.fixed` off a row that - * hasn't got one is a type error rather than `undefined` — the widening is here so the - * checks below can ask the questions a reader would. */ +/** The shipped table widened from `as const`, so optional fields like `.fixed` can be read. */ const SHIPPED: readonly Binding[] = DEFAULT_BINDINGS; let passed = 0; @@ -71,17 +53,13 @@ function press(key: string, mods: Partial = {}): KeyEventLike { // --- the table itself --------------------------------------------------------------- check("every chord in the table parses", () => { - // parseChord throws on an unknown key or modifier, so a typo — "Mod-Delete" for - // "Mod-Backspace", "Cmd+f" for "Mod-f" — fails here rather than shipping a dead chord - // that nothing reports because nothing was ever bound to notice. for (const b of DEFAULT_BINDINGS) { for (const spec of allKeys(b)) parseChord(spec); } }); check("command ids are unique", () => { - // The lookup is a Map, so a duplicated id wouldn't error — the later row would quietly - // win and the earlier command would stop responding. + // The lookup is a Map, so a duplicate would silently win over the earlier row. const seen = new Set(); for (const b of DEFAULT_BINDINGS) { assert(!seen.has(b.id), `duplicate command id: ${b.id}`); @@ -90,9 +68,6 @@ check("command ids are unique", () => { }); check("every command carries a label a human could read", () => { - // The recorder names what it is rebinding and the conflict messages name what they - // clash with, so an empty or id-shaped label is a sentence that reads like a stack - // trace — "⌘F already runs pane.discovery here." for (const b of SHIPPED) { assert(b.label.trim() !== "", `${b.id} has no label`); assert(b.label !== b.id, `${b.id}'s label is just its id`); @@ -100,11 +75,7 @@ check("every command carries a label a human could read", () => { }); check("the chords that can't be rebound are exactly the platform's own reflexes", () => { - // `fixed` is a promise about *why* something can't be moved, not a convenience list, so - // it is pinned rather than left to accumulate: every row here is ⏎/Esc in a text field, - // ⏎ on a dialog's default button, the ⏎/Space a x", arguments: "", seeded: false }]), "Looked up with B2 tools: x", @@ -230,24 +218,19 @@ test("an answer says which B2 tools it was built from", () => { test("the tool-call cap field is untouched, cleared, set, or refused — before it is sent", () => { const cap = { in_force: 64, default: 64, ceiling: 4096 }; - // Untouched: the field paints the cap in force, so saving it back unchanged must not - // *store* it — that would pin today's default over tomorrow's environment variable. + // Untouched must not store the shown cap, or it pins today's default over the env var. assert.deepEqual(toolCapInput("64", cap), { send: null }); assert.deepEqual(toolCapInput(" 64 ", cap), { send: null }); - // Cleared: back to the environment/default, which is the host's `""`. assert.deepEqual(toolCapInput("", cap), { send: "" }); assert.deepEqual(toolCapInput(" ", cap), { send: "" }); - // Set, at both ends of the range the host quoted. assert.deepEqual(toolCapInput("1", cap), { send: "1" }); assert.deepEqual(toolCapInput("4096", cap), { send: "4096" }); - // Refused here, with a sentence, rather than sent to a host that would ignore it and - // leave *Save* looking like it did nothing. + // Refused here with a sentence, not sent to a host that would ignore it. for (const bad of ["0", "4097", "-2", "6.5", "lots", "1e3", "12abc"]) { const verdict = toolCapInput(bad, cap); assert.ok("error" in verdict, `${bad} should be refused`); assert.match(verdict.error, /1 to 4096/); } - // The range is the host's, not a constant here. assert.ok("error" in toolCapInput("200", { in_force: 64, default: 64, ceiling: 100 })); }); diff --git a/ui/src/chat.ts b/ui/src/chat.ts index 4d1f5a8..a0a66d1 100644 --- a/ui/src/chat.ts +++ b/ui/src/chat.ts @@ -1,33 +1,9 @@ -// The chat pane's pure logic — no DOM, no IPC — so node runs its test straight off the -// source (`npm test`), like sidenav.ts / treenav.ts / settingstabs.ts. +// The chat pane's pure logic (no DOM, no IPC), flow ④ in the GUI (GH #155). // -// Chat is flow ④ (GH #151/#153) reaching the GUI (GH #155): ask a question, watch the -// answer stream in, click a citation to open the note it came from. Three things about it -// are decisions rather than details, and they are why this module exists at all. -// -// **Where it lives: the right column.** Chat replaces discovery there rather than taking -// the centre pane, and the reason is the citations. An answer's whole value is that you -// can go *read* the note behind it, and the centre pane is where notes are read — so chat -// beside it means a citation opens the note **without the conversation leaving the -// screen**. Chat in the centre would make every citation a choice between the answer and -// the evidence. (Search takes the column too; opening either closes the other — see -// main.ts. One column, one thing in it.) -// -// **The transcript is a row list, like discovery's.** `chatRows` emits sidenav.ts's own -// `SideRow` shape, so the pane inherits the discovery pane's whole ARIA `tree` walk — -// ↑↓ between rows, Home/End, a roving `tabindex`, and (crucially) focus restoration by row -// key across the `innerHTML` swap every streamed answer causes. Reusing the shape rather -// than the *state* is what keeps that free: `sideRows` delegates here when the pane is in -// chat mode, and everything downstream (the paint, the arrows, `paintSide`'s focus -// capture) carries on not knowing which mode it is in. A turn is a row you can land on; a -// citation is a row *under* it that ⏎ opens — the only actionable rows in the transcript, -// which is exactly K1's "a row you can see is a row you can reach". -// -// **History is session-only** (invariant S4). `chatHistory` is what the next ask carries, -// and it is derived from the transcript in memory — never persisted, not even to -// `localStorage` where the theme and the keymap live. A saved transcript would be -// B2-derived state outside the Markdown; "save this chat as a note" would be an explicit -// human-invoked export, and is not MVP (GH #151's open question 2). +// Chat lives in the right column so a citation opens its note in the centre pane without +// the conversation leaving the screen. The transcript emits sidenav.ts's `SideRow`, so it +// inherits the discovery pane's keyboard walk and focus restoration. History is +// session-only (S4): never persisted, not even to `localStorage`. import type { SideRow } from "./sidenav.ts"; import { coverage } from "./coverage.ts"; @@ -42,26 +18,20 @@ import type { } from "./types"; /** - * One entry in the transcript. A `user` message is what was typed; an `assistant` message - * is what came back — with its citations, whether it was stopped mid-answer, and, when the - * call failed outright, the generic message the host gave instead of an answer. - * - * A **failed** turn is still a turn: it stays on screen (the question is still there to - * retry) but contributes nothing to `chatHistory`, because there is no answer to carry - * forward as context. + * One transcript entry. A failed turn stays on screen but contributes nothing to + * `chatHistory`. */ export interface ChatMessage { role: "user" | "assistant"; - /** The text — the question, the answer, or "" when `error` says why there isn't one. */ + /** The question, the answer, or "" when `error` is set. */ text: string; /** Resolved `[n]` markers; empty for a user message or a failed turn. */ citations: Citation[]; - /** The stream was stopped: the text is an honest prefix, and the pane says so. */ + /** The stream was stopped; the text is a prefix. */ cancelled: boolean; - /** The B2 tools the answer was built from; absent or empty for a plain ask or a user - * message. */ + /** The B2 tools the answer was built from. */ tools?: ToolUse[]; - /** The generic, actionable failure this turn produced instead of an answer. */ + /** The host's failure message, in place of an answer. */ error?: string; } @@ -82,12 +52,8 @@ export function answerMessage(view: AnswerView): ChatMessage { } /** - * The question a click on a *Similar & unlinked* card's **Why?** puts in the transcript. - * - * Display text, and the turn a follow-up is condensed against — not the prompt. What the - * model is actually asked, and the evidence it is handed, is assembled host-side by - * `Vault::why_similar` (b2-core's `chat.rs`); this names the two notes the way the pane - * names them everywhere else, by title with the path as the fallback. + * The question a Similar card's **Why?** puts in the transcript. Display text only: the + * prompt is assembled host-side by `Vault::why_similar`. */ export function whyQuestion( candidate: { path: string; title: string | null }, @@ -98,10 +64,8 @@ export function whyQuestion( } /** - * The line under an answer that names the B2 tools it was built from, or "" when there - * were none. Each tool once, in first-use order, with the `b2_` prefix and underscores - * dropped so it reads as words. **Not escaped here**: a tool name is whatever the model - * sent, so it is untrusted like the answer, and the paint runs it through `escapeHtml`. + * The line naming the B2 tools an answer used, or "". Not escaped here: a tool name is + * untrusted model output, and the paint escapes it. */ export function toolsLine(tools: readonly ToolUse[] = []): string { const names = [...new Set(tools.map((t) => t.name.replace(/^b2_/, "").replaceAll("_", " ")))]; @@ -109,17 +73,9 @@ export function toolsLine(tools: readonly ToolUse[] = []): string { } /** - * What Settings → Chat's **Tool calls per reply** field should send with a save — the - * host's three-state rule (`apply_tool_cap`), decided here so a bad value is a sentence - * beside the field rather than a save that appears to do nothing. - * - * `{ send: null }` untouched — the field still shows the cap in force, and storing - * that would pin today's default over a later environment variable - * `{ send: "" }` cleared — back to `B2_LLM_MAX_TOOL_CALLS` or the default - * `{ send: "128" }` set - * `{ error }` refused: not a whole number in the host's range - * - * The range is `cap`'s, which the host sent; nothing about it is spelled here. + * What the **Tool calls per reply** field sends with a save, per the host's three-state + * rule (`apply_tool_cap`): `null` when untouched (storing the shown value would pin today's + * default over a later env var), `""` when cleared, the number when set, or an `error`. */ export function toolCapInput( raw: string, @@ -135,18 +91,14 @@ export function toolCapInput( return { send: String(n) }; } -/** A turn that failed — the host's generic message stands in for the answer. */ +/** A turn that failed. */ export function errorMessage(error: string): ChatMessage { return { role: "assistant", text: "", citations: [], cancelled: false, tools: [], error }; } /** - * The conversation as the next ask should see it (`b2-core`'s `ChatTurn`), oldest first. - * - * A **cancelled** answer is included, deliberately and for the reason `b2 chat` includes - * it: the human read that text, so a follow-up like "go on" has to be condensed against - * what was actually said, not against a turn we pretend never happened. A **failed** turn - * is excluded — there is nothing it said. + * The conversation as the next ask sees it, oldest first. A cancelled answer is included + * (the human read it, so "go on" must condense against it); a failed turn is not. */ export function chatHistory(messages: readonly ChatMessage[]): ChatTurn[] { return messages @@ -154,38 +106,23 @@ export function chatHistory(messages: readonly ChatMessage[]): ChatTurn[] { .map((m) => ({ role: m.role, content: m.text })); } -/** A transcript row's key — the identity that survives the pane's `innerHTML` swap. - * Position-based because two identical questions are two different rows. */ +/** A transcript row's key. Position-based: two identical questions are two rows. */ export function turnRowKey(index: number): string { return `chat:turn:${index}`; } -/** A citation row's key: its turn, its marker, and what it points at — the same - * belt-and-braces as `cardRowKey` (position makes it unique, target makes a stale key - * fail to match rather than resolve to whatever now sits there). - * - * What the row *opens* is not derived from this key: a citation paints as a `data-open` - * button, so the mouse and ⏎ share the one activation path the rest of the app uses (K1), - * and the note it names lives in the markup rather than in a lookup that could go stale - * between the paint and the keystroke. render.test.ts pins that it is a button and never - * an `href` — in-app navigation, never the webview (E5). */ +/** A citation row's key. Like `cardRowKey`, position makes it unique and the target makes + * a stale key fail to match. What it opens comes from the markup's `data-open` (E5). */ export function citationRowKey(turn: number, marker: number, path: string): string { return `chat:cite:${turn}:${marker}:${path}`; } -/** The row key of the answer currently streaming — a row like any other, so the keyboard - * can sit on it while it fills, and so focus restoration has something to name. */ +/** The streaming answer's row key, so focus can sit on it while it fills. */ export const STREAMING_ROW_KEY = "chat:streaming"; /** - * Every row the chat pane paints, in paint order: each turn, with its citations nested - * under it, then the in-flight answer if one is streaming. - * - * Emits sidenav.ts's `SideRow` so the discovery pane's walk drives this one unchanged. - * Nothing here **folds** (`fold: null`): a turn's citations are always shown with it — a - * conversation is read, not browsed. Folding and having children are different facts, - * though, and only the second one moves →, so an answer with citations still steps into - * them and ← still steps back out to the turn (sidenav.ts's `side.row.in` says so). + * Every row the chat pane paints, in order: each turn with its citations under it, then + * the streaming answer. Nothing folds, but → still steps into a turn's citations. */ export function chatRows(messages: readonly ChatMessage[], streaming: boolean): SideRow[] { const rows: SideRow[] = []; @@ -219,19 +156,8 @@ export function chatRows(messages: readonly ChatMessage[], streaming: boolean): } /** - * Which state the chat pane is in before (or between) questions — the empty-state - * selection the spec asks for, as one pure decision rather than a chain of `if`s spread - * through the paint. - * - * no-vault nothing to ground an answer in - * loading the setup probe hasn't answered yet - * no-server nothing is listening (the daemon isn't running, or the URL is wrong) - * no-model a server is there but doesn't serve the configured model - * ready chat works — the pane shows its prompt, or the conversation - * - * `unembedded` is deliberately **not** one of these: a projected-but-unembedded vault - * still answers, BM25-only (M4), so blocking on it would break a working thing. It is a - * quiet note beside the composer instead — [`retrievalNote`]. + * The chat pane's empty state. An unembedded vault is not one: it still answers, + * keyword-only (M4), so it gets a note instead ([`retrievalNote`]). */ export type ChatEmptyState = "no-vault" | "loading" | "no-server" | "no-model" | "ready"; @@ -246,45 +172,30 @@ export function chatEmptyState(s: { return "no-server"; case "model_missing": return "no-model"; - // The fake provider answers deterministically — chat "works", and the pane says - // what is answering rather than pretending a model is (`b2 chat`'s own note). case "fake": case "ready": return "ready"; } } -/** The app's side of `chatEmptyState`: the vault and the probe, read off the app state. */ +/** `chatEmptyState` read off the app state. */ export function chatStateOf(s: { vaultRoot: string | null; chatSetup: ChatSetup | null }): ChatEmptyState { return chatEmptyState({ hasVault: s.vaultRoot !== null, setup: s.chatSetup }); } -/** Can a question be asked right now? What the composer's presence and *Why?* both wait on. */ +/** Can a question be asked right now? */ export function chatReady(s: { vaultRoot: string | null; chatSetup: ChatSetup | null }): boolean { return chatStateOf(s) === "ready"; } -/** Ollama's OpenAI-compatible endpoint — the **Local** configuration's starting point, and - * the only place the frontend spells it: the Endpoint field's placeholder, and its seed - * when the user presses *Local* after typing a cloud URL. The host's - * `b2_llm::DEFAULT_BASE_URL` is the authority (it is what an unset endpoint resolves - * to) — change them together. */ +/** Ollama's OpenAI-compatible endpoint, the Local configuration's seed. Mirrors + * `b2_llm::DEFAULT_BASE_URL`; change them together. */ export const LOCAL_CHAT_ENDPOINT = "http://localhost:11434/v1"; /** - * The honest note about *retrieval* under the composer, or "" when there is nothing to - * say — the search caveat (#26) applied to chat. - * - * Chat retrieves through the same hybrid search everything else does, so an unembedded - * vault answers from keyword matches alone. That is a real difference in answer quality - * and it must be said, quietly, in place — never as a blocker (E4: chat degrading never - * degrades anything else, and a degraded chat is still chat). - * - * **It reports a state, not an activity.** A partial fraction is not evidence that a run - * is under way: it is equally the residue of a cancelled or crashed reindex, and it sits - * there until someone acts. The note used to read "while this vault embeds", which asked - * the reader to wait for something that might never happen — so it names the gap and the - * gesture that closes it, the way the search caveat and the ghost hint already do. + * The retrieval note under the composer, or "": the search caveat (#26) applied to chat, + * never a blocker (E4). It reports a state, not an activity: a partial fraction may be + * the residue of a cancelled reindex, so it names the gesture that closes the gap. */ export function retrievalNote(s: { semantic: boolean; @@ -305,41 +216,30 @@ export function retrievalNote(s: { } } -/** The one command the setup card tells a human to run, spelled here so the label, the - * copy button's payload and any message agree (`b2-llm`'s `pull_command` mirror). */ +/** The setup card's command, spelled once (mirrors `b2-llm`'s `pull_command`). */ export function pullCommand(model: string): string { return `ollama pull ${model}`; } -/** The placeholder a human substitutes into `ollama pull` — the *shape* of the command, - * shown where there is no particular model to name. Angle brackets, so it can never be - * mistaken for a model that exists; it goes through `escapeHtml` like everything else. */ +/** The `ollama pull` placeholder where no model is named; angle brackets so it can't be + * mistaken for a real model. */ export const PULL_PLACEHOLDER = ""; /** - * Where B2 sends someone who has to go and *do* something in Ollama. - * - * The quickstart, not the product page: a reader of either link has already been told - * that nothing is listening or that a model isn't there, so the page they need is the one - * carrying the install command and `ollama pull`. `b2-llm`'s `OLLAMA_INSTALL_URL` is the - * same link on the host side (it rides inside the probe's own sentence) — **change them - * together**, the way `pullCommand` mirrors `pull_command`. + * Where B2 sends someone who has to install or pull in Ollama. Mirrors `b2-llm`'s + * `OLLAMA_INSTALL_URL`; change them together. */ export const OLLAMA_QUICKSTART_URL = "https://docs.ollama.com/quickstart"; -/** Ollama's hosted models — the **Cloud models** configuration's one concrete example, - * offered where that decision is made. Named rather than defaulted to: picking a - * provider is the explicit act M5 is about, so this is a link, never a pre-filled URL. */ +/** Ollama's hosted models, offered as a link, never a pre-filled URL (M5). */ export const OLLAMA_CLOUD_URL = "https://docs.ollama.com/cloud"; -/** An installed model's one-line detail — its parameter label and size, whichever it - * has ("3.2B · 2.0 GB") — as both the setup card and the Settings picker print it. */ +/** An installed model's detail, e.g. "3.2B · 2.0 GB". */ export function modelDetail(m: OllamaModel): string { return [m.parameters ?? "", formatModelSize(m.size)].filter(Boolean).join(" · "); } -/** A model's on-disk size, for the installed list. Whole GB past a gigabyte, one decimal - * below — an inventory line, not a measurement. */ +/** A model's on-disk size, for the installed list. */ export function formatModelSize(bytes: number): string { if (!Number.isFinite(bytes) || bytes <= 0) return ""; const gb = bytes / 1_073_741_824; diff --git a/ui/src/chatview.ts b/ui/src/chatview.ts index 38a6ffc..181a73c 100644 --- a/ui/src/chatview.ts +++ b/ui/src/chatview.ts @@ -23,23 +23,13 @@ import { sideTab } from "./widgets.ts"; // --- chat (flow ④, GH #151/#153/#155) ----------------------------------------------- // -// The right column's third mode: ask a question, watch the answer stream, click a -// citation to open the note behind it — *without* the conversation leaving the screen, -// which is why chat lives here rather than in the centre pane (chat.ts's header). +// The right column's chat mode, so a clicked citation opens beside the conversation. // -// Three rules this builder holds, all of them invariants rather than styling: -// -// • **E5 — model output is untrusted content.** An answer is a string from a model that -// was itself fed note content (which anyone can author), so it renders through the one -// sanitizing `renderMarkdown` seam like every other document. The *streaming* half is -// stronger still: it is written as `textContent` by `paintChatStream` (main.ts), so a -// half-arrived answer is never parsed as markup at all. -// • **A citation navigates in-app, never the webview.** Each one is a `data-open` button -// — the same delegation search results and discovery cards use — so the mouse and ⏎ -// share one activation path (K1), and no `href` ever exists to be followed. -// • **K1 — the pane is keyboard-complete.** Rows are `role="treeitem"` over chat.ts's -// row order (the same order the arrows walk), with a roving tabstop; every control has -// a stable `id` so `paintSide` can hand focus back across the repaint each token causes. +// • E5: an answer is untrusted, so it renders through the sanitizing `renderMarkdown` +// seam; the streaming half is written as `textContent` (`paintChatStream`), never markup. +// • A citation is a `data-open` button, never an `href`, so it navigates in-app (K1). +// • K1: rows are `role="treeitem"` in chat.ts's row order with a roving tabstop; every +// control has a stable `id` so focus survives the per-token repaint. export function chatPaneHtml(state: AppState, roving: string | null): string { const setup = state.chatSetup; const streaming = state.chatStreaming !== null; @@ -55,14 +45,11 @@ export function chatPaneHtml(state: AppState, roving: string | null): string { state.chatMessages.length === 0 || streaming ? " disabled" : "" } data-chat-new title="Start a new conversation — nothing here is saved">new `; - // No composer until there is something to answer with: a disabled field under a card - // that says why is chrome with nothing behind it, and one more stop for a keyboard user - // to walk past on the way to the fix the card is pointing at. + // No composer until chat is ready: a disabled field is just another Tab stop. return head + chatStageHtml(state, roving) + (chatReady(state) ? chatComposerHtml(state) : ""); } -/** The conversation, or the state that stands in for one (chat.ts's `chatEmptyState` - * makes that choice once, so the paint doesn't re-derive it branch by branch). */ +/** The conversation, or the state that stands in for one (chosen by `chatStateOf`). */ function chatStageHtml(state: AppState, roving: string | null): string { switch (chatStateOf(state)) { case "no-vault": @@ -97,9 +84,8 @@ function chatLogHtml(state: AppState, roving: string | null): string { const turns = state.chatMessages .map((m, i) => chatTurnHtml(m, i, roving)) .join(""); - // The in-flight answer is a row of its own so the keyboard can sit on it while it - // fills. `#chat-stream` is the element `paintChatStream` writes tokens into — as text, - // never markup — which is what keeps a streaming answer off the full-render path. + // The in-flight answer is its own row, so the keyboard can sit on it. `paintChatStream` + // writes tokens into `#chat-stream` as text, off the full-render path. const live = state.chatStreaming === null ? "" @@ -140,8 +126,7 @@ function chatTurnHtml(m: ChatMessage, index: number, roving: string | null): str const stopped = m.cancelled ? `

Stopped — this answer is partial.

` : ""; - // Which B2 tools the answer was built from (a *Why?* turn). Tool names are the - // model's own words, so they are escaped like every other value in the chrome (E5). + // Which B2 tools built the answer (a Why? turn). Model-chosen names, so escaped (E5). const line = toolsLine(m.tools); const used = line ? `

${escapeHtml(line)}

` : ""; const cites = m.citations @@ -168,16 +153,9 @@ function chatTurnHtml(m: ChatMessage, index: number, roving: string | null): str } /** - * The composer. A ` `; } - // Every chip here carries a stable `id`: pressing one *is* a note-pane repaint, and the - // `innerHTML` swap destroys the button under the keyboard — the id is the identity - // `paintNote` puts focus back by (GH #91, crates/b2-desktop/CLAUDE.md). Each is unique - // because the pane paints exactly one of its three modes at a time. + // Every chip carries a stable `id`: pressing one repaints the pane, and `paintNote` + // restores focus by id (GH #91). return `
` : `

No notes link to this file yet.

`; const name = baseName(r.path); - // The alt text is the filename: the `

` right above already says it, so a screen - // reader that reads both is repeating itself rather than being told nothing — and B2 - // has no description of the picture to offer that would be truer than its name. + // Alt text is the filename: B2 has no truer description of the picture. const viewer = image ? `${escapeHtml(name)}` : `

No viewer available for this file type yet.

`; @@ -511,41 +428,28 @@ export function notePaneHtml(state: AppState): string { } /** - * The right column: search results, or the open note's discovery. - * - * Its rows follow the file tree's ARIA `tree` pattern (K1, GH #78) — `role="treeitem"`, - * `aria-level`, and a **roving `tabindex`**, so the pane is one Tab stop and ↑↓→← move - * within it. Like the tree, the DOM is flat: rows are siblings and `aria-level` is what - * tells a screen reader that a card sits under its section head, so the `.cards` - * wrappers are `role="none"` rather than groups. The row keys come from sidenav.ts, - * which both this paint and the arrow keys derive from the same state — the one place - * their order can't drift apart. + * The right column: search results, or the open note's discovery. Follows the tree's ARIA + * pattern (K1, GH #78): flat rows, `aria-level`, roving `tabindex`, `.cards` wrappers as + * `role="none"`. Row keys come from sidenav.ts, shared with the arrow keys. */ export function sidePaneHtml(state: AppState): string { const roving = rovingSideKey(sideRows(state), state.sideFocus); - // Chat owns the column outright when it's open (chat.ts's header says why it lives - // here at all): one column, one thing in it, and opening chat or running a search - // closes the other (main.ts). + // Chat owns the column when open; opening chat or running a search closes the other. if (state.chatOpen) return chatPaneHtml(state, roving); return state.searchQuery ? searchSectionHtml(state, roving) : discoverySectionHtml(state, roving); } -// The honest search-ranking caveat (#26). Search always answers over the keyword (BM25) -// index; this says how much *semantic* ranking is mixed in, so a projected-but-unembedded -// vault never silently under-ranks: -// • no real model → "keyword only (run `b2 init`)" -// • model, nothing embedded → "keyword-only for now (0/M embedded — Reindex)" -// • model, partly embedded → "keyword-first (N/M embedded)" (vector half still filling) -// • model, fully embedded → "" (ranking is fully semantic; no caveat) +// The search-ranking caveat (#26): how much semantic ranking is mixed into keyword search, +// so an unembedded vault never silently under-ranks. function searchCaveat(state: AppState): string { const c = coverage(state); if (!c.model) return " · keyword only (run b2 init for semantic)"; switch (c.embedded) { case "empty": case "all": - return ""; // empty vault, or every note embedded — semantic is live + return ""; case "none": return ` · keyword-only for now (0/${c.m} embedded — Reindex)`; case "partial": @@ -553,16 +457,9 @@ function searchCaveat(state: AppState): string { } } -// The install banner — the prominent, persistent counterpart to the small search caveat -// above (#26). On a fresh install with no model, the vault still gets its keyword index, -// but embedding is silently skipped (`autoIndexOnOpen` bails on `!semantic`), so semantic -// ranking and discovery are off with almost no visible sign — the reported gap. This -// surfaces that state as a dismissible strip under the top bar, pointing at Settings → -// Download. Gating is the pure, tested `shouldPromptEmbedInstall`; the controls are wired -// in main.ts: -// • Open Settings → opens the model picker + Download (the in-app `b2 init`) -// • ✕ → hide for this session (returns next launch — a gentle nag) -// • Don't remind me again → persist the opt-out (a keyword-only user, for good) +// The install banner (#26): with no model installed, semantic search and discovery are off +// with little visible sign, so say so. ✕ hides it for the session; the checkbox opts out +// for good. export function embedBannerHtml(state: AppState): string { const show = shouldPromptEmbedInstall({ hasVault: state.vaultRoot !== null, @@ -591,9 +488,8 @@ export function embedBannerHtml(state: AppState): string {

`; } -// Search mode. `clear` is the one focusable in this pane that is *not* a row, so it carries -// a stable `id` — `paintSide` restores a row by its row key and everything else by id, and -// without one the button was dropped to `` by any repaint (GH #91). +// `clear` is the pane's one non-row focusable, so it needs a stable `id` for `paintSide` +// to restore focus (GH #91). function searchSectionHtml(state: AppState, roving: string | null): string { const head = `

Results

@@ -602,7 +498,6 @@ function searchSectionHtml(state: AppState, roving: string | null): string {

for “${escapeHtml(state.searchQuery)}”${searchCaveat(state)}

`; if (state.loading) return head + `

Searching…

`; if (state.searchResults.length === 0) return head + searchEmptyHtml(state); - // A result card is the whole button, so it *is* the row — no fold, one level. const items = state.searchResults .map((r, i) => { const key = cardRowKey("search", i, r.path); @@ -621,17 +516,9 @@ function searchSectionHtml(state: AppState, roving: string | null): string { ); } -// The search pane's two empty states — the same blank list, two different reasons, -// and saying so is the whole of D2's honesty on this surface (GH #202). -// -// • `searchVouched === false` — the vault holds no evidence for this query: no -// lexical anchor, and nothing near enough by meaning at the active model's -// calibrated bar. The engine had rows and `doSearch` dropped them, strictly, so -// there is nothing to reveal and nothing to count. The copy says "no matches" -// because that is a claim about the *query*, which is the claim we can support. -// • anything else — the list is simply empty (an unbuilt index, a `null` verdict -// with nothing retrieved). That is not a judgment about the query, so the copy -// doesn't make one. +// The two empty states (D2, GH #202): `searchVouched === false` means the vault holds no +// evidence for the query, so the copy says so; otherwise the list is just empty (e.g. an +// unbuilt index) and the copy makes no claim about the query. function searchEmptyHtml(state: AppState): string { return state.searchVouched === false ? `

No matches. Nothing in this vault matches “${escapeHtml( @@ -651,10 +538,8 @@ function discoverySectionHtml(state: AppState, roving: string | null): string { )}${similarSectionHtml(state, roving)}

`; } -// A collapsible discovery-section header (chevron + title + count) — the same fold -// idiom the file tree and the Frontmatter drawer use, so the right column reads the -// same way. Collapsing is a sticky viewing preference (`collapsedSections`); the count -// is shown only when non-zero so an empty section stays quiet. +// A collapsible discovery-section header. Collapsing is sticky (`collapsedSections`); the +// count shows only when non-zero. function sideFoldHead( section: SideSection, label: string, @@ -675,14 +560,9 @@ function sideFoldHead( `; } -// The card's own fold chevron. Cards default expanded (the snippet is the signal you -// link on); this collapses the body (path + snippet) to just the title row. Kept out of -// the `.card-open` button so a click on the chevron folds without opening the note -// (nested buttons aren't allowed — the chevron and the open-region are siblings). -// -// A **mouse** affordance only, hence `tabindex="-1"` + `aria-hidden`: the row it sits on -// is the treeitem, it carries the `aria-expanded` a screen reader reads, and →← is the -// keyboard's fold (K1) — exactly as the file tree's caret is a span, not a tab stop. +// The card's fold chevron, a sibling of `.card-open` so folding doesn't open the note. +// Mouse-only (`tabindex="-1"`, `aria-hidden`): the row carries `aria-expanded` and →← folds +// (K1). function cardFold(key: string, collapsed: boolean): string { return ` `; - // *Why?* hands the pair to chat (main.ts's `askWhy`): the snippet above is the - // passage that matched, and this is the question it raises. `tabindex="-1"` keeps - // the card one Tab stop; the keyboard reaches the same action through the card - // menu's *Why was this suggested?* (⇧F10 — K1). - // - // `data-card-path`/`-title` on the root feed the right-click menu (Open / Link…); - // the whole card is the target now that the inline Link button is gone. The card is - // also the keyboard's *row* (`data-side-row`), which is why the whole box — title, - // path, and snippet — is what a screen reader reads and what the ring wraps. - // - // Draggable **only while the note is being edited** (droplink.ts): the drop lands a - // `[[wikilink]]` in the buffer, so with no buffer open there is nowhere for the drag - // to go, and an affordance that starts a drag nothing can accept is a lie the OS - // cursor has to walk back. The `title` says what the gesture does, since a drag is - // the one affordance with no visible label; the keyboard's half of it is the card - // menu's *Insert link at cursor* (⇧F10 — K1), below. + // The action buttons are `tabindex="-1"` to keep the card one Tab stop; the card + // menu (⇧F10, K1) reaches them. `data-card-path`/`-title` feed the right-click menu. + // Draggable only while editing (droplink.ts): with no buffer, nothing can accept it. return `
{ @@ -898,9 +749,7 @@ function unresolvedCardsHtml(state: AppState, roving: string | null): string { .join(""); } -/** One menu row. `chord` names the direct shortcut where one exists — a menu is where - * a keyboard user *learns* the chord that lets them skip the menu next time — so it - * comes out of the live registry (`displayKeys`), never spelled here. */ +/** One menu row. `chord` is the direct shortcut, from the live registry (`displayKeys`). */ function contextItemHtml(attr: string, label: string, chord = "", danger = false): string { const hint = chord ? `${escapeHtml(chord)}` : ""; return ``; } -// The right-click menu — one overlay, two surfaces (state.ts `ContextMenuState`): -// a discovery card (Open note / Link…, replacing the old inline "Link…" button) or -// the file tree (New note / New folder in the folder under the cursor, named in a -// muted context line). Anchored at the cursor via inline left/top — the coords are -// set + clamped on-screen in main.ts, and are plain numbers, so no escaping is -// needed. Rendered into its own overlay root so it floats above the panes; an -// outside click / Escape / scroll dismisses it (main.ts). +// The right-click menu for a discovery card or the file tree (state.ts `ContextMenuState`), +// anchored at the cursor. The coords are numbers clamped in main.ts, so need no escaping. export function contextMenuHtml(state: AppState): string { const m = state.contextMenu; if (!m) return ""; let items: string; if (m.kind === "tree") { - // Over a concrete row the menu targets that node (Rename / Move… — renaming - // acts on the file path, never a frontmatter title); the create pair keeps - // targeting the folder context either way. - // - // The two copy items are the row's *read* actions, sitting between the ones - // that change the file and the one that destroys it (Delete stays last, where - // a destructive item belongs). They copy the two paths that name this row — - // vault-relative and absolute (copypath.ts says why both) — and they are the - // only way to get either: the context line above shows the vault path but is - // muted text, and the system path is nowhere in the UI at all. + // Over a row the menu targets that node; the create items target the folder context. + // Delete stays last. The copy items are the only way to get either path (copypath.ts). const node = m.node ? `
${escapeHtml(m.node.path)}
${contextItemHtml("data-ctx-rename", "Rename", displayKeys(["tree.rename"]))} @@ -939,36 +775,25 @@ export function contextMenuHtml(state: AppState): string { ${contextItemHtml("data-ctx-delete", "Delete", displayKeys(["delete.focused"]), true)} ` : `
${escapeHtml(m.dir ? `${m.dir}/` : "vault root")}
`; - // Import files… is the drop gesture's keyboard half (K1): dragging a file in from - // Finder is a pointer-only action, so the same import runs from here — via ⇧F10, - // the keyboard's right-click — with an OS picker instead of a drag. It targets the - // folder context like the create pair, so it reads as the third way to put - // something in this folder. + // Import files… is the Finder drop's keyboard half (K1). items = `${node} ${contextItemHtml("data-ctx-new-note", "New note", displayKeys(["tree.new-note"]))} ${contextItemHtml("data-ctx-new-folder", "New folder", displayKeys(["tree.new-folder"]))} ${contextItemHtml("data-ctx-import", "Import files…")}`; } else { - // *Insert link at cursor* is the card drag's keyboard half (K1), the same shape - // *Import files…* is the Finder drop's: dragging a card onto a line is pointer-only, - // so the identical insertion runs from here — via ⇧F10 — aimed at the line the caret - // is already on. Only while editing, because that is the only time there is a buffer - // to insert into; the drag is withheld on the same condition (the card's `draggable`). + // *Insert link at cursor* is the card drag's keyboard half (K1); only while editing. items = `${contextItemHtml("data-ctx-open", "Open note", "⏎")} ${state.editing ? contextItemHtml("data-ctx-insert", "Insert link at cursor") : ""} ${contextItemHtml("data-ctx-link", "Link…")} ${contextItemHtml("data-ctx-explain", "Explain this suggestion")} ${contextItemHtml("data-ctx-why", "Why was this suggested?")}`; } - // `tabindex="-1"` on the menu itself makes the container focusable-by-script but not - // by Tab: main.ts moves focus to the first item on open and traps ↑↓/⏎/Esc inside. + // main.ts focuses the first item on open and traps ↑↓/⏎/Esc inside. return ``; } -/** The Move… modal: pick a destination folder for the targeted tree node. Every - * folder the tree knows renders as a row; an invalid destination (the node's - * current folder, or a folder inside the folder being moved) renders disabled - * with the reason, so the modal teaches the same rule the host enforces. */ +/** The Move… modal. An invalid destination renders disabled with the reason, teaching the + * rule the host enforces. */ function moveModalHtml(state: AppState): string { const t = state.moveTarget; if (!t) return ""; @@ -996,9 +821,7 @@ function moveModalHtml(state: AppState): string {
`; } -/** The folder-delete confirm — the one destructive gesture that asks first: a - * whole subtree (unindexed files included) leaves the disk. Files delete - * without a dialog; the tree gesture itself is the intent. */ +/** The folder-delete confirm, the one delete that asks first: a whole subtree leaves disk. */ function deleteModalHtml(state: AppState): string { const t = state.deleteTarget; if (!t) return ""; @@ -1015,10 +838,8 @@ function deleteModalHtml(state: AppState): string { `; } -// The overlay layer, in precedence order (main.ts's `currentOverlay` ranks them the same -// way). Settings is the odd one out since it went full-window — same `role="dialog"` and -// the same trap, no backdrop and no box — so it is first here for the reason it is first -// there: it paints *over* a Move/Delete/Link target left set behind it. +// The overlay layer, in the same precedence as main.ts's `currentOverlay`. Settings is +// first: it paints over a Move/Delete/Link target left set behind it. export function modalHtml(state: AppState): string { if (state.settingsOpen) return settingsScreenHtml(state); if (state.moveTarget) return moveModalHtml(state); @@ -1029,9 +850,8 @@ export function modalHtml(state: AppState): string { const opts = RELATION_VERBS.map( (v) => ``, ).join(""); - // The backdrop carries no cancel attr (a click on it closes only when it is the - // exact target — see main.ts); the Cancel button uses `data-cancel`. This keeps a - // click *inside* the modal from bubbling into an accidental close. + // The backdrop carries no cancel attr, so a click inside the modal can't bubble into a + // close (main.ts closes only when the backdrop is the exact target). return `