From 7805ed32499953ffc61074ca0fc4bf8c02b65d60 Mon Sep 17 00:00:00 2001 From: Sam Keen Date: Wed, 23 Sep 2026 10:44:13 -0700 Subject: [PATCH 1/2] feat: Explain on a Similar card shows the evidence behind it, model-free (#236) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A card's Explain (and `b2 similar NOTE --explain OTHER`) reads one note's place in the anchor's discovery field from the same computation that ranked the list, so an explanation can never disagree with the card: same rank, same z, same winning pair. - core: `discover` computes one field that `candidates` serves a prefix of and `explain` reads a row of; `Vault::explain_similar` adds the standing (ranked, past the list, linked, unembedded, not shortlisted), whole-note rank beside best-passage rank, the z population, every passage pair graded on the dots' yardstick, identical-text pairs, and shared neighbors. Row order and z unchanged. - cli: `b2 similar NOTE --explain OTHER [--json]`. - desktop: `explain_similar` command; a Compare view in the centre pane with a summary label, a where-it-sits strip (σ axis, key, "?" for the long account), side-by-side passages, and hand-offs to Open and Ask chat why. Keyboard-complete. - docs: quickstart, search-and-similarity, architecture. Co-Authored-By: Claude Opus 5.5 --- crates/b2-cli/src/main.rs | 107 ++++++- crates/b2-cli/tests/cli.rs | 75 +++++ crates/b2-core/src/discover.rs | 301 ++++++++++++++---- crates/b2-core/src/vault.rs | 202 +++++++++++- crates/b2-core/tests/common/mod.rs | 92 +++++- crates/b2-core/tests/discover_surfacing.rs | 86 +---- crates/b2-core/tests/explain_similar.rs | 347 +++++++++++++++++++++ crates/b2-desktop/src/commands.rs | 16 +- crates/b2-desktop/src/main.rs | 1 + docs/architecture.md | 4 +- docs/quickstart.md | 20 +- docs/search-and-similarity.md | 9 + ui/scripts/gen-icons.ts | 4 + ui/src/api.ts | 7 + ui/src/explain.test.ts | 169 ++++++++++ ui/src/explain.ts | 181 +++++++++++ ui/src/icons.gen.ts | 1 + ui/src/main.ts | 96 +++++- ui/src/render.test.ts | 105 +++++++ ui/src/render.ts | 196 +++++++++++- ui/src/state.ts | 18 ++ ui/src/strength.ts | 9 +- ui/src/types.ts | 41 +++ ui/style.css | 255 +++++++++++++++ 24 files changed, 2170 insertions(+), 172 deletions(-) create mode 100644 crates/b2-core/tests/explain_similar.rs create mode 100644 ui/src/explain.test.ts create mode 100644 ui/src/explain.ts diff --git a/crates/b2-cli/src/main.rs b/crates/b2-cli/src/main.rs index 15aa2d3..44f6db8 100644 --- a/crates/b2-cli/src/main.rs +++ b/crates/b2-cli/src/main.rs @@ -12,7 +12,7 @@ use b2_core::embed::Embedder; use b2_core::llm::{ChatTurn, FakeLlm, LlmProvider}; use b2_core::resource::{doc_kind, DocKind}; -use b2_core::vault::{AnswerView, SearchEvidenceView, Vault}; +use b2_core::vault::{AnswerView, SearchEvidenceView, SimilarStanding, Vault}; use b2_embed::{provision, EmbedConfig, EmbedError, LocalEmbedder}; use b2_llm::{ is_ollama, refusal_message, LlmConfig, LlmError, OpenAiCompatProvider, OLLAMA_INSTALL_URL, @@ -205,6 +205,11 @@ enum Command { /// Maximum number of similar notes to return. #[arg(long, default_value_t = 10)] limit: usize, + /// Explain one note against NOTE's list instead of printing the list: where it + /// stands (and why, if it is not a card), and the passage pairs behind it. No + /// model call. Read at `--limit`, so the rank is the one the list shows. + #[arg(long, value_name = "OTHER")] + explain: Option, }, /// Commit a typed connection SRC → DST into SRC's frontmatter `b2_relations:`. /// SRC and DST are each a vault-relative path. @@ -399,7 +404,16 @@ fn dispatch(cli: &Cli) -> Result<(), CliError> { limit, exclude, } => cmd_search(cli, query, *limit, exclude), - Command::Similar { note, limit } => cmd_similar(cli, note, *limit), + Command::Similar { + note, + limit, + explain: Some(other), + } => cmd_explain_similar(cli, note, other, *limit), + Command::Similar { + note, + limit, + explain: None, + } => cmd_similar(cli, note, *limit), Command::Link { src, dst, @@ -1003,6 +1017,95 @@ fn cmd_similar(cli: &Cli, note: &str, limit: usize) -> Result<(), CliError> { Ok(()) } +/// Longest passage excerpt `similar --explain` prints per side, so a pair stays readable +/// in a terminal. `--json` carries the full text. +const EXPLAIN_EXCERPT_CHARS: usize = 200; + +fn cmd_explain_similar(cli: &Cli, note: &str, other: &str, limit: usize) -> Result<(), CliError> { + // A pure read over stored vectors, like `similar`: the fake is enough to open with. + let vault = open_vault(cli.vault_or_cwd(), false)?; + let ex = vault.explain_similar(note, other, limit)?; + if cli.json { + return print_json(&ex); + } + let anchor = display_name(ex.anchor.title.as_deref(), &ex.anchor.path); + let candidate = display_name(ex.candidate.title.as_deref(), &ex.candidate.path); + println!( + "{candidate} ({}) against {anchor} ({})", + ex.candidate.path, ex.anchor.path + ); + let z = ex.z.map(|z| format!(", z {z:.2}")).unwrap_or_default(); + let standing = match ex.standing { + SimilarStanding::SameNote => "That is the same note.".to_string(), + SimilarStanding::AnchorUnembedded => format!( + "{anchor} has no stored vectors yet, so there is nothing to compare from. Run `b2 reindex`." + ), + SimilarStanding::Linked => { + "Already linked, so Similar leaves it out: it shows what you haven't connected." + .to_string() + } + SimilarStanding::Unembedded => { + "Not embedded yet, so it can't be compared. Run `b2 reindex`.".to_string() + } + SimilarStanding::NotShortlisted { shortlist } => format!( + "Never scored: judged as a whole note it ranks #{}, past the first-pass shortlist of {shortlist}.", + ex.centroid_rank.unwrap_or(0) + ), + SimilarStanding::Ranked { + rank, + of, + served: true, + } => format!("Shown at #{rank} of {limit} ({of} notes scored{z})."), + SimilarStanding::Ranked { + rank, + of, + served: false, + } => format!("Ranked #{rank} of {of} scored{z}: past the {limit} shown."), + }; + println!("{standing}"); + if let (Some(c), SimilarStanding::Ranked { rank, .. }) = (ex.centroid_rank, ex.standing) { + println!("Judged as a whole note it ranks #{c}; by its best passage, #{rank}."); + } + if !ex.shared_neighbors.is_empty() { + let names: Vec = ex + .shared_neighbors + .iter() + .map(|n| format!("{} ({})", display_name(n.title.as_deref(), &n.path), n.path)) + .collect(); + println!("Both link with: {}", names.join(", ")); + } + if !ex.pairs.is_empty() { + println!("\nPassage pairs, nearest first:"); + } + for (i, p) in ex.pairs.iter().enumerate() { + let grade = p.z.map(|z| format!(" z {z:.2}")).unwrap_or_default(); + let same = if p.identical { " identical text" } else { "" }; + println!("{:>3}.{grade}{same}", i + 1); + for (side, passage) in [("this", &p.anchor), ("that", &p.candidate)] { + let heading = passage + .heading_path + .as_deref() + .map(|h| format!("[{h}] ")) + .unwrap_or_default(); + println!( + " {side}: {heading}{}", + excerpt(&passage.text, EXPLAIN_EXCERPT_CHARS) + ); + } + } + Ok(()) +} + +/// A passage flattened to one line and cut at `max` characters. +fn excerpt(text: &str, max: usize) -> String { + let flat = text.split_whitespace().collect::>().join(" "); + if flat.chars().count() <= max { + flat + } else { + format!("{}…", flat.chars().take(max).collect::().trim_end()) + } +} + fn cmd_link( cli: &Cli, src: &str, diff --git a/crates/b2-cli/tests/cli.rs b/crates/b2-cli/tests/cli.rs index 5972658..022a5c4 100644 --- a/crates/b2-cli/tests/cli.rs +++ b/crates/b2-cli/tests/cli.rs @@ -1005,6 +1005,81 @@ fn similar_limit_zero_prints_nothing_rather_than_a_claim() { ); } +/// `b2 similar NOTE --explain OTHER` (GH #236): the model-free explanation of one +/// card, read from the same computation as the list. JSON names both notes, the +/// standing (tagged by `kind`), and the passage pairs; the card it explains stands at +/// the rank the list showed it. +#[test] +fn similar_explain_describes_the_served_row() { + let (_g, root) = discovery_vault(); + let list = similar(&root, "alpha.md"); + let second = list[1]["path"].as_str().unwrap().to_string(); + + let out = run_in( + &root, + &["--json", "similar", "alpha.md", "--explain", &second], + ); + assert!(out.status.success(), "{}", stderr(&out)); + let ex: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(ex["anchor"]["path"], "alpha.md"); + assert_eq!(ex["candidate"]["path"], second.as_str()); + assert_eq!(ex["standing"]["kind"], "ranked"); + assert_eq!(ex["standing"]["rank"], 2, "the rank the card was shown at"); + assert_eq!(ex["standing"]["served"], true); + assert_eq!(ex["limit"], 10); + let pairs = ex["pairs"].as_array().unwrap(); + assert!(!pairs.is_empty(), "an embedded pair has passages"); + assert!(pairs[0]["anchor"]["text"] + .as_str() + .unwrap() + .contains("alpha")); + assert!( + (pairs[0]["score"].as_f64().unwrap() - list[1]["score"].as_f64().unwrap()).abs() < 1e-9 + ); + + // Human: the standing in words, then the pairs. + let human = run_in(&root, &["similar", "alpha.md", "--explain", &second]); + assert!(human.status.success(), "{}", stderr(&human)); + let text = stdout(&human); + assert!(text.contains("#2"), "says where it stands: {text}"); + assert!(text.contains("alpha"), "shows this note's passage: {text}"); +} + +#[test] +fn similar_explain_says_why_a_linked_note_is_not_a_card() { + let (_g, root) = discovery_vault(); + let out = run_in( + &root, + &["link", "alpha.md", "beta.md", "--type", "supports"], + ); + assert!(out.status.success(), "{}", stderr(&out)); + + let out = run_in( + &root, + &["--json", "similar", "alpha.md", "--explain", "beta.md"], + ); + assert!(out.status.success(), "{}", stderr(&out)); + let ex: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(ex["standing"]["kind"], "linked"); + + let human = run_in(&root, &["similar", "alpha.md", "--explain", "beta.md"]); + assert!( + stdout(&human).to_lowercase().contains("already linked"), + "{}", + stdout(&human) + ); +} + +#[test] +fn similar_explain_unknown_note_fails_cleanly() { + let (_g, root) = discovery_vault(); + let out = run_in(&root, &["similar", "alpha.md", "--explain", "nope.md"]); + assert!(!out.status.success(), "unknown note must be a nonzero exit"); + let err = stderr(&out).to_lowercase(); + assert!(err.contains("not found"), "actionable message: {err}"); + assert!(!err.contains("panicked"), "no stack trace: {err}"); +} + #[test] fn link_writes_frontmatter_and_shows_in_both_directions() { let (_g, root) = discovery_vault(); diff --git a/crates/b2-core/src/discover.rs b/crates/b2-core/src/discover.rs index 7c920f5..3db701a 100644 --- a/crates/b2-core/src/discover.rs +++ b/crates/b2-core/src/discover.rs @@ -83,29 +83,51 @@ pub struct CandidateNote { pub z: Option, } -/// Generate up to `limit` discovery candidates for `anchor`, strongest first by -/// best-passage distance (ties on `note_path`, for determinism). That is one order with -/// three names — the stage-2 `score`, the `z`, and the strength band an adapter paints, -/// each strictly monotonic in the others — so none of them can disagree with the row -/// order. **The ranked list is always served** (ADR-0014): `limit` is a cap, and the -/// list under-fills only for want of scorable notes. -/// -/// `grade` asks for the per-candidate z beside the ranking, computed when the scored -/// population is at least [`STATS_MIN_POPULATION`] with nonzero variance. The façade -/// passes `false` for a fake-embedded space, whose hash vectors have no semantic -/// geometry to claim a statistic over. Grading changes what the rows *carry*, never -/// which rows exist or their order. -/// -/// Returns empty when the vault has no embedding space, when the anchor has no stored -/// vectors, or when `limit` is 0 — there is nothing to search from. -pub fn candidates( - conn: &Connection, - anchor: &str, - limit: usize, - grade: bool, -) -> Result> { - if limit == 0 || !db::embedding_space_exists(conn)? { - return Ok(Vec::new()); +/// The statistic behind the strength band: the mean and spread of the scored +/// population's best-passage **squared** distances. It grades; it never gates +/// (ADR-0014). Any distance in the same space can be read against it, which is how a +/// passage pair in the explain view is graded on the same yardstick as the cards. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Population { + /// How many notes were scored: the size of the population the z is read against. + pub n: usize, + mean: f64, + sd: f64, +} + +impl Population { + /// The z of one squared distance, oriented so nearer is higher. + pub fn z(&self, dist_sq: f32) -> f64 { + (self.mean - dist_sq as f64) / self.sd + } +} + +/// One anchor's whole discovery field: the stage-1 order, the stage-2 scores, and the +/// statistic. [`candidates`] serves a prefix of it and [`explain`] reads one note's place +/// in it, so a card and its explanation are two reads of one computation and can never +/// disagree. +struct Field { + /// Every unlinked note with a centroid, nearest centroid first (ties by path), + /// before the shortlist cut. + coarse: Vec, + /// How many of `coarse` stage 2 was allowed to score. + shortlist: usize, + /// Stage 2: `(squared distance, note, evidence chunk)` for every shortlisted note + /// with stored vectors, nearest first (ties by path). + scored: Vec<(f32, String, i64)>, + /// `None` when ungraded: not asked for, a pool under [`STATS_MIN_POPULATION`], or + /// zero variance. + population: Option, + /// The anchor and its 1-hop neighbours: excluded from discovery. + excluded: std::collections::HashSet, +} + +/// Compute `anchor`'s field at the shortlist a `limit`-sized list uses. `None` when +/// there is nothing to search from: no embedding space, or an anchor with no stored +/// vectors. +fn field(conn: &Connection, anchor: &str, limit: usize, grade: bool) -> Result> { + if !db::embedding_space_exists(conn)? { + return Ok(None); } // The anchor's own stored vectors, loaded once (re-embeds nothing — index-engine.md §3); // none ⇒ nothing to search from. Its centroid is computed in-process from them @@ -115,19 +137,19 @@ pub fn candidates( .map(|(_, v)| v) .collect(); let Some(anchor_centroid) = centroid_of(&anchor_vecs) else { - return Ok(Vec::new()); + return Ok(None); }; // The only use of the graph in generation: subtract what's already linked — the // anchor and everything within 1 hop (self + direct neighbors). - let exclude = graph::reachable_within(conn, anchor, EXCLUDE_HOPS)?; + let excluded = graph::reachable_within(conn, anchor, EXCLUDE_HOPS)?; // Stage 1 — coarse shortlist over note centroids: one O(notes) scan, excluded // notes skipped up front so they never occupy a shortlist slot. let mut coarse: Vec<(f32, String)> = Vec::new(); let mut scratch: Vec = Vec::new(); db::for_each_note_centroid(conn, |note, blob| { - if exclude.contains(note) { + if excluded.contains(note) { return; // the anchor or a direct neighbor — already connected } unpack_f32_into(blob, &mut scratch); @@ -138,26 +160,25 @@ pub fn candidates( .unwrap_or(std::cmp::Ordering::Equal) .then(a.1.cmp(&b.1)) }); + let coarse: Vec = coarse.into_iter().map(|(_, note)| note).collect(); // Stage 1 ends here, and nothing judges it: the shortlist is a recall device, never // a quality gate (GH #192). It used to be both — cutting the coarse list on centroid // z meant max-sim could never rescue a candidate whose best passage is far nearer // than its centroid suggests, which is exactly a multi-topic note's shape (ADR-0014). - coarse.truncate( - limit - .saturating_mul(SHORTLIST_PER_RESULT) - .max(SHORTLIST_MIN), - ); + let shortlist = limit + .saturating_mul(SHORTLIST_PER_RESULT) + .max(SHORTLIST_MIN); // Stage 2 — exact max-sim over the whole shortlist: per note, the best (smallest // squared-L2) pair across the anchor's chunks and its own. Squared L2 is the same // ranking key without the per-comparison `sqrt`, applied once per surfaced candidate - // below. Strictly-less keeps the earliest chunk on ties. A shortlisted note with no - // stored chunk vectors (possible mid-embed) scores nothing and drops out. + // by the readers. Strictly-less keeps the earliest chunk on ties. A shortlisted note + // with no stored chunk vectors (possible mid-embed) scores nothing and drops out. let mut scored: Vec<(f32, String, i64)> = Vec::new(); - for (_, note_path) in coarse { + for note_path in coarse.iter().take(shortlist) { let mut best: Option<(f32, i64)> = None; - for (chunk_id, v) in db::note_chunk_vectors(conn, ¬e_path)? { + for (chunk_id, v) in db::note_chunk_vectors(conn, note_path)? { for a in &anchor_vecs { let dist_sq = l2_sq(a, &v); if best.is_none_or(|(cur, _)| dist_sq < cur) { @@ -166,7 +187,7 @@ pub fn candidates( } } if let Some((dist_sq, evidence_chunk_id)) = best { - scored.push((dist_sq, note_path, evidence_chunk_id)); + scored.push((dist_sq, note_path.clone(), evidence_chunk_id)); } } // Nearest-first, ties by path: the served order, and — because z below is affine in @@ -183,7 +204,7 @@ pub fn candidates( // a personal-scale vault is every unlinked note there is. Above SHORTLIST_MIN it is // the anchor's centroid-nearest slice, a bias the dogfooding obligation owns. z is // oriented so nearer = higher, and travels to the output as the band's input. - let mut zs: Option> = None; + let mut population = None; if grade && scored.len() >= STATS_MIN_POPULATION { let n = scored.len() as f64; let mean = scored.iter().map(|(d, _, _)| *d as f64).sum::() / n; @@ -194,28 +215,171 @@ pub fn candidates( / (n - 1.0); let sd = var.sqrt(); if sd > 0.0 { - zs = Some( - scored - .iter() - .map(|(d, _, _)| (mean - *d as f64) / sd) - .collect(), - ); + population = Some(Population { + n: scored.len(), + mean, + sd, + }); } } - let out = scored + Ok(Some(Field { + coarse, + shortlist, + scored, + population, + excluded, + })) +} + +/// Generate up to `limit` discovery candidates for `anchor`, strongest first by +/// best-passage distance (ties on `note_path`, for determinism). That is one order with +/// three names — the stage-2 `score`, the `z`, and the strength band an adapter paints, +/// each strictly monotonic in the others — so none of them can disagree with the row +/// order. **The ranked list is always served** (ADR-0014): `limit` is a cap, and the +/// list under-fills only for want of scorable notes. +/// +/// `grade` asks for the per-candidate z beside the ranking, computed when the scored +/// population is at least [`STATS_MIN_POPULATION`] with nonzero variance. The façade +/// passes `false` for a fake-embedded space, whose hash vectors have no semantic +/// geometry to claim a statistic over. Grading changes what the rows *carry*, never +/// which rows exist or their order. +/// +/// Returns empty when the vault has no embedding space, when the anchor has no stored +/// vectors, or when `limit` is 0 — there is nothing to search from. +pub fn candidates( + conn: &Connection, + anchor: &str, + limit: usize, + grade: bool, +) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let Some(field) = field(conn, anchor, limit, grade)? else { + return Ok(Vec::new()); + }; + let population = field.population; + Ok(field + .scored .into_iter() - .enumerate() .take(limit) - .map( - |(i, (dist_sq, note_path, evidence_chunk_id))| CandidateNote { - note_path, - score: -(dist_sq.sqrt() as f64), // nearer = higher, matching Hit's -L2 - evidence_chunk_id, - z: zs.as_ref().and_then(|z| z.get(i)).copied(), - }, - ) + .map(|(dist_sq, note_path, evidence_chunk_id)| CandidateNote { + note_path, + score: -(dist_sq.sqrt() as f64), // nearer = higher, matching Hit's -L2 + evidence_chunk_id, + z: population.map(|p| p.z(dist_sq)), + }) + .collect()) +} + +/// Where one note stands in an anchor's discovery field, which is the first thing an +/// explanation says: why it is a card, or why it is not. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Standing { + /// The candidate is the anchor. + SameNote, + /// The anchor has no stored vectors, so there is nothing to compare from. + AnchorUnembedded, + /// Within one link of the anchor: excluded from discovery as already known. + Linked, + /// The candidate has no stored vectors (or no centroid) yet. + Unembedded, + /// Its whole-note (centroid) rank fell past the stage-1 shortlist, so stage 2 + /// never scored it. + NotShortlisted { shortlist: usize }, + /// Scored in stage 2: `rank` (1-based, the served order) among `of` scored notes. + Ranked { rank: usize, of: usize }, +} + +/// [`PassagePair`] graded on the same yardstick as the cards. +#[derive(Debug, Clone, PartialEq)] +pub struct GradedPair { + pub pair: PassagePair, + /// The pair's z against the anchor's population, or `None` when ungraded. The + /// winning pair's z is exactly the row's z: the same squared distance, read against + /// the same population. + pub z: Option, +} + +/// Everything [`explain`] reads for one (anchor, candidate) pair. +#[derive(Debug, Clone, PartialEq)] +pub struct Explanation { + pub standing: Standing, + /// The candidate's rank by whole-note centroid in stage 1 (1-based), when it entered + /// stage 1 at all. Beside a best-passage rank it shows a buried gem: a note whose + /// one section matches far better than the note as a whole. + pub centroid_rank: Option, + /// The candidate's z, when it was scored in a graded field. + pub z: Option, + /// Every scored note's z in served order (so descending): the field the band is + /// relative to. Empty when ungraded. + pub population: Vec, + /// One pair per candidate passage, nearest first ([`passage_pairs`]), each graded. + pub pairs: Vec, +} + +/// Explain one note's place in `anchor`'s discovery field, at the shortlist a +/// `limit`-sized list uses. The same computation [`candidates`] serves from, so a +/// served row's rank, z and winning pair are exactly the card's. Answers for any note, +/// not only a served one: a linked, unembedded or unshortlisted note says which, and its +/// passages are still compared when both sides have vectors. A pure read over stored +/// vectors, re-embedding nothing. +pub fn explain( + conn: &Connection, + anchor: &str, + candidate: &str, + limit: usize, + grade: bool, +) -> Result { + let mut out = Explanation { + standing: Standing::SameNote, + centroid_rank: None, + z: None, + population: Vec::new(), + pairs: Vec::new(), + }; + if anchor == candidate { + return Ok(out); + } + let Some(field) = field(conn, anchor, limit, grade)? else { + out.standing = Standing::AnchorUnembedded; + return Ok(out); + }; + let population = field.population; + out.population = population + .map(|p| field.scored.iter().map(|(d, _, _)| p.z(*d)).collect()) + .unwrap_or_default(); + out.pairs = passage_pairs_sq(conn, anchor, candidate, usize::MAX)? + .into_iter() + .map(|(dist_sq, pair)| GradedPair { + pair, + z: population.map(|p| p.z(dist_sq)), + }) .collect(); + out.centroid_rank = field + .coarse + .iter() + .position(|n| n == candidate) + .map(|i| i + 1); + let scored_at = field.scored.iter().position(|(_, n, _)| n == candidate); + out.standing = if field.excluded.contains(candidate) { + Standing::Linked + } else if let Some(i) = scored_at { + out.z = population.map(|p| p.z(field.scored[i].0)); + Standing::Ranked { + rank: i + 1, + of: field.scored.len(), + } + } else if out.centroid_rank.is_some_and(|r| r > field.shortlist) { + Standing::NotShortlisted { + shortlist: field.shortlist, + } + } else { + // Not scored and not past the shortlist: it has no centroid or no chunk + // vectors yet. + Standing::Unembedded + }; Ok(out) } @@ -250,6 +414,20 @@ pub fn passage_pairs( candidate: &str, limit: usize, ) -> Result> { + Ok(passage_pairs_sq(conn, anchor, candidate, limit)? + .into_iter() + .map(|(_, pair)| pair) + .collect()) +} + +/// [`passage_pairs`] with each pair's squared distance kept beside it, the unit the +/// population statistic reads. +fn passage_pairs_sq( + conn: &Connection, + anchor: &str, + candidate: &str, + limit: usize, +) -> Result> { if limit == 0 || !db::embedding_space_exists(conn)? { return Ok(Vec::new()); } @@ -273,12 +451,15 @@ pub fn passage_pairs( Ok(pairs .into_iter() .take(limit) - .map( - |(dist_sq, anchor_chunk_id, candidate_chunk_id)| PassagePair { - anchor_chunk_id, - candidate_chunk_id, - score: -(dist_sq.sqrt() as f64), - }, - ) + .map(|(dist_sq, anchor_chunk_id, candidate_chunk_id)| { + ( + dist_sq, + PassagePair { + anchor_chunk_id, + candidate_chunk_id, + score: -(dist_sq.sqrt() as f64), + }, + ) + }) .collect()) } diff --git a/crates/b2-core/src/vault.rs b/crates/b2-core/src/vault.rs index ef717ea..8041af3 100644 --- a/crates/b2-core/src/vault.rs +++ b/crates/b2-core/src/vault.rs @@ -482,6 +482,81 @@ pub struct SimilarView { pub z: Option, } +/// **Explain** for a *Similar & unlinked* card (GH #236): where one note stands in an +/// anchor's discovery field, and the passage pairs behind it. Model-free and read from +/// the same computation as [`Vault::similar`], so a served row's rank, z and best pair are +/// exactly the card's. Raw distances are never the point: every grade is a z on the same +/// yardstick as the strength band. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SimilarExplainView { + pub anchor: NoteSummary, + pub candidate: NoteSummary, + /// The list length the explanation is read at: the surface's own, so `served` + /// answers "was this a card?". + pub limit: usize, + pub standing: SimilarStanding, + /// The candidate's z when it was scored in a graded field (the card's band input). + pub z: Option, + /// The candidate's rank judged by whole-note average (stage 1), when it entered + /// stage 1. Far worse than a ranked standing's `rank` means one section carries the + /// match: a buried gem. + pub centroid_rank: Option, + /// Every scored note's z, nearest first: the field the band is relative to. Empty + /// when ungraded (a fake space, a small pool, no spread). + pub population: Vec, + /// One pair per candidate passage, nearest first, each matched to the anchor passage + /// nearest to it. Empty when either side has no vectors. + pub pairs: Vec, + /// Notes both sides already link with (either direction). + pub shared_neighbors: Vec, +} + +/// Where a note stands in an anchor's discovery field: why it is a card, or why not. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SimilarStanding { + /// The candidate is the anchor itself. + SameNote, + /// The anchor has no stored vectors yet: nothing to compare from. + AnchorUnembedded, + /// Directly linked (either direction), so discovery leaves it out as already known. + Linked, + /// The candidate has no stored vectors yet. + Unembedded, + /// Its whole-note rank fell past the first-pass shortlist of `shortlist` notes, so + /// it was never scored passage by passage. + NotShortlisted { shortlist: usize }, + /// Scored: `rank` (1-based) of `of` scored notes; `served` when `rank <= limit`. + Ranked { + rank: usize, + of: usize, + served: bool, + }, +} + +/// One passage pair: a candidate passage and the anchor passage nearest to it. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PassagePairView { + pub anchor: PassageView, + pub candidate: PassageView, + /// Negated L2, higher is nearer: [`SimilarView::score`]'s unit. + pub score: f64, + /// The pair's z on the card's yardstick, `None` when ungraded. + pub z: Option, + /// Both passages hold the same text (a template, a copy): a perfect match that says + /// nothing about what the notes are about. + pub identical: bool, +} + +/// One passage, as stored in the index. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct PassageView { + /// The chunk's heading breadcrumb (`"Fermentation > Vegetables"`), when it has one. + pub heading_path: Option, + /// The chunk's stored text, verbatim. + pub text: String, +} + /// What [`write`](Vault::write) did: the saved note's path and the **new revision** /// (blake3 of the final on-disk bytes) — the token the editor chains its next save /// on, so sequential saves never self-conflict. @@ -1763,14 +1838,9 @@ impl Vault { pub fn similar(&self, note_ref: &str, limit: usize) -> Result> { let _op = tracing::debug_span!(target: "b2::vault", "similar", note = note_ref, limit).entered(); - // Never claim a statistic over a fake-embedded space: hash vectors have no - // semantic geometry, so a z there would be noise wearing a band. Judged by the - // RECORDED identity — the space being searched — not the injected embedder. - // Grading changes what the rows carry, never which rows exist (ADR-0014). - let grade = !matches!( - db::recorded_embedder(&self.conn)?, - Some((model, _)) if model == crate::embed::FAKE_MODEL_ID - ); + // Never claim a statistic over a fake-embedded space (`grades`). Grading changes + // what the rows carry, never which rows exist (ADR-0014). + let grade = self.grades()?; // A resource anchor is honest, not silent: resources become discoverable once // they have chunks + centroids. Until then an inventoried resource errs "not // yet" — never an empty result — and an unknown path falls through to @@ -1798,6 +1868,122 @@ impl Vault { Ok(out) } + /// **Explain** one note against an anchor's *Similar & unlinked* list (GH #236): + /// where it stands (ranked, past the list, linked, not embedded, not shortlisted), + /// its z and whole-note rank, the field's z population, every passage pair graded on + /// the card's yardstick, and the neighbors both notes share. `limit` is the list + /// length the surface showed, so the rank and z are the card's. Any note can be + /// explained, not only a served one. A pure read over stored vectors, no model call. + /// [`Error::NoteNotFound`] for an unknown ref on either side; + /// [`Error::ResourceUnsupported`] for a resource anchor, as [`similar`](Self::similar). + pub fn explain_similar( + &self, + anchor_ref: &str, + candidate_ref: &str, + limit: usize, + ) -> Result { + let _op = tracing::debug_span!( + target: "b2::vault", + "explain_similar", + anchor = anchor_ref, + candidate = candidate_ref, + limit + ) + .entered(); + if crate::resource::doc_kind(anchor_ref) == crate::resource::DocKind::Resource + && db::resource_detail(&self.conn, anchor_ref)?.is_some() + { + return Err(Error::ResourceUnsupported(anchor_ref.to_string())); + } + let anchor = self.resolve_ref(anchor_ref)?; + let candidate = self.resolve_ref(candidate_ref)?; + let ex = discover::explain(&self.conn, &anchor, &candidate, limit, self.grades()?)?; + + let standing = match ex.standing { + discover::Standing::SameNote => SimilarStanding::SameNote, + discover::Standing::AnchorUnembedded => SimilarStanding::AnchorUnembedded, + discover::Standing::Linked => SimilarStanding::Linked, + discover::Standing::Unembedded => SimilarStanding::Unembedded, + discover::Standing::NotShortlisted { shortlist } => { + SimilarStanding::NotShortlisted { shortlist } + } + discover::Standing::Ranked { rank, of } => SimilarStanding::Ranked { + rank, + of, + served: rank <= limit, + }, + }; + let mut pairs = Vec::with_capacity(ex.pairs.len()); + for g in ex.pairs { + let anchor_side = self.passage(g.pair.anchor_chunk_id)?; + let candidate_side = self.passage(g.pair.candidate_chunk_id)?; + // A chunk row that vanished mid-read (a concurrent reindex, C1) is skipped + // rather than shown half-empty. + let (Some(a), Some(c)) = (anchor_side, candidate_side) else { + continue; + }; + pairs.push(PassagePairView { + identical: a.text == c.text, + anchor: a, + candidate: c, + score: g.pair.score, + z: g.z, + }); + } + let shared_neighbors = self + .shared_neighbors(&anchor, &candidate)? + .into_iter() + .map(|path| { + let title = db::note_title(&self.conn, &path)?; + Ok(NoteSummary { path, title }) + }) + .collect::>>()?; + Ok(SimilarExplainView { + anchor: NoteSummary { + title: db::note_title(&self.conn, &anchor)?, + path: anchor, + }, + candidate: NoteSummary { + title: db::note_title(&self.conn, &candidate)?, + path: candidate, + }, + limit, + standing, + z: ex.z, + centroid_rank: ex.centroid_rank, + population: ex.population, + pairs, + shared_neighbors, + }) + } + + /// Whether this vault's discovery is graded: never over a fake-embedded space, whose + /// hash vectors have no semantic geometry for a z to describe. Judged by the RECORDED + /// identity (the space being searched), not the injected embedder. + fn grades(&self) -> Result { + Ok(!matches!( + db::recorded_embedder(&self.conn)?, + Some((model, _)) if model == crate::embed::FAKE_MODEL_ID + )) + } + + /// One stored chunk as a [`PassageView`], `None` if the row is gone. + fn passage(&self, chunk_id: i64) -> Result> { + Ok(db::chunk_detail(&self.conn, chunk_id)? + .map(|(heading_path, text)| PassageView { heading_path, text })) + } + + /// The notes both `a` and `b` are linked with, in either direction, by path. + fn shared_neighbors(&self, a: &str, b: &str) -> Result> { + let of = |note: &str| -> Result> { + Ok(graph::neighbors(&self.conn, note)? + .into_iter() + .map(|n| n.other) + .collect()) + }; + Ok(of(a)?.intersection(&of(b)?).cloned().collect()) + } + /// Commit a typed connection `src --type--> dst` (`b2 link`, flow ③): append a /// typed-link string to the **source note's frontmatter `b2_relations:`** — never /// the body (ADR-0010) — and re-project it as an `origin='frontmatter'` active diff --git a/crates/b2-core/tests/common/mod.rs b/crates/b2-core/tests/common/mod.rs index 16bfdef..b550b3a 100644 --- a/crates/b2-core/tests/common/mod.rs +++ b/crates/b2-core/tests/common/mod.rs @@ -7,10 +7,11 @@ //! link `tracing-subscriber` to serve two that already need their own binary anyway. #![allow(dead_code)] -use b2_core::embed::FakeEmbedder; +use b2_core::embed::{Embedder, FakeEmbedder}; use b2_core::ingest::ingest_vault; use b2_core::open; use b2_core::vault::Vault; +use b2_core::Result; use rusqlite::Connection; use std::fs; use std::path::{Path, PathBuf}; @@ -92,3 +93,92 @@ pub fn ingest_golden(dir: &Path, embedder: &FakeEmbedder) -> Connection { ingest_vault(&conn, &vault, embedder).unwrap(); conn } + +// --- a geometric embedder, for the suites that need real distances -------------- +// +// The fake embedder's hash vectors have no geometry, so discovery's statistics can't be +// exercised on them. These hand-placed vectors can. Shared by `discover_surfacing.rs` and +// `explain_similar.rs`. + +/// Hand-placed unit vectors keyed by a `VEC:` marker in the chunk text. +/// Dim 4: axis 0 is the "anchor topic", axis 2 the "noise topic"; small designed +/// offsets on axes 1/3 give the noise cloud a nonzero, controlled variance. +pub struct GeometricEmbedder; + +impl GeometricEmbedder { + pub fn vector_for(text: &str) -> Vec { + let tag_start = text.find("VEC:").map(|i| i + 4); + let tag: String = tag_start + .map(|s| { + text[s..] + .chars() + .take_while(|c| c.is_ascii_alphanumeric()) + .collect() + }) + .unwrap_or_default(); + let v: Vec = match tag.as_str() { + // The anchor and its one true mate: nearly parallel. + "ANCHOR" => vec![1.0, 0.0, 0.0, 0.0], + "MATE" => vec![0.995, 0.0998, 0.0, 0.0], + // A diffuse anchor living inside the noise cloud itself. + "DIFFUSE" => vec![0.0, 0.0, 1.0, 0.0], + // The buried-gem pair (see `a_buried_gem_outranks_and_is_served`): + // `MID` is one middling chunk, nearer the anchor than a split note's + // *centroid* but further than that note's best *chunk* (`NEAR`, whose + // other half `FAR` drags the centroid away). + "MID" => vec![0.8, 0.6, 0.0, 0.0], + "NEAR" => vec![0.995, 0.0998, 0.0, 0.0], + "FAR" => vec![0.0, 0.0, 1.0, 0.0], + // The noise cloud: all near axis 2, fanned evenly on axis 3 so the + // diffuse anchor sees one smooth spread of distances (max-z ≈ 1.6 + // for 13 evenly spaced values — under the retired leader gate by + // construction, which is what made that anchor's pane dark). + t if t.starts_with('N') => { + let i: f32 = t[1..].parse().unwrap_or(0.0); + vec![0.0, 0.0, 1.0, 0.05 + i * 0.03] + } + _ => vec![0.0, 1.0, 0.0, 0.0], + }; + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + v.into_iter().map(|x| x / norm).collect() + } +} + +impl Embedder for GeometricEmbedder { + fn model_id(&self) -> &str { + "test-geometric-v1" + } + fn dim(&self) -> usize { + 4 + } + fn embed(&self, text: &str) -> Result> { + Ok(Self::vector_for(text)) + } +} + +/// One geometric note: a single passage carrying the `VEC:` marker +/// [`GeometricEmbedder`] places. +pub fn write_geometric_note(vault: &Path, name: &str, tag: &str) { + fs::write( + vault.join(name), + format!("---\ntype: note\ntitle: {name}\n---\ntopic marker VEC:{tag} body.\n"), + ) + .unwrap(); +} + +/// A note long enough to chunk in two, each half carrying its own `VEC:` tag. +/// ~1400 chars per half against the 450-token (≈1800-char) target: short enough +/// that the two halves don't make a third chunk, long enough that the H2 between +/// them is inside the backscan and becomes the boundary. The 15% overlap re-shares +/// only filler, leaving each chunk's *first* `VEC:` its own. +pub fn write_split_note(vault: &Path, name: &str, first: &str, second: &str) { + let filler = "alpha beta gamma delta epsilon zeta eta theta iota kappa. ".repeat(24); + fs::write( + vault.join(name), + format!( + "---\ntype: note\ntitle: {name}\n---\nmarker VEC:{first} {filler}\n\n\ + ## Second section\n\nmarker VEC:{second} {filler}\n" + ), + ) + .unwrap(); +} diff --git a/crates/b2-core/tests/discover_surfacing.rs b/crates/b2-core/tests/discover_surfacing.rs index 0527f5b..739613b 100644 --- a/crates/b2-core/tests/discover_surfacing.rs +++ b/crates/b2-core/tests/discover_surfacing.rs @@ -16,82 +16,19 @@ mod common; +use common::{write_geometric_note as write_note, write_split_note, GeometricEmbedder}; + use b2_core::discover; -use b2_core::embed::{Embedder, FakeEmbedder}; +use b2_core::embed::FakeEmbedder; use b2_core::ingest::ingest_vault; use b2_core::open; use b2_core::vault::Vault; -use b2_core::Result; use rusqlite::Connection; use std::fs; use std::path::Path; -/// Hand-placed unit vectors keyed by a `VEC:` marker in the chunk text. -/// Dim 4: axis 0 is the "anchor topic", axis 2 the "noise topic"; small designed -/// offsets on axes 1/3 give the noise cloud a nonzero, controlled variance. -struct GeometricEmbedder; - -impl GeometricEmbedder { - fn vector_for(text: &str) -> Vec { - let tag_start = text.find("VEC:").map(|i| i + 4); - let tag: String = tag_start - .map(|s| { - text[s..] - .chars() - .take_while(|c| c.is_ascii_alphanumeric()) - .collect() - }) - .unwrap_or_default(); - let v: Vec = match tag.as_str() { - // The anchor and its one true mate: nearly parallel. - "ANCHOR" => vec![1.0, 0.0, 0.0, 0.0], - "MATE" => vec![0.995, 0.0998, 0.0, 0.0], - // A diffuse anchor living inside the noise cloud itself. - "DIFFUSE" => vec![0.0, 0.0, 1.0, 0.0], - // The buried-gem pair (see `a_buried_gem_outranks_and_is_served`): - // `MID` is one middling chunk, nearer the anchor than a split note's - // *centroid* but further than that note's best *chunk* (`NEAR`, whose - // other half `FAR` drags the centroid away). - "MID" => vec![0.8, 0.6, 0.0, 0.0], - "NEAR" => vec![0.995, 0.0998, 0.0, 0.0], - "FAR" => vec![0.0, 0.0, 1.0, 0.0], - // The noise cloud: all near axis 2, fanned evenly on axis 3 so the - // diffuse anchor sees one smooth spread of distances (max-z ≈ 1.6 - // for 13 evenly spaced values — under the retired leader gate by - // construction, which is what made that anchor's pane dark). - t if t.starts_with('N') => { - let i: f32 = t[1..].parse().unwrap_or(0.0); - vec![0.0, 0.0, 1.0, 0.05 + i * 0.03] - } - _ => vec![0.0, 1.0, 0.0, 0.0], - }; - let norm = v.iter().map(|x| x * x).sum::().sqrt(); - v.into_iter().map(|x| x / norm).collect() - } -} - -impl Embedder for GeometricEmbedder { - fn model_id(&self) -> &str { - "test-geometric-v1" - } - fn dim(&self) -> usize { - 4 - } - fn embed(&self, text: &str) -> Result> { - Ok(Self::vector_for(text)) - } -} - const NOISE_NOTES: usize = 13; -fn write_note(vault: &Path, name: &str, tag: &str) { - fs::write( - vault.join(name), - format!("---\ntype: note\ntitle: {name}\n---\ntopic marker VEC:{tag} body.\n"), - ) - .unwrap(); -} - /// anchor + mate + a 13-note noise cloud + a diffuse anchor inside the cloud: /// 16 notes, so every anchor's candidate pool (15) clears STATS_MIN_POPULATION /// and the z statistics exist to travel. @@ -203,23 +140,6 @@ fn ungraded_serving_changes_banding_never_membership() { ); } -/// A note long enough to chunk in two, each half carrying its own `VEC:` tag. -/// ~1400 chars per half against the 450-token (≈1800-char) target: short enough -/// that the two halves don't make a third chunk, long enough that the H2 between -/// them is inside the backscan and becomes the boundary. The 15% overlap re-shares -/// only filler, leaving each chunk's *first* `VEC:` its own. -fn write_split_note(vault: &Path, name: &str, first: &str, second: &str) { - let filler = "alpha beta gamma delta epsilon zeta eta theta iota kappa. ".repeat(24); - fs::write( - vault.join(name), - format!( - "---\ntype: note\ntitle: {name}\n---\nmarker VEC:{first} {filler}\n\n\ - ## Second section\n\nmarker VEC:{second} {filler}\n" - ), - ) - .unwrap(); -} - #[test] fn a_buried_gem_outranks_and_is_served() { // The multi-topic shape the GH #192 reorder exists for: `split.md` holds a passage diff --git a/crates/b2-core/tests/explain_similar.rs b/crates/b2-core/tests/explain_similar.rs new file mode 100644 index 0000000..cd680d2 --- /dev/null +++ b/crates/b2-core/tests/explain_similar.rs @@ -0,0 +1,347 @@ +//! **Explain** for a *Similar & unlinked* card (GH #236): `Vault::explain_similar`, the +//! model-free read behind the Compare view. It must describe the row the surface served, +//! from the same numbers that ranked it, so the explanation can never disagree with the +//! list: the same rank, the same z, the same winning passage. And it must answer for any +//! note, not only a served one: a linked note, one past the list, one not embedded yet, +//! each says why it is not a card. +//! +//! Distances need geometry, so most of these run on the hand-placed +//! [`GeometricEmbedder`]; the fake embedder is used only to pin that a hash space is +//! never graded. + +mod common; + +use b2_core::embed::FakeEmbedder; +use b2_core::vault::{SimilarStanding, Vault}; +use b2_core::Error; +use common::{write_geometric_note, write_split_note, GeometricEmbedder}; +use std::fs; +use std::path::{Path, PathBuf}; + +const NOISE_NOTES: usize = 13; + +/// anchor + mate + diffuse + a 13-note noise cloud: every anchor's pool (15) is big +/// enough to grade. +fn geometric_vault(dir: &Path) -> (Vault, PathBuf) { + let root = dir.join("vault"); + fs::create_dir_all(&root).unwrap(); + write_geometric_note(&root, "anchor.md", "ANCHOR"); + write_geometric_note(&root, "mate.md", "MATE"); + write_geometric_note(&root, "diffuse.md", "DIFFUSE"); + for i in 0..NOISE_NOTES { + write_geometric_note(&root, &format!("noise{i}.md"), &format!("N{i}")); + } + let v = Vault::open_with_embedder(&root, Box::new(GeometricEmbedder)).unwrap(); + v.reindex().unwrap(); + (v, root) +} + +fn ranked(s: &SimilarStanding) -> (usize, usize, bool) { + match s { + SimilarStanding::Ranked { rank, of, served } => (*rank, *of, *served), + other => panic!("expected a ranked standing, got {other:?}"), + } +} + +#[test] +fn every_served_row_is_explained_with_its_own_rank_z_and_evidence() { + let tmp = tempfile::TempDir::new().unwrap(); + let (v, _) = geometric_vault(tmp.path()); + let served = v.similar("anchor.md", 10).unwrap(); + assert_eq!(served.len(), 10); + for (i, row) in served.iter().enumerate() { + let ex = v.explain_similar("anchor.md", &row.path, 10).unwrap(); + let (rank, of, is_served) = ranked(&ex.standing); + assert_eq!(rank, i + 1, "{}: the rank the card was shown at", row.path); + assert_eq!(of, 15, "the whole scored pool: every unlinked note"); + assert!(is_served, "a top-10 row at limit 10 is served"); + assert_eq!(ex.z, row.z, "{}: the same z as the card's band", row.path); + let best = ex.pairs.first().expect("an embedded pair has passages"); + assert_eq!( + best.z, row.z, + "the best pair is the one the row was scored on, so it carries the row's z" + ); + assert!((best.score - row.score).abs() < 1e-9, "same score"); + assert_eq!( + row.evidence, + best.candidate + .text + .split_whitespace() + .collect::>() + .join(" "), + "the best pair's candidate passage is the card's evidence" + ); + } +} + +#[test] +fn the_population_is_every_scored_candidates_z_nearest_first() { + let tmp = tempfile::TempDir::new().unwrap(); + let (v, _) = geometric_vault(tmp.path()); + let ex = v.explain_similar("anchor.md", "mate.md", 10).unwrap(); + assert_eq!( + ex.population.len(), + 15, + "one z per scored note, served or not" + ); + assert!( + ex.population.windows(2).all(|w| w[0] >= w[1]), + "in served order, which is descending z: {:?}", + ex.population + ); + assert_eq!( + Some(ex.population[0]), + ex.z, + "the mate leads, and is marked by its z" + ); +} + +#[test] +fn a_row_past_the_list_reports_its_rank_and_is_not_served() { + let tmp = tempfile::TempDir::new().unwrap(); + let (v, _) = geometric_vault(tmp.path()); + let full = v.similar("anchor.md", 100).unwrap(); + let eighth = &full[7].path; + let ex = v.explain_similar("anchor.md", eighth, 3).unwrap(); + let (rank, of, served) = ranked(&ex.standing); + assert_eq!((rank, of), (8, 15)); + assert!(!served, "rank 8 is past a 3-card list"); + assert!( + ex.z.is_some(), + "a ranked note is graded whether shown or not" + ); +} + +#[test] +fn a_buried_gem_reports_its_whole_note_rank_beside_its_best_passage_rank() { + let tmp = tempfile::TempDir::new().unwrap(); + let root = tmp.path().join("vault"); + fs::create_dir_all(&root).unwrap(); + write_geometric_note(&root, "anchor.md", "ANCHOR"); + write_geometric_note(&root, "mid.md", "MID"); + write_split_note(&root, "split.md", "NEAR", "FAR"); + for i in 0..NOISE_NOTES { + write_geometric_note(&root, &format!("noise{i}.md"), &format!("N{i}")); + } + let v = Vault::open_with_embedder(&root, Box::new(GeometricEmbedder)).unwrap(); + v.reindex().unwrap(); + + let gem = v.explain_similar("anchor.md", "split.md", 10).unwrap(); + assert_eq!( + ranked(&gem.standing).0, + 1, + "its best passage ranks it first" + ); + assert_eq!( + gem.centroid_rank, + Some(2), + "judged as a whole note, `mid.md` is nearer: the gem is buried" + ); + assert_eq!(gem.pairs.len(), 2, "one pair per candidate passage"); + assert!( + gem.pairs[0].z > gem.pairs[1].z, + "the near half pairs far better than the far half: {:?}", + gem.pairs + ); + assert!( + gem.pairs[0].candidate.text.contains("VEC:NEAR") + && gem.pairs[1].candidate.text.contains("VEC:FAR"), + "each pair carries its passages' text, so the view can show which section matched" + ); + assert!(gem + .pairs + .iter() + .all(|p| p.anchor.text.contains("VEC:ANCHOR"))); + + let mid = v.explain_similar("anchor.md", "mid.md", 10).unwrap(); + assert_eq!(ranked(&mid.standing).0, 2); + assert_eq!(mid.centroid_rank, Some(1)); +} + +#[test] +fn a_linked_note_is_explained_as_already_linked() { + let tmp = tempfile::TempDir::new().unwrap(); + let root = tmp.path().join("vault"); + fs::create_dir_all(&root).unwrap(); + fs::write( + root.join("anchor.md"), + "---\ntitle: anchor\n---\ntopic marker VEC:ANCHOR body. See [[mate]].\n", + ) + .unwrap(); + write_geometric_note(&root, "mate.md", "MATE"); + write_geometric_note(&root, "other.md", "N1"); + let v = Vault::open_with_embedder(&root, Box::new(GeometricEmbedder)).unwrap(); + v.reindex().unwrap(); + + let ex = v.explain_similar("anchor.md", "mate.md", 10).unwrap(); + assert_eq!(ex.standing, SimilarStanding::Linked); + assert_eq!(ex.centroid_rank, None, "a linked note never enters stage 1"); + assert!( + !ex.pairs.is_empty(), + "the passages are still facts worth showing, even for a linked note" + ); +} + +#[test] +fn shared_neighbors_are_reported_with_titles() { + let tmp = tempfile::TempDir::new().unwrap(); + let root = tmp.path().join("vault"); + fs::create_dir_all(&root).unwrap(); + fs::write( + root.join("anchor.md"), + "---\ntitle: anchor\n---\ntopic marker VEC:ANCHOR body. See [[hub]].\n", + ) + .unwrap(); + fs::write( + root.join("mate.md"), + "---\ntitle: mate\n---\ntopic marker VEC:MATE body. See [[hub]].\n", + ) + .unwrap(); + write_geometric_note(&root, "hub.md", "N1"); + let v = Vault::open_with_embedder(&root, Box::new(GeometricEmbedder)).unwrap(); + v.reindex().unwrap(); + + let ex = v.explain_similar("anchor.md", "mate.md", 10).unwrap(); + assert_eq!(ranked(&ex.standing).0, 1); + assert_eq!(ex.shared_neighbors.len(), 1); + assert_eq!(ex.shared_neighbors[0].path, "hub.md"); + assert_eq!( + ex.shared_neighbors[0].title, + v.read("hub.md").unwrap().title, + "with the title the rest of the app shows" + ); +} + +#[test] +fn identical_passages_are_flagged() { + // The template case from a real vault: the same text in two notes shares one + // content-addressed vector, so the pair is a perfect match that says nothing about + // what the notes are about. The view must be able to say so. + let tmp = tempfile::TempDir::new().unwrap(); + let (v, root) = geometric_vault(tmp.path()); + fs::write( + root.join("twin.md"), + fs::read_to_string(root.join("anchor.md")).unwrap(), + ) + .unwrap(); + v.reindex().unwrap(); + + let twin = v.explain_similar("anchor.md", "twin.md", 10).unwrap(); + assert!( + twin.pairs[0].identical, + "the same passage text on both sides" + ); + let mate = v.explain_similar("anchor.md", "mate.md", 10).unwrap(); + assert!(!mate.pairs[0].identical, "near is not identical"); +} + +#[test] +fn an_unembedded_candidate_says_so() { + let tmp = tempfile::TempDir::new().unwrap(); + let (v, root) = geometric_vault(tmp.path()); + // Text no other note holds: vectors are content-addressed, so a copy of an existing + // passage would arrive already embedded. + fs::write(root.join("fresh.md"), "fresh words VEC:MATE unseen.\n").unwrap(); + v.project(false).unwrap(); // indexed for keywords, no vectors yet + + let ex = v.explain_similar("anchor.md", "fresh.md", 10).unwrap(); + assert_eq!(ex.standing, SimilarStanding::Unembedded); + assert!(ex.pairs.is_empty(), "no vectors, no pairs"); + assert_eq!(ex.z, None); +} + +#[test] +fn an_unembedded_anchor_says_so() { + let tmp = tempfile::TempDir::new().unwrap(); + let (v, root) = geometric_vault(tmp.path()); + fs::write(root.join("fresh.md"), "fresh words VEC:ANCHOR unseen.\n").unwrap(); + v.project(false).unwrap(); + + let ex = v.explain_similar("fresh.md", "mate.md", 10).unwrap(); + assert_eq!(ex.standing, SimilarStanding::AnchorUnembedded); + assert!(ex.population.is_empty()); +} + +#[test] +fn a_note_outside_the_stage_one_shortlist_says_so() { + // Stage 1 keeps max(limit × 20, 200) notes by centroid. Past that, a note is never + // scored at all, which is a different answer from "ranked too low". + let tmp = tempfile::TempDir::new().unwrap(); + let root = tmp.path().join("vault"); + fs::create_dir_all(&root).unwrap(); + write_geometric_note(&root, "anchor.md", "DIFFUSE"); + for i in 0..205 { + // The N-tagged notes fan away from the DIFFUSE axis as `i` grows, so their + // whole-note order is their index: `n204.md` is last, at rank 205. + write_geometric_note(&root, &format!("n{i:03}.md"), &format!("N{i}")); + } + let v = Vault::open_with_embedder(&root, Box::new(GeometricEmbedder)).unwrap(); + v.reindex().unwrap(); + + let far = v.explain_similar("anchor.md", "n204.md", 1).unwrap(); + assert_eq!( + far.standing, + SimilarStanding::NotShortlisted { shortlist: 200 } + ); + assert_eq!( + far.centroid_rank, + Some(205), + "its whole-note rank is still known" + ); + assert!(far.z.is_none(), "never scored, so never graded"); + assert!(!far.pairs.is_empty(), "its passages can still be compared"); +} + +#[test] +fn a_fake_space_is_explained_but_never_graded() { + let tmp = tempfile::TempDir::new().unwrap(); + let root = tmp.path().join("vault"); + fs::create_dir_all(&root).unwrap(); + write_geometric_note(&root, "anchor.md", "ANCHOR"); + for i in 0..NOISE_NOTES { + write_geometric_note(&root, &format!("noise{i}.md"), &format!("N{i}")); + } + let v = Vault::open_with_embedder(&root, Box::new(FakeEmbedder::new(64))).unwrap(); + v.reindex().unwrap(); + + let first = &v.similar("anchor.md", 10).unwrap()[0].path; + let ex = v.explain_similar("anchor.md", first, 10).unwrap(); + assert_eq!(ranked(&ex.standing).0, 1, "ranked all the same"); + assert!(ex.z.is_none() && ex.population.is_empty()); + assert!( + ex.pairs.iter().all(|p| p.z.is_none()), + "no pair is graded either" + ); +} + +#[test] +fn a_note_against_itself_and_unknown_notes() { + let tmp = tempfile::TempDir::new().unwrap(); + let (v, _) = geometric_vault(tmp.path()); + let same = v.explain_similar("anchor.md", "anchor.md", 10).unwrap(); + assert_eq!(same.standing, SimilarStanding::SameNote); + assert!(same.pairs.is_empty()); + + assert!(matches!( + v.explain_similar("nope.md", "mate.md", 10), + Err(Error::NoteNotFound(_)) + )); + assert!(matches!( + v.explain_similar("anchor.md", "nope.md", 10), + Err(Error::NoteNotFound(_)) + )); +} + +#[test] +fn the_view_names_both_notes() { + let tmp = tempfile::TempDir::new().unwrap(); + let (v, _) = geometric_vault(tmp.path()); + let ex = v.explain_similar("anchor", "mate", 10).unwrap(); + assert_eq!( + ex.anchor.path, "anchor.md", + "refs resolve like every other command" + ); + assert_eq!(ex.candidate.path, "mate.md"); + assert_eq!(ex.candidate.title, v.read("mate.md").unwrap().title); + assert_eq!(ex.limit, 10); +} diff --git a/crates/b2-desktop/src/commands.rs b/crates/b2-desktop/src/commands.rs index 0f2fae7..7331120 100644 --- a/crates/b2-desktop/src/commands.rs +++ b/crates/b2-desktop/src/commands.rs @@ -23,7 +23,7 @@ use b2_core::vault::{ AnswerView, DeleteReport, DirCreateReport, DirDeleteReport, DirMoveReport, EmbedReport, ExplainView, ImportReport, LinkReport, MoveReport, NeighborView, NoteSummary, NoteView, ProjectReport, ResourceDeleteReport, ResourceExplainView, ResourceMoveReport, ResourceSummary, - SearchEvidenceView, SimilarView, Vault, WriteReport, + SearchEvidenceView, SimilarExplainView, SimilarView, Vault, WriteReport, }; use b2_embed::{EmbedConfig, ModelChoice}; use b2_llm::ChatSetup; @@ -391,6 +391,20 @@ pub fn similar( Ok(vault.similar(¬e, limit)?) } +/// **Explain** one Similar card (GH #236): the model-free Compare view's data, the GUI +/// sibling of `b2 similar --explain`. `limit` is the list length the side pane asked +/// `similar` for, so the rank is the card's. +#[tauri::command(async)] +pub fn explain_similar( + state: State<'_, AppState>, + anchor: String, + candidate: String, + limit: usize, +) -> Result { + let vault = open_read(state.inner())?; + Ok(vault.explain_similar(&anchor, &candidate, limit)?) +} + /// Hybrid search **with its evidence reading** (invariants.md D2, GH #202) — the /// GUI sibling of `b2 search`, which changed to the same façade op in the same /// change. The rows are `search`'s, in `search`'s order; what rides beside them diff --git a/crates/b2-desktop/src/main.rs b/crates/b2-desktop/src/main.rs index dcc8ae3..f306bdf 100644 --- a/crates/b2-desktop/src/main.rs +++ b/crates/b2-desktop/src/main.rs @@ -467,6 +467,7 @@ fn main() { commands::delete_resource, commands::delete_dir, commands::similar, + commands::explain_similar, commands::search, commands::neighbors, commands::explain, diff --git a/docs/architecture.md b/docs/architecture.md index 505ecc3..4bc1133 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -88,7 +88,9 @@ an error. Details: [index-engine.md §4](index-engine.md). **Flow ③: similar, then link** (`discover.rs`, `note.rs`). `b2 similar ` surfaces the nearest notes you haven't linked yet: a two-stage scan over stored vectors, no model call, no network. The machine finds candidates; you supply the judgment and the type (ADR-0009). The -ranked list is always served; `limit` is a cap, not a promise (D1, ADR-0014). Committing is +ranked list is always served; `limit` is a cap, not a promise (D1, ADR-0014). `b2 similar +--explain` (the desktop's **Explain**) reads one note's place in the same computation, so an +explanation can never disagree with the card it explains. Committing is `b2 link`, which appends one typed-link line to the source note's frontmatter and re-projects it, or a `[[link]]` you write in the body yourself. Details: [index-engine.md §3 and §4](index-engine.md). diff --git a/docs/quickstart.md b/docs/quickstart.md index 08073c9..bc23d5f 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -269,6 +269,19 @@ related, and the top of the list is still the right answer, so B2 serves the ran leaves the judgment to you. An empty list (at any nonzero `--limit`) means only that there was nothing to compare, not that nothing relates. +To see why one note is on the list, explain it against the note you asked about: + +``` +$ b2 similar notes/spaced-repetition --explain notes/memory-palace +``` + +It prints where the note stands (its card number, or why it isn't a card: already linked, +not embedded yet, ranked past the list), its rank judged as a whole note beside its rank by +its best passage, the notes both already link with, and every passage pair behind the +match, graded on the same scale as the strength dots. It reads the same numbers the list was +ranked on, so it can't disagree with it, and it calls no model. `--json` hands an agent the +whole view. + You are the precision gate: B2 finds the candidates; you supply the judgment and the type. There is no review queue and nothing "inert until accepted". A connection exists only once you author it. @@ -394,8 +407,9 @@ What the window adds over the terminal: not a command you remember to run: always the ranked nearest, with a strength band grading each candidate within the list where a statistic exists (a vault under a dozen candidates is served ungraded, and says so), and a one-click typed Link that writes the same - `b2_relations:` line `b2 link` does. There is also a graph view of the connections around - the open note. + `b2_relations:` line `b2 link` does. A card's **Explain** opens a compare view in the + centre (the `--explain` read, no model), and **Why?** asks chat to explain it with + citations. There is also a graph view of the connections around the open note. - **Grounded chat is a pane (⌘J):** the same cited, streamed answers as `b2 ask`, with Esc to stop a stream. The model endpoint and its Keychain-held key live in Settings. - **An editor.** CodeMirror 6 with live-preview Markdown, autosave, syntax-highlighted code @@ -427,7 +441,7 @@ your notes. Nothing locks you in. | `b2 reindex` | Re-project the vault; incremental. `--dry-run`, `--force`, `--cancel` (stop a run backgrounded with `&`; `b2 status` names its pid) | embeds | | `b2 status` | Embedding coverage (is semantic ranking live?) and whether a reindex is running | no | | `b2 search ` | Hybrid keyword + semantic search. `--limit N`; `--exclude ` subtracts already-inspected notes (for agent loops) | embeds query | -| `b2 similar ` | Surface the semantically nearest notes you haven't linked yet: the ranked list, always. `--limit N` | reads vectors | +| `b2 similar ` | Surface the semantically nearest notes you haven't linked yet: the ranked list, always. `--limit N`; `--explain ` explains one note against that list instead (where it stands, and the passage pairs behind it) | reads vectors | | `b2 link ` | Commit a typed relation into the source note's frontmatter. `--type` (references/supports/contradicts), `--explanation` | re-embeds note | | `b2 neighbors ` | List a note's links, in and out; flags ones that resolve to nothing | no | | `b2 explain ` | Every connection with its provenance and "why"; flags orphans | no | diff --git a/docs/search-and-similarity.md b/docs/search-and-similarity.md index 368c73e..3237f24 100644 --- a/docs/search-and-similarity.md +++ b/docs/search-and-similarity.md @@ -187,6 +187,15 @@ connected. Anything one link away from the anchor is excluded, because you alrea it. Notes two links away stay in, since a related-but-not-directly-linked note is exactly the connection worth finding. +**Every card can be explained.** The card's **Explain** (or `b2 similar --explain +`) shows the evidence the ranking used, read from the same computation, so it can +never disagree with the list: where the note stands among everything compared, its rank as +a whole note beside its rank by its best passage (far apart means a buried gem), and every +one of its passages paired with your note's nearest, each graded on the dots' scale. It +also flags the one case where a high rank means little: the best pair is *identical text*, +such as a daily-note template shared by both notes. No model is involved; **Why?** is the +chat model's account, and Explain is how you check it. + **B2 suggests; you decide.** Nothing in this panel changes your notes. A connection exists only when you author it, with `b2 link` or by dragging a card into your note in the app. There is no suggestion queue and no pending state: you are the precision gate. diff --git a/ui/scripts/gen-icons.ts b/ui/scripts/gen-icons.ts index dc6ed84..8f87d1d 100644 --- a/ui/scripts/gen-icons.ts +++ b/ui/scripts/gen-icons.ts @@ -73,6 +73,10 @@ const MANIFEST: Record = { // rather than faults. status: ["exclamation-triangle", "info-circle", "stars"], + // The Explain view's "?" (GH #236): the longer account of a figure, one click away + // rather than printed under it. + help: ["question-circle"], + // A connection's direction, on the discovery card that names it. edges: ["arrow-left", "arrow-right"], }; diff --git a/ui/src/api.ts b/ui/src/api.ts index e149ac0..74a573d 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -33,6 +33,7 @@ import type { ResourceMoveReport, ResourceSummary, SearchEvidenceView, + SimilarExplainView, SimilarView, VaultInfo, WriteReport, @@ -149,6 +150,12 @@ export const api = { similar: (note: string, limit = 10): Promise => invoke("similar", { note, limit }), + /** **Explain** one Similar card (GH #236): where `candidate` stands in `anchor`'s list + * and the passage pairs behind it. Model-free; `limit` is the list length the pane + * asked `similar` for, so the rank is the card's. */ + explainSimilar: (anchor: string, candidate: string, limit: number): Promise => + invoke("explain_similar", { anchor, candidate, limit }), + /** Hybrid keyword+semantic search across the vault, **with its evidence * reading** (invariants.md D2, GH #202): the same rows in the same order, plus * the query-level verdict that says whether the vault holds evidence for the diff --git a/ui/src/explain.test.ts b/ui/src/explain.test.ts new file mode 100644 index 0000000..d0a29b0 --- /dev/null +++ b/ui/src/explain.test.ts @@ -0,0 +1,169 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + explainSummary, + fieldCaption, + fieldHelp, + fieldStrip, + standingText, + wholeNoteText, +} from "./explain.ts"; +import type { PassagePairView, SimilarExplainView } from "./types.ts"; + +function pair(z: number | null, identical = false): PassagePairView { + return { + anchor: { heading_path: null, text: "a" }, + candidate: { heading_path: null, text: identical ? "a" : "b" }, + score: -0.5, + z, + identical, + }; +} + +function view(over: Partial = {}): SimilarExplainView { + return { + anchor: { path: "a.md", title: "A" }, + candidate: { path: "b.md", title: "B" }, + limit: 10, + standing: { kind: "ranked", rank: 2, of: 200, served: true }, + z: 2.1, + centroid_rank: 2, + population: [3, 2.1, 1, 0, -1], + pairs: [pair(2.1)], + shared_neighbors: [], + ...over, + }; +} + +test("explain: identical text wins over every other reading", () => { + const s = explainSummary(view({ pairs: [pair(3, true), pair(2.9)] })); + assert.equal(s?.label, "Identical text"); +}); + +test("explain: an ungraded field gets no label, only facts", () => { + assert.equal(explainSummary(view({ pairs: [pair(null), pair(null)] })), null); + assert.equal(explainSummary(view({ pairs: [] })), null); +}); + +test("explain: the label counts passages at the ●●○ landmark", () => { + assert.equal(explainSummary(view({ pairs: [pair(1.9), pair(1)] }))?.label, "No passage stands out"); + assert.equal(explainSummary(view({ pairs: [pair(2.0)] }))?.label, "A clear match"); + assert.equal( + explainSummary(view({ pairs: [pair(3.2), pair(1.1), pair(0.2)] }))?.label, + "One section matches", + ); + assert.equal( + explainSummary(view({ pairs: [pair(3.2), pair(2.0), pair(0.2), pair(0)] }))?.label, + "Broadly similar", + ); + assert.equal( + explainSummary(view({ pairs: [pair(3.2), pair(2.0), pair(0.2), pair(0), pair(-1)] }))?.label, + "A few sections match", + ); + // The landmark is inclusive, as the band's is. + assert.equal(explainSummary(view({ pairs: [pair(1.96), pair(0)] }))?.label, "One section matches"); +}); + +test("explain: the standing says why a note is, or isn't, a card", () => { + assert.equal(standingText(view()), "Card #2 of the 10 shown · 200 notes compared"); + assert.equal( + standingText(view({ standing: { kind: "ranked", rank: 34, of: 200, served: false } })), + "Ranked #34 of 200 compared, past the 10 shown", + ); + assert.match(standingText(view({ standing: { kind: "linked" } })), /Already linked/); + assert.match(standingText(view({ standing: { kind: "unembedded" } })), /Not embedded/); + assert.match( + standingText(view({ standing: { kind: "not_shortlisted", shortlist: 200 }, centroid_rank: 412 })), + /#412, past the first 200/, + ); +}); + +test("explain: whole-note rank is pointed out only when it differs", () => { + assert.equal(wholeNoteText(view()), null); + assert.equal( + wholeNoteText(view({ centroid_rank: 23, standing: { kind: "ranked", rank: 1, of: 200, served: true } })), + "Judged as a whole note it ranks #23. Its best passage ranks it #1.", + ); + assert.equal(wholeNoteText(view({ standing: { kind: "linked" }, centroid_rank: null })), null); +}); + +test("explain: the strip always spans both landmarks, so a compressed field reads as one", () => { + const s = fieldStrip([1.2, 1.1, 1.0, 0.9], 1.2); + assert.ok(s); + assert.ok(s.strong < 1 && s.clear < s.strong, "both marks are on the strip"); + assert.ok(s.dots.every((d) => d < s.clear), "a field with nothing clear sits left of ●●○"); + assert.equal(s.marker, s.dots[0]); + assert.ok([...s.dots, s.marker ?? 0, s.clear, s.strong].every((x) => x >= 0 && x <= 1)); +}); + +test("explain: a wide field stretches the strip instead of spilling past it", () => { + const s = fieldStrip([6.5, 1, -2.5], 6.5); + assert.ok(s); + assert.equal(s.marker, 1); + assert.equal(Math.min(...s.dots), 0); +}); + +test("explain: no population, no strip", () => { + assert.equal(fieldStrip([], null), null); +}); + +test("explain: the caption names whose strip it is and counts each band once", () => { + assert.equal( + fieldCaption([3, 2.6, 2.0, 1.0, 0], "12 Factor App"), + "The 5 notes closest to 12 Factor App · 2 at ●●● · 1 at ●●○ · 2 below", + ); +}); + +test("explain: the ? names both notes and says what the dashed lines are", () => { + const help = fieldHelp( + view({ + anchor: { path: "12.md", title: "12 Factor App" }, + candidate: { path: "ab.md", title: "Architecture bible" }, + }), + ).join(" "); + assert.match(help, /closest to 12 Factor App/); + assert.match(help, /blue line is Architecture bible/); + assert.match(help, /fixed cut-offs, the same on every note/); + assert.match(help, /1\.96σ/); + assert.match(help, /2\.52σ/); + assert.match(help, /don’t decide what is listed/); + assert.match(help, /from Architecture bible’s side/); +}); + +test("explain: two notes with one title are told apart by path", () => { + const help = fieldHelp( + view({ + anchor: { path: "a/polish.md", title: "polish" }, + candidate: { path: "b/polish.md", title: "polish" }, + }), + ).join(" "); + assert.match(help, /closest to a\/polish\.md/); + assert.match(help, /blue line is b\/polish\.md/); +}); + +test("explain: an untitled note is named by its path", () => { + const help = fieldHelp(view({ anchor: { path: "a.md", title: null } })).join(" "); + assert.match(help, /closest to a\.md/); +}); + +test("explain: the strip labels whole σ steps, with 0 (the average) always on it", () => { + const s = fieldStrip([4.0, 3.1, 1.2, 0.3, -0.4, -1.7], 4.0); + assert.ok(s); + assert.deepEqual( + s.ticks.map((t) => t.label), + ["−1σ", "0σ", "1σ", "2σ", "3σ", "4σ"], + ); + const zero = s.ticks.find((t) => t.label === "0σ"); + assert.ok(zero && zero.at > 0 && zero.at < 1); + assert.ok(s.ticks.every((t, i, all) => i === 0 || t.at > all[i - 1].at), "left to right"); + assert.ok(s.ticks.every((t) => t.at >= 0 && t.at <= 1), "on the strip"); +}); + +test("explain: a very wide strip labels every other σ so the labels don't collide", () => { + const s = fieldStrip([9.5, 0, -2.2], 9.5); + assert.ok(s); + assert.deepEqual( + s.ticks.map((t) => t.label), + ["−2σ", "0σ", "2σ", "4σ", "6σ", "8σ"], + ); +}); diff --git a/ui/src/explain.ts b/ui/src/explain.ts new file mode 100644 index 0000000..731bba8 --- /dev/null +++ b/ui/src/explain.ts @@ -0,0 +1,181 @@ +// The Explain view's words and geometry (GH #236): pure functions over a +// `SimilarExplainView`, so node tests them straight off the source and render.ts only +// paints. The engine decides the facts (rank, z, pairs, standing); this module decides +// how they read. It never decides what a card *is*. +// +// Every grade here is on the strength dots' yardstick (strength.ts): a pair's z is its +// squared distance read against the same population the card's z is, so "a passage pair +// at ●●○" means "as near as a clear candidate's best pair". No raw distance is shown, +// for the reason search-and-similarity.md §2 gives: its meaning depends on the vault. +// +// The summary label is chosen by fixed rules, in order, and grades nothing: +// +// 1. *Identical text*: the best pair is the same passage in both notes (a template or a +// copy). First, because it is the one case where a high rank says nothing about +// what the notes are about, and a real vault has dozens of them (GH #235). +// 2. No label when the field is ungraded: no z, so no landmark to count against. +// 3. Count the candidate's passages whose pair reaches the ●●○ landmark (CLEAR_Z): +// none → *No passage stands out*; its only passage → *A clear match*; exactly one +// of several → *One section matches*; at least half → *Broadly similar*; +// otherwise → *A few sections match*. + +import type { SimilarExplainView } from "./types.ts"; +import { CLEAR_Z, STRONG_Z } from "./strength.ts"; + +/** How many passage pairs the view shows before *Show all*. */ +export const EXPLAIN_PAIRS_SHOWN = 5; + +export interface ExplainSummary { + label: string; + detail: string; +} + +const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; + +/** The one-line reading of the pairs, or null when there is nothing to say honestly. */ +export function explainSummary(v: SimilarExplainView): ExplainSummary | null { + const pairs = v.pairs; + if (pairs.length === 0) return null; + if (pairs[0].identical) { + return { + label: "Identical text", + detail: + "The closest match is the same passage in both notes, often a template or a copy. It says little about what the two notes are about.", + }; + } + if (pairs.some((p) => p.z === null)) return null; + const n = pairs.length; + const clear = pairs.filter((p) => (p.z as number) >= CLEAR_Z).length; + if (clear === 0) { + return { + label: "No passage stands out", + detail: `None of its ${plural(n, "passage")} reaches the ●●○ mark. It is here because it is among the nearest notes, not because one passage is a clear match.`, + }; + } + if (n === 1) { + return { label: "A clear match", detail: "Its only passage matches clearly." }; + } + if (clear === 1) { + return { + label: "One section matches", + detail: `1 of its ${n} passages matches clearly. The rest of the note is about other things.`, + }; + } + return { + label: clear * 2 >= n ? "Broadly similar" : "A few sections match", + detail: `${clear} of its ${n} passages match clearly.`, + }; +} + +/** Where the note stands, in words: why it is a card, or why it is not. */ +export function standingText(v: SimilarExplainView): string { + const s = v.standing; + switch (s.kind) { + case "ranked": + return s.served + ? `Card #${s.rank} of the ${v.limit} shown · ${s.of} notes compared` + : `Ranked #${s.rank} of ${s.of} compared, past the ${v.limit} shown`; + case "linked": + return "Already linked, so it isn’t suggested"; + case "unembedded": + return "Not embedded yet, so it can’t be compared"; + case "anchor_unembedded": + return "This note isn’t embedded yet, so there is nothing to compare from"; + case "not_shortlisted": + return `Never compared passage by passage: as a whole note it ranks #${ + v.centroid_rank ?? "?" + }, past the first ${s.shortlist}`; + case "same_note": + return "This is the note itself"; + } +} + +/** + * Whole-note rank beside best-passage rank, when they differ: the buried-gem signal. + * Null when there is no ranked standing or the two agree (nothing to point out). + */ +export function wholeNoteText(v: SimilarExplainView): string | null { + if (v.standing.kind !== "ranked" || v.centroid_rank === null) return null; + if (v.centroid_rank === v.standing.rank) return null; + return `Judged as a whole note it ranks #${v.centroid_rank}. Its best passage ranks it #${v.standing.rank}.`; +} + +/** The where-it-sits strip, as fractions of its width in [0, 1]. */ +export interface FieldStrip { + /** Every compared note's z. */ + dots: number[]; + /** This note's z, when it has one. */ + marker: number | null; + /** The ●●○ and ●●● landmarks. */ + clear: number; + strong: number; + /** Axis labels at whole σ steps (every other step on a very wide strip). 0σ, the + * average of the notes compared, is always among them. */ + ticks: { at: number; label: string }[]; +} + +/** Past this many σ of width, label every other step so the labels don't collide. */ +const TICK_WIDE_SIGMA = 8; + +/** + * Lay the population out on one axis. The axis always spans both landmarks, so a + * compressed field (every candidate middling, GH #196) reads as a cluster left of + * the marks, not as a spread that fills the strip. Null when ungraded. + */ +export function fieldStrip(population: number[], z: number | null): FieldStrip | null { + const finite = population.filter(Number.isFinite); + if (finite.length === 0) return null; + const lo = Math.min(0, ...finite, z ?? 0); + const hi = Math.max(STRONG_Z + 0.5, ...finite, z ?? 0); + const at = (x: number) => (x - lo) / (hi - lo); + const step = hi - lo > TICK_WIDE_SIGMA ? 2 : 1; + const ticks = []; + for (let t = Math.ceil(lo / step) * step; t <= hi; t += step) { + ticks.push({ at: at(t), label: `${t < 0 ? "−" : ""}${Math.abs(t)}σ` }); + } + return { + dots: finite.map(at), + marker: z === null || !Number.isFinite(z) ? null : at(z), + clear: at(CLEAR_Z), + strong: at(STRONG_Z), + ticks, + }; +} + +/** The two notes' display names: titles, else paths, and paths for both when the titles + * are the same (a vault of `polish.md` files), because this view's whole job is to say + * which side is which. */ +export function explainNames(v: SimilarExplainView): { anchor: string; candidate: string } { + const anchor = v.anchor.title ?? v.anchor.path; + const candidate = v.candidate.title ?? v.candidate.path; + return anchor === candidate + ? { anchor: v.anchor.path, candidate: v.candidate.path } + : { anchor, candidate }; +} + +/** The strip's caption: whose strip it is, and how many compared notes reach each band. */ +export function fieldCaption(population: number[], anchor: string): string { + const n = population.length; + const strong = population.filter((z) => z >= STRONG_Z).length; + const clear = population.filter((z) => z >= CLEAR_Z).length - strong; + return `The ${plural(n, "note")} closest to ${anchor} · ${strong} at ●●● · ${clear} at ●●○ · ${ + n - strong - clear + } below`; +} + +/** The strip's longer account, behind its "?". It names both notes, because the strip + * is drawn from the anchor's side and that is the thing most easily misread: what a dot + * is, what the axis measures, what the dashed lines are (and aren't), and what not to + * read into the spread. */ +export function fieldHelp(v: SimilarExplainView): string[] { + const n = v.population.length; + const { anchor, candidate } = explainNames(v); + return [ + `This strip is drawn from ${anchor}’s side. Each dot is one of the ${n} notes closest to ${anchor}, placed by how close its best passage gets to any passage in ${anchor}. The blue line is ${candidate}.`, + `The axis is σ (standard deviations) from the average of those ${n}. 0σ is their average; further right is closer.`, + `The dashed lines are fixed cut-offs, the same on every note’s strip: ●●○ starts at ${CLEAR_Z}σ and ●●● at ${STRONG_Z}σ. They were measured on B2’s test notes, where a person marked which pairs really belong together, not on your vault.`, + "They set the dots on each card in the Similar list. They don’t decide what is listed: the closest notes are shown whatever their grade.", + `The ${n} are the closest to ${anchor} judged as whole notes, out of every note not already linked. So 0σ is the average of the closest, not of the whole vault, and every note’s strip has its own scale. The same pair can grade differently from ${candidate}’s side.`, + "Gaps between dots are ordinary spread, not groups of topics.", + ]; +} diff --git a/ui/src/icons.gen.ts b/ui/src/icons.gen.ts index ee820e8..80666ae 100644 --- a/ui/src/icons.gen.ts +++ b/ui/src/icons.gen.ts @@ -55,6 +55,7 @@ export const ICON_BODIES = { "folder2-open": "", "gear": " ", "info-circle": " ", + "question-circle": " ", "search": "", "stars": "", "x-lg": "", diff --git a/ui/src/main.ts b/ui/src/main.ts index 754b788..7d02f21 100644 --- a/ui/src/main.ts +++ b/ui/src/main.ts @@ -30,7 +30,7 @@ import { tooltips, } from "@codemirror/view"; import { api, errText, isWriteConflict } from "./api"; -import { state, type SideSection, type ThemePref, type TreeNodeRef } from "./state"; +import { state, type AppState, type SideSection, type ThemePref, type TreeNodeRef } from "./state"; import { dirChain, joinPath, normalizeName, parentDir } from "./newentry"; import { systemPath } from "./copypath"; import { bytesToBase64, importSummary, planImport } from "./importfiles"; @@ -53,7 +53,7 @@ import { visibleRows, type TreeRow, } from "./treenav"; -import { sideArrowMove, sideNavFor, sideRowIndex, sideRows } from "./sidenav"; +import { cardRowKey, sideArrowMove, sideNavFor, sideRowIndex, sideRows } from "./sidenav"; import { answerMessage, chatEmptyState, @@ -569,6 +569,7 @@ async function loadNote(ref: string, commit: (path: string) => void): Promise { function toggleGraph(): void { if (!state.current) return; // the graph anchors on an open note if (!fmEditGuard()) return; // the graph takes the pane the mini-editor holds + state.explainCard = null; // the graph and Explain share the pane; the toggle picks the graph state.graphOpen = !state.graphOpen; render(); } @@ -2252,6 +2254,55 @@ async function askWhy(candidate: { path: string; title: string | null }): Promis ); } +/** + * **Explain** — a candidate card's model-free explanation (GH #236): the centre pane + * compares the open note with `candidate`, from the same computation that ranked the + * card (`Vault::explain_similar`). The card's *Explain*, and the card menu's item (its + * keyboard half, K1). The pane paints at once with a spinner, and a read that arrives + * after the user has moved on is dropped. The keyboard lands on *Back to note*. + */ +async function openExplain(candidate: string): Promise { + const anchor = state.current; + if (!anchor) return; + // The note pane belongs to the live editor while editing (the carve-out), so the + // Compare view can't take it. Say so rather than drop the click. + if (state.editing) { + flash(`Leave edit mode (${displayKeys(["edit.toggle"])}) to see Explain.`); + return; + } + if (!fmEditGuard()) return; // the view takes the pane the mini-editor holds + const current: NonNullable = { + anchor: anchor.path, + candidate, + view: null, + error: null, + allPairs: false, + help: false, + }; + state.explainCard = current; + render(); + document.getElementById("explain-close")?.focus(); + try { + const view = await api.explainSimilar(anchor.path, candidate, SIMILAR_LIMIT); + if (state.explainCard !== current) return; // superseded or closed meanwhile + current.view = view; + } catch (e) { + if (state.explainCard !== current) return; + current.error = errText(e); + } + render(); +} + +/** Back out of the Explain view, handing the keyboard back to the card it explained. */ +function closeExplain(): void { + const ec = state.explainCard; + if (!ec) return; + state.explainCard = null; + render(); + const i = state.similar.findIndex((c) => c.path === ec.candidate); + if (i >= 0) focusSideRow(cardRowKey("similar", i, ec.candidate)); +} + /** * The shared body of a chat turn, whichever host call answers it: `shown` joins the * transcript, `call` streams tokens into the live row, and the resolved answer (or the @@ -3348,6 +3399,7 @@ function enterEdit(): void { const n = state.current; if (!n || state.editing || state.loading) return; if (!fmEditGuard()) return; // one editor at a time — resolve the drawer first + state.explainCard = null; // editing takes the pane back to the note state.editing = true; state.editConflict = false; render(); @@ -4558,6 +4610,12 @@ function wireEvents(): void { openLinkModal(path, title ?? ""); return; } + if (target.closest("[data-ctx-explain]")) { + const p = menu.path; + closeContextMenu(); + void openExplain(p); + return; + } if (target.closest("[data-ctx-why]")) { const { path, title } = menu; closeContextMenu(); @@ -4898,6 +4956,29 @@ function wireEvents(): void { return; } + // A candidate card's *Explain*, and the Explain view's own controls — before + // `data-open`, which the same card also carries. + const explain = target.closest("[data-explain]"); + if (explain) { + const p = explain.dataset.explain; + if (p) void openExplain(p); + return; + } + if (target.closest("[data-explain-close]")) { + closeExplain(); + return; + } + if (target.closest("[data-explain-help]")) { + if (state.explainCard) state.explainCard.help = !state.explainCard.help; + render(); + return; + } + if (target.closest("[data-explain-all]")) { + if (state.explainCard) state.explainCard.allPairs = !state.explainCard.allPairs; + render(); + return; + } + // A candidate card's *Why?* — before `data-open`, which the same card also carries. const why = target.closest("[data-why]"); if (why) { @@ -5518,7 +5599,12 @@ function wireEvents(): void { closeChat(); return; } - // With nothing else to dismiss, Escape backs out of the graph into reading. + // With nothing else to dismiss, Escape backs out of Explain, then the graph, + // into reading. + if (state.explainCard && !state.editing) { + closeExplain(); + return; + } if (state.graphOpen && state.current && !state.editing) toggleGraph(); return; } diff --git a/ui/src/render.test.ts b/ui/src/render.test.ts index 3be125e..f26218d 100644 --- a/ui/src/render.test.ts +++ b/ui/src/render.test.ts @@ -24,6 +24,7 @@ import type { OllamaSetup, ResourceLink, EvidencedResult, + SimilarExplainView, SimilarView, UnresolvedLink, } from "./types.ts"; @@ -1040,4 +1041,108 @@ check("an answer built with tools says so, escaped like everything a model names assert(!plain.includes("chat-tools"), "a plain ask has no tools line"); }); +// --- Explain (GH #236) ------------------------------------------------------------------- + +function explainView(over: Partial = {}): SimilarExplainView { + return { + anchor: { path: "notes/anchor.md", title: "anchor" }, + candidate: { path: "notes/ghost.md", title: "ghost" }, + limit: 10, + standing: { kind: "ranked", rank: 1, of: 20, served: true }, + z: 2.8, + centroid_rank: 7, + population: [2.8, 1.0, 0.5, 0.1, -0.2, -0.4, -0.6, -0.8, -1, -1.2, -1.4, -1.6], + pairs: [ + { + anchor: { heading_path: "Intro", text: " anchor side" }, + candidate: { heading_path: null, text: "**not markdown** candidate side" }, + score: -0.4, + z: 2.8, + identical: false, + }, + ], + shared_neighbors: [{ path: "notes/hub.md", title: "hub" }], + ...over, + }; +} + +const explaining = (over: Partial = {}) => + app({ + current: note(), + similar: [ghost()], + explainCard: { + anchor: "notes/anchor.md", + candidate: "notes/ghost.md", + view: explainView(), + error: null, + allPairs: false, + help: false, + }, + ...over, + }); + +check("a Similar card offers Explain beside Why?", () => { + const html = sidePaneHtml(app({ current: note(), similar: [ghost()] })); + assert(html.includes('data-explain="notes/ghost.md"'), "the card's Explain names its note"); + assert(html.includes('data-why="notes/ghost.md"'), "Why? stays"); + const menu = contextMenuHtml( + app({ contextMenu: { kind: "card", x: 0, y: 0, path: "notes/ghost.md", title: "ghost" } }), + ); + assert(menu.includes("data-ctx-explain"), "the card menu carries Explain: its keyboard half (K1)"); +}); + +check("Explain takes the note pane, and passages are text, never markup (E5)", () => { + const html = notePaneHtml(explaining()); + assert(html.includes("explain-view"), "the Compare view is painted"); + assert(!html.includes(" { + const html = notePaneHtml(explaining()); + for (const marker of ["data-explain-close", 'data-open="notes/ghost.md"', "data-why"]) { + assert(hasId(tagWith(html, marker)), `the Explain view's ${marker} control carries an id`); + } + const loading = notePaneHtml( + explaining({ + explainCard: { + anchor: "notes/anchor.md", + candidate: "notes/ghost.md", + view: null, + error: null, + allPairs: false, + help: false, + }, + }), + ); + assert(hasId(tagWith(loading, "data-explain-close")), "Back is there while the read runs"); +}); + +check("Explain belongs to the note it was opened on, and editing takes the pane back", () => { + const other = notePaneHtml(explaining({ current: note({ path: "notes/other.md" }) })); + assert(!other.includes("explain-view"), "another note never shows a stale explanation"); + const editing = notePaneHtml(explaining({ editing: true })); + assert(!editing.includes("explain-view"), "the editor owns the pane while editing"); +}); + +check("the strip is labelled in σ, and its longer account waits behind the ?", () => { + const closed = notePaneHtml(explaining()); + assert(closed.includes(">0σ<"), "the average is labelled on the axis"); + assert(hasId(tagWith(closed, "data-explain-help")), "the ? carries an id for focus"); + assert(tagWith(closed, "data-explain-help").includes('aria-expanded="false"'), "closed by default"); + assert(!closed.includes('id="explain-help-text"'), "the account is not painted until asked for"); + const ec = explaining().explainCard!; + const open = notePaneHtml(explaining({ explainCard: { ...ec, help: true } })); + assert(tagWith(open, "data-explain-help").includes('aria-expanded="true"'), "says it is open"); + assert(open.includes('id="explain-help-text"'), "the account is painted"); + assert(open.includes("standard deviations"), "and says what the axis is"); + assert(closed.includes("strip-key"), "the strip carries a key for its three marks"); + assert(closed.includes("where ●●○ and ●●● start"), "the dashed lines are named as cut-offs"); + assert(closed.includes("The 12 notes closest to anchor"), "the caption says whose strip it is"); +}); + console.log(`render: ${passed} checks passed`); diff --git a/ui/src/render.ts b/ui/src/render.ts index 93bcf11..9e9a8b0 100644 --- a/ui/src/render.ts +++ b/ui/src/render.ts @@ -28,6 +28,16 @@ import { RELATION_VERBS, type AppState, type SideSection } from "./state.ts"; import { allDirs, canMoveInto, renamePrefill } from "./move.ts"; import { shouldPromptEmbedInstall } from "./embedreminder.ts"; import { STRENGTH_MIN_CANDIDATES, strengthBand } from "./strength.ts"; +import { + EXPLAIN_PAIRS_SHOWN, + explainSummary, + explainNames, + fieldCaption, + fieldHelp, + fieldStrip, + standingText, + wholeNoteText, +} from "./explain.ts"; import { type ShortcutKey, shortcuts } from "./shortcuts.ts"; import { cmdShortcuts } from "./cmdhold.ts"; import { @@ -78,7 +88,7 @@ import { toolsLine, turnRowKey, } from "./chat.ts"; -import type { ChatSetup, NoteView, ResourceExplainView } from "./types.ts"; +import type { ChatSetup, NoteView, PassageView, ResourceExplainView } from "./types.ts"; import { buildScene, NODE_R, @@ -613,6 +623,7 @@ function resourceCardHtml(r: ResourceExplainView, image: string | null): string export function notePaneHtml(state: AppState): string { if (state.currentResource) return resourceCardHtml(state.currentResource, state.resourceImage); const n = state.current; + if (n && state.explainCard?.anchor === n.path && !state.editing) return explainPaneHtml(state); if (n && state.graphOpen) return graphPaneHtml(state, n); if (n) { const metaBits = [n.type, n.created].filter(Boolean).map((s) => escapeHtml(s as string)); @@ -919,11 +930,16 @@ function similarSectionHtml(state: AppState, roving: string | null): string { : `
${escapeHtml(c.path)}
${c.evidence ? `
${escapeHtml(c.evidence)}
` : ""} - +
+ + +
`; // *Why?* hands the pair to chat (main.ts's `askWhy`): the snippet above is the // passage that matched, and this is the question it raises. `tabindex="-1"` keeps @@ -1608,6 +1624,173 @@ function graphPaneHtml(state: AppState, n: NoteView): string { `; } +// --- Explain: the Compare view for one Similar card (GH #236) ------------------------ +// +// The centre pane's fourth mode: the open note against one of its Similar cards, read +// model-free from the same computation as the list (`Vault::explain_similar`), so its +// rank, grade and best passage are the card's. The words and the strip's geometry are +// explain.ts's; this is only paint. Passage text is note content, so it is escaped and +// shown as text (E5), never rendered as Markdown. Every control is a real button in tab +// order, and Escape backs out, as it does from the graph (K1). + +function explainPaneHtml(state: AppState): string { + const ec = state.explainCard; + const bar = `
+ +
`; + if (!ec) return bar; + if (ec.error !== null) { + return `
${bar}

${escapeHtml(ec.error)}

`; + } + const v = ec.view; + if (v === null) { + return `
${bar}
`; + } + const name = (x: { path: string; title: string | null }) => escapeHtml(x.title ?? x.path); + const names = explainNames(v); + const band = strengthBand(v.z); + const summary = explainSummary(v); + const whole = wholeNoteText(v); + + const strip = fieldStrip(v.population, v.z); + const W = 600; + const H = 44; + const help = strip + ? `` + : ""; + const fieldHtml = strip + ? `
+

Where it sits ${help}

+ ${ + ec.help + ? `
${fieldHelp(v) + .map((t) => `

${escapeHtml(t)}

`) + .join("")}
` + : "" + } + + + + ${strip.ticks + .map( + (t) => + ``, + ) + .join("")} + ${[strip.clear, strip.strong] + .map((x) => ``) + .join("")} + ${strip.dots + .map((x) => ``) + .join("")} + ${ + strip.marker !== null + ? `` + : "" + } + + + +

${escapeHtml( + fieldCaption(v.population, names.anchor), + )} · σ from their average

+
` + : `

Ungraded: ranked by nearness only. Grading needs ${STRENGTH_MIN_CANDIDATES} or more notes to compare against.

`; + + const passage = (who: string, p: PassageView) => + `
+
${escapeHtml(who)}${ + p.heading_path ? ` · ${escapeHtml(p.heading_path)}` : "" + }
+
${escapeHtml(p.text)}
+
`; + const shown = ec.allPairs ? v.pairs : v.pairs.slice(0, EXPLAIN_PAIRS_SHOWN); + const pairsHtml = v.pairs.length + ? `
+

Passage pairs · each of its passages, with this note’s nearest

+
    ${shown + .map( + (p, i) => `
  1. +
    #${i + 1} ${strengthHtml(p.z ?? undefined)}${ + p.identical ? `identical text` : "" + }
    +
    ${passage("This note", p.anchor)}${passage( + "Suggested", + p.candidate, + )}
    +
    2. `, + ) + .join("")}
+ ${ + v.pairs.length > EXPLAIN_PAIRS_SHOWN + ? `` + : "" + } +
` + : ""; + + const links = v.shared_neighbors.length + ? `

Both link with ${v.shared_neighbors + .map( + (s) => + ``, + ) + .join(", ")}.

` + : `

They share no linked notes.

`; + + return `
${bar} +
+
+
Explain · suggested for ${name(v.anchor)}
+

${name(v.candidate)}

+
${escapeHtml(v.candidate.path)}
+

${band ? `${strengthHtml(v.z ?? undefined)} ` : ""}${escapeHtml( + standingText(v), + )}

+ ${whole ? `

${escapeHtml(whole)}

` : ""} +
+ ${ + summary + ? `
${escapeHtml(summary.label)}. ${escapeHtml( + summary.detail, + )}
` + : "" + } + ${fieldHtml} + ${pairsHtml} +

Links

${links}
+
+ + +
+
+
`; +} + /** A cumulative-duration label from milliseconds: "3h 25m", "12m 04s", "45s", "0s". */ function formatDuration(ms: number): string { const totalSec = Math.round(ms / 1000); @@ -2403,6 +2586,7 @@ export function contextMenuHtml(state: AppState): string { items = `${contextItemHtml("data-ctx-open", "Open note", "⏎")} ${state.editing ? contextItemHtml("data-ctx-insert", "Insert link at cursor") : ""} ${contextItemHtml("data-ctx-link", "Link…")} + ${contextItemHtml("data-ctx-explain", "Explain this suggestion")} ${contextItemHtml("data-ctx-why", "Why was this suggested?")}`; } // `tabindex="-1"` on the menu itself makes the container focusable-by-script but not diff --git a/ui/src/state.ts b/ui/src/state.ts index cdd4fad..a5254ec 100644 --- a/ui/src/state.ts +++ b/ui/src/state.ts @@ -15,6 +15,7 @@ import type { ResourceLink, ResourceSummary, EvidencedResult, + SimilarExplainView, SimilarView, UnresolvedLink, } from "./types"; @@ -225,6 +226,22 @@ export interface AppState { * `unresolved`/`similar`), so toggling costs no IPC. */ graphOpen: boolean; + /** + * The Explain view (GH #236): the centre pane compares the open note with one of its + * Similar cards, model-free. Set while it is open; `view` is null while the read is in + * flight, `error` holds a failed read's message. Not sticky: any navigation, the graph + * toggle and entering edit mode close it, because it explains one card of one note. + */ + explainCard: { + anchor: string; + candidate: string; + view: SimilarExplainView | null; + error: string | null; + /** Every passage pair shown, not only the first few. */ + allPairs: boolean; + /** The strip's "?" is open: its longer account is shown. */ + help: boolean; + } | null; /** * Discovery sections the user has collapsed (foldable headers, Obsidian-style). * Sticky across notes — a viewing preference — so a collapsed section stays folded @@ -413,6 +430,7 @@ export const state: AppState = { connections: [], resourceLinks: [], graphOpen: false, + explainCard: null, collapsedSections: new Set(), collapsedCards: new Set(), contextMenu: null, diff --git a/ui/src/strength.ts b/ui/src/strength.ts index dc49190..24cdb11 100644 --- a/ui/src/strength.ts +++ b/ui/src/strength.ts @@ -65,11 +65,16 @@ export interface StrengthBand { value: string; } +/** The ●●● landmark: the labelled-mate population's upper quartile (rounded down). */ +export const STRONG_Z = 2.52; +/** The ●●○ landmark: where the corpus's labelled leaders read. */ +export const CLEAR_Z = 1.96; + export function strengthBand(z: number | undefined | null): StrengthBand | null { if (z === undefined || z === null || !Number.isFinite(z)) return null; - const [glyph, label] = z >= 2.52 + const [glyph, label] = z >= STRONG_Z ? ["●●●", "strong match"] - : z >= 1.96 + : z >= CLEAR_Z ? ["●●○", "clear match"] : ["●○○", "near match"]; const value = `${z.toFixed(1)}σ`; diff --git a/ui/src/types.ts b/ui/src/types.ts index 19ef32c..5731be1 100644 --- a/ui/src/types.ts +++ b/ui/src/types.ts @@ -286,6 +286,47 @@ export interface SimilarView { z?: number; } +/** One passage as stored in the index. */ +export interface PassageView { + heading_path: string | null; + text: string; +} + +/** A candidate passage and the anchor passage nearest to it (`Vault::explain_similar`). */ +export interface PassagePairView { + anchor: PassageView; + candidate: PassageView; + score: number; + /** The pair's z on the card's yardstick; null when the field is ungraded. */ + z: number | null; + /** Both passages hold the same text (a template, a copy). */ + identical: boolean; +} + +/** Where a note stands in an anchor's discovery field: why it is a card, or why not. */ +export type SimilarStanding = + | { kind: "same_note" } + | { kind: "anchor_unembedded" } + | { kind: "linked" } + | { kind: "unembedded" } + | { kind: "not_shortlisted"; shortlist: number } + | { kind: "ranked"; rank: number; of: number; served: boolean }; + +/** `Vault::explain_similar` — the model-free Explain view for one Similar card (GH #236). */ +export interface SimilarExplainView { + anchor: NoteSummary; + candidate: NoteSummary; + limit: number; + standing: SimilarStanding; + z: number | null; + /** Rank judged by whole-note average (stage 1), when it entered stage 1. */ + centroid_rank: number | null; + /** Every scored note's z, nearest first. Empty when ungraded. */ + population: number[]; + pairs: PassagePairView[]; + shared_neighbors: NoteSummary[]; +} + /** `Vault::search` — one hybrid-search hit. */ export interface SearchResult { path: string; diff --git a/ui/style.css b/ui/style.css index 6e4e2fa..1d0477e 100644 --- a/ui/style.css +++ b/ui/style.css @@ -3598,3 +3598,258 @@ button.move-dest:hover { .chat-setup-inline { overflow: visible; } + +/* --- Explain: the Compare view for one Similar card (GH #236) --------------------- + The centre pane's fourth mode (render.ts `explainPaneHtml`). A reading column like a + note's, under a sticky bar with the one way back. Passages are note text shown as + text, so they keep their line breaks and scroll inside a bounded box rather than + letting one 11k-token passage push the rest of the view off the page. */ +.card-actions { + display: flex; + gap: 12px; +} + +.explain-bar { + margin: 0 calc(-1 * var(--note-pad-x)) 28px; + padding: 10px var(--note-pad-x); + position: sticky; + top: 0; + z-index: 3; + display: flex; + align-items: center; + background: var(--bg); + border-bottom: 1px solid var(--border); +} +.explain-close { + font-family: inherit; + letter-spacing: 0; +} + +.explain-kicker { + color: var(--muted); + font-size: 12px; + font-weight: 600; + letter-spacing: 0.06em; + text-transform: uppercase; + margin-bottom: 6px; +} +.explain-standing { + margin: 10px 0 0; + font-size: 14px; +} +.explain-standing .card-score { + margin-right: 4px; +} + +.explain-summary { + margin: 0 0 24px; + padding: 12px 14px; + border-radius: var(--radius); + background: var(--accent-soft); + font-size: 14px; + line-height: 1.5; +} + +.explain-section { + margin: 0 0 28px; +} +.explain-section h2 { + margin: 0 0 10px; + font-size: 15px; +} +.explain-section h2 .muted { + font-weight: 400; + font-size: 13px; +} +.explain-caption { + margin: 6px 0 0; + color: var(--muted); + font-size: 13px; +} + +.explain-strip { + display: block; + width: 100%; + height: 44px; + overflow: visible; +} +.strip-axis { + stroke: var(--border); + stroke-width: 1; +} +.strip-mark { + stroke: var(--muted); + stroke-width: 1; + stroke-dasharray: 3 3; + vector-effect: non-scaling-stroke; +} +.strip-dot { + fill: var(--muted); + fill-opacity: 0.45; +} +.strip-this { + stroke: var(--accent); + stroke-width: 3; + vector-effect: non-scaling-stroke; +} +.strip-legend { + position: relative; + height: 16px; + font-size: 11px; + color: var(--muted); +} +.strip-legend span { + position: absolute; + transform: translateX(-50%); +} +.strip-bands { + height: 14px; +} +.strip-ticks { + font-variant-numeric: tabular-nums; +} +.strip-tick { + stroke: var(--muted); + stroke-width: 1; + vector-effect: non-scaling-stroke; +} + +/* The strip's key: the three marks it draws, drawn the same way, so nobody has to guess + that the dashed lines are grade cut-offs rather than notes. */ +.strip-key { + display: flex; + flex-wrap: wrap; + gap: 4px 16px; + margin-top: 6px; + font-size: 12px; + color: var(--muted); +} +.strip-key > span { + display: inline-flex; + align-items: center; + gap: 6px; +} +.key-this { + width: 3px; + height: 12px; + background: var(--accent); +} +.key-mark { + width: 0; + height: 12px; + border-left: 1px dashed var(--muted); +} +.key-dot { + width: 6px; + height: 6px; + border-radius: 50%; + background: var(--muted); + opacity: 0.6; +} + +/* The strip's "?": the longer account, one click away rather than printed under it. */ +.explain-h2-help { + display: flex; + align-items: center; + gap: 6px; +} +.explain-help-btn { + display: inline-flex; + align-items: center; + padding: 2px; + border: none; + border-radius: 50%; + background: none; + color: var(--muted); + cursor: pointer; +} +.explain-help-btn:hover, +.explain-help-btn[aria-expanded="true"] { + color: var(--accent); +} +.explain-help { + margin: 0 0 14px; + padding: 10px 14px; + border-left: 3px solid var(--accent-soft); + font-size: 13px; + line-height: 1.5; + color: var(--muted); +} +.explain-help p { + margin: 0 0 6px; +} +.explain-help p:last-child { + margin: 0; +} + +.explain-pairs { + list-style: none; + margin: 0 0 12px; + padding: 0; + display: flex; + flex-direction: column; + gap: 14px; +} +.explain-pair { + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface); + padding: 10px 12px; +} +.explain-pair-head { + display: flex; + align-items: center; + gap: 8px; + font-size: 12px; + color: var(--muted); + margin-bottom: 8px; +} +.explain-pill { + padding: 1px 6px; + border-radius: 999px; + background: var(--surface-2); + color: var(--danger); + font-weight: 600; +} +.explain-pair-body { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 12px; +} +@media (max-width: 900px) { + .explain-pair-body { + grid-template-columns: 1fr; + } +} +.explain-passage { + min-width: 0; +} +.explain-passage-head { + font-size: 12px; + color: var(--muted); + margin-bottom: 4px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.explain-who { + color: var(--text); + font-weight: 600; +} +.explain-passage-text { + font-size: 13px; + line-height: 1.5; + white-space: pre-wrap; + overflow-wrap: anywhere; + max-height: 14em; + overflow-y: auto; + padding: 8px 10px; + border-radius: 6px; + background: var(--surface-2); +} + +.explain-actions { + display: flex; + gap: 10px; + margin-bottom: 40px; +} From e3a414a18dfd040688471deb1629ff4fe511904e Mon Sep 17 00:00:00 2001 From: Sam Keen Date: Wed, 23 Sep 2026 11:15:53 -0700 Subject: [PATCH 2/2] fix: Explain's card count, ungraded pairs and --limit agreement say what is true From the PR #242 review: - The standing counts the cards the list actually shows: min(limit, scored), so a small vault no longer reads "Card #2 of the 10 shown" beside a five-card list. - `similar --explain` says once why its pairs carry no grade (too few notes, or a vault indexed without the real model) instead of printing bare pairs. - The docs qualify "can't disagree with the list": at the same `--limit`. The desktop always passes the list's own; the CLI user is told to. Co-Authored-By: Claude Opus 5.5 --- crates/b2-cli/src/main.rs | 7 +++++++ crates/b2-cli/tests/cli.rs | 6 ++++++ docs/architecture.md | 4 ++-- docs/quickstart.md | 8 +++++--- docs/search-and-similarity.md | 9 +++++---- ui/src/explain.test.ts | 5 +++++ ui/src/explain.ts | 2 +- 7 files changed, 31 insertions(+), 10 deletions(-) diff --git a/crates/b2-cli/src/main.rs b/crates/b2-cli/src/main.rs index 44f6db8..c215e86 100644 --- a/crates/b2-cli/src/main.rs +++ b/crates/b2-cli/src/main.rs @@ -1076,6 +1076,13 @@ fn cmd_explain_similar(cli: &Cli, note: &str, other: &str, limit: usize) -> Resu } if !ex.pairs.is_empty() { println!("\nPassage pairs, nearest first:"); + // Pairs carry a grade only when the list does; say why they don't, once, rather + // than print bare pairs that read as unmeasured by accident. + if ex.pairs.iter().all(|p| p.z.is_none()) { + println!( + " Ungraded: too few notes to compare against, or a vault indexed without the real model." + ); + } } for (i, p) in ex.pairs.iter().enumerate() { let grade = p.z.map(|z| format!(" z {z:.2}")).unwrap_or_default(); diff --git a/crates/b2-cli/tests/cli.rs b/crates/b2-cli/tests/cli.rs index 022a5c4..08fab2d 100644 --- a/crates/b2-cli/tests/cli.rs +++ b/crates/b2-cli/tests/cli.rs @@ -1043,6 +1043,12 @@ fn similar_explain_describes_the_served_row() { let text = stdout(&human); assert!(text.contains("#2"), "says where it stands: {text}"); assert!(text.contains("alpha"), "shows this note's passage: {text}"); + // Five notes under the fake embedder: nothing to grade against, and the output says + // so rather than printing pairs with no grade and no reason. + assert!( + text.contains("Ungraded"), + "an ungraded explanation says so: {text}" + ); } #[test] diff --git a/docs/architecture.md b/docs/architecture.md index 4bc1133..15ba484 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,8 +89,8 @@ an error. Details: [index-engine.md §4](index-engine.md). nearest notes you haven't linked yet: a two-stage scan over stored vectors, no model call, no network. The machine finds candidates; you supply the judgment and the type (ADR-0009). The ranked list is always served; `limit` is a cap, not a promise (D1, ADR-0014). `b2 similar ---explain` (the desktop's **Explain**) reads one note's place in the same computation, so an -explanation can never disagree with the card it explains. Committing is +--explain` (the desktop's **Explain**) reads one note's place in the same computation, so at +the same `limit` it can't disagree with the card; the desktop always passes the list's own. Committing is `b2 link`, which appends one typed-link line to the source note's frontmatter and re-projects it, or a `[[link]]` you write in the body yourself. Details: [index-engine.md §3 and §4](index-engine.md). diff --git a/docs/quickstart.md b/docs/quickstart.md index bc23d5f..1da1261 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -278,9 +278,11 @@ $ b2 similar notes/spaced-repetition --explain notes/memory-palace It prints where the note stands (its card number, or why it isn't a card: already linked, not embedded yet, ranked past the list), its rank judged as a whole note beside its rank by its best passage, the notes both already link with, and every passage pair behind the -match, graded on the same scale as the strength dots. It reads the same numbers the list was -ranked on, so it can't disagree with it, and it calls no model. `--json` hands an agent the -whole view. +match, graded on the same scale as the strength dots when the list is graded (a vault with +too few notes to compare, or one indexed without the real model, says "Ungraded"). It reads +the same numbers the list was ranked on, so it agrees with the list you ran at the same +`--limit`; pass the list's `--limit` when explaining one of its cards. It calls no model, and +`--json` hands an agent the whole view. You are the precision gate: B2 finds the candidates; you supply the judgment and the type. There is no review queue and nothing "inert until accepted". A connection exists only once diff --git a/docs/search-and-similarity.md b/docs/search-and-similarity.md index 3237f24..1c327f5 100644 --- a/docs/search-and-similarity.md +++ b/docs/search-and-similarity.md @@ -188,10 +188,11 @@ it. Notes two links away stay in, since a related-but-not-directly-linked note i connection worth finding. **Every card can be explained.** The card's **Explain** (or `b2 similar --explain -`) shows the evidence the ranking used, read from the same computation, so it can -never disagree with the list: where the note stands among everything compared, its rank as -a whole note beside its rank by its best passage (far apart means a buried gem), and every -one of its passages paired with your note's nearest, each graded on the dots' scale. It +`, at the same `--limit` as the list) shows the evidence the ranking used, read from +the same computation, so it can't disagree with the list: where the note stands among +everything compared, its rank as a whole note beside its rank by its best passage (far apart +means a buried gem), and every one of its passages paired with your note's nearest, each +graded on the dots' scale when the list is graded, and marked ungraded when it isn't. It also flags the one case where a high rank means little: the best pair is *identical text*, such as a daily-note template shared by both notes. No model is involved; **Why?** is the chat model's account, and Explain is how you check it. diff --git a/ui/src/explain.test.ts b/ui/src/explain.test.ts index d0a29b0..0cb7705 100644 --- a/ui/src/explain.test.ts +++ b/ui/src/explain.test.ts @@ -66,6 +66,11 @@ test("explain: the label counts passages at the ●●○ landmark", () => { test("explain: the standing says why a note is, or isn't, a card", () => { assert.equal(standingText(view()), "Card #2 of the 10 shown · 200 notes compared"); + // A small vault shows fewer cards than the limit: the count is what the list shows. + assert.equal( + standingText(view({ standing: { kind: "ranked", rank: 2, of: 5, served: true } })), + "Card #2 of the 5 shown · 5 notes compared", + ); assert.equal( standingText(view({ standing: { kind: "ranked", rank: 34, of: 200, served: false } })), "Ranked #34 of 200 compared, past the 10 shown", diff --git a/ui/src/explain.ts b/ui/src/explain.ts index 731bba8..3c96337 100644 --- a/ui/src/explain.ts +++ b/ui/src/explain.ts @@ -73,7 +73,7 @@ export function standingText(v: SimilarExplainView): string { switch (s.kind) { case "ranked": return s.served - ? `Card #${s.rank} of the ${v.limit} shown · ${s.of} notes compared` + ? `Card #${s.rank} of the ${Math.min(v.limit, s.of)} shown · ${s.of} notes compared` : `Ranked #${s.rank} of ${s.of} compared, past the ${v.limit} shown`; case "linked": return "Already linked, so it isn’t suggested";