diff --git a/README.md b/README.md index a805da2..094c1dc 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,15 @@ tree-ring integrations certify --source-root . `target/tree-ring-certification/harness/` and updates `target/tree-ring-certification/evidence-index.json`. Pass, fail, and skip states are evidence records, not broad compatibility claims. +- `recall-quality` writes non-private recall diagnostics under + `target/tree-ring-certification/recall-quality/default-fixture-v1.json` + and merges the result into `evidence-index.json`. It uses deterministic + safe fixture memories, records returned ids, rank positions, score factors, + and latency, and marks each query as `pass`, `fail`, or `needs_review`. + +```bash +tree-ring recall-quality --source-root . +``` It looks for local markers for DOX, Revolve, Codex, Claude Code, Agent Zero/A0, Goose, OpenCode, Hermes, and Pi, then suggests next steps without editing those @@ -381,8 +390,9 @@ for the synthetic workload. `scripts/certify-tree-ring.sh` runs the fuller local certification suite: formatting, tests, Clippy, release build, isolated project/global installs, CLI -JSON smokes, DOX/Revolve adapter smokes, integration-scan origin checks, import -throughput, and 10k/30k recall timing. It writes +JSON smokes, DOX/Revolve adapter smokes, integration-scan origin checks, +install-size, recall-speed, recall-quality diagnostics, CLI, adapter, and +harness checks, import throughput, and 10k/30k recall timing. It writes `target/tree-ring-certification/summary.md` and `metrics.json`. Latest local certification run, generated at `2026-07-09T02:42:24Z` with diff --git a/crates/tree-ring-memory-cli/src/evidence.rs b/crates/tree-ring-memory-cli/src/evidence.rs index a6a068e..0f68cd7 100644 --- a/crates/tree-ring-memory-cli/src/evidence.rs +++ b/crates/tree-ring-memory-cli/src/evidence.rs @@ -12,6 +12,7 @@ pub enum EvidenceStatus { Skip, Missing, Stale, + NeedsReview, Error, } @@ -23,6 +24,7 @@ impl EvidenceStatus { Self::Skip => "skip", Self::Missing => "missing", Self::Stale => "stale", + Self::NeedsReview => "needs_review", Self::Error => "error", } } @@ -70,12 +72,39 @@ pub struct CertificationEvidence { pub agent_zero_note: Option, } +#[derive(Debug, Clone, PartialEq)] +pub struct RecallQualityEvidence { + pub status: EvidenceStatus, + pub generated_at: String, + pub record_path: PathBuf, + pub query_set_id: String, + pub query_count: u64, + pub pass_count: u64, + pub fail_count: u64, + pub needs_review_count: u64, + pub avg_latency_ms: Option, + pub max_latency_ms: Option, + pub queries: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RecallQualityQueryEvidence { + pub query_id: String, + pub query: String, + pub status: String, + pub expected_top_id: Option, + pub expected_rank: Option, + pub latency_ms: Option, + pub returned_ids: Vec, +} + #[derive(Debug, Clone, PartialEq)] pub struct EvidenceSnapshot { pub root: PathBuf, pub index_path: PathBuf, pub index: Option, pub certification: Option, + pub recall_quality: Option, pub status: EvidenceStatus, pub message: String, } @@ -87,8 +116,11 @@ pub fn certification_dir_for_project(project_root: &Path) -> PathBuf { pub fn load_snapshot(evidence_dir: &Path) -> EvidenceSnapshot { let index_path = evidence_dir.join("evidence-index.json"); match load_index(&index_path) { - Ok(index) => match load_index_certification(evidence_dir, &index) { - Ok(certification) => { + Ok(index) => match ( + load_index_certification(evidence_dir, &index), + load_index_recall_quality(evidence_dir, &index), + ) { + (Ok(certification), Ok(recall_quality)) => { let message = match &certification { Some(certification) => format!( "certification {} at {}", @@ -103,14 +135,16 @@ pub fn load_snapshot(evidence_dir: &Path) -> EvidenceSnapshot { status: index.overall_status, index: Some(index), certification, + recall_quality, message, } } - Err(error) => EvidenceSnapshot { + (Err(error), _) | (_, Err(error)) => EvidenceSnapshot { root: evidence_dir.to_path_buf(), index_path, index: Some(index), certification: None, + recall_quality: None, status: EvidenceStatus::Error, message: error, }, @@ -127,12 +161,14 @@ pub fn load_snapshot(evidence_dir: &Path) -> EvidenceSnapshot { certification.generated_at ), certification: Some(certification), + recall_quality: None, }, Err(_) if !metrics_path_for_dir(evidence_dir).exists() => EvidenceSnapshot { root: evidence_dir.to_path_buf(), index_path, index: None, certification: None, + recall_quality: None, status: EvidenceStatus::Missing, message: format!( "no evidence index found; run sh scripts/certify-tree-ring.sh to generate {}", @@ -144,6 +180,7 @@ pub fn load_snapshot(evidence_dir: &Path) -> EvidenceSnapshot { index_path, index: None, certification: None, + recall_quality: None, status: EvidenceStatus::Error, message: error, }, @@ -153,6 +190,7 @@ pub fn load_snapshot(evidence_dir: &Path) -> EvidenceSnapshot { index_path, index: None, certification: None, + recall_quality: None, status: EvidenceStatus::Error, message: error, }, @@ -164,6 +202,74 @@ fn load_index(index_path: &Path) -> Result { serde_json::from_str(&input).map_err(|err| err.to_string()) } +pub(crate) fn read_or_create_index( + evidence_dir: &Path, + generated_at: &str, +) -> Result { + let index_path = evidence_dir.join("evidence-index.json"); + if index_path.exists() { + let input = fs::read_to_string(&index_path).map_err(|err| err.to_string())?; + return serde_json::from_str(&input).map_err(|err| err.to_string()); + } + Ok(EvidenceIndex { + generated_at: generated_at.to_string(), + overall_status: EvidenceStatus::Missing, + certification: certification_record_from_metrics(evidence_dir, generated_at), + harness: BTreeMap::new(), + recall_quality: None, + missing: vec!["harness".to_string(), "recall_quality".to_string()], + stale: Vec::new(), + }) +} + +pub(crate) fn write_index(evidence_dir: &Path, index: &EvidenceIndex) -> Result { + fs::create_dir_all(evidence_dir).map_err(|err| err.to_string())?; + let index_path = evidence_dir.join("evidence-index.json"); + let json = serde_json::to_string_pretty(index).map_err(|err| err.to_string())?; + fs::write(&index_path, json).map_err(|err| err.to_string())?; + Ok(index_path) +} + +pub(crate) fn rollup_index_status(index: &EvidenceIndex) -> EvidenceStatus { + if index + .harness + .values() + .any(|record| record.status == EvidenceStatus::Fail) + || index + .recall_quality + .as_ref() + .is_some_and(|record| record.status == EvidenceStatus::Fail) + { + return EvidenceStatus::Fail; + } + if index + .harness + .values() + .any(|record| record.status == EvidenceStatus::Error) + || index + .recall_quality + .as_ref() + .is_some_and(|record| record.status == EvidenceStatus::Error) + { + return EvidenceStatus::Error; + } + if index + .recall_quality + .as_ref() + .is_some_and(|record| record.status == EvidenceStatus::NeedsReview) + { + return EvidenceStatus::NeedsReview; + } + if let Some(certification) = &index.certification { + return certification.status; + } + if index.harness.is_empty() && index.recall_quality.is_none() { + EvidenceStatus::Missing + } else { + EvidenceStatus::Skip + } +} + fn load_certification( evidence_dir: &Path, record: &EvidenceRecordRef, @@ -218,6 +324,27 @@ fn load_index_certification( } } +fn load_index_recall_quality( + evidence_dir: &Path, + index: &EvidenceIndex, +) -> Result, String> { + match index.recall_quality.as_ref() { + Some(record) => { + let record_path = resolve_evidence_path(evidence_dir, &record.path); + load_recall_quality(evidence_dir, record) + .map(Some) + .map_err(|error| { + format!( + "failed to load recall quality payload {}: {}", + record_path.display(), + error + ) + }) + } + None => Ok(None), + } +} + fn load_metrics_only_certification(evidence_dir: &Path) -> Result { let metrics_path = metrics_path_for_dir(evidence_dir); let input = fs::read_to_string(&metrics_path).map_err(|err| err.to_string())?; @@ -244,6 +371,75 @@ fn load_metrics_only_certification(evidence_dir: &Path) -> Result Option { + let metrics_path = evidence_dir.join("metrics.json"); + if !metrics_path.exists() { + return None; + } + let summary_path = evidence_dir.join("summary.md"); + Some(EvidenceRecordRef { + category: "certification".to_string(), + status: EvidenceStatus::Pass, + label: "Local certification".to_string(), + path: PathBuf::from("metrics.json"), + summary_path: summary_path.exists().then_some(PathBuf::from("summary.md")), + generated_at: generated_at.to_string(), + }) +} + +fn load_recall_quality( + evidence_dir: &Path, + record: &EvidenceRecordRef, +) -> Result { + let record_path = resolve_evidence_path(evidence_dir, &record.path); + let input = fs::read_to_string(&record_path).map_err(|err| err.to_string())?; + let value: Value = serde_json::from_str(&input).map_err(|err| err.to_string())?; + let queries = value + .get("queries") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .map(|query| RecallQualityQueryEvidence { + query_id: get_string(query, &["query_id"]).unwrap_or_default(), + query: get_string(query, &["query"]).unwrap_or_default(), + status: get_string(query, &["status"]).unwrap_or_default(), + expected_top_id: get_string(query, &["expected_top_id"]), + expected_rank: get_u64(query, &["expected_rank"]), + latency_ms: get_f64(query, &["latency_ms"]), + returned_ids: query + .get("returned") + .and_then(Value::as_array) + .map(|returned| { + returned + .iter() + .filter_map(|item| get_string(item, &["id"])) + .collect() + }) + .unwrap_or_default(), + }) + .collect() + }) + .unwrap_or_default(); + Ok(RecallQualityEvidence { + status: get_status(&value, &["status"]).unwrap_or(record.status), + generated_at: get_string(&value, &["generated_at"]) + .unwrap_or_else(|| record.generated_at.clone()), + record_path, + query_set_id: get_string(&value, &["query_set_id"]).unwrap_or_default(), + query_count: get_u64(&value, &["summary", "query_count"]).unwrap_or(0), + pass_count: get_u64(&value, &["summary", "pass_count"]).unwrap_or(0), + fail_count: get_u64(&value, &["summary", "fail_count"]).unwrap_or(0), + needs_review_count: get_u64(&value, &["summary", "needs_review_count"]).unwrap_or(0), + avg_latency_ms: get_f64(&value, &["summary", "avg_latency_ms"]), + max_latency_ms: get_f64(&value, &["summary", "max_latency_ms"]), + queries, + }) +} + fn metrics_path_for_dir(evidence_dir: &Path) -> PathBuf { evidence_dir.join("metrics.json") } @@ -275,6 +471,19 @@ fn get_string(value: &Value, path: &[&str]) -> Option { .map(ToString::to_string) } +fn get_status(value: &Value, path: &[&str]) -> Option { + match get_value(value, path).and_then(Value::as_str)? { + "pass" => Some(EvidenceStatus::Pass), + "fail" => Some(EvidenceStatus::Fail), + "skip" => Some(EvidenceStatus::Skip), + "missing" => Some(EvidenceStatus::Missing), + "stale" => Some(EvidenceStatus::Stale), + "needs_review" => Some(EvidenceStatus::NeedsReview), + "error" => Some(EvidenceStatus::Error), + _ => None, + } +} + fn get_value<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> { path.iter() .try_fold(value, |current, key| current.get(*key)) @@ -426,4 +635,194 @@ mod tests { .contains("failed to load certification payload")); assert!(snapshot.message.contains("metrics.json")); } + + #[test] + fn evidence_snapshot_loads_recall_quality_record_from_index() { + let dir = tempdir().unwrap(); + let evidence_dir = certification_dir_for_project(dir.path()); + fs::create_dir_all(evidence_dir.join("recall-quality")).unwrap(); + fs::write( + evidence_dir.join("metrics.json"), + r#"{"ok":true,"created_at":"2026-07-09T04:22:38Z"}"#, + ) + .unwrap(); + fs::write( + evidence_dir.join("recall-quality/default-fixture-v1.json"), + r#"{ + "schema_version": 1, + "generated_at": "2026-07-09T06:00:00Z", + "query_set_id": "default-fixture-v1", + "status": "pass", + "summary": { + "query_count": 2, + "pass_count": 2, + "fail_count": 0, + "needs_review_count": 0, + "avg_latency_ms": 1.25, + "max_latency_ms": 2.5 + }, + "queries": [ + { + "query_id": "scar-stale-cache", + "query": "failure stale cache", + "status": "pass", + "expected_top_id": "rq_scar_stale_cache", + "expected_rank": 1, + "latency_ms": 0.9, + "returned": [ + { + "id": "rq_scar_stale_cache", + "rank": 1, + "ring": "scar", + "source_ref": "recall-quality/scar-stale-cache", + "score": 1.2, + "ranking": {"textual_match": 1.0} + } + ], + "notes": [] + } + ] + }"#, + ) + .unwrap(); + fs::write( + evidence_dir.join("evidence-index.json"), + r#"{ + "generated_at": "2026-07-09T06:00:00Z", + "overall_status": "pass", + "certification": { + "category": "certification", + "status": "pass", + "label": "Local certification", + "path": "metrics.json", + "summary_path": null, + "generated_at": "2026-07-09T04:22:38Z" + }, + "harness": {}, + "recall_quality": { + "category": "recall_quality", + "status": "pass", + "label": "Recall quality", + "path": "recall-quality/default-fixture-v1.json", + "summary_path": null, + "generated_at": "2026-07-09T06:00:00Z" + }, + "missing": [], + "stale": [] + }"#, + ) + .unwrap(); + + let snapshot = load_snapshot(&evidence_dir); + + let recall_quality = snapshot.recall_quality.unwrap(); + assert_eq!(recall_quality.status, EvidenceStatus::Pass); + assert_eq!(recall_quality.query_set_id, "default-fixture-v1"); + assert_eq!(recall_quality.query_count, 2); + assert_eq!(recall_quality.pass_count, 2); + assert_eq!(recall_quality.fail_count, 0); + assert_eq!(recall_quality.needs_review_count, 0); + assert_eq!(recall_quality.avg_latency_ms, Some(1.25)); + assert_eq!(recall_quality.max_latency_ms, Some(2.5)); + assert_eq!(recall_quality.queries[0].query_id, "scar-stale-cache"); + assert_eq!( + recall_quality.queries[0].returned_ids, + vec!["rq_scar_stale_cache"] + ); + } + + #[test] + fn evidence_snapshot_uses_recall_quality_payload_status_as_source_of_truth() { + let dir = tempdir().unwrap(); + let evidence_dir = certification_dir_for_project(dir.path()); + fs::create_dir_all(evidence_dir.join("recall-quality")).unwrap(); + fs::write( + evidence_dir.join("metrics.json"), + r#"{"ok":true,"created_at":"2026-07-09T04:22:38Z"}"#, + ) + .unwrap(); + fs::write( + evidence_dir.join("recall-quality/default-fixture-v1.json"), + r#"{ + "schema_version": 1, + "generated_at": "2026-07-09T06:00:00Z", + "query_set_id": "default-fixture-v1", + "status": "needs_review", + "summary": { + "query_count": 1, + "pass_count": 0, + "fail_count": 0, + "needs_review_count": 1, + "avg_latency_ms": 1.25, + "max_latency_ms": 2.5 + }, + "queries": [] + }"#, + ) + .unwrap(); + fs::write( + evidence_dir.join("evidence-index.json"), + r#"{ + "generated_at": "2026-07-09T06:00:00Z", + "overall_status": "pass", + "certification": { + "category": "certification", + "status": "pass", + "label": "Local certification", + "path": "metrics.json", + "summary_path": null, + "generated_at": "2026-07-09T04:22:38Z" + }, + "harness": {}, + "recall_quality": { + "category": "recall_quality", + "status": "pass", + "label": "Recall quality", + "path": "recall-quality/default-fixture-v1.json", + "summary_path": null, + "generated_at": "2026-07-09T06:00:00Z" + }, + "missing": [], + "stale": [] + }"#, + ) + .unwrap(); + + let snapshot = load_snapshot(&evidence_dir); + + assert_eq!( + snapshot.recall_quality.unwrap().status, + EvidenceStatus::NeedsReview + ); + } + + #[test] + fn rollup_index_status_marks_recall_quality_needs_review() { + let mut index = EvidenceIndex { + generated_at: "2026-07-09T06:00:00Z".to_string(), + overall_status: EvidenceStatus::Pass, + certification: Some(EvidenceRecordRef { + category: "certification".to_string(), + status: EvidenceStatus::Pass, + label: "Local certification".to_string(), + path: PathBuf::from("metrics.json"), + summary_path: None, + generated_at: "2026-07-09T06:00:00Z".to_string(), + }), + harness: BTreeMap::new(), + recall_quality: Some(EvidenceRecordRef { + category: "recall_quality".to_string(), + status: EvidenceStatus::NeedsReview, + label: "Recall quality".to_string(), + path: PathBuf::from("recall-quality/default-fixture-v1.json"), + summary_path: None, + generated_at: "2026-07-09T06:00:00Z".to_string(), + }), + missing: Vec::new(), + stale: Vec::new(), + }; + assert_eq!(rollup_index_status(&index), EvidenceStatus::NeedsReview); + index.recall_quality.as_mut().unwrap().status = EvidenceStatus::Fail; + assert_eq!(rollup_index_status(&index), EvidenceStatus::Fail); + } } diff --git a/crates/tree-ring-memory-cli/src/harness_evidence.rs b/crates/tree-ring-memory-cli/src/harness_evidence.rs index 5425c61..d75a699 100644 --- a/crates/tree-ring-memory-cli/src/harness_evidence.rs +++ b/crates/tree-ring-memory-cli/src/harness_evidence.rs @@ -1,8 +1,9 @@ -use crate::evidence::{EvidenceRecordRef, EvidenceStatus}; +use crate::evidence::{ + read_or_create_index, rollup_index_status, write_index, EvidenceRecordRef, EvidenceStatus, +}; use crate::integrations::{scan_integrations, IntegrationMarker, MarkerOrigin}; use chrono::{SecondsFormat, Utc}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; @@ -216,8 +217,6 @@ fn merge_harness_index( generated_at: &str, records: &[HarnessProbeRecord], ) -> Result { - fs::create_dir_all(evidence_dir).map_err(|err| err.to_string())?; - let index_path = evidence_dir.join("evidence-index.json"); let mut index = read_or_create_index(evidence_dir, generated_at)?; index.generated_at = generated_at.to_string(); for record in records { @@ -240,80 +239,15 @@ fn merge_harness_index( } index.missing.sort(); index.missing.dedup(); - index.overall_status = rollup_status(&index); - let json = serde_json::to_string_pretty(&index).map_err(|err| err.to_string())?; - fs::write(&index_path, json).map_err(|err| err.to_string())?; - Ok(index_path) -} - -fn read_or_create_index( - evidence_dir: &Path, - generated_at: &str, -) -> Result { - let index_path = evidence_dir.join("evidence-index.json"); - if index_path.exists() { - let input = fs::read_to_string(&index_path).map_err(|err| err.to_string())?; - return serde_json::from_str(&input).map_err(|err| err.to_string()); - } - Ok(crate::evidence::EvidenceIndex { - generated_at: generated_at.to_string(), - overall_status: EvidenceStatus::Missing, - certification: certification_record_from_metrics(evidence_dir, generated_at), - harness: BTreeMap::new(), - recall_quality: None, - missing: vec!["recall_quality".to_string()], - stale: Vec::new(), - }) -} - -fn certification_record_from_metrics( - evidence_dir: &Path, - generated_at: &str, -) -> Option { - let metrics_path = evidence_dir.join("metrics.json"); - if !metrics_path.exists() { - return None; - } - let summary_path = evidence_dir.join("summary.md"); - Some(EvidenceRecordRef { - category: "certification".to_string(), - status: EvidenceStatus::Pass, - label: "Local certification".to_string(), - path: PathBuf::from("metrics.json"), - summary_path: summary_path.exists().then_some(PathBuf::from("summary.md")), - generated_at: generated_at.to_string(), - }) -} - -fn rollup_status(index: &crate::evidence::EvidenceIndex) -> EvidenceStatus { - if index - .harness - .values() - .any(|record| record.status == EvidenceStatus::Fail) - { - return EvidenceStatus::Fail; - } - if index - .harness - .values() - .any(|record| record.status == EvidenceStatus::Error) - { - return EvidenceStatus::Error; - } - if let Some(certification) = &index.certification { - return certification.status; - } - if index.harness.is_empty() { - EvidenceStatus::Missing - } else { - EvidenceStatus::Skip - } + index.overall_status = rollup_index_status(&index); + write_index(evidence_dir, &index) } #[cfg(test)] mod tests { use super::*; use crate::evidence::certification_dir_for_project; + use std::collections::BTreeMap; use tempfile::tempdir; #[test] @@ -550,7 +484,7 @@ mod tests { stale: Vec::new(), }; - assert_eq!(rollup_status(&index), EvidenceStatus::Pass); + assert_eq!(rollup_index_status(&index), EvidenceStatus::Pass); } #[test] @@ -575,6 +509,6 @@ mod tests { stale: Vec::new(), }; - assert_eq!(rollup_status(&index), EvidenceStatus::Skip); + assert_eq!(rollup_index_status(&index), EvidenceStatus::Skip); } } diff --git a/crates/tree-ring-memory-cli/src/main.rs b/crates/tree-ring-memory-cli/src/main.rs index 9fc1576..68c5d92 100644 --- a/crates/tree-ring-memory-cli/src/main.rs +++ b/crates/tree-ring-memory-cli/src/main.rs @@ -24,6 +24,7 @@ use actions::remember::{remember as remember_action, RememberRequest}; use harness_evidence::{ certify_harnesses, HarnessCertificationReport, HarnessCertificationRequest, }; +use recall_quality::{run_recall_quality, RecallQualityReport, RecallQualityRequest}; use serde_json::json; mod actions; @@ -32,6 +33,7 @@ mod commands; mod evidence; mod harness_evidence; mod integrations; +mod recall_quality; mod ring_mark; mod tui; mod welcome; @@ -199,6 +201,20 @@ enum Command { #[arg(long, help = "print a stable onboarding screen without animation")] no_animation: bool, }, + #[command(about = "write non-private recall quality evidence")] + RecallQuality { + #[arg( + long, + default_value = ".", + help = "project root used for default evidence output" + )] + source_root: PathBuf, + #[arg( + long, + help = "evidence output directory; defaults to /target/tree-ring-certification" + )] + out_dir: Option, + }, #[command(about = "summarize DOX-style AGENTS.md guidance into memory")] Dox { #[command(subcommand)] @@ -358,6 +374,22 @@ fn run(cli: Cli) -> Result<(), String> { return Ok(()); } + if let Command::RecallQuality { + source_root, + out_dir, + } = &cli.command + { + let evidence_dir = out_dir + .clone() + .unwrap_or_else(|| evidence::certification_dir_for_project(source_root)); + let report = run_recall_quality(RecallQualityRequest { + source_root: source_root.clone(), + evidence_dir, + })?; + print_recall_quality_report(&report, cli.json)?; + return Ok(()); + } + if let Command::Tui { event_stream, tick_ms, @@ -677,6 +709,9 @@ fn run(cli: Cli) -> Result<(), String> { Command::Welcome { .. } => { unreachable!("welcome returns before opening the scriptable store") } + Command::RecallQuality { .. } => { + unreachable!("recall-quality returns before opening the scriptable store") + } Command::Integrations { .. } => { unreachable!("integrations commands return before opening the scriptable store") } @@ -1045,6 +1080,14 @@ fn print_harness_certification_report( Ok(()) } +fn print_recall_quality_report( + report: &RecallQualityReport, + json_output: bool, +) -> Result<(), String> { + println!("{}", format_recall_quality_report(report, json_output)); + Ok(()) +} + fn format_harness_certification_report( report: &HarnessCertificationReport, json_output: bool, @@ -1076,6 +1119,43 @@ fn format_harness_certification_report( } } +fn format_recall_quality_report(report: &RecallQualityReport, json_output: bool) -> String { + if json_output { + json!({ + "ok": true, + "report": report, + }) + .to_string() + } else { + let mut lines = vec![format!( + "Tree Ring Memory recall quality: status={} queries={} pass={} fail={} needs_review={} avg={:.3}ms max={:.3}ms evidence={}", + report.status.as_str(), + report.summary.query_count, + report.summary.pass_count, + report.summary.fail_count, + report.summary.needs_review_count, + report.summary.avg_latency_ms, + report.summary.max_latency_ms, + report.evidence_dir.display() + )]; + for query in &report.queries { + lines.push(format!( + "{} [{:?}] latency={:.3}ms returned={}", + query.query_id, + query.status, + query.latency_ms, + query + .returned + .iter() + .map(|item| item.id.as_str()) + .collect::>() + .join(",") + )); + } + lines.join("\n") + } +} + fn print_agent_awareness_summary(report: &agent_awareness::AgentAwarenessReport) { if !report.created.is_empty() { println!("Agent awareness files created:"); @@ -1291,6 +1371,131 @@ mod tests { .exists()); } + #[test] + fn recall_quality_json_output_contract() { + let report = RecallQualityReport { + schema_version: 1, + generated_at: "2026-07-09T00:00:00Z".to_string(), + query_set_id: "default-fixture-v1".to_string(), + status: crate::evidence::EvidenceStatus::Pass, + source_root: PathBuf::from("/tmp/project"), + evidence_dir: PathBuf::from("/tmp/project/target/tree-ring-certification"), + record_path: PathBuf::from( + "/tmp/project/target/tree-ring-certification/recall-quality/default-fixture-v1.json", + ), + summary: crate::recall_quality::RecallQualitySummary { + query_count: 4, + pass_count: 4, + fail_count: 0, + needs_review_count: 0, + avg_latency_ms: 1.25, + max_latency_ms: 2.5, + fixture_memory_count: 5, + sensitive_fixture_count: 1, + private_payloads_used: false, + }, + queries: vec![crate::recall_quality::RecallQualityQueryRecord { + query_id: "scar-stale-cache".to_string(), + query: "failure stale cache".to_string(), + status: crate::recall_quality::RecallQualityQueryStatus::Pass, + expected_top_id: Some("rq_scar_stale_cache".to_string()), + expected_rank: Some(1), + latency_ms: 1.25, + returned: vec![crate::recall_quality::RecallQualityReturnedMemory { + id: "rq_scar_stale_cache".to_string(), + rank: 1, + ring: "scar".to_string(), + source_ref: "fixture://recall-quality/scar-stale-cache".to_string(), + score: 0.98, + ranking: std::collections::BTreeMap::from([ + ("fts".to_string(), 0.75), + ("ring".to_string(), 0.23), + ]), + }], + notes: Vec::new(), + }], + }; + + let output = format_recall_quality_report(&report, true); + let parsed: serde_json::Value = serde_json::from_str(&output).unwrap(); + + assert_eq!(parsed["ok"], true); + assert_eq!(parsed["report"]["query_set_id"], "default-fixture-v1"); + assert_eq!(parsed["report"]["status"], "pass"); + assert_eq!(parsed["report"]["summary"]["query_count"], 4); + assert_eq!(parsed["report"]["summary"]["sensitive_fixture_count"], 1); + assert_eq!(parsed["report"]["summary"]["private_payloads_used"], false); + assert_eq!( + parsed["report"]["queries"][0]["query_id"], + "scar-stale-cache" + ); + } + + #[test] + fn recall_quality_human_output_contract() { + let report = RecallQualityReport { + schema_version: 1, + generated_at: "2026-07-09T00:00:00Z".to_string(), + query_set_id: "default-fixture-v1".to_string(), + status: crate::evidence::EvidenceStatus::Pass, + source_root: PathBuf::from("/tmp/project"), + evidence_dir: PathBuf::from("/tmp/project/target/tree-ring-certification"), + record_path: PathBuf::from( + "/tmp/project/target/tree-ring-certification/recall-quality/default-fixture-v1.json", + ), + summary: crate::recall_quality::RecallQualitySummary { + query_count: 4, + pass_count: 4, + fail_count: 0, + needs_review_count: 0, + avg_latency_ms: 1.25, + max_latency_ms: 2.5, + fixture_memory_count: 5, + sensitive_fixture_count: 1, + private_payloads_used: false, + }, + queries: vec![ + crate::recall_quality::RecallQualityQueryRecord { + query_id: "scar-stale-cache".to_string(), + query: "failure stale cache".to_string(), + status: crate::recall_quality::RecallQualityQueryStatus::Pass, + expected_top_id: Some("rq_scar_stale_cache".to_string()), + expected_rank: Some(1), + latency_ms: 1.25, + returned: vec![crate::recall_quality::RecallQualityReturnedMemory { + id: "rq_scar_stale_cache".to_string(), + rank: 1, + ring: "scar".to_string(), + source_ref: "fixture://recall-quality/scar-stale-cache".to_string(), + score: 0.98, + ranking: std::collections::BTreeMap::new(), + }], + notes: Vec::new(), + }, + crate::recall_quality::RecallQualityQueryRecord { + query_id: "sensitive-filter".to_string(), + query: "health private payload".to_string(), + status: crate::recall_quality::RecallQualityQueryStatus::Pass, + expected_top_id: None, + expected_rank: None, + latency_ms: 2.5, + returned: Vec::new(), + notes: Vec::new(), + }, + ], + }; + + let output = format_recall_quality_report(&report, false); + + assert!(output.contains( + "Tree Ring Memory recall quality: status=pass queries=4 pass=4 fail=0 needs_review=0 avg=1.250ms max=2.500ms evidence=/tmp/project/target/tree-ring-certification" + )); + assert!( + output.contains("scar-stale-cache [Pass] latency=1.250ms returned=rq_scar_stale_cache") + ); + assert!(output.contains("sensitive-filter [Pass] latency=2.500ms returned=")); + } + #[test] fn harness_certification_json_output_contract() { let report = HarnessCertificationReport { diff --git a/crates/tree-ring-memory-cli/src/recall_quality.rs b/crates/tree-ring-memory-cli/src/recall_quality.rs new file mode 100644 index 0000000..65588af --- /dev/null +++ b/crates/tree-ring-memory-cli/src/recall_quality.rs @@ -0,0 +1,607 @@ +use crate::evidence::{ + read_or_create_index, rollup_index_status, write_index, EvidenceRecordRef, EvidenceStatus, +}; +use chrono::{SecondsFormat, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Instant; +use tree_ring_memory_core::{MemoryEvent, MemorySource}; +use tree_ring_memory_sqlite::{MemoryRetriever, SQLiteMemoryStore}; + +pub const RECALL_QUALITY_QUERY_SET_ID: &str = "default-fixture-v1"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecallQualityRequest { + pub source_root: PathBuf, + pub evidence_dir: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualityReport { + pub schema_version: u8, + pub generated_at: String, + pub query_set_id: String, + pub status: EvidenceStatus, + pub source_root: PathBuf, + pub evidence_dir: PathBuf, + pub record_path: PathBuf, + pub summary: RecallQualitySummary, + pub queries: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualitySummary { + pub query_count: usize, + pub pass_count: usize, + pub fail_count: usize, + pub needs_review_count: usize, + pub avg_latency_ms: f64, + pub max_latency_ms: f64, + pub fixture_memory_count: usize, + pub sensitive_fixture_count: usize, + pub private_payloads_used: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecallQualityQueryStatus { + Pass, + Fail, + NeedsReview, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualityQueryRecord { + pub query_id: String, + pub query: String, + pub status: RecallQualityQueryStatus, + pub expected_top_id: Option, + pub expected_rank: Option, + pub latency_ms: f64, + pub returned: Vec, + pub notes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualityReturnedMemory { + pub id: String, + pub rank: usize, + pub ring: String, + pub source_ref: String, + pub score: f64, + pub ranking: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct QueryCase { + query_id: &'static str, + query: &'static str, + expected_top_id: Option<&'static str>, + max_expected_rank: Option, + forbidden_ids: &'static [&'static str], +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct QueryEvaluation { + status: RecallQualityQueryStatus, + notes: Vec, +} + +pub fn run_recall_quality(request: RecallQualityRequest) -> Result { + let generated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let mut store = + SQLiteMemoryStore::open(Path::new(":memory:")).map_err(|err| err.to_string())?; + let fixtures = fixture_memories()?; + store.put_many(&fixtures).map_err(|err| err.to_string())?; + + let retriever = MemoryRetriever::new(&store); + let mut queries = Vec::new(); + for case in query_cases() { + queries.push(run_case(&retriever, &case)?); + } + let summary = summarize(&queries, &fixtures); + let status = report_status(&summary); + let record_path = request + .evidence_dir + .join("recall-quality") + .join(format!("{RECALL_QUALITY_QUERY_SET_ID}.json")); + let report = RecallQualityReport { + schema_version: 1, + generated_at: generated_at.clone(), + query_set_id: RECALL_QUALITY_QUERY_SET_ID.to_string(), + status, + source_root: request.source_root, + evidence_dir: request.evidence_dir.clone(), + record_path: record_path.clone(), + summary, + queries, + }; + write_report_and_index(&request.evidence_dir, &generated_at, &report)?; + Ok(report) +} + +fn run_case( + retriever: &MemoryRetriever<'_>, + case: &QueryCase, +) -> Result { + let started_at = Instant::now(); + let returned = retriever + .recall( + case.query, None, None, None, None, None, false, false, 5, true, + ) + .map_err(|err| err.to_string())?; + let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let returned: Vec<_> = returned + .into_iter() + .enumerate() + .map(|(index, item)| RecallQualityReturnedMemory { + id: item.memory.id, + rank: index + 1, + ring: item.memory.ring, + source_ref: item.memory.source.ref_, + score: item.score, + ranking: item.ranking, + }) + .collect(); + let evaluation = evaluate_query( + case.expected_top_id, + case.max_expected_rank, + case.forbidden_ids, + &returned, + ); + + Ok(RecallQualityQueryRecord { + query_id: case.query_id.to_string(), + query: case.query.to_string(), + status: evaluation.status, + expected_top_id: case.expected_top_id.map(str::to_string), + expected_rank: case.max_expected_rank, + latency_ms, + returned, + notes: evaluation.notes, + }) +} + +fn evaluate_query( + expected_top_id: Option<&str>, + max_expected_rank: Option, + forbidden_ids: &[&str], + returned: &[RecallQualityReturnedMemory], +) -> QueryEvaluation { + let mut status = RecallQualityQueryStatus::Pass; + let mut notes = Vec::new(); + + if let Some(expected_top_id) = expected_top_id { + match returned.iter().find(|item| item.id == expected_top_id) { + Some(item) => { + if max_expected_rank.is_some_and(|expected_rank| item.rank > expected_rank) { + status = RecallQualityQueryStatus::NeedsReview; + notes.push(format!( + "expected memory {expected_top_id} returned at rank {} which is worse than allowed rank {}", + item.rank, + max_expected_rank.unwrap_or(item.rank) + )); + } + } + None => { + status = RecallQualityQueryStatus::Fail; + notes.push(format!("missing expected memory {expected_top_id}")); + } + } + } + + let forbidden_hits: Vec<_> = returned + .iter() + .filter(|item| forbidden_ids.contains(&item.id.as_str())) + .map(|item| item.id.clone()) + .collect(); + if !forbidden_hits.is_empty() { + status = RecallQualityQueryStatus::Fail; + notes.push(format!( + "forbidden memories returned: {}", + forbidden_hits.join(", ") + )); + } + + QueryEvaluation { status, notes } +} + +fn summarize( + queries: &[RecallQualityQueryRecord], + fixtures: &[MemoryEvent], +) -> RecallQualitySummary { + let pass_count = queries + .iter() + .filter(|query| query.status == RecallQualityQueryStatus::Pass) + .count(); + let fail_count = queries + .iter() + .filter(|query| query.status == RecallQualityQueryStatus::Fail) + .count(); + let needs_review_count = queries + .iter() + .filter(|query| query.status == RecallQualityQueryStatus::NeedsReview) + .count(); + let total_latency_ms: f64 = queries.iter().map(|query| query.latency_ms).sum(); + let avg_latency_ms = if queries.is_empty() { + 0.0 + } else { + total_latency_ms / queries.len() as f64 + }; + let max_latency_ms = queries + .iter() + .map(|query| query.latency_ms) + .fold(0.0, f64::max); + + RecallQualitySummary { + query_count: queries.len(), + pass_count, + fail_count, + needs_review_count, + avg_latency_ms, + max_latency_ms, + fixture_memory_count: fixtures.len(), + sensitive_fixture_count: fixtures + .iter() + .filter(|fixture| fixture.sensitivity != "normal") + .count(), + private_payloads_used: false, + } +} + +fn report_status(summary: &RecallQualitySummary) -> EvidenceStatus { + if summary.fail_count > 0 { + EvidenceStatus::Fail + } else if summary.needs_review_count > 0 { + EvidenceStatus::NeedsReview + } else { + EvidenceStatus::Pass + } +} + +fn write_report_and_index( + evidence_dir: &Path, + generated_at: &str, + report: &RecallQualityReport, +) -> Result<(), String> { + let report_dir = evidence_dir.join("recall-quality"); + fs::create_dir_all(&report_dir).map_err(|err| err.to_string())?; + let report_json = serde_json::to_string_pretty(report).map_err(|err| err.to_string())?; + fs::write(&report.record_path, report_json).map_err(|err| err.to_string())?; + + let mut index = read_or_create_index(evidence_dir, generated_at)?; + index.generated_at = generated_at.to_string(); + index.recall_quality = Some(EvidenceRecordRef { + category: "recall_quality".to_string(), + status: report.status, + label: "Recall quality".to_string(), + path: PathBuf::from(format!("recall-quality/{RECALL_QUALITY_QUERY_SET_ID}.json")), + summary_path: None, + generated_at: generated_at.to_string(), + }); + index.missing.retain(|item| item != "recall_quality"); + if index.harness.is_empty() { + if !index.missing.iter().any(|item| item == "harness") { + index.missing.push("harness".to_string()); + } + } else { + index.missing.retain(|item| item != "harness"); + } + index.missing.sort(); + index.missing.dedup(); + index.overall_status = rollup_index_status(&index); + write_index(evidence_dir, &index)?; + Ok(()) +} + +fn query_cases() -> Vec { + vec![ + QueryCase { + query_id: "scar-stale-cache", + query: "failure stale cache", + expected_top_id: Some("rq_scar_stale_cache"), + max_expected_rank: Some(1), + forbidden_ids: &[], + }, + QueryCase { + query_id: "heartwood-sqlite-decision", + query: "durable local sqlite decision", + expected_top_id: Some("rq_heartwood_sqlite_decision"), + max_expected_rank: Some(1), + forbidden_ids: &[], + }, + QueryCase { + query_id: "seed-harness-experiment", + query: "planning experiment agent harness", + expected_top_id: Some("rq_seed_harness_experiment"), + max_expected_rank: Some(1), + forbidden_ids: &[], + }, + QueryCase { + query_id: "sensitive-filter", + query: "health private payload", + expected_top_id: None, + max_expected_rank: None, + forbidden_ids: &["rq_sensitive_health_note"], + }, + ] +} + +fn fixture_memories() -> Result, String> { + Ok(vec![ + fixture_memory( + "rq_scar_stale_cache", + "scar", + "warning", + "Failure stale cache regression after deploy.", + "The cache stayed stale until invalidation was added to the failure path.", + "fixture://recall-quality/scar-stale-cache", + "normal", + )?, + fixture_memory( + "rq_heartwood_sqlite_decision", + "heartwood", + "decision", + "Durable local sqlite decision for recall storage.", + "Choose local sqlite as the durable storage decision for offline recall.", + "fixture://recall-quality/heartwood-sqlite-decision", + "normal", + )?, + fixture_memory( + "rq_seed_harness_experiment", + "seed", + "hypothesis", + "Planning experiment for agent harness diagnostics.", + "Run a planning experiment around the agent harness diagnostic runner.", + "fixture://recall-quality/seed-harness-experiment", + "normal", + )?, + fixture_memory( + "rq_outer_release_guardrail", + "outer", + "lesson", + "Release guardrail for evidence output before shipping.", + "Keep release guardrail checks separate from recall-quality diagnostics.", + "fixture://recall-quality/outer-release-guardrail", + "normal", + )?, + fixture_memory( + "rq_sensitive_health_note", + "cambium", + "lesson", + "Health private payload stays filtered from safe recall evidence.", + "This fixture proves sensitive health payload filtering without exposing details.", + "fixture://recall-quality/sensitive-health-note", + "health", + )?, + ]) +} + +fn fixture_memory( + id: &str, + ring: &str, + event_type: &str, + summary: &str, + details: &str, + source_ref: &str, + sensitivity: &str, +) -> Result { + let mut event = MemoryEvent::new(summary, event_type).map_err(|err| err.to_string())?; + event.id = id.to_string(); + event.project = Some("recall-quality-fixture".to_string()); + event.scope = "project".to_string(); + event.ring = ring.to_string(); + event.details = details.to_string(); + event.source = MemorySource { + source_type: "fixture".to_string(), + ref_: source_ref.to_string(), + quote: String::new(), + }; + event.tags = vec![ + "recall-quality".to_string(), + RECALL_QUALITY_QUERY_SET_ID.to_string(), + ]; + event.salience = 0.92; + event.confidence = 0.91; + event.retention = "durable".to_string(); + event.sensitivity = sensitivity.to_string(); + event.validated().map_err(|err| err.to_string()) +} + +#[cfg(test)] +fn returned_memory_for_test(id: &str, rank: usize) -> RecallQualityReturnedMemory { + RecallQualityReturnedMemory { + id: id.to_string(), + rank, + ring: "cambium".to_string(), + source_ref: format!("fixture://{id}"), + score: 0.5, + ranking: BTreeMap::from([("fts".to_string(), 0.5)]), + } +} + +#[cfg(test)] +fn evaluate_query_for_test( + expected_top_id: Option<&str>, + max_expected_rank: Option, + forbidden_ids: &[&str], + returned: &[RecallQualityReturnedMemory], +) -> QueryEvaluation { + evaluate_query(expected_top_id, max_expected_rank, forbidden_ids, returned) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::evidence::{EvidenceIndex, EvidenceStatus}; + use std::path::PathBuf; + use tempfile::tempdir; + + #[test] + fn recall_quality_writes_safe_evidence_and_updates_index() { + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("target/tree-ring-certification"); + + let report = run_recall_quality(RecallQualityRequest { + source_root: dir.path().to_path_buf(), + evidence_dir: evidence_dir.clone(), + }) + .unwrap(); + + assert_eq!(report.query_set_id, "default-fixture-v1"); + assert_eq!(report.status, EvidenceStatus::Pass); + assert_eq!(report.summary.fail_count, 0); + assert!(report.summary.pass_count >= 3); + assert!(evidence_dir + .join("recall-quality/default-fixture-v1.json") + .exists()); + let json = + std::fs::read_to_string(evidence_dir.join("recall-quality/default-fixture-v1.json")) + .unwrap(); + assert!(json.contains("\"ranking\"")); + assert!(json.contains("\"latency_ms\"")); + assert!(json.contains("\"sensitive_fixture_count\": 1")); + assert!(json.contains("\"private_payloads_used\": false")); + assert!(!json.contains("Private bank account note")); + + let index: crate::evidence::EvidenceIndex = serde_json::from_str( + &std::fs::read_to_string(evidence_dir.join("evidence-index.json")).unwrap(), + ) + .unwrap(); + assert!(index.recall_quality.is_some()); + assert!(!index.missing.iter().any(|item| item == "recall_quality")); + } + + #[test] + fn query_evaluation_distinguishes_fail_and_needs_review() { + let passing_returned = vec![ + returned_memory_for_test("other", 1), + returned_memory_for_test("expected", 2), + ]; + let passing = evaluate_query_for_test(Some("expected"), Some(3), &[], &passing_returned); + assert_eq!(passing.status, RecallQualityQueryStatus::Pass); + + let review_returned = vec![ + returned_memory_for_test("other", 1), + returned_memory_for_test("expected", 4), + ]; + let review = evaluate_query_for_test(Some("expected"), Some(3), &[], &review_returned); + assert_eq!(review.status, RecallQualityQueryStatus::NeedsReview); + + let failed = + evaluate_query_for_test(Some("missing"), Some(1), &["other"], &passing_returned); + assert_eq!(failed.status, RecallQualityQueryStatus::Fail); + assert!(failed.notes.iter().any(|note| note.contains("missing"))); + assert!(failed.notes.iter().any(|note| note.contains("forbidden"))); + } + + #[test] + fn recall_quality_index_merge_preserves_existing_certification_and_harness_entries() { + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("target/tree-ring-certification"); + std::fs::create_dir_all(&evidence_dir).unwrap(); + std::fs::write( + evidence_dir.join("metrics.json"), + r#"{"ok":true,"created_at":"2026-07-09T05:44:48Z"}"#, + ) + .unwrap(); + std::fs::write( + evidence_dir.join("evidence-index.json"), + r#"{ + "generated_at": "2026-07-09T05:44:48Z", + "overall_status": "pass", + "certification": { + "category": "certification", + "status": "pass", + "label": "Local certification", + "path": "metrics.json", + "summary_path": "summary.md", + "generated_at": "2026-07-09T05:44:48Z" + }, + "harness": { + "codex": { + "category": "harness", + "status": "pass", + "label": "Codex", + "path": "harness/codex.json", + "summary_path": null, + "generated_at": "2026-07-09T05:44:48Z" + }, + "manual-harness": { + "category": "harness", + "status": "skip", + "label": "Manual Harness", + "path": "harness/manual.json", + "summary_path": null, + "generated_at": "2026-07-09T05:44:48Z" + } + }, + "recall_quality": null, + "missing": ["harness", "recall_quality"], + "stale": [] + }"#, + ) + .unwrap(); + + let report = RecallQualityReport { + schema_version: 1, + generated_at: "2026-07-09T06:00:00Z".to_string(), + query_set_id: RECALL_QUALITY_QUERY_SET_ID.to_string(), + status: EvidenceStatus::Pass, + source_root: dir.path().to_path_buf(), + evidence_dir: evidence_dir.clone(), + record_path: evidence_dir + .join(format!("recall-quality/{RECALL_QUALITY_QUERY_SET_ID}.json")), + summary: RecallQualitySummary { + query_count: 4, + pass_count: 4, + fail_count: 0, + needs_review_count: 0, + avg_latency_ms: 1.0, + max_latency_ms: 2.0, + fixture_memory_count: 5, + sensitive_fixture_count: 1, + private_payloads_used: false, + }, + queries: Vec::new(), + }; + + write_report_and_index(&evidence_dir, &report.generated_at, &report).unwrap(); + + let index: EvidenceIndex = serde_json::from_str( + &std::fs::read_to_string(evidence_dir.join("evidence-index.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + index + .certification + .as_ref() + .map(|record| record.path.clone()), + Some(PathBuf::from("metrics.json")) + ); + assert_eq!( + index.harness.get("codex").map(|record| record.path.clone()), + Some(PathBuf::from("harness/codex.json")) + ); + assert_eq!( + index + .harness + .get("manual-harness") + .map(|record| record.path.clone()), + Some(PathBuf::from("harness/manual.json")) + ); + assert_eq!( + index + .recall_quality + .as_ref() + .map(|record| record.path.clone()), + Some(PathBuf::from(format!( + "recall-quality/{RECALL_QUALITY_QUERY_SET_ID}.json" + ))) + ); + assert_eq!(index.missing, Vec::::new()); + assert_eq!(index.overall_status, EvidenceStatus::Pass); + } +} diff --git a/crates/tree-ring-memory-cli/src/tui/render.rs b/crates/tree-ring-memory-cli/src/tui/render.rs index 856c880..4946af9 100644 --- a/crates/tree-ring-memory-cli/src/tui/render.rs +++ b/crates/tree-ring-memory-cli/src/tui/render.rs @@ -326,11 +326,11 @@ fn render_evidence_list(frame: &mut Frame<'_>, area: Rect, app: &App) { Span::styled(format!(" {status}"), theme::dim()), ]))); - let (harness_status, recall_status) = snapshot + let harness_status = snapshot .index .as_ref() .map(|index| { - let harness = if index.harness.is_empty() { + if index.harness.is_empty() { if index.missing.iter().any(|item| item == "harness") { " missing" } else { @@ -338,17 +338,26 @@ fn render_evidence_list(frame: &mut Frame<'_>, area: Rect, app: &App) { } } else { " loaded" - }; - let recall = if index.recall_quality.is_some() { - " loaded" - } else if index.missing.iter().any(|item| item == "recall_quality") { - " missing" - } else { - " none" - }; - (harness, recall) + } }) - .unwrap_or((" missing", " missing")); + .unwrap_or(" missing"); + let recall_status = if let Some(recall_quality) = snapshot.recall_quality.as_ref() { + format!(" {}", recall_quality.status.as_str()) + } else { + snapshot + .index + .as_ref() + .map(|index| { + if let Some(record) = &index.recall_quality { + format!(" {}", record.status.as_str()) + } else if index.missing.iter().any(|item| item == "recall_quality") { + " missing".to_string() + } else { + " none".to_string() + } + }) + .unwrap_or(" missing".to_string()) + }; if let Some(index) = snapshot.index.as_ref() { if index.harness.is_empty() { @@ -562,12 +571,15 @@ fn render_evidence_detail(frame: &mut Frame<'_>, area: Rect, app: &App) { if let Some(certification) = &snapshot.certification { lines.push(Line::from(vec![ Span::styled("Local certification ", theme::brand()), - Span::styled(certification.status.as_str(), theme::dim()), + Span::styled( + format!( + "{} generated {}", + certification.status.as_str(), + certification.generated_at + ), + theme::dim(), + ), ])); - lines.push(Line::from(format!( - "generated {}", - certification.generated_at - ))); let mut install_parts = Vec::new(); if let Some(bytes) = certification.release_binary_bytes { install_parts.push(format!("release {bytes} bytes")); @@ -604,7 +616,6 @@ fn render_evidence_detail(frame: &mut Frame<'_>, area: Rect, app: &App) { certification.agent_zero_note.as_deref().unwrap_or("") ))); } - lines.push(Line::from("")); lines.push(Line::from(vec![ Span::styled("root ", theme::dim()), Span::raw(truncate(&snapshot.root.display().to_string(), 56)), @@ -647,7 +658,53 @@ fn render_evidence_detail(frame: &mut Frame<'_>, area: Rect, app: &App) { } } } - lines.push(Line::from("")); + if let Some(recall_quality) = &snapshot.recall_quality { + lines.push(Line::from(vec![ + Span::styled("Recall quality ", theme::brand()), + Span::styled(recall_quality.status.as_str(), theme::dim()), + ])); + lines.push(Line::from(format!( + "{} queries {} pass {} fail {} review {}", + recall_quality.query_set_id, + recall_quality.query_count, + recall_quality.pass_count, + recall_quality.fail_count, + recall_quality.needs_review_count + ))); + if let Some(avg) = recall_quality.avg_latency_ms { + let max = recall_quality.max_latency_ms.unwrap_or(avg); + lines.push(Line::from(format!("avg {avg:.3} ms max {max:.3} ms"))); + } + lines.push(Line::from(vec![ + Span::styled("record ", theme::dim()), + Span::raw(truncate( + &recall_quality.record_path.display().to_string(), + 52, + )), + ])); + for query in recall_quality.queries.iter().take(4) { + let returned = if query.returned_ids.is_empty() { + "-".to_string() + } else { + query.returned_ids.join(",") + }; + let rank = query + .expected_rank + .map(|rank| rank.to_string()) + .unwrap_or_else(|| "-".to_string()); + lines.push(Line::from(format!( + "{} [{}] rank {} {:.3}ms", + truncate(&query.query_id, 28), + query.status, + rank, + query.latency_ms.unwrap_or(0.0) + ))); + lines.push(Line::from(Span::styled( + truncate(&format!("returned {returned}"), 70), + theme::dim(), + ))); + } + } lines.push(Line::from("Actions: /evidence refresh | /integrations")); } else { lines.push(Line::from("Run /evidence to load certification proof.")); @@ -925,6 +982,106 @@ mod tests { assert!(output.contains("codex.json")); } + #[test] + fn render_evidence_mode_shows_recall_quality_records() { + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("target/tree-ring-certification"); + std::fs::create_dir_all(evidence_dir.join("recall-quality")).unwrap(); + std::fs::write( + evidence_dir.join("metrics.json"), + r#"{"ok":true,"created_at":"2026-07-09T05:44:48Z"}"#, + ) + .unwrap(); + std::fs::write( + evidence_dir.join("recall-quality/default-fixture-v1.json"), + r#"{ + "schema_version": 1, + "generated_at": "2026-07-09T06:00:00Z", + "query_set_id": "default-fixture-v1", + "status": "needs_review", + "summary": { + "query_count": 4, + "pass_count": 3, + "fail_count": 0, + "needs_review_count": 1, + "avg_latency_ms": 0.5, + "max_latency_ms": 1.0 + }, + "queries": [ + { + "query_id": "scar-stale-cache", + "query": "failure stale cache", + "status": "pass", + "expected_top_id": "rq_scar_stale_cache", + "expected_rank": 1, + "latency_ms": 0.25, + "returned": [ + {"id":"rq_scar_stale_cache","rank":1,"ring":"scar","source_ref":"recall-quality/scar-stale-cache","score":1.2,"ranking":{"textual_match":1.0}} + ], + "notes": [] + } + , + { + "query_id": "scar-no-hit", + "query": "cache miss", + "status": "needs_review", + "expected_top_id": null, + "expected_rank": null, + "latency_ms": 0.75, + "returned": [], + "notes": [] + } + ] + }"#, + ) + .unwrap(); + std::fs::write( + evidence_dir.join("evidence-index.json"), + r#"{ + "generated_at": "2026-07-09T06:00:00Z", + "overall_status": "pass", + "certification": { + "category": "certification", + "status": "pass", + "label": "Local certification", + "path": "metrics.json", + "summary_path": null, + "generated_at": "2026-07-09T05:44:48Z" + }, + "harness": {}, + "recall_quality": { + "category": "recall_quality", + "status": "pass", + "label": "Recall quality", + "path": "recall-quality/default-fixture-v1.json", + "summary_path": null, + "generated_at": "2026-07-09T06:00:00Z" + }, + "missing": [], + "stale": [] + }"#, + ) + .unwrap(); + let mut app = App::new(dir.path().join(".tree-ring"), None).unwrap(); + app.execute_slash_command("/evidence").unwrap(); + let backend = TestBackend::new(120, 36); + let mut terminal = Terminal::new(backend).unwrap(); + + terminal.draw(|frame| render(frame, &app)).unwrap(); + let output = terminal.backend().to_string(); + + assert!(output.contains("Recall quality needs_review")); + assert!(!output.contains("Recall quality pass")); + assert!(output.contains("default-fixture-v1")); + assert!(output.contains("queries 4")); + assert!(output.contains("avg 0.500 ms")); + assert!(output.contains("scar-stale-cache")); + assert!(output.contains("rank 1")); + assert!(output.contains("scar-no-hit")); + assert!(output.contains("rank -")); + assert!(output.contains("rq_scar_stale_cache")); + } + #[test] fn render_evidence_mode_keeps_status_without_certification() { let dir = tempdir().unwrap(); diff --git a/docs/superpowers/plans/2026-07-09-tree-ring-recall-quality-dashboard-implementation-plan.md b/docs/superpowers/plans/2026-07-09-tree-ring-recall-quality-dashboard-implementation-plan.md new file mode 100644 index 0000000..c3d3a1b --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-tree-ring-recall-quality-dashboard-implementation-plan.md @@ -0,0 +1,1082 @@ +# Tree Ring Recall Quality Dashboard Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build Phase 3 of the evidence spine: deterministic recall-quality diagnostics that prove recall speed and relevance without using private memories, then surface the evidence in `/evidence`. + +**Architecture:** Add a non-mutating recall-quality evidence producer in the CLI crate. It creates a deterministic safe fixture store in memory, runs curated recall queries with ranking explanations enabled, writes a compact evidence record under `target/tree-ring-certification/recall-quality/default-fixture-v1.json`, merges that record into `evidence-index.json`, and lets the TUI render the record through the existing evidence browser. + +**Tech Stack:** Rust 2021, Clap, serde/serde_json, chrono, existing `tree-ring-memory-core`, existing `tree-ring-memory-sqlite`, Ratatui, shell certification script, cargo tests. + +## Global Constraints + +- Do not use real user memory or private payloads for recall-quality diagnostics. +- The default evidence path is `target/tree-ring-certification/recall-quality/default-fixture-v1.json`. +- Recall-quality evidence must record returned ids, rank positions, score breakdowns, and latency. +- Query results must support `pass`, `fail`, and `needs_review` statuses. +- The TUI `/evidence` view must show recall-quality evidence without leaking memory summaries or details. +- Do not add new runtime dependencies. +- Do not change recall ranking semantics except where a test explicitly proves an existing bug. +- Do not run hidden background certification from the TUI; `/evidence refresh` remains an explicit confirmation-only workflow. +- Run `cargo fmt --check`, `cargo test --locked`, `cargo clippy --locked --all-targets`, `git diff --check`, and `sh scripts/certify-tree-ring.sh` before PR handoff. + +--- + +## File Structure + +- Modify `crates/tree-ring-memory-cli/src/evidence.rs`: add `EvidenceStatus::NeedsReview`, shared index helpers, recall-quality payload summary loading, and tests. +- Modify `crates/tree-ring-memory-cli/src/harness_evidence.rs`: reuse shared evidence-index helpers and shared rollup logic. +- Create `crates/tree-ring-memory-cli/src/recall_quality.rs`: deterministic fixture, query runner, evaluation logic, evidence JSON writer, index merge, and focused tests. +- Modify `crates/tree-ring-memory-cli/src/main.rs`: expose `tree-ring recall-quality`, print JSON/human reports, and add CLI contract tests. +- Modify `crates/tree-ring-memory-cli/src/tui/render.rs`: render recall-quality list/detail rows and add a render test. +- Modify `scripts/certify-tree-ring.sh`: run the recall-quality command and assert the evidence record/index are present. +- Modify `README.md`: document the new recall-quality evidence command and certification artifact. + +--- + +### Task 1: Shared Evidence Index And Recall-Quality Reader + +**Files:** +- Modify: `crates/tree-ring-memory-cli/src/evidence.rs` +- Modify: `crates/tree-ring-memory-cli/src/harness_evidence.rs` + +**Interfaces:** +- Produces: `EvidenceStatus::NeedsReview`. +- Produces: `pub(crate) fn read_or_create_index(evidence_dir: &Path, generated_at: &str) -> Result`. +- Produces: `pub(crate) fn write_index(evidence_dir: &Path, index: &EvidenceIndex) -> Result`. +- Produces: `pub(crate) fn rollup_index_status(index: &EvidenceIndex) -> EvidenceStatus`. +- Produces: `pub struct RecallQualityEvidence` and `pub struct RecallQualityQueryEvidence`. +- Produces: `EvidenceSnapshot { recall_quality: Option, ... }`. +- Consumes: existing `EvidenceIndex`, `EvidenceRecordRef`, and certification reader behavior. + +- [ ] **Step 1: Add the failing evidence reader tests** + +Add tests in `crates/tree-ring-memory-cli/src/evidence.rs`: + +```rust +#[test] +fn evidence_snapshot_loads_recall_quality_record_from_index() { + let dir = tempdir().unwrap(); + let evidence_dir = certification_dir_for_project(dir.path()); + fs::create_dir_all(evidence_dir.join("recall-quality")).unwrap(); + fs::write(evidence_dir.join("metrics.json"), r#"{"ok":true,"created_at":"2026-07-09T04:22:38Z"}"#).unwrap(); + fs::write( + evidence_dir.join("recall-quality/default-fixture-v1.json"), + r#"{ + "schema_version": 1, + "generated_at": "2026-07-09T06:00:00Z", + "query_set_id": "default-fixture-v1", + "status": "pass", + "summary": { + "query_count": 2, + "pass_count": 2, + "fail_count": 0, + "needs_review_count": 0, + "avg_latency_ms": 1.25, + "max_latency_ms": 2.5 + }, + "queries": [ + { + "query_id": "scar-stale-cache", + "query": "failure stale cache", + "status": "pass", + "expected_top_id": "rq_scar_stale_cache", + "expected_rank": 1, + "latency_ms": 0.9, + "returned": [ + { + "id": "rq_scar_stale_cache", + "rank": 1, + "ring": "scar", + "source_ref": "recall-quality/scar-stale-cache", + "score": 1.2, + "ranking": {"textual_match": 1.0} + } + ], + "notes": [] + } + ] + }"#, + ) + .unwrap(); + fs::write( + evidence_dir.join("evidence-index.json"), + r#"{ + "generated_at": "2026-07-09T06:00:00Z", + "overall_status": "pass", + "certification": { + "category": "certification", + "status": "pass", + "label": "Local certification", + "path": "metrics.json", + "summary_path": null, + "generated_at": "2026-07-09T04:22:38Z" + }, + "harness": {}, + "recall_quality": { + "category": "recall_quality", + "status": "pass", + "label": "Recall quality", + "path": "recall-quality/default-fixture-v1.json", + "summary_path": null, + "generated_at": "2026-07-09T06:00:00Z" + }, + "missing": [], + "stale": [] + }"#, + ) + .unwrap(); + + let snapshot = load_snapshot(&evidence_dir); + + let recall_quality = snapshot.recall_quality.unwrap(); + assert_eq!(recall_quality.status, EvidenceStatus::Pass); + assert_eq!(recall_quality.query_set_id, "default-fixture-v1"); + assert_eq!(recall_quality.query_count, 2); + assert_eq!(recall_quality.pass_count, 2); + assert_eq!(recall_quality.fail_count, 0); + assert_eq!(recall_quality.needs_review_count, 0); + assert_eq!(recall_quality.avg_latency_ms, Some(1.25)); + assert_eq!(recall_quality.max_latency_ms, Some(2.5)); + assert_eq!(recall_quality.queries[0].query_id, "scar-stale-cache"); + assert_eq!(recall_quality.queries[0].returned_ids, vec!["rq_scar_stale_cache"]); +} + +#[test] +fn rollup_index_status_marks_recall_quality_needs_review() { + let mut index = EvidenceIndex { + generated_at: "2026-07-09T06:00:00Z".to_string(), + overall_status: EvidenceStatus::Pass, + certification: Some(EvidenceRecordRef { + category: "certification".to_string(), + status: EvidenceStatus::Pass, + label: "Local certification".to_string(), + path: PathBuf::from("metrics.json"), + summary_path: None, + generated_at: "2026-07-09T06:00:00Z".to_string(), + }), + harness: BTreeMap::new(), + recall_quality: Some(EvidenceRecordRef { + category: "recall_quality".to_string(), + status: EvidenceStatus::NeedsReview, + label: "Recall quality".to_string(), + path: PathBuf::from("recall-quality/default-fixture-v1.json"), + summary_path: None, + generated_at: "2026-07-09T06:00:00Z".to_string(), + }), + missing: Vec::new(), + stale: Vec::new(), + }; + assert_eq!(rollup_index_status(&index), EvidenceStatus::NeedsReview); + index.recall_quality.as_mut().unwrap().status = EvidenceStatus::Fail; + assert_eq!(rollup_index_status(&index), EvidenceStatus::Fail); +} +``` + +- [ ] **Step 2: Run the focused failing tests** + +Run: + +```bash +cargo test -p tree-ring-memory-cli evidence_snapshot_loads_recall_quality_record_from_index rollup_index_status_marks_recall_quality_needs_review --locked +``` + +Expected: FAIL because the recall-quality reader and `NeedsReview` status do not exist yet. + +- [ ] **Step 3: Add shared status, reader, and index helper implementation** + +Update `EvidenceStatus`: + +```rust +pub enum EvidenceStatus { + Pass, + Fail, + Skip, + Missing, + Stale, + NeedsReview, + Error, +} +``` + +Add to `EvidenceStatus::as_str()`: + +```rust +Self::NeedsReview => "needs_review", +``` + +Add these structs near `CertificationEvidence`: + +```rust +#[derive(Debug, Clone, PartialEq)] +pub struct RecallQualityEvidence { + pub status: EvidenceStatus, + pub generated_at: String, + pub record_path: PathBuf, + pub query_set_id: String, + pub query_count: u64, + pub pass_count: u64, + pub fail_count: u64, + pub needs_review_count: u64, + pub avg_latency_ms: Option, + pub max_latency_ms: Option, + pub queries: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RecallQualityQueryEvidence { + pub query_id: String, + pub query: String, + pub status: String, + pub expected_top_id: Option, + pub expected_rank: Option, + pub latency_ms: Option, + pub returned_ids: Vec, +} +``` + +Add `pub recall_quality: Option` to `EvidenceSnapshot` and initialize it in every `EvidenceSnapshot` literal. When an index is present, load both certification and recall quality: + +```rust +let certification = load_index_certification(evidence_dir, &index)?; +let recall_quality = load_index_recall_quality(evidence_dir, &index)?; +``` + +Add helpers: + +```rust +pub(crate) fn read_or_create_index( + evidence_dir: &Path, + generated_at: &str, +) -> Result { + let index_path = evidence_dir.join("evidence-index.json"); + if index_path.exists() { + let input = fs::read_to_string(&index_path).map_err(|err| err.to_string())?; + return serde_json::from_str(&input).map_err(|err| err.to_string()); + } + Ok(EvidenceIndex { + generated_at: generated_at.to_string(), + overall_status: EvidenceStatus::Missing, + certification: certification_record_from_metrics(evidence_dir, generated_at), + harness: BTreeMap::new(), + recall_quality: None, + missing: vec!["harness".to_string(), "recall_quality".to_string()], + stale: Vec::new(), + }) +} + +pub(crate) fn write_index(evidence_dir: &Path, index: &EvidenceIndex) -> Result { + fs::create_dir_all(evidence_dir).map_err(|err| err.to_string())?; + let index_path = evidence_dir.join("evidence-index.json"); + let json = serde_json::to_string_pretty(index).map_err(|err| err.to_string())?; + fs::write(&index_path, json).map_err(|err| err.to_string())?; + Ok(index_path) +} + +pub(crate) fn rollup_index_status(index: &EvidenceIndex) -> EvidenceStatus { + if index.harness.values().any(|record| record.status == EvidenceStatus::Fail) + || index + .recall_quality + .as_ref() + .is_some_and(|record| record.status == EvidenceStatus::Fail) + { + return EvidenceStatus::Fail; + } + if index.harness.values().any(|record| record.status == EvidenceStatus::Error) + || index + .recall_quality + .as_ref() + .is_some_and(|record| record.status == EvidenceStatus::Error) + { + return EvidenceStatus::Error; + } + if index + .recall_quality + .as_ref() + .is_some_and(|record| record.status == EvidenceStatus::NeedsReview) + { + return EvidenceStatus::NeedsReview; + } + if let Some(certification) = &index.certification { + return certification.status; + } + if index.harness.is_empty() && index.recall_quality.is_none() { + EvidenceStatus::Missing + } else { + EvidenceStatus::Skip + } +} +``` + +Move the private `certification_record_from_metrics` behavior from `harness_evidence.rs` into `evidence.rs` as a private helper used by `read_or_create_index`. + +Add `load_index_recall_quality()` and `load_recall_quality()` using `serde_json::Value`, `get_u64`, `get_f64`, `get_string`, and `resolve_evidence_path()`. Parse only safe metadata: `query_set_id`, summary counts, latency, `query_id`, `query`, `status`, `expected_top_id`, `expected_rank`, and returned `id`s. Do not parse or display memory summaries/details. + +- [ ] **Step 4: Update harness evidence to use shared helpers** + +In `crates/tree-ring-memory-cli/src/harness_evidence.rs`, replace private `read_or_create_index`, `certification_record_from_metrics`, and `rollup_status` with imports from `crate::evidence`: + +```rust +use crate::evidence::{ + read_or_create_index, rollup_index_status, write_index, EvidenceRecordRef, EvidenceStatus, +}; +``` + +In `merge_harness_index()`, replace the final rollup/write block with: + +```rust +index.overall_status = rollup_index_status(&index); +write_index(evidence_dir, &index) +``` + +Update existing harness rollup tests to call `rollup_index_status(&index)`. + +- [ ] **Step 5: Run focused tests** + +Run: + +```bash +cargo test -p tree-ring-memory-cli evidence::tests harness_evidence::tests --locked +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/tree-ring-memory-cli/src/evidence.rs crates/tree-ring-memory-cli/src/harness_evidence.rs +git commit -m "Add recall quality evidence reader" +``` + +--- + +### Task 2: Recall-Quality Diagnostic Runner And CLI + +**Files:** +- Create: `crates/tree-ring-memory-cli/src/recall_quality.rs` +- Modify: `crates/tree-ring-memory-cli/src/main.rs` + +**Interfaces:** +- Consumes: `read_or_create_index`, `write_index`, `rollup_index_status`, `EvidenceRecordRef`, and `EvidenceStatus` from Task 1. +- Produces: `pub struct RecallQualityRequest { pub source_root: PathBuf, pub evidence_dir: PathBuf }`. +- Produces: `pub struct RecallQualityReport`. +- Produces: `pub fn run_recall_quality(request: RecallQualityRequest) -> Result`. +- Produces CLI command: `tree-ring recall-quality --source-root --out-dir `. + +- [ ] **Step 1: Write failing runner tests** + +Create `crates/tree-ring-memory-cli/src/recall_quality.rs` with test scaffolding and tests: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn recall_quality_writes_safe_evidence_and_updates_index() { + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("target/tree-ring-certification"); + + let report = run_recall_quality(RecallQualityRequest { + source_root: dir.path().to_path_buf(), + evidence_dir: evidence_dir.clone(), + }) + .unwrap(); + + assert_eq!(report.query_set_id, "default-fixture-v1"); + assert_eq!(report.status, EvidenceStatus::Pass); + assert_eq!(report.summary.fail_count, 0); + assert!(report.summary.pass_count >= 3); + assert!(evidence_dir.join("recall-quality/default-fixture-v1.json").exists()); + let json = std::fs::read_to_string(evidence_dir.join("recall-quality/default-fixture-v1.json")).unwrap(); + assert!(json.contains("\"ranking\"")); + assert!(json.contains("\"latency_ms\"")); + assert!(!json.contains("Private bank account note")); + + let index: crate::evidence::EvidenceIndex = + serde_json::from_str(&std::fs::read_to_string(evidence_dir.join("evidence-index.json")).unwrap()).unwrap(); + assert!(index.recall_quality.is_some()); + assert!(!index.missing.iter().any(|item| item == "recall_quality")); + } + + #[test] + fn query_evaluation_distinguishes_fail_and_needs_review() { + let returned = vec![ + returned_memory_for_test("other", 1), + returned_memory_for_test("expected", 2), + ]; + let review = evaluate_query_for_test( + Some("expected"), + Some(3), + &[], + &returned, + ); + assert_eq!(review.status, RecallQualityQueryStatus::NeedsReview); + + let failed = evaluate_query_for_test( + Some("missing"), + Some(1), + &["other"], + &returned, + ); + assert_eq!(failed.status, RecallQualityQueryStatus::Fail); + assert!(failed.notes.iter().any(|note| note.contains("missing"))); + assert!(failed.notes.iter().any(|note| note.contains("forbidden"))); + } +} +``` + +The test helper names can be `pub(crate)` or private test-only helpers. They must exercise the same evaluation function used by the production runner. + +- [ ] **Step 2: Run failing runner tests** + +Run: + +```bash +cargo test -p tree-ring-memory-cli recall_quality --locked +``` + +Expected: FAIL because the module and CLI command are not wired yet. + +- [ ] **Step 3: Implement the diagnostic types and deterministic fixture** + +Add production structs with `Serialize`, `Deserialize`, and `PartialEq` where useful: + +```rust +pub const RECALL_QUALITY_QUERY_SET_ID: &str = "default-fixture-v1"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecallQualityRequest { + pub source_root: PathBuf, + pub evidence_dir: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualityReport { + pub schema_version: u8, + pub generated_at: String, + pub query_set_id: String, + pub status: EvidenceStatus, + pub source_root: PathBuf, + pub evidence_dir: PathBuf, + pub record_path: PathBuf, + pub summary: RecallQualitySummary, + pub queries: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualitySummary { + pub query_count: usize, + pub pass_count: usize, + pub fail_count: usize, + pub needs_review_count: usize, + pub avg_latency_ms: f64, + pub max_latency_ms: f64, + pub fixture_memory_count: usize, + pub private_payloads_used: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecallQualityQueryStatus { + Pass, + Fail, + NeedsReview, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualityQueryRecord { + pub query_id: String, + pub query: String, + pub status: RecallQualityQueryStatus, + pub expected_top_id: Option, + pub expected_rank: Option, + pub latency_ms: f64, + pub returned: Vec, + pub notes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallQualityReturnedMemory { + pub id: String, + pub rank: usize, + pub ring: String, + pub source_ref: String, + pub score: f64, + pub ranking: BTreeMap, +} +``` + +Build fixture memories with stable ids, safe text, stable project `recall-quality-fixture`, stable source refs, and normal sensitivity. Include at least: + +```rust +rq_scar_stale_cache +rq_heartwood_sqlite_decision +rq_seed_harness_experiment +rq_outer_release_guardrail +rq_sensitive_health_note +``` + +Set `rq_sensitive_health_note.sensitivity = "health"` so the sensitive-filter query can prove it is absent without writing or displaying its details. + +- [ ] **Step 4: Implement the runner and index merge** + +Implement `run_recall_quality()`: + +```rust +pub fn run_recall_quality(request: RecallQualityRequest) -> Result { + let generated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let mut store = SQLiteMemoryStore::open(Path::new(":memory:")).map_err(|err| err.to_string())?; + let fixtures = fixture_memories()?; + store.put_many(&fixtures).map_err(|err| err.to_string())?; + + let mut queries = Vec::new(); + for case in query_cases() { + queries.push(run_case(&store, case)?); + } + let summary = summarize(&queries, fixtures.len()); + let status = report_status(&summary); + let record_path = request + .evidence_dir + .join("recall-quality") + .join(format!("{RECALL_QUALITY_QUERY_SET_ID}.json")); + let report = RecallQualityReport { + schema_version: 1, + generated_at: generated_at.clone(), + query_set_id: RECALL_QUALITY_QUERY_SET_ID.to_string(), + status, + source_root: request.source_root, + evidence_dir: request.evidence_dir.clone(), + record_path: record_path.clone(), + summary, + queries, + }; + write_report_and_index(&request.evidence_dir, &generated_at, &report)?; + Ok(report) +} +``` + +Use `MemoryRetriever::new(&store).recall(..., include_sensitive=false, include_superseded=false, limit=5, explain_ranking=true)` for every case. Use `Instant::now()` for latency. + +Use this default query set: + +```rust +QueryCase { + query_id: "scar-stale-cache", + query: "failure stale cache", + expected_top_id: Some("rq_scar_stale_cache"), + max_expected_rank: Some(1), + forbidden_ids: &[], +} +QueryCase { + query_id: "heartwood-sqlite-decision", + query: "durable local sqlite decision", + expected_top_id: Some("rq_heartwood_sqlite_decision"), + max_expected_rank: Some(1), + forbidden_ids: &[], +} +QueryCase { + query_id: "seed-harness-experiment", + query: "planning experiment agent harness", + expected_top_id: Some("rq_seed_harness_experiment"), + max_expected_rank: Some(1), + forbidden_ids: &[], +} +QueryCase { + query_id: "sensitive-filter", + query: "health private payload", + expected_top_id: None, + max_expected_rank: None, + forbidden_ids: &["rq_sensitive_health_note"], +} +``` + +Write the report with pretty JSON. Merge the index: + +```rust +let mut index = read_or_create_index(evidence_dir, generated_at)?; +index.generated_at = generated_at.to_string(); +index.recall_quality = Some(EvidenceRecordRef { + category: "recall_quality".to_string(), + status: report.status, + label: "Recall quality".to_string(), + path: PathBuf::from(format!("recall-quality/{RECALL_QUALITY_QUERY_SET_ID}.json")), + summary_path: None, + generated_at: generated_at.to_string(), +}); +index.missing.retain(|item| item != "recall_quality"); +if index.harness.is_empty() && !index.missing.iter().any(|item| item == "harness") { + index.missing.push("harness".to_string()); +} +index.missing.sort(); +index.missing.dedup(); +index.overall_status = rollup_index_status(&index); +write_index(evidence_dir, &index)?; +``` + +- [ ] **Step 5: Wire the CLI command** + +In `main.rs`, add: + +```rust +use recall_quality::{run_recall_quality, RecallQualityReport, RecallQualityRequest}; +mod recall_quality; +``` + +Add a command variant: + +```rust +#[command(about = "write non-private recall quality evidence")] +RecallQuality { + #[arg(long, default_value = ".", help = "project root used for default evidence output")] + source_root: PathBuf, + #[arg( + long, + help = "evidence output directory; defaults to /target/tree-ring-certification" + )] + out_dir: Option, +}, +``` + +Handle it before opening the writable memory store: + +```rust +if let Command::RecallQuality { + source_root, + out_dir, +} = &cli.command +{ + let evidence_dir = out_dir + .clone() + .unwrap_or_else(|| evidence::certification_dir_for_project(source_root)); + let report = run_recall_quality(RecallQualityRequest { + source_root: source_root.clone(), + evidence_dir, + })?; + print_recall_quality_report(&report, cli.json)?; + return Ok(()); +} +``` + +Add a printer: + +```rust +fn print_recall_quality_report( + report: &RecallQualityReport, + json_output: bool, +) -> Result<(), String> { + if json_output { + println!("{}", json!({"ok": true, "report": report})); + } else { + println!( + "Tree Ring Memory recall quality: status={} queries={} pass={} fail={} needs_review={} avg={:.3}ms max={:.3}ms evidence={}", + report.status.as_str(), + report.summary.query_count, + report.summary.pass_count, + report.summary.fail_count, + report.summary.needs_review_count, + report.summary.avg_latency_ms, + report.summary.max_latency_ms, + report.evidence_dir.display() + ); + for query in &report.queries { + println!( + "{} [{:?}] latency={:.3}ms returned={}", + query.query_id, + query.status, + query.latency_ms, + query.returned.iter().map(|item| item.id.as_str()).collect::>().join(",") + ); + } + } + Ok(()) +} +``` + +Add command tests in `main.rs` for JSON and human output, mirroring the existing harness certification tests. + +- [ ] **Step 6: Run focused tests and manual command** + +Run: + +```bash +cargo test -p tree-ring-memory-cli recall_quality --locked +cargo test -p tree-ring-memory-cli recall_quality_json_output_contract recall_quality_human_output_contract --locked +cargo run -p tree-ring-memory-cli -- --json recall-quality --source-root . --out-dir target/tree-ring-certification +``` + +Expected: PASS and command JSON contains `"query_set_id":"default-fixture-v1"` plus `"status":"pass"`. + +- [ ] **Step 7: Commit** + +```bash +git add crates/tree-ring-memory-cli/src/recall_quality.rs crates/tree-ring-memory-cli/src/main.rs +git commit -m "Add recall quality diagnostic runner" +``` + +--- + +### Task 3: Certification Script And Documentation + +**Files:** +- Modify: `scripts/certify-tree-ring.sh` +- Modify: `README.md` + +**Interfaces:** +- Consumes: `tree-ring recall-quality --source-root --out-dir `. +- Produces: certification artifacts `recall-quality.json` and `recall-quality/default-fixture-v1.json`. + +- [ ] **Step 1: Add script assertions** + +In `scripts/certify-tree-ring.sh`, after the harness certification assertions, add: + +```sh +"$BIN" --json recall-quality --source-root "$scan_root" --out-dir "$OUT_DIR" \ + > "$OUT_DIR/recall-quality.json" +require_file "$OUT_DIR/recall-quality/default-fixture-v1.json" +grep -E '"status"[[:space:]]*:[[:space:]]*"pass"' "$OUT_DIR/recall-quality.json" > /dev/null \ + || fail "recall quality command did not report pass status" +grep -E '"fail_count"[[:space:]]*:[[:space:]]*0' "$OUT_DIR/recall-quality.json" > /dev/null \ + || fail "recall quality command reported failing queries" +grep -F '"recall_quality": {' "$INDEX" > /dev/null \ + || fail "evidence index did not include recall quality record" +grep -F '"missing": []' "$INDEX" > /dev/null \ + || fail "evidence index still reports missing evidence categories" +``` + +Update the final output: + +```sh +printf 'Recall quality evidence: %s\n' "$OUT_DIR/recall-quality/default-fixture-v1.json" +``` + +- [ ] **Step 2: Update README command docs** + +In `README.md`, near the `integrations certify` text, add: + +```markdown +- `recall-quality` writes non-private recall diagnostics under + `target/tree-ring-certification/recall-quality/default-fixture-v1.json` + and merges the result into `evidence-index.json`. It uses deterministic + safe fixture memories, records returned ids, rank positions, score factors, + and latency, and marks each query as `pass`, `fail`, or `needs_review`. + +```bash +tree-ring recall-quality --source-root . +``` +``` + +In the certification script section, add that `scripts/certify-tree-ring.sh` now runs the recall-quality diagnostics in addition to install-size, recall-speed, CLI, adapter, and harness checks. + +- [ ] **Step 3: Run focused checks** + +Run: + +```bash +cargo run -p tree-ring-memory-cli -- --json recall-quality --source-root . --out-dir target/tree-ring-certification +sh scripts/certify-tree-ring.sh +``` + +Expected: both commands pass. Certification output includes `Recall quality evidence:`. + +- [ ] **Step 4: Commit** + +```bash +git add scripts/certify-tree-ring.sh README.md +git commit -m "Certify recall quality evidence" +``` + +--- + +### Task 4: TUI Recall-Quality Dashboard Rendering + +**Files:** +- Modify: `crates/tree-ring-memory-cli/src/tui/render.rs` + +**Interfaces:** +- Consumes: `EvidenceSnapshot.recall_quality` from Task 1. +- Produces: `/evidence` list and detail rendering for recall-quality status, query counts, latency, and returned ids. + +- [ ] **Step 1: Add failing render test** + +Add a test near the existing evidence render tests: + +```rust +#[test] +fn render_evidence_mode_shows_recall_quality_records() { + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("target/tree-ring-certification"); + std::fs::create_dir_all(evidence_dir.join("recall-quality")).unwrap(); + std::fs::write(evidence_dir.join("metrics.json"), r#"{"ok":true,"created_at":"2026-07-09T05:44:48Z"}"#).unwrap(); + std::fs::write( + evidence_dir.join("recall-quality/default-fixture-v1.json"), + r#"{ + "schema_version": 1, + "generated_at": "2026-07-09T06:00:00Z", + "query_set_id": "default-fixture-v1", + "status": "pass", + "summary": { + "query_count": 4, + "pass_count": 4, + "fail_count": 0, + "needs_review_count": 0, + "avg_latency_ms": 0.5, + "max_latency_ms": 1.0 + }, + "queries": [ + { + "query_id": "scar-stale-cache", + "query": "failure stale cache", + "status": "pass", + "expected_top_id": "rq_scar_stale_cache", + "expected_rank": 1, + "latency_ms": 0.25, + "returned": [ + {"id":"rq_scar_stale_cache","rank":1,"ring":"scar","source_ref":"recall-quality/scar-stale-cache","score":1.2,"ranking":{"textual_match":1.0}} + ], + "notes": [] + } + ] + }"#, + ) + .unwrap(); + std::fs::write( + evidence_dir.join("evidence-index.json"), + r#"{ + "generated_at": "2026-07-09T06:00:00Z", + "overall_status": "pass", + "certification": { + "category": "certification", + "status": "pass", + "label": "Local certification", + "path": "metrics.json", + "summary_path": null, + "generated_at": "2026-07-09T05:44:48Z" + }, + "harness": {}, + "recall_quality": { + "category": "recall_quality", + "status": "pass", + "label": "Recall quality", + "path": "recall-quality/default-fixture-v1.json", + "summary_path": null, + "generated_at": "2026-07-09T06:00:00Z" + }, + "missing": [], + "stale": [] + }"#, + ) + .unwrap(); + let mut app = App::new(dir.path().join(".tree-ring"), None).unwrap(); + app.execute_slash_command("/evidence").unwrap(); + let backend = TestBackend::new(120, 36); + let mut terminal = Terminal::new(backend).unwrap(); + + terminal.draw(|frame| render(frame, &app)).unwrap(); + let output = terminal.backend().to_string(); + + assert!(output.contains("Recall quality")); + assert!(output.contains("default-fixture-v1")); + assert!(output.contains("queries 4")); + assert!(output.contains("avg 0.500 ms")); + assert!(output.contains("scar-stale-cache")); + assert!(output.contains("rq_scar_stale_cache")); +} +``` + +- [ ] **Step 2: Run failing render test** + +Run: + +```bash +cargo test -p tree-ring-memory-cli render_evidence_mode_shows_recall_quality_records --locked +``` + +Expected: FAIL because the TUI still only shows recall quality as loaded/missing. + +- [ ] **Step 3: Render recall-quality status in the evidence list** + +In `render_evidence_list()`, when `index.recall_quality` is present, show the actual status instead of only `loaded`: + +```rust +let recall = if let Some(record) = &index.recall_quality { + format!(" {}", record.status.as_str()) +} else if index.missing.iter().any(|item| item == "recall_quality") { + " missing".to_string() +} else { + " none".to_string() +}; +``` + +Update the list row to use owned `String` status text. + +- [ ] **Step 4: Render recall-quality details** + +In `render_evidence_detail()`, after the harness matrix block, add: + +```rust +if let Some(recall_quality) = &snapshot.recall_quality { + lines.push(Line::from("")); + lines.push(Line::from(vec![ + Span::styled("Recall quality ", theme::brand()), + Span::styled(recall_quality.status.as_str(), theme::dim()), + ])); + lines.push(Line::from(format!( + "{} queries {} pass {} fail {} review {}", + recall_quality.query_set_id, + recall_quality.query_count, + recall_quality.pass_count, + recall_quality.fail_count, + recall_quality.needs_review_count + ))); + if let Some(avg) = recall_quality.avg_latency_ms { + let max = recall_quality.max_latency_ms.unwrap_or(avg); + lines.push(Line::from(format!("avg {avg:.3} ms max {max:.3} ms"))); + } + lines.push(Line::from(vec![ + Span::styled("record ", theme::dim()), + Span::raw(truncate(&recall_quality.record_path.display().to_string(), 52)), + ])); + for query in recall_quality.queries.iter().take(4) { + let returned = if query.returned_ids.is_empty() { + "-".to_string() + } else { + query.returned_ids.join(",") + }; + lines.push(Line::from(format!( + "{} [{}] rank {:?} {:.3}ms", + truncate(&query.query_id, 28), + query.status, + query.expected_rank, + query.latency_ms.unwrap_or(0.0) + ))); + lines.push(Line::from(Span::styled( + truncate(&format!("returned {returned}"), 70), + theme::dim(), + ))); + } +} +``` + +Do not render memory summaries or details from the diagnostic payload. + +- [ ] **Step 5: Run focused TUI tests** + +Run: + +```bash +cargo test -p tree-ring-memory-cli render_evidence_mode --locked +``` + +Expected: PASS with the new recall-quality render test and existing certification/harness evidence render tests passing. + +- [ ] **Step 6: Commit** + +```bash +git add crates/tree-ring-memory-cli/src/tui/render.rs +git commit -m "Render recall quality evidence in TUI" +``` + +--- + +### Task 5: Final Verification And PR Handoff + +**Files:** +- Modify only if verification reveals a real defect in files touched by Tasks 1-4. + +**Interfaces:** +- Consumes completed recall-quality runner, certification script, docs, and TUI rendering. +- Produces PR-ready branch with verification evidence. + +- [ ] **Step 1: Run formatting** + +Run: + +```bash +cargo fmt --check +``` + +Expected: PASS. + +- [ ] **Step 2: Run full tests** + +Run: + +```bash +cargo test --locked +``` + +Expected: PASS. + +- [ ] **Step 3: Run clippy** + +Run: + +```bash +cargo clippy --locked --all-targets +``` + +Expected: PASS with no new warnings. + +- [ ] **Step 4: Run whitespace diff check** + +Run: + +```bash +git diff --check +``` + +Expected: no output. + +- [ ] **Step 5: Run certification** + +Run: + +```bash +sh scripts/certify-tree-ring.sh +``` + +Expected: PASS. Output includes `Evidence index:`, `Harness evidence:`, and `Recall quality evidence:`. + +- [ ] **Step 6: Inspect evidence artifacts** + +Run: + +```bash +test -f target/tree-ring-certification/recall-quality/default-fixture-v1.json +grep -F '"recall_quality": {' target/tree-ring-certification/evidence-index.json +grep -F '"missing": []' target/tree-ring-certification/evidence-index.json +``` + +Expected: all commands pass. + +- [ ] **Step 7: Commit verification fixes if any** + +If Tasks 1-4 did not need follow-up fixes, skip this step. Otherwise: + +```bash +git add +git commit -m "Stabilize recall quality certification" +``` + +- [ ] **Step 8: Push and open PR to main** + +Run: + +```bash +git push -u origin codex/recall-quality-dashboard +gh pr create --base main --head codex/recall-quality-dashboard --title "Add recall quality evidence dashboard" --body-file +``` + +Expected: PR created against `main`. + +--- + +## Self-Review + +- Spec coverage: Task 2 creates curated recall query sets, deterministic fixture memory, returned ids, rank positions, score breakdowns, latency, and pass/fail/needs_review query status. Task 1 and Task 2 merge recall-quality into `/evidence` artifacts. Task 4 surfaces ranking evidence in the TUI without payload leakage. Task 3 wires certification. Task 5 verifies. +- Privacy check: The default runner uses safe fixture memory only; TUI parsing renders ids, query ids, statuses, rank, and latency only. +- Scope check: No real memory store, background daemon, hidden TUI refresh execution, vector search changes, or ranking semantic changes are included. +- Type consistency: `EvidenceStatus::NeedsReview`, `RecallQualityReport`, `RecallQualitySummary`, `RecallQualityQueryRecord`, `RecallQualityReturnedMemory`, and `EvidenceSnapshot.recall_quality` are introduced before use. +- Placeholder scan: No task uses placeholder tokens, "similar to", or unspecified validation language. diff --git a/scripts/certify-tree-ring.sh b/scripts/certify-tree-ring.sh index 86ac2fc..609288e 100755 --- a/scripts/certify-tree-ring.sh +++ b/scripts/certify-tree-ring.sh @@ -82,6 +82,45 @@ extract_metrics_json() { sed -n 's/^METRICS_JSON=//p' "$1" | tail -n 1 } +assert_recall_quality_evidence() { + command -v python3 >/dev/null 2>&1 \ + || fail "python3 is required for recall quality evidence validation" + python3 - "$1" "$2" <<'PY' +import json +import sys + +report_path, index_path = sys.argv[1:3] +with open(report_path, encoding="utf-8") as handle: + payload = json.load(handle) +with open(index_path, encoding="utf-8") as handle: + index = json.load(handle) + +report = payload.get("report") or {} +summary = report.get("summary") or {} +recall_quality = index.get("recall_quality") or {} +errors = [] + +if report.get("status") != "pass": + errors.append(f"report.status={report.get('status')!r}") +if summary.get("fail_count") != 0: + errors.append(f"summary.fail_count={summary.get('fail_count')!r}") +if summary.get("private_payloads_used") is not False: + errors.append( + f"summary.private_payloads_used={summary.get('private_payloads_used')!r}" + ) +if recall_quality.get("status") != "pass": + errors.append(f"index.recall_quality.status={recall_quality.get('status')!r}") +if recall_quality.get("path") != "recall-quality/default-fixture-v1.json": + errors.append(f"index.recall_quality.path={recall_quality.get('path')!r}") +if index.get("missing") != []: + errors.append(f"index.missing={index.get('missing')!r}") + +if errors: + print("; ".join(errors), file=sys.stderr) + sys.exit(1) +PY +} + mkdir -p "$OUT_DIR" SUMMARY="$OUT_DIR/summary.md" METRICS="$OUT_DIR/metrics.json" @@ -325,9 +364,15 @@ grep -F '"harness"' "$INDEX" > /dev/null \ || fail "evidence index did not include harness records" grep -F '"codex"' "$INDEX" > /dev/null \ || fail "evidence index did not include Codex harness record" +"$BIN" --json recall-quality --source-root "$scan_root" --out-dir "$OUT_DIR" \ + > "$OUT_DIR/recall-quality.json" +require_file "$OUT_DIR/recall-quality/default-fixture-v1.json" +assert_recall_quality_evidence "$OUT_DIR/recall-quality.json" "$INDEX" \ + || fail "recall quality evidence validation failed" log "certification passed" printf 'Summary: %s\n' "$SUMMARY" printf 'Metrics: %s\n' "$METRICS" printf 'Evidence index: %s\n' "$INDEX" printf 'Harness evidence: %s\n' "$OUT_DIR/harness" +printf 'Recall quality evidence: %s\n' "$OUT_DIR/recall-quality/default-fixture-v1.json"