diff --git a/README.md b/README.md index 98a8b22..fd08cc7 100644 --- a/README.md +++ b/README.md @@ -390,6 +390,13 @@ Agent Zero plugin smoke enabled: - 30k recall max: 14.705 ms. - Agent Zero plugin smoke: passed. +Certification also runs the default memory quality scenario pack under +`fixtures/quality/`. Those scenarios prove recall gates, spam rejection, +stale-truth suppression, evidence requirements, and behavior-proof outcomes. +The quality report is written to +`target/tree-ring-certification/quality/quality-report.json` with a readable +summary at `target/tree-ring-certification/quality/quality-summary.md`. + `scripts/package-release.sh` builds the Rust CLI in release mode, creates a platform tarball under `dist/`, and writes a SHA-256 checksum file. Tag pushes run the release artifact workflow for Linux and macOS. diff --git a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs new file mode 100644 index 0000000..1df3fa1 --- /dev/null +++ b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs @@ -0,0 +1,542 @@ +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use tree_ring_memory_core::{ + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, + QualityRunError, QualityRunReport, QualityScenario, QualityScenarioReport, TreeRingError, +}; +use tree_ring_memory_sqlite::{MemoryRetriever, SQLiteMemoryStore}; + +const USAGE: &str = "usage: quality_scenarios "; +const RECALL_LIMIT: usize = 8; +static TEMP_ROOT_COUNTER: AtomicU64 = AtomicU64::new(0); + +fn main() { + if let Err(err) = run() { + eprintln!("quality scenarios failed: {err}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let mut args = std::env::args_os().skip(1); + let fixture_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| USAGE.to_string())?; + let output_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| USAGE.to_string())?; + if args.next().is_some() { + return Err(USAGE.to_string()); + } + + let run_report = run_quality_command(&fixture_dir, &output_dir)?; + + println!( + "quality scenarios passed: {} scenario(s)", + run_report.scenario_count + ); + Ok(()) +} + +fn run_quality_command(fixture_dir: &Path, output_dir: &Path) -> Result { + let run_report = run_quality_scenarios(fixture_dir, output_dir)?; + verify_persisted_quality_report(output_dir, &run_report)?; + Ok(run_report) +} + +fn verify_persisted_quality_report( + output_dir: &Path, + expected: &QualityRunReport, +) -> Result<(), String> { + let json = fs::read_to_string(output_dir.join("quality-report.json")) + .map_err(|_| "quality_report_read_error".to_string())?; + let persisted = serde_json::from_str::(&json) + .map_err(|_| "quality_report_parse_error".to_string())?; + if persisted != *expected { + return Err("quality_report_mismatch_error".to_string()); + } + if !persisted.ok || !persisted.quality_pass { + return Err(format!( + "quality run failed: {} scenarios evaluated", + persisted.scenario_count + )); + } + Ok(()) +} + +fn run_quality_scenarios( + fixture_dir: &Path, + output_dir: &Path, +) -> Result { + fs::create_dir_all(output_dir).map_err(|_| "output_directory_create_error".to_string())?; + + let paths = match fixture_paths(fixture_dir) { + Ok(paths) if !paths.is_empty() => paths, + Ok(_) => { + return write_failed_report( + output_dir, + Vec::new(), + QualityRunError { + scenario: None, + path: Some(fixture_dir.display().to_string()), + stage: "fixture_directory".to_string(), + message: "no quality scenario JSON files found".to_string(), + }, + ); + } + Err(error) => return write_failed_report(output_dir, Vec::new(), error), + }; + + let mut reports = Vec::with_capacity(paths.len()); + for path in paths { + let input = match fs::read_to_string(&path) { + Ok(input) => input, + Err(_) => { + return write_failed_report( + output_dir, + reports, + QualityRunError { + scenario: None, + path: Some(path.display().to_string()), + stage: "file_read".to_string(), + message: "fixture_file_read_error".to_string(), + }, + ); + } + }; + let scenario = match parse_quality_scenario(&input) { + Ok(scenario) => scenario, + Err(err) => { + return write_failed_report( + output_dir, + reports, + QualityRunError { + scenario: None, + path: Some(path.display().to_string()), + stage: "parse".to_string(), + message: classify_parse_error(&err).to_string(), + }, + ); + } + }; + match run_scenario(&scenario) { + Ok(report) => reports.push(report), + Err(failure) => { + return write_failed_report( + output_dir, + reports, + QualityRunError { + scenario: Some(scenario.name), + path: Some(path.display().to_string()), + stage: failure.stage.to_string(), + message: failure.message, + }, + ); + } + } + } + + let report = summarize_quality_run(reports); + write_reports(output_dir, &report)?; + Ok(report) +} + +fn fixture_paths(fixture_dir: &Path) -> Result, QualityRunError> { + let entries = fs::read_dir(fixture_dir).map_err(|_| QualityRunError { + scenario: None, + path: Some(fixture_dir.display().to_string()), + stage: "fixture_directory".to_string(), + message: "fixture_directory_read_error".to_string(), + })?; + let mut paths = entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(|_| QualityRunError { + scenario: None, + path: Some(fixture_dir.display().to_string()), + stage: "fixture_directory".to_string(), + message: "fixture_directory_read_error".to_string(), + }) + }) + .collect::, _>>()?; + paths.retain(|path| path.extension() == Some(OsStr::new("json"))); + paths.sort(); + Ok(paths) +} + +fn write_failed_report( + output_dir: &Path, + reports: Vec, + error: QualityRunError, +) -> Result { + let mut report = summarize_quality_run(reports); + report.ok = false; + report.quality_pass = false; + report.errors.push(error); + write_reports(output_dir, &report)?; + Ok(report) +} + +struct ScenarioRunFailure { + stage: &'static str, + message: String, +} + +fn run_scenario(scenario: &QualityScenario) -> Result { + let root = TemporaryRoot::new(&scenario.name).map_err(|message| ScenarioRunFailure { + stage: "store_open", + message, + })?; + let db_path = root.path().join("memory.sqlite"); + let mut store = SQLiteMemoryStore::open(&db_path).map_err(|_| ScenarioRunFailure { + stage: "store_open", + message: "sqlite_store_open_error".to_string(), + })?; + store + .put_many(&scenario.seed_memories) + .map_err(|_| ScenarioRunFailure { + stage: "seed", + message: "seed_memory_write_error".to_string(), + })?; + + let prompt = scenario.prompt().ok_or_else(|| ScenarioRunFailure { + stage: "evaluation", + message: "scenario_prompt_missing".to_string(), + })?; + let recalls = MemoryRetriever::new(&store) + .recall( + prompt, + None, + None, + None, + None, + None, + false, + false, + RECALL_LIMIT, + false, + ) + .map_err(|_| ScenarioRunFailure { + stage: "recall", + message: "recall_execution_error".to_string(), + })? + .into_iter() + .map(|result| QualityRecall { + memory: result.memory, + score: result.score, + }) + .collect::>(); + + evaluate_quality_scenario(scenario, &recalls).map_err(|err| ScenarioRunFailure { + stage: "evaluation", + message: classify_evaluation_error(&err).to_string(), + }) +} + +fn write_reports(output_dir: &Path, report: &QualityRunReport) -> Result<(), String> { + let json = + serde_json::to_string_pretty(report).map_err(|_| "report_json_encode_error".to_string())?; + fs::write(output_dir.join("quality-report.json"), json) + .map_err(|_| "report_json_write_error".to_string())?; + fs::write( + output_dir.join("quality-summary.md"), + markdown_summary(report), + ) + .map_err(|_| "report_markdown_write_error".to_string())?; + Ok(()) +} + +fn classify_parse_error(error: &TreeRingError) -> &'static str { + match error { + TreeRingError::Json(_) => "scenario_parse_error", + TreeRingError::Validation(_) => "scenario_validation_error", + _ => "scenario_parse_error", + } +} + +fn classify_evaluation_error(error: &TreeRingError) -> &'static str { + match error { + TreeRingError::Validation(_) => "scenario_validation_error", + _ => "scenario_evaluation_error", + } +} + +fn markdown_summary(report: &QualityRunReport) -> String { + let mut lines = vec![ + "# Tree Ring Memory Quality Summary".to_string(), + String::new(), + format!("- quality pass: {}", report.quality_pass), + format!("- scenarios: {}", report.scenario_count), + format!( + "- constraint recall rate: {}", + format_rate(report.constraint_recall_rate) + ), + format!( + "- forbidden recall rate: {}", + format_rate(report.forbidden_recall_rate) + ), + format!( + "- spam rejection rate: {}", + format_rate(report.spam_rejection_rate) + ), + format!( + "- evidence required rate: {}", + format_rate(report.evidence_required_rate) + ), + format!( + "- behavior proof pass rate: {}", + format_rate(report.behavior_proof_pass_rate) + ), + String::new(), + "## Scenarios".to_string(), + String::new(), + ]; + + for scenario in &report.scenarios { + lines.push(format!( + "- `{}` [{}]: pass={}", + scenario.name, scenario.category, scenario.quality_pass + )); + } + + if !report.errors.is_empty() { + lines.extend([String::new(), "## Errors".to_string(), String::new()]); + for error in &report.errors { + let scenario = error.scenario.as_deref().unwrap_or("unknown"); + let path = error.path.as_deref().unwrap_or("unknown"); + lines.push(format!( + "- stage={} scenario=`{scenario}` path=`{path}`: {}", + error.stage, error.message + )); + } + } + + lines.push(String::new()); + lines.join("\n") +} + +fn format_rate(rate: Option) -> String { + rate.map_or_else(|| "n/a".to_string(), |rate| format!("{rate:.3}")) +} + +struct TemporaryRoot { + path: PathBuf, +} + +impl TemporaryRoot { + fn new(name: &str) -> Result { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| "temporary_root_clock_error".to_string())? + .as_nanos(); + let unique = TEMP_ROOT_COUNTER.fetch_add(1, Ordering::Relaxed); + let safe_name = name + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) + .collect::(); + let path = std::env::temp_dir().join(format!( + "tree-ring-quality-{}-{}-{}-{safe_name}", + std::process::id(), + nanos, + unique + )); + if path.exists() { + fs::remove_dir_all(&path).map_err(|_| "temporary_root_cleanup_error".to_string())?; + } + fs::create_dir_all(&path).map_err(|_| "temporary_root_create_error".to_string())?; + Ok(Self { path }) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TemporaryRoot { + fn drop(&mut self) { + if self.path.exists() { + let _ = fs::remove_dir_all(&self.path); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + const SECRET_MARKER: &str = "EDGE-SECRET-MARKER-7f1c2e"; + + fn copy_valid_fixture(fixture_dir: &Path, name: &str) { + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join("fixtures/quality/no-background-writer-constraint.json"); + fs::copy(source, fixture_dir.join(name)).unwrap(); + } + + fn write_invalid_memory_fixture(fixture_dir: &Path) { + fs::write( + fixture_dir.join("invalid-memory.json"), + format!( + r#"{{ + "name": "invalid-memory-{SECRET_MARKER}", + "category": "constraint_recall", + "query": "quality proof", + "seed_memories": [ + {{ + "id": "mem_invalid_secret", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "{SECRET_MARKER}", + "event_type": "decision", + "summary": "Invalid memory", + "details": "", + "source": {{"type": "evidence", "ref": "docs/spec.md", "quote": ""}}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {{"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null}} + }} + ], + "expected_recall": [ + {{"memory_id": "mem_invalid_secret"}} + ] + }}"# + ), + ) + .unwrap(); + } + + #[test] + fn writes_partial_artifacts_when_a_later_fixture_fails_to_parse() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + copy_valid_fixture(fixtures.path(), "01-valid.json"); + fs::write( + fixtures.path().join("02-private-invalid.json"), + r#"{ + "name": "invalid-private-fixture", + "category": "constraint_recall", + "query": "private-memory-payload", + "expected_recal": [] + }"#, + ) + .unwrap(); + + let report = run_quality_scenarios(fixtures.path(), output.path()).unwrap(); + + assert!(!report.ok); + assert!(!report.quality_pass); + assert_eq!(report.scenario_count, 1); + assert_eq!(report.errors.len(), 1); + assert_eq!(report.errors[0].stage, "parse"); + assert_eq!(report.errors[0].message, "scenario_parse_error"); + assert!(report.errors[0].scenario.is_none()); + assert!(report.errors[0] + .path + .as_deref() + .is_some_and(|path| path.ends_with("02-private-invalid.json"))); + + let json = fs::read_to_string(output.path().join("quality-report.json")).unwrap(); + let markdown = fs::read_to_string(output.path().join("quality-summary.md")).unwrap(); + assert!(!json.contains("private-memory-payload")); + assert!(!markdown.contains("private-memory-payload")); + assert!(markdown.contains("## Errors")); + assert!(markdown.contains("stage=parse")); + } + + #[test] + fn sanitizes_validation_failures_before_writing_reports() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + copy_valid_fixture(fixtures.path(), "01-valid.json"); + write_invalid_memory_fixture(fixtures.path()); + + let report = run_quality_scenarios(fixtures.path(), output.path()).unwrap(); + let json = fs::read_to_string(output.path().join("quality-report.json")).unwrap(); + let markdown = fs::read_to_string(output.path().join("quality-summary.md")).unwrap(); + + assert!(!report.ok); + assert!(!report.quality_pass); + assert_eq!(report.errors.len(), 1); + assert_eq!(report.errors[0].stage, "parse"); + assert_eq!(report.errors[0].message, "scenario_validation_error"); + assert!(report.errors[0].scenario.is_none()); + assert!(!report.errors[0].message.contains(SECRET_MARKER)); + assert!(!json.contains(SECRET_MARKER)); + assert!(!markdown.contains(SECRET_MARKER)); + } + + #[test] + fn sanitizes_validation_failures_in_returned_command_error() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + write_invalid_memory_fixture(fixtures.path()); + + let error = run_quality_command(fixtures.path(), output.path()).unwrap_err(); + + assert_eq!(error, "quality run failed: 0 scenarios evaluated"); + assert!(!error.contains(SECRET_MARKER)); + } + + #[test] + fn persisted_gate_rejects_nested_pass_when_top_level_fails() { + let output = tempdir().unwrap(); + let scenario = QualityScenarioReport { + name: "nested pass".to_string(), + category: "constraint_recall".to_string(), + constraint_recall_rate: Some(1.0), + forbidden_recall_rate: None, + spam_rejection_rate: None, + evidence_required_rate: None, + behavior_proof_pass: None, + behavior_expectation: None, + quality_pass: true, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_decisions: Vec::new(), + }; + let mut report = summarize_quality_run(vec![scenario]); + report.ok = false; + report.quality_pass = false; + write_reports(output.path(), &report).unwrap(); + + let error = verify_persisted_quality_report(output.path(), &report).unwrap_err(); + + assert_eq!(error, "quality run failed: 1 scenarios evaluated"); + } + + #[test] + fn writes_failed_artifacts_for_an_empty_fixture_directory() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + + let report = run_quality_scenarios(fixtures.path(), output.path()).unwrap(); + + assert!(!report.ok); + assert!(!report.quality_pass); + assert_eq!(report.scenario_count, 0); + assert_eq!(report.errors.len(), 1); + assert_eq!(report.errors[0].stage, "fixture_directory"); + assert!(output.path().join("quality-report.json").is_file()); + let markdown = fs::read_to_string(output.path().join("quality-summary.md")).unwrap(); + assert!(markdown.contains("constraint recall rate: n/a")); + assert!(markdown.contains("## Errors")); + } +} diff --git a/crates/tree-ring-memory-cli/src/agent_awareness.rs b/crates/tree-ring-memory-cli/src/agent_awareness.rs index 66d1fd7..735696d 100644 --- a/crates/tree-ring-memory-cli/src/agent_awareness.rs +++ b/crates/tree-ring-memory-cli/src/agent_awareness.rs @@ -1,9 +1,19 @@ use serde::Serialize; +use std::borrow::Cow; use std::fs; use std::path::{Path, PathBuf}; const SKILL_TEMPLATE: &str = include_str!("../../../skills/tree-ring-memory/SKILL.md"); const DOX_TEMPLATE: &str = include_str!("../../../templates/dox/AGENTS.md"); +const AGENT_HEADER: &str = "# Tree Ring Memory Agent Instructions"; +const CLI_HEADER: &str = "# Tree Ring Memory CLI Quick Reference"; +const SKILL_FRONT_MATTER_MARKER: &str = "name: tree-ring-memory"; +const AGENT_QUALITY_GATES_HEADING: &str = "## Memory Quality Gates"; +const AGENT_QUALITY_GATES_ANCHOR: &str = "## DOX Integration"; +const CLI_QUALITY_GATES_HEADING: &str = "Memory quality gates:"; +const CLI_QUALITY_GATES_ANCHOR: &str = "Safety rules:"; +const SKILL_QUALITY_GATES_HEADING: &str = "## Memory Quality Gates"; +const SKILL_QUALITY_GATES_ANCHOR: &str = "## Ring Selection"; const CLI_REFERENCE: &str = r#"# Tree Ring Memory CLI Quick Reference Tree Ring Memory is a local-first memory lifecycle layer for AI agents. @@ -51,6 +61,18 @@ Adapter rules: - Run adapter commands with `--dry-run` before writing memory. - `tree-ring integrations scan` is read-only; add harness bridge references manually until a link command is available. +Memory quality gates: + +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. +- Before trusting memory, prefer source-linked, non-superseded, high-confidence results. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Before writing memory, reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + Safety rules: - Do not store secrets, credentials, private keys, or raw chain-of-thought. @@ -73,19 +95,61 @@ pub fn ensure_agent_awareness(root: &Path) -> Result bool, + section_heading: &str, + anchor: &str, + section: Option<&str>, ) -> Result<(), String> { if path.exists() { + maybe_backfill_generated_file(path, recognizer, section_heading, anchor, section)?; report.existing.push(path.to_path_buf()); return Ok(()); } @@ -97,9 +161,81 @@ fn write_if_missing( Ok(()) } +fn maybe_backfill_generated_file( + path: &Path, + recognizer: fn(&str) -> bool, + section_heading: &str, + anchor: &str, + section: Option<&str>, +) -> Result<(), String> { + let Some(section) = section else { + return Ok(()); + }; + + let existing = fs::read_to_string(path).map_err(|err| err.to_string())?; + if !recognizer(&existing) || existing.contains(section_heading) { + return Ok(()); + } + + let Some(updated) = insert_section_before_anchor(&existing, section, anchor) else { + return Ok(()); + }; + if updated != existing { + fs::write(path, updated).map_err(|err| err.to_string())?; + } + Ok(()) +} + +fn extract_section<'a>(content: &'a str, heading: &str, anchor: &str) -> Option<&'a str> { + let (start, end) = managed_section_range(content, heading, anchor)?; + Some(&content[start..end]) +} + +#[cfg(test)] +fn remove_managed_section(content: &str, heading: &str, anchor: &str) -> Option { + let (start, end) = managed_section_range(content, heading, anchor)?; + let mut stale = String::with_capacity(content.len() - (end - start)); + stale.push_str(&content[..start]); + stale.push_str(&content[end..]); + Some(stale) +} + +fn managed_section_range(content: &str, heading: &str, anchor: &str) -> Option<(usize, usize)> { + let start = content.find(heading)?; + let end = content[start..].find(anchor)? + start; + Some((start, end)) +} + +fn insert_section_before_anchor(content: &str, section: &str, anchor: &str) -> Option { + let anchor_index = content.find(anchor)?; + let section = if content.contains("\r\n") { + Cow::Owned(section.replace('\n', "\r\n")) + } else { + Cow::Borrowed(section) + }; + let mut updated = String::with_capacity(content.len() + section.len()); + updated.push_str(&content[..anchor_index]); + updated.push_str(§ion); + updated.push_str(&content[anchor_index..]); + Some(updated) +} + +fn is_generated_agents_file(content: &str) -> bool { + content.starts_with(AGENT_HEADER) +} + +fn is_generated_cli_file(content: &str) -> bool { + content.starts_with(CLI_HEADER) +} + +fn is_generated_skill_file(content: &str) -> bool { + (content.starts_with("---\n") || content.starts_with("---\r\n")) + && content.contains(SKILL_FRONT_MATTER_MARKER) +} + fn agent_contract(root: &Path) -> String { format!( - r#"# Tree Ring Memory Agent Instructions + r#"{AGENT_HEADER} This directory contains Tree Ring Memory's local project memory store. @@ -154,6 +290,28 @@ call `tree-ring recall`, `tree-ring remember`, `tree-ring evidence`, `tree-ring forget`, `tree-ring consolidate --dry-run`, or `tree-ring maintain`. They do not authorize hidden transcript scraping or autonomous durable writes. +## Memory Quality Gates + +Recall gates: + +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. + +Trust gates: + +- Prefer source-linked, non-superseded, high-confidence memories. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Do not treat sensitive or hidden-by-default memory as ordinary recall context. + +Write gates: + +- Remember only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes. +- Reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + ## DOX Integration If this project uses DOX-style `AGENTS.md` traversal, merge the relevant @@ -181,6 +339,7 @@ outcomes become seeds, and observed outcomes become outer-ring evidence. {DOX_TEMPLATE} "#, + AGENT_HEADER = AGENT_HEADER, root = shell_path(root) ) } @@ -199,6 +358,33 @@ mod tests { use super::*; use tempfile::tempdir; + #[test] + fn generated_canonical_managed_section_can_be_removed_and_reinserted() { + let root = Path::new("/tmp/tree-ring-quality-fixture"); + let canonical = agent_contract(root); + let section = extract_section( + &canonical, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + + let stale = remove_managed_section( + &canonical, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let restored = + insert_section_before_anchor(&stale, section, AGENT_QUALITY_GATES_ANCHOR).unwrap(); + + assert_eq!(restored, canonical); + } + + fn stale_fixture(content: &str, heading: &str, anchor: &str) -> String { + remove_managed_section(content, heading, anchor).unwrap() + } + #[test] fn init_writes_agent_awareness_files_without_overwriting() { let dir = tempdir().unwrap(); @@ -234,6 +420,322 @@ mod tests { assert!(agents.contains("Tree Ring Memory Project Contract")); } + #[test] + fn generated_agents_file_mentions_quality_gates() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + + ensure_agent_awareness(&root).unwrap(); + let agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + + assert!(agents.contains("Memory Quality Gates")); + assert!(agents.contains("Recall gates")); + assert!(agents.contains("Trust gates")); + assert!(agents.contains("Write gates")); + assert!(agents.contains("Reject transient planning chatter")); + } + + #[test] + fn generated_cli_reference_mentions_quality_gates() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + + ensure_agent_awareness(&root).unwrap(); + let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + + assert!(cli.contains("Memory quality gates")); + assert!(cli.contains("Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds.")); + assert!(cli + .contains("Before risky changes, recall warnings and evidence-linked prior failures.")); + assert!(cli + .contains("Before repeating a workflow, recall prior errors and accepted procedures.")); + assert!(cli.contains("Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons.")); + assert!(cli.contains("Before trusting memory, prefer source-linked")); + assert!(cli.contains("Before writing memory, reject transient planning chatter")); + } + + #[test] + fn generated_backfills_quality_gates_into_recognized_stale_generated_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + let canonical_agents = agent_contract(&root); + let agent_section = extract_section( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let cli_section = extract_section( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let skill_section = extract_section( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + + fs::write( + root.join("AGENTS.md"), + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + + let report = ensure_agent_awareness(&root).unwrap(); + assert_eq!(report.created.len(), 0); + assert_eq!(report.existing.len(), 3); + + let agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + assert!(agents.contains(agent_section.trim())); + assert!(cli.contains(cli_section.trim())); + assert!(skill.contains(skill_section.trim())); + } + + #[test] + fn generated_backfill_recognizes_and_preserves_crlf_skill_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + let stale_skill = stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + .replace('\n', "\r\n"); + fs::write(root.join("SKILL.md"), stale_skill).unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + assert!(skill.contains("## Memory Quality Gates\r\n")); + assert!(!skill.replace("\r\n", "").contains('\n')); + } + + #[test] + fn generated_backfill_inserts_quality_gates_at_canonical_anchors() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + let canonical_agents = agent_contract(&root); + let agent_section = extract_section( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let cli_section = extract_section( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let skill_section = extract_section( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + + fs::write( + root.join("AGENTS.md"), + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + let agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + assert!(agents.contains(&format!("{}{}", agent_section, AGENT_QUALITY_GATES_ANCHOR))); + assert!(cli.contains(&format!("{}{}", cli_section, CLI_QUALITY_GATES_ANCHOR))); + assert!(skill.contains(&format!("{}{}", skill_section, SKILL_QUALITY_GATES_ANCHOR))); + } + + #[test] + fn generated_backfill_preserves_custom_content_in_recognized_generated_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + + let agents_custom = "\nCustom note: keep me.\n"; + let cli_custom = "\nCustom alias note: keep me.\n"; + let skill_custom = "\nCustom workflow note: keep me.\n"; + let canonical_agents = agent_contract(&root); + + fs::write( + root.join("AGENTS.md"), + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + agents_custom, + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ) + cli_custom, + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + skill_custom, + ) + .unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + assert!(fs::read_to_string(root.join("AGENTS.md")) + .unwrap() + .ends_with(agents_custom)); + assert!(fs::read_to_string(root.join("CLI.md")) + .unwrap() + .ends_with(cli_custom)); + assert!(fs::read_to_string(root.join("SKILL.md")) + .unwrap() + .ends_with(skill_custom)); + } + + #[test] + fn generated_backfill_leaves_arbitrary_custom_files_untouched() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + + fs::write(root.join("AGENTS.md"), "# Custom project contract\n").unwrap(); + fs::write(root.join("CLI.md"), "# Custom CLI guide\n").unwrap(); + fs::write(root.join("SKILL.md"), "# Custom skill\n").unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + assert_eq!( + fs::read_to_string(root.join("AGENTS.md")).unwrap(), + "# Custom project contract\n" + ); + assert_eq!( + fs::read_to_string(root.join("CLI.md")).unwrap(), + "# Custom CLI guide\n" + ); + assert_eq!( + fs::read_to_string(root.join("SKILL.md")).unwrap(), + "# Custom skill\n" + ); + } + + #[test] + fn generated_backfill_is_idempotent_for_recognized_stale_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + let canonical_agents = agent_contract(&root); + + fs::write( + root.join("AGENTS.md"), + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ), + ) + .unwrap(); + + ensure_agent_awareness(&root).unwrap(); + let first_agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let first_cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let first_skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + let second_agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let second_cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let second_skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + assert_eq!(first_agents, second_agents); + assert_eq!(first_cli, second_cli); + assert_eq!(first_skill, second_skill); + assert_eq!(second_agents.matches("## Memory Quality Gates").count(), 1); + assert_eq!(second_cli.matches("Memory quality gates:").count(), 1); + assert_eq!(second_skill.matches("## Memory Quality Gates").count(), 1); + } + #[test] fn generated_agents_file_quotes_roots_with_spaces() { let dir = tempdir().unwrap(); diff --git a/crates/tree-ring-memory-cli/src/tui/model.rs b/crates/tree-ring-memory-cli/src/tui/model.rs index 578905a..fbc1a25 100644 --- a/crates/tree-ring-memory-cli/src/tui/model.rs +++ b/crates/tree-ring-memory-cli/src/tui/model.rs @@ -14,6 +14,7 @@ pub struct RingStats { pub average_confidence: f64, pub newest_at: Option, pub oldest_at: Option, + pub fullness_level: f64, pub pulse_level: f64, pub warning_level: f64, } @@ -30,6 +31,7 @@ impl RingStats { average_confidence: 0.0, newest_at: None, oldest_at: None, + fullness_level: 0.0, pulse_level: 0.0, warning_level: 0.0, } @@ -106,6 +108,11 @@ impl DashboardStats { stats.average_salience /= stats.total as f64; stats.average_confidence /= stats.total as f64; } + stats.fullness_level = if dashboard.total == 0 { + 0.0 + } else { + stats.total as f64 / dashboard.total as f64 + }; stats.warning_level = ring_warning_level(stats); if let Some(previous_stats) = previous.and_then(|previous| previous.ring(&stats.ring)) { if stats.total != previous_stats.total { @@ -213,4 +220,23 @@ mod tests { assert_eq!(second.ring("cambium").unwrap().pulse_level, 1.0); } + + #[test] + fn derives_relative_ring_fullness() { + let dashboard = DashboardStats::from_memories( + &[ + event("Fresh one", "cambium", "lesson"), + event("Fresh two", "cambium", "lesson"), + event("Durable", "heartwood", "decision"), + ], + None, + ); + + assert_eq!(dashboard.ring("cambium").unwrap().fullness_level, 2.0 / 3.0); + assert_eq!( + dashboard.ring("heartwood").unwrap().fullness_level, + 1.0 / 3.0 + ); + assert_eq!(dashboard.ring("outer").unwrap().fullness_level, 0.0); + } } diff --git a/crates/tree-ring-memory-cli/src/tui/rings.rs b/crates/tree-ring-memory-cli/src/tui/rings.rs index 90a68eb..d2fc91a 100644 --- a/crates/tree-ring-memory-cli/src/tui/rings.rs +++ b/crates/tree-ring-memory-cli/src/tui/rings.rs @@ -321,13 +321,17 @@ fn animated_color(ring: &str, stats: Option<&RingStats>, tick: u64, offset: u64) if stats.total == 0 { return dim_color(base, 0.34); } + let fullness = stats.fullness_level.clamp(0.0, 1.0); if stats.pulse_level > 0.05 && (tick + offset) % 6 < 3 { - return brighten_color(base, 0.34 + (stats.pulse_level * 0.18)); + return brighten_color(base, 0.28 + fullness * 0.16 + stats.pulse_level * 0.18); } if (tick + offset) % 18 < 3 { - return brighten_color(base, 0.18); + return brighten_color(base, 0.08 + fullness * 0.16); } - base + if fullness < 0.18 { + return dim_color(base, 0.42 + fullness * 1.35); + } + brighten_color(base, fullness * 0.20) } fn brighten_color(color: Color, amount: f64) -> Color { @@ -422,6 +426,20 @@ mod tests { use super::*; use tree_ring_memory_core::MemoryEvent; + fn brightness_sum(color: Color) -> u16 { + match color { + Color::Rgb(red, green, blue) => red as u16 + green as u16 + blue as u16, + _ => 0, + } + } + + fn stats_with_fullness(ring: &str, total: usize, fullness_level: f64) -> RingStats { + let mut stats = RingStats::empty(ring); + stats.total = total; + stats.fullness_level = fullness_level; + stats + } + #[test] fn ambient_lines_include_all_core_ring_labels() { let mut memory = MemoryEvent::new("Durable", "lesson").unwrap(); @@ -485,4 +503,15 @@ mod tests { assert_ne!(early, later); } + + #[test] + fn fullness_brightens_ambient_ring_without_pulse() { + let low = stats_with_fullness("cambium", 1, 0.10); + let high = stats_with_fullness("cambium", 8, 0.80); + + let low_color = animated_color("cambium", Some(&low), 10, 0); + let high_color = animated_color("cambium", Some(&high), 10, 0); + + assert!(brightness_sum(high_color) > brightness_sum(low_color)); + } } diff --git a/crates/tree-ring-memory-core/src/lib.rs b/crates/tree-ring-memory-core/src/lib.rs index 10c8759..f92f60b 100644 --- a/crates/tree-ring-memory-core/src/lib.rs +++ b/crates/tree-ring-memory-core/src/lib.rs @@ -4,6 +4,7 @@ pub mod dox; pub mod import_export; pub mod maintenance; pub mod models; +pub mod quality; pub mod recall; pub mod revolve; pub mod sensitivity; @@ -26,6 +27,12 @@ pub use maintenance::{ pub use models::{ now_iso, MemoryEvent, MemoryLink, MemoryReview, MemorySource, TreeRingError, TreeRingResult, }; +pub use quality::{ + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, BehaviorExpectation, + BehaviorExpectationReport, QualityRecall, QualityRunError, QualityRunReport, QualityScenario, + QualityScenarioReport, QualityThresholds, RecallExpectation, RecallExpectationReport, + WriteDecisionExpectation, WriteDecisionReport, QUALITY_CATEGORIES, WRITE_DECISIONS, +}; pub use recall::{RecallRanking, RecallScore, RecallScorer}; pub use revolve::{collect_revolve_memories, RevolveSyncReport, RevolveSyncRequest}; pub use sensitivity::{SensitivityGuard, SensitivityResult}; diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs new file mode 100644 index 0000000..d553206 --- /dev/null +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -0,0 +1,2020 @@ +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +use crate::models::{MemoryEvent, TreeRingError, TreeRingResult}; + +pub const QUALITY_CATEGORIES: &[&str] = &[ + "constraint_recall", + "spam_prevention", + "stale_truth_suppression", + "behavior_proof", + "evidence_preservation", +]; + +pub const WRITE_DECISIONS: &[&str] = &[ + "accept", + "reject", + "require_evidence", + "require_user_confirmation", +]; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct QualityScenario { + pub name: String, + pub category: String, + #[serde(default)] + pub seed_memories: Vec, + #[serde(default)] + pub query: Option, + #[serde(default)] + pub workflow_prompt: Option, + #[serde(default)] + pub expected_recall: Vec, + #[serde(default)] + pub forbidden_recall: Vec, + #[serde(default)] + pub write_candidates: Vec, + #[serde(default)] + pub expected_write_decisions: Vec, + #[serde(default)] + pub evidence_refs: Vec, + #[serde(default)] + pub behavior_expectation: Option, + #[serde(default)] + pub thresholds: QualityThresholds, +} + +impl QualityScenario { + pub fn prompt(&self) -> Option<&str> { + if let Some(query) = self.query.as_deref() { + if !query.trim().is_empty() { + return Some(query); + } + } + self.workflow_prompt + .as_deref() + .filter(|value| !value.trim().is_empty()) + } + + pub fn validate(&self) -> TreeRingResult<()> { + if self.name.trim().is_empty() { + return Err(TreeRingError::Validation( + "quality scenario name is required".to_string(), + )); + } + validate_member("quality category", &self.category, QUALITY_CATEGORIES)?; + if self + .prompt() + .map(|value| value.trim().is_empty()) + .unwrap_or(true) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} requires query or workflow_prompt", + self.name + ))); + } + for memory in self + .seed_memories + .iter() + .chain(self.write_candidates.iter()) + { + memory.validate()?; + } + for expectation in &self.expected_recall { + expectation.validate("expected_recall", &self.name)?; + } + for (index, expectation) in self.forbidden_recall.iter().enumerate() { + expectation.validate("forbidden_recall", &self.name)?; + if !has_matching_seed_memory(expectation, &self.seed_memories) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} forbidden_recall[{}] does not match a seed_memory", + self.name, index + ))); + } + } + for decision in &self.expected_write_decisions { + decision.validate(&self.name)?; + } + validate_write_decision_coverage(self)?; + for (index, evidence_ref) in self.evidence_refs.iter().enumerate() { + if evidence_ref.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {} evidence_refs[{}] must not be blank", + self.name, index + ))); + } + } + if self.category == "behavior_proof" && self.behavior_expectation.is_none() { + return Err(TreeRingError::Validation(format!( + "quality scenario {} category behavior_proof requires behavior_expectation", + self.name + ))); + } + if let Some(expectation) = &self.behavior_expectation { + expectation.validate(self)?; + } + self.validate_primary_observation_contract()?; + self.thresholds + .validate(&self.name, self.metric_applicability())?; + Ok(()) + } + + fn validate_primary_observation_contract(&self) -> TreeRingResult<()> { + match self.category.as_str() { + "constraint_recall" if self.expected_recall.is_empty() => { + Err(TreeRingError::Validation(format!( + "quality scenario {} category constraint_recall requires at least one expected_recall", + self.name + ))) + } + "spam_prevention" + if !self + .expected_write_decisions + .iter() + .any(|decision| decision.decision == "reject") => + { + Err(TreeRingError::Validation(format!( + "quality scenario {} category spam_prevention requires at least one expected_write_decision of reject", + self.name + ))) + } + "stale_truth_suppression" if self.forbidden_recall.is_empty() => { + Err(TreeRingError::Validation(format!( + "quality scenario {} category stale_truth_suppression requires at least one forbidden_recall", + self.name + ))) + } + "evidence_preservation" if !self.has_evaluation_write_candidate() => { + Err(TreeRingError::Validation(format!( + "quality scenario {} category evidence_preservation requires at least one evaluation write_candidate", + self.name + ))) + } + _ => Ok(()), + } + } + + fn metric_applicability(&self) -> MetricApplicability { + MetricApplicability { + constraint_recall: !self.expected_recall.is_empty(), + forbidden_recall: !self.forbidden_recall.is_empty(), + spam_rejection: self + .expected_write_decisions + .iter() + .any(|decision| decision.decision == "reject"), + evidence_required: self.has_evaluation_write_candidate(), + behavior_proof: self.behavior_expectation.is_some(), + } + } + + fn has_evaluation_write_candidate(&self) -> bool { + self.write_candidates + .iter() + .any(|candidate| candidate.event_type.starts_with("evaluation_")) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct MetricApplicability { + constraint_recall: bool, + forbidden_recall: bool, + spam_rejection: bool, + evidence_required: bool, + behavior_proof: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecallExpectation { + #[serde(default)] + pub memory_id: Option, + #[serde(default)] + pub ring: Option, + #[serde(default)] + pub tag: Option, + #[serde(default)] + pub source_ref: Option, + #[serde(default)] + pub reason: String, +} + +impl RecallExpectation { + fn validate(&self, field: &str, scenario_name: &str) -> TreeRingResult<()> { + let has_nonblank_selector = [ + self.memory_id.as_deref(), + self.ring.as_deref(), + self.tag.as_deref(), + self.source_ref.as_deref(), + ] + .into_iter() + .flatten() + .any(|value| !value.trim().is_empty()); + + if !has_nonblank_selector { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} {field} entry must include a nonblank memory_id, ring, tag, or source_ref" + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WriteDecisionExpectation { + pub memory_id: String, + pub decision: String, + #[serde(default)] + pub reason: String, +} + +impl WriteDecisionExpectation { + fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + if self.memory_id.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} write decision requires memory_id" + ))); + } + validate_member("write decision", &self.decision, WRITE_DECISIONS) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BehaviorExpectation { + pub required_memory_id: String, + pub baseline_decision: String, + pub memory_informed_decision: String, + pub expected_decision: String, + #[serde(default)] + pub reason: Option, +} + +impl BehaviorExpectation { + fn validate(&self, scenario: &QualityScenario) -> TreeRingResult<()> { + for (field, value) in [ + ("required_memory_id", self.required_memory_id.as_str()), + ("baseline_decision", self.baseline_decision.as_str()), + ( + "memory_informed_decision", + self.memory_informed_decision.as_str(), + ), + ("expected_decision", self.expected_decision.as_str()), + ] { + if value.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {} behavior_expectation {field} must not be blank", + scenario.name + ))); + } + } + if self + .reason + .as_deref() + .is_some_and(|reason| reason.trim().is_empty()) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} behavior_expectation reason must not be blank when provided", + scenario.name + ))); + } + if !scenario + .seed_memories + .iter() + .any(|memory| memory.id == self.required_memory_id) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} behavior_expectation required_memory_id {} does not match a seed_memory", + scenario.name, self.required_memory_id + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct QualityThresholds { + #[serde(default)] + pub min_constraint_recall_rate: Option, + #[serde(default)] + pub max_forbidden_recall_rate: Option, + #[serde(default)] + pub min_spam_rejection_rate: Option, + #[serde(default)] + pub min_evidence_required_rate: Option, + #[serde(default)] + pub min_behavior_proof_pass_rate: Option, +} + +impl QualityThresholds { + fn validate( + &self, + scenario_name: &str, + applicability: MetricApplicability, + ) -> TreeRingResult<()> { + for (field, value) in [ + ( + "min_constraint_recall_rate", + self.min_constraint_recall_rate, + ), + ("max_forbidden_recall_rate", self.max_forbidden_recall_rate), + ("min_spam_rejection_rate", self.min_spam_rejection_rate), + ( + "min_evidence_required_rate", + self.min_evidence_required_rate, + ), + ( + "min_behavior_proof_pass_rate", + self.min_behavior_proof_pass_rate, + ), + ] { + if value.is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} threshold {field} must be between 0 and 1" + ))); + } + } + for (field, configured, applicable, required_observation) in [ + ( + "min_constraint_recall_rate", + self.min_constraint_recall_rate.is_some(), + applicability.constraint_recall, + "expected_recall observations", + ), + ( + "max_forbidden_recall_rate", + self.max_forbidden_recall_rate.is_some(), + applicability.forbidden_recall, + "forbidden_recall observations", + ), + ( + "min_spam_rejection_rate", + self.min_spam_rejection_rate.is_some(), + applicability.spam_rejection, + "expected_write_decision reject observations", + ), + ( + "min_evidence_required_rate", + self.min_evidence_required_rate.is_some(), + applicability.evidence_required, + "evaluation write_candidate observations", + ), + ( + "min_behavior_proof_pass_rate", + self.min_behavior_proof_pass_rate.is_some(), + applicability.behavior_proof, + "behavior_expectation observations", + ), + ] { + if configured && !applicable { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} threshold {field} is inapplicable without {required_observation}" + ))); + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRecall { + pub memory: MemoryEvent, + pub score: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallExpectationReport { + pub expectation: RecallExpectation, + pub matched_memory_ids: Vec, + pub passed: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WriteDecisionReport { + pub memory_id: String, + pub expected: String, + pub actual: String, + pub passed: bool, + pub reason: String, + #[serde(default)] + pub evidence_applicable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BehaviorExpectationReport { + pub expectation: BehaviorExpectation, + pub required_memory_recalled: bool, + pub decision_changed: bool, + pub expected_decision_reached: bool, + pub passed: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityScenarioReport { + pub name: String, + pub category: String, + pub constraint_recall_rate: Option, + pub forbidden_recall_rate: Option, + pub spam_rejection_rate: Option, + pub evidence_required_rate: Option, + pub behavior_proof_pass: Option, + #[serde(default)] + pub behavior_expectation: Option, + pub quality_pass: bool, + pub expected_recall: Vec, + pub forbidden_recall: Vec, + pub write_decisions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct QualityRunError { + #[serde(default)] + pub scenario: Option, + #[serde(default)] + pub path: Option, + pub stage: String, + pub message: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRunReport { + pub ok: bool, + pub scenario_count: usize, + pub constraint_recall_rate: Option, + pub forbidden_recall_rate: Option, + pub spam_rejection_rate: Option, + pub evidence_required_rate: Option, + pub behavior_proof_pass_rate: Option, + pub quality_pass: bool, + pub scenarios: Vec, + #[serde(default)] + pub errors: Vec, +} + +pub fn evaluate_quality_scenario( + scenario: &QualityScenario, + recalls: &[QualityRecall], +) -> TreeRingResult { + scenario.validate()?; + + let expected_recall = scenario + .expected_recall + .iter() + .map(|expectation| expected_recall_report(expectation, recalls)) + .collect::>(); + let forbidden_recall = scenario + .forbidden_recall + .iter() + .map(|expectation| forbidden_recall_report(expectation, recalls)) + .collect::>(); + let write_decisions = scenario + .expected_write_decisions + .iter() + .map(|expectation| write_decision_report(expectation, scenario)) + .collect::>>()?; + + let constraint_recall_rate = pass_rate(&expected_recall); + let forbidden_recall_rate = failure_rate(&forbidden_recall); + let spam_rejection_rate = + decision_pass_rate(&write_decisions, |report| report.expected == "reject"); + let evidence_required_rate = + decision_pass_rate(&write_decisions, |report| report.evidence_applicable); + let behavior_expectation = scenario + .behavior_expectation + .as_ref() + .map(|expectation| behavior_expectation_report(expectation, recalls)); + let behavior_proof_pass = behavior_expectation.as_ref().map(|report| report.passed); + let behavior_rate = behavior_proof_pass.map(|passed| if passed { 1.0 } else { 0.0 }); + let thresholds = scenario + .metric_applicability() + .thresholds(&scenario.thresholds); + let quality_pass = + minimum_met( + constraint_recall_rate, + thresholds.min_constraint_recall_rate, + ) && maximum_met(forbidden_recall_rate, thresholds.max_forbidden_recall_rate) + && write_decisions.iter().all(|report| report.passed) + && minimum_met(spam_rejection_rate, thresholds.min_spam_rejection_rate) + && minimum_met( + evidence_required_rate, + thresholds.min_evidence_required_rate, + ) + && minimum_met(behavior_rate, thresholds.min_behavior_proof_pass_rate); + + Ok(QualityScenarioReport { + name: scenario.name.clone(), + category: scenario.category.clone(), + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass, + behavior_expectation, + quality_pass, + expected_recall, + forbidden_recall, + write_decisions, + }) +} + +pub fn summarize_quality_run(reports: Vec) -> QualityRunReport { + let scenario_count = reports.len(); + let constraint_recall_rate = aggregate_rate( + reports + .iter() + .flat_map(|report| report.expected_recall.iter().map(|item| item.passed)), + ); + let forbidden_recall_rate = aggregate_rate( + reports + .iter() + .flat_map(|report| report.forbidden_recall.iter().map(|item| !item.passed)), + ); + let spam_rejection_rate = aggregate_rate(reports.iter().flat_map(|report| { + report + .write_decisions + .iter() + .filter(|item| item.expected == "reject") + .map(|item| item.passed) + })); + let evidence_required_rate = aggregate_rate(reports.iter().flat_map(|report| { + report + .write_decisions + .iter() + .filter(|item| item.evidence_applicable) + .map(|item| item.passed) + })); + let behavior_proof_pass_rate = aggregate_rate( + reports + .iter() + .filter_map(|report| report.behavior_proof_pass), + ); + let quality_pass = scenario_count > 0 && reports.iter().all(|report| report.quality_pass); + + QualityRunReport { + ok: quality_pass, + scenario_count, + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass_rate, + quality_pass, + scenarios: reports, + errors: Vec::new(), + } +} + +pub fn parse_quality_scenario(input: &str) -> TreeRingResult { + let scenario: QualityScenario = serde_json::from_str(input)?; + scenario.validate()?; + Ok(scenario) +} + +fn validate_member(field: &str, value: &str, allowed: &[&str]) -> TreeRingResult<()> { + if allowed.contains(&value) { + Ok(()) + } else { + Err(TreeRingError::Validation(format!( + "invalid {field}: {value}" + ))) + } +} + +fn validate_write_decision_coverage(scenario: &QualityScenario) -> TreeRingResult<()> { + let mut candidate_ids = HashSet::new(); + for (index, candidate) in scenario.write_candidates.iter().enumerate() { + if !candidate_ids.insert(candidate.id.as_str()) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} write_candidates[{index}] duplicates memory_id {}", + scenario.name, candidate.id + ))); + } + } + + let mut decision_index_by_id = HashMap::new(); + + for (index, decision) in scenario.expected_write_decisions.iter().enumerate() { + if let Some(previous_index) = + decision_index_by_id.insert(decision.memory_id.as_str(), index) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} expected_write_decisions[{index}] duplicates memory_id {} from expected_write_decisions[{previous_index}]", + scenario.name, decision.memory_id + ))); + } + if !candidate_ids.contains(decision.memory_id.as_str()) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} expected_write_decisions[{index}] memory_id {} does not match any write_candidate", + scenario.name, decision.memory_id + ))); + } + } + + for (index, candidate) in scenario.write_candidates.iter().enumerate() { + if !decision_index_by_id.contains_key(candidate.id.as_str()) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} write_candidates[{index}] id {} is missing expected_write_decision coverage", + scenario.name, candidate.id + ))); + } + } + + Ok(()) +} + +fn expected_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: !matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn forbidden_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn behavior_expectation_report( + expectation: &BehaviorExpectation, + recalls: &[QualityRecall], +) -> BehaviorExpectationReport { + let required_memory_recalled = recalls + .iter() + .any(|recall| recall.memory.id == expectation.required_memory_id); + let decision_changed = expectation.baseline_decision != expectation.memory_informed_decision; + let expected_decision_reached = + expectation.memory_informed_decision == expectation.expected_decision; + BehaviorExpectationReport { + expectation: expectation.clone(), + required_memory_recalled, + decision_changed, + expected_decision_reached, + passed: required_memory_recalled && decision_changed && expected_decision_reached, + } +} + +fn matching_recall_ids(expectation: &RecallExpectation, recalls: &[QualityRecall]) -> Vec { + recalls + .iter() + .filter(|recall| expectation_matches_memory(expectation, &recall.memory)) + .map(|recall| recall.memory.id.clone()) + .collect() +} + +fn has_matching_seed_memory( + expectation: &RecallExpectation, + seed_memories: &[MemoryEvent], +) -> bool { + seed_memories + .iter() + .any(|memory| expectation_matches_memory(expectation, memory)) +} + +fn expectation_matches_memory(expectation: &RecallExpectation, memory: &MemoryEvent) -> bool { + if expectation + .memory_id + .as_deref() + .is_some_and(|id| memory.id != id) + { + return false; + } + if expectation + .ring + .as_deref() + .is_some_and(|ring| memory.ring != ring) + { + return false; + } + if expectation + .tag + .as_deref() + .is_some_and(|tag| !memory.tags.iter().any(|candidate| candidate == tag)) + { + return false; + } + if expectation + .source_ref + .as_deref() + .is_some_and(|source_ref| memory.source.ref_ != source_ref) + { + return false; + } + true +} + +fn write_decision_report( + expectation: &WriteDecisionExpectation, + scenario: &QualityScenario, +) -> TreeRingResult { + let candidate = scenario + .write_candidates + .iter() + .find(|candidate| candidate.id == expectation.memory_id) + .ok_or_else(|| { + TreeRingError::Validation(format!( + "quality scenario {} expected write decision for memory_id {} could not resolve a write_candidate", + scenario.name, expectation.memory_id + )) + })?; + let actual = classify_write_candidate(candidate, &scenario.evidence_refs); + Ok(WriteDecisionReport { + memory_id: expectation.memory_id.clone(), + expected: expectation.decision.clone(), + passed: actual == expectation.decision, + actual, + reason: expectation.reason.clone(), + evidence_applicable: candidate.event_type.starts_with("evaluation_"), + }) +} + +fn classify_write_candidate(candidate: &MemoryEvent, required_evidence_refs: &[String]) -> String { + if candidate.sensitivity == "secret" { + return "reject".to_string(); + } + if candidate.retention == "ephemeral" + || candidate.tags.iter().any(|tag| { + matches!( + tag.as_str(), + "transient" | "planning-chatter" | "scratchpad" | "tool-noise" + ) + }) + { + return "reject".to_string(); + } + if candidate.event_type.starts_with("evaluation_") && candidate.source.ref_.trim().is_empty() { + return "require_evidence".to_string(); + } + if !required_evidence_refs.is_empty() + && candidate.event_type.starts_with("evaluation_") + && !required_evidence_refs.contains(&candidate.source.ref_) + { + return "require_evidence".to_string(); + } + if candidate.ring == "heartwood" && candidate.source.ref_.trim().is_empty() { + return "require_user_confirmation".to_string(); + } + "accept".to_string() +} + +fn pass_rate(reports: &[RecallExpectationReport]) -> Option { + aggregate_rate(reports.iter().map(|report| report.passed)) +} + +fn failure_rate(reports: &[RecallExpectationReport]) -> Option { + aggregate_rate(reports.iter().map(|report| !report.passed)) +} + +fn decision_pass_rate( + reports: &[WriteDecisionReport], + applicable: impl Fn(&WriteDecisionReport) -> bool, +) -> Option { + aggregate_rate( + reports + .iter() + .filter(|report| applicable(report)) + .map(|report| report.passed), + ) +} + +fn aggregate_rate(observations: impl Iterator) -> Option { + let (passed, total) = observations.fold((0usize, 0usize), |(passed, total), observation| { + (passed + usize::from(observation), total + 1) + }); + (total > 0).then_some(passed as f64 / total as f64) +} + +#[derive(Debug, Clone, Copy, PartialEq)] +struct EffectiveThresholds { + min_constraint_recall_rate: Option, + max_forbidden_recall_rate: Option, + min_spam_rejection_rate: Option, + min_evidence_required_rate: Option, + min_behavior_proof_pass_rate: Option, +} + +impl MetricApplicability { + fn thresholds(&self, configured: &QualityThresholds) -> EffectiveThresholds { + EffectiveThresholds { + min_constraint_recall_rate: configured + .min_constraint_recall_rate + .or(self.constraint_recall.then_some(1.0)), + max_forbidden_recall_rate: configured + .max_forbidden_recall_rate + .or(self.forbidden_recall.then_some(0.0)), + min_spam_rejection_rate: configured + .min_spam_rejection_rate + .or(self.spam_rejection.then_some(1.0)), + min_evidence_required_rate: configured + .min_evidence_required_rate + .or(self.evidence_required.then_some(1.0)), + min_behavior_proof_pass_rate: configured + .min_behavior_proof_pass_rate + .or(self.behavior_proof.then_some(1.0)), + } + } +} + +fn minimum_met(rate: Option, threshold: Option) -> bool { + threshold.is_none_or(|threshold| rate.is_some_and(|rate| rate >= threshold)) +} + +fn maximum_met(rate: Option, threshold: Option) -> bool { + threshold.is_none_or(|threshold| rate.is_some_and(|rate| rate <= threshold)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn memory(id: &str, summary: &str, ring: &str) -> MemoryEvent { + let mut event = MemoryEvent::new(summary, "lesson").unwrap(); + event.id = id.to_string(); + event.created_at = "2026-07-09T00:00:00Z".to_string(); + event.updated_at = "2026-07-09T00:00:00Z".to_string(); + event.scope = "project".to_string(); + event.project = Some("tree-ring".to_string()); + event.ring = ring.to_string(); + event.source.source_type = "evidence".to_string(); + event.source.ref_ = "docs/spec.md".to_string(); + event.salience = 0.8; + event.confidence = 0.8; + event + } + + fn scenario(name: &str, category: &str) -> QualityScenario { + QualityScenario { + name: name.to_string(), + category: category.to_string(), + seed_memories: Vec::new(), + query: Some("quality proof".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + } + } + + fn report_json(scenario: &QualityScenario, recalls: &[QualityRecall]) -> serde_json::Value { + serde_json::to_value(evaluate_quality_scenario(scenario, recalls).unwrap()).unwrap() + } + + #[test] + fn rejects_unknown_top_level_fixture_field() { + let error = parse_quality_scenario( + r#"{ + "name": "misspelled recall", + "category": "constraint_recall", + "query": "quality proof", + "expected_recal": [] + }"#, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("unknown field `expected_recal`"), "{error}"); + } + + #[test] + fn rejects_unknown_threshold_field() { + let error = parse_quality_scenario( + r#"{ + "name": "misspelled threshold", + "category": "constraint_recall", + "query": "quality proof", + "thresholds": {"min_constraint_recal_rate": 1.0} + }"#, + ) + .unwrap_err() + .to_string(); + + assert!( + error.contains("unknown field `min_constraint_recal_rate`"), + "{error}" + ); + } + + #[test] + fn rejects_unknown_nested_expectation_field() { + let error = parse_quality_scenario( + r#"{ + "name": "misspelled nested field", + "category": "constraint_recall", + "query": "quality proof", + "expected_recall": [{"memoryid": "mem_required"}] + }"#, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("unknown field `memoryid`"), "{error}"); + } + + #[test] + fn behavior_proof_requires_explicit_expectation() { + let error = parse_quality_scenario( + r#"{ + "name": "implicit behavior proof", + "category": "behavior_proof", + "query": "quality proof" + }"#, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("requires behavior_expectation"), "{error}"); + } + + #[test] + fn constraint_recall_requires_expected_recall_observations() { + let error = parse_quality_scenario( + r#"{ + "name": "empty constraint recall", + "category": "constraint_recall", + "query": "proof loop" + }"#, + ) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one expected_recall"), + "{error}" + ); + } + + #[test] + fn spam_prevention_requires_reject_observations() { + let input = serde_json::json!({ + "name": "empty spam prevention", + "category": "spam_prevention", + "query": "memory spam", + "write_candidates": [{ + "id": "mem_non_reject_candidate", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "lesson", + "summary": "Durable note", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_non_reject_candidate", + "decision": "accept" + }] + }); + + let error = parse_quality_scenario(&input.to_string()) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one expected_write_decision of reject"), + "{error}" + ); + } + + #[test] + fn stale_truth_suppression_requires_forbidden_recall_observations() { + let error = parse_quality_scenario( + r#"{ + "name": "empty stale truth suppression", + "category": "stale_truth_suppression", + "query": "stale rule" + }"#, + ) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one forbidden_recall"), + "{error}" + ); + } + + #[test] + fn evidence_preservation_requires_evaluation_write_candidates() { + let input = serde_json::json!({ + "name": "empty evidence preservation", + "category": "evidence_preservation", + "query": "preserve evidence", + "write_candidates": [{ + "id": "mem_non_evaluation_candidate", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "lesson", + "summary": "Regular lesson", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_non_evaluation_candidate", + "decision": "accept" + }] + }); + + let error = parse_quality_scenario(&input.to_string()) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one evaluation write_candidate"), + "{error}" + ); + } + + #[test] + fn rejects_explicit_thresholds_for_inapplicable_metrics() { + let cases = [ + ( + serde_json::json!({ + "name": "irrelevant constraint threshold", + "category": "stale_truth_suppression", + "query": "stale rule", + "seed_memories": [{ + "id": "mem_stale", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "Stale instruction", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "forbidden_recall": [{"memory_id": "mem_stale"}], + "thresholds": {"min_constraint_recall_rate": 1.0} + }), + "min_constraint_recall_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant forbidden threshold", + "category": "constraint_recall", + "query": "proof loop", + "expected_recall": [{"memory_id": "mem_required"}], + "thresholds": {"max_forbidden_recall_rate": 0.0} + }), + "max_forbidden_recall_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant spam threshold", + "category": "evidence_preservation", + "query": "preserve evidence", + "write_candidates": [{ + "id": "mem_eval", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "evaluation_outcome", + "summary": "Evidence-backed evaluation", + "details": "", + "source": {"type": "evidence", "ref": "evals/run-001", "quote": ""}, + "tags": ["evaluation"], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_eval", + "decision": "accept" + }], + "evidence_refs": ["evals/run-001"], + "thresholds": {"min_spam_rejection_rate": 1.0} + }), + "min_spam_rejection_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant evidence threshold", + "category": "spam_prevention", + "query": "memory spam", + "write_candidates": [{ + "id": "mem_spam", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "Transient planning chatter", + "details": "", + "source": {"type": "manual", "ref": "", "quote": ""}, + "tags": ["transient"], + "salience": 0.2, + "confidence": 0.2, + "sensitivity": "normal", + "retention": "ephemeral", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_spam", + "decision": "reject" + }], + "thresholds": {"min_evidence_required_rate": 1.0} + }), + "min_evidence_required_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant behavior threshold", + "category": "constraint_recall", + "query": "proof loop", + "expected_recall": [{"memory_id": "mem_required"}], + "thresholds": {"min_behavior_proof_pass_rate": 1.0} + }), + "min_behavior_proof_pass_rate", + ), + ]; + + for (input, field) in cases { + let error = parse_quality_scenario(&input.to_string()) + .unwrap_err() + .to_string(); + assert!(error.contains(field), "{field}: {error}"); + assert!(error.contains("inapplicable"), "{field}: {error}"); + } + } + + #[test] + fn older_quality_reports_deserialize_without_behavior_expectation() { + let report = serde_json::from_str::( + r#"{ + "name": "legacy report", + "category": "constraint_recall", + "constraint_recall_rate": 1.0, + "forbidden_recall_rate": null, + "spam_rejection_rate": null, + "evidence_required_rate": null, + "behavior_proof_pass": null, + "quality_pass": true, + "expected_recall": [], + "forbidden_recall": [], + "write_decisions": [] + }"#, + ) + .unwrap(); + + assert!(report.behavior_expectation.is_none()); + } + + #[test] + fn behavior_proof_reports_an_observed_decision_change() { + let required = memory( + "mem_required_behavior", + "Rollback instead of retrying the stale cache migration.", + "scar", + ); + let input = serde_json::json!({ + "name": "explicit behavior proof", + "category": "behavior_proof", + "query": "stale cache migration failed", + "seed_memories": [required], + "behavior_expectation": { + "required_memory_id": "mem_required_behavior", + "baseline_decision": "retry the migration unchanged", + "memory_informed_decision": "rollback and inspect cache state", + "expected_decision": "rollback and inspect cache state", + "reason": "the scar changes the recovery decision" + } + }); + let scenario = parse_quality_scenario(&input.to_string()).unwrap(); + let recalls = [QualityRecall { + memory: scenario.seed_memories[0].clone(), + score: 0.9, + }]; + + let report = report_json(&scenario, &recalls); + + assert_eq!(report["behavior_proof_pass"], true); + assert_eq!( + report["behavior_expectation"]["required_memory_recalled"], + true + ); + assert_eq!(report["behavior_expectation"]["decision_changed"], true); + assert_eq!( + report["behavior_expectation"]["expected_decision_reached"], + true + ); + assert_eq!(report["behavior_expectation"]["passed"], true); + } + + #[test] + fn behavior_threshold_applies_to_an_applicable_failure() { + let required = memory("mem_behavior_threshold", "Use rollback recovery.", "scar"); + let input = serde_json::json!({ + "name": "behavior threshold", + "category": "behavior_proof", + "query": "failed recovery", + "seed_memories": [required], + "behavior_expectation": { + "required_memory_id": "mem_behavior_threshold", + "baseline_decision": "retry unchanged", + "memory_informed_decision": "retry unchanged", + "expected_decision": "rollback" + }, + "thresholds": {"min_behavior_proof_pass_rate": 0.0} + }); + let mut scenario = parse_quality_scenario(&input.to_string()).unwrap(); + let recalls = [QualityRecall { + memory: scenario.seed_memories[0].clone(), + score: 0.9, + }]; + + assert_eq!(report_json(&scenario, &recalls)["quality_pass"], true); + scenario.thresholds.min_behavior_proof_pass_rate = Some(0.5); + assert_eq!(report_json(&scenario, &recalls)["quality_pass"], false); + } + + #[test] + fn inapplicable_metrics_are_null_when_a_primary_observation_exists() { + let mut scenario = scenario("constraint only", "constraint_recall"); + scenario.expected_recall = vec![RecallExpectation { + memory_id: Some("mem_missing".to_string()), + ..Default::default() + }]; + let report = report_json(&scenario, &[]); + + assert_eq!(report["constraint_recall_rate"], 0.0); + assert_eq!(report["quality_pass"], false); + assert!(report["forbidden_recall_rate"].is_null()); + assert!(report["spam_rejection_rate"].is_null()); + assert!(report["evidence_required_rate"].is_null()); + assert!(report["behavior_proof_pass"].is_null()); + } + + #[test] + fn run_metrics_weight_only_applicable_expectations() { + let mut failing = scenario("missed constraint", "constraint_recall"); + failing.expected_recall = vec![RecallExpectation { + memory_id: Some("mem_missing".to_string()), + ..Default::default() + }]; + let mut unrelated = scenario("unrelated", "evidence_preservation"); + let mut candidate = memory("mem_eval", "Evidence-backed evaluation.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-001".to_string(); + unrelated.write_candidates = vec![candidate]; + unrelated.expected_write_decisions = vec![WriteDecisionExpectation { + memory_id: "mem_eval".to_string(), + decision: "accept".to_string(), + reason: "valid unrelated evidence scenario".to_string(), + }]; + unrelated.evidence_refs = vec!["evals/run-001".to_string()]; + let reports = vec![ + evaluate_quality_scenario(&failing, &[]).unwrap(), + evaluate_quality_scenario(&unrelated, &[]).unwrap(), + ]; + + let run = serde_json::to_value(summarize_quality_run(reports)).unwrap(); + + assert_eq!(run["constraint_recall_rate"], 0.0); + assert!(run["spam_rejection_rate"].is_null()); + } + + #[test] + fn forbidden_run_rate_uses_failures_over_applicable_expectations() { + let mut mixed = scenario("mixed forbidden recall", "stale_truth_suppression"); + mixed.seed_memories = vec![ + memory("mem_forbidden_recalled", "Stale instruction.", "heartwood"), + memory( + "mem_forbidden_hidden", + "Hidden stale instruction.", + "heartwood", + ), + ]; + mixed.forbidden_recall = vec![ + RecallExpectation { + memory_id: Some("mem_forbidden_recalled".to_string()), + ..Default::default() + }, + RecallExpectation { + memory_id: Some("mem_forbidden_hidden".to_string()), + ..Default::default() + }, + ]; + let recalls = [QualityRecall { + memory: memory("mem_forbidden_recalled", "Stale instruction.", "heartwood"), + score: 0.8, + }]; + let mut unrelated = scenario("unrelated", "evidence_preservation"); + let mut candidate = memory("mem_eval", "Evidence-backed evaluation.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-001".to_string(); + unrelated.write_candidates = vec![candidate]; + unrelated.expected_write_decisions = vec![WriteDecisionExpectation { + memory_id: "mem_eval".to_string(), + decision: "accept".to_string(), + reason: "valid unrelated evidence scenario".to_string(), + }]; + unrelated.evidence_refs = vec!["evals/run-001".to_string()]; + let reports = vec![ + evaluate_quality_scenario(&mixed, &recalls).unwrap(), + evaluate_quality_scenario(&unrelated, &[]).unwrap(), + ]; + + let run = serde_json::to_value(summarize_quality_run(reports)).unwrap(); + + assert_eq!(run["forbidden_recall_rate"], 0.5); + } + + #[test] + fn applicable_thresholds_gate_scenario_quality() { + let required = memory("mem_recalled", "Required constraint.", "heartwood"); + let mut recall = scenario("threshold gates", "constraint_recall"); + recall.expected_recall = vec![ + RecallExpectation { + memory_id: Some("mem_recalled".to_string()), + ..Default::default() + }, + RecallExpectation { + memory_id: Some("mem_missing".to_string()), + ..Default::default() + }, + ]; + recall.thresholds.min_constraint_recall_rate = Some(0.5); + let recalls = [QualityRecall { + memory: required, + score: 0.9, + }]; + assert_eq!(report_json(&recall, &recalls)["quality_pass"], true); + recall.thresholds.min_constraint_recall_rate = Some(0.6); + assert_eq!(report_json(&recall, &recalls)["quality_pass"], false); + + let mut forbidden = scenario("forbidden threshold", "stale_truth_suppression"); + forbidden.seed_memories = vec![memory("mem_stale", "Stale instruction.", "heartwood")]; + forbidden.forbidden_recall = vec![RecallExpectation { + memory_id: Some("mem_stale".to_string()), + ..Default::default() + }]; + forbidden.thresholds.max_forbidden_recall_rate = Some(0.0); + let stale = [QualityRecall { + memory: memory("mem_stale", "Stale instruction.", "heartwood"), + score: 0.9, + }]; + assert_eq!(report_json(&forbidden, &stale)["quality_pass"], false); + } + + #[test] + fn evaluation_write_decisions_are_evidence_applicable() { + let mut candidate = memory( + "mem_evaluation", + "Preserve the evaluated outcome with its evidence.", + "cambium", + ); + candidate.event_type = "evaluation_result".to_string(); + candidate.source.ref_.clear(); + let mut scenario = scenario("evidence applicability", "evidence_preservation"); + scenario.write_candidates = vec![candidate]; + scenario.expected_write_decisions = vec![WriteDecisionExpectation { + memory_id: "mem_evaluation".to_string(), + decision: "accept".to_string(), + reason: "accepted outcomes preserve evidence".to_string(), + }]; + + let report = report_json(&scenario, &[]); + + assert_eq!(report["evidence_required_rate"], 0.0); + assert_eq!(report["write_decisions"][0]["evidence_applicable"], true); + } + + #[test] + fn zero_scenario_run_is_failed_and_has_null_metrics() { + let report = serde_json::to_value(summarize_quality_run(Vec::new())).unwrap(); + + assert_eq!(report["ok"], false); + assert_eq!(report["quality_pass"], false); + assert!(report["constraint_recall_rate"].is_null()); + assert!(report["behavior_proof_pass_rate"].is_null()); + } + + #[test] + fn parses_valid_quality_scenario() { + let input = r#"{ + "name": "constraint recall", + "category": "constraint_recall", + "query": "proof loop background writer", + "seed_memories": [ + { + "id": "mem_quality_constraint", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background writer to proof loop work.", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": ["proof-loop"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + } + ], + "expected_recall": [ + {"memory_id": "mem_quality_constraint", "reason": "must recall proof-loop constraint"} + ] + }"#; + + let scenario = parse_quality_scenario(input).unwrap(); + + assert_eq!(scenario.name, "constraint recall"); + assert_eq!(scenario.category, "constraint_recall"); + assert_eq!(scenario.seed_memories.len(), 1); + assert_eq!( + scenario.expected_recall[0].memory_id.as_deref(), + Some("mem_quality_constraint") + ); + } + + #[test] + fn rejects_scenario_without_prompt() { + let input = r#"{"name":"bad","category":"constraint_recall"}"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("requires query or workflow_prompt")); + } + + #[test] + fn rejects_unknown_write_decision() { + let input = r#"{ + "name": "bad decision", + "category": "spam_prevention", + "query": "planning chatter", + "expected_write_decisions": [ + {"memory_id": "mem_candidate", "decision": "maybe"} + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("invalid write decision")); + } + + #[test] + fn rejects_blank_evidence_refs() { + let input = r#"{ + "name": "bad evidence refs", + "category": "constraint_recall", + "query": "prove behavior", + "evidence_refs": ["docs/spec.md", " "] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("evidence_refs")); + assert!(error.contains("must not be blank")); + } + + #[test] + fn rejects_blank_recall_selectors() { + let input = r#"{ + "name": "blank recall selectors", + "category": "constraint_recall", + "query": "proof loop", + "expected_recall": [ + {"memory_id": " ", "ring": "", "tag": " ", "source_ref": "\t"} + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("expected_recall")); + assert!(error.contains("must include a nonblank")); + } + + #[test] + fn uses_workflow_prompt_when_query_is_blank() { + let input = r#"{ + "name": "workflow prompt fallback", + "category": "constraint_recall", + "query": " ", + "workflow_prompt": " validate behavior proof ", + "expected_recall": [{"memory_id": "mem_required"}] + }"#; + + let scenario = parse_quality_scenario(input).unwrap(); + + assert_eq!(scenario.prompt(), Some(" validate behavior proof ")); + } + + #[test] + fn rejects_uncovered_write_candidate() { + let input = r#"{ + "name": "missing decision", + "category": "spam_prevention", + "query": "write gates", + "write_candidates": [ + { + "id": "mem_candidate", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Candidate without expected decision.", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + } + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("write_candidates[0]")); + assert!(error.contains("missing expected_write_decision")); + } + + #[test] + fn rejects_orphan_write_decision_mapping() { + let scenario = QualityScenario { + name: "orphan decision".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory("mem_real", "Real candidate.", "heartwood")], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_missing".to_string(), + decision: "reject".to_string(), + reason: "orphan mapping".to_string(), + }], + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let error = scenario.validate().unwrap_err().to_string(); + + assert!(error.contains("expected_write_decisions[0]")); + assert!(error.contains("does not match any write_candidate")); + } + + #[test] + fn parses_default_quality_fixtures() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join("fixtures") + .join("quality"); + let mut parsed = 0usize; + for entry in std::fs::read_dir(&root).unwrap() { + let path = entry.unwrap().path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let input = std::fs::read_to_string(&path).unwrap(); + parse_quality_scenario(&input) + .unwrap_or_else(|err| panic!("{}: {err}", path.display())); + parsed += 1; + } + + assert_eq!(parsed, 7); + } + + #[test] + fn rejects_duplicate_write_decision_mapping() { + let scenario = QualityScenario { + name: "duplicate decision".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory("mem_real", "Real candidate.", "heartwood")], + expected_write_decisions: vec![ + WriteDecisionExpectation { + memory_id: "mem_real".to_string(), + decision: "reject".to_string(), + reason: "first mapping".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_real".to_string(), + decision: "accept".to_string(), + reason: "duplicate mapping".to_string(), + }, + ], + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let error = scenario.validate().unwrap_err().to_string(); + + assert!(error.contains("expected_write_decisions[1]")); + assert!(error.contains("duplicate")); + } + + #[test] + fn evaluates_required_and_forbidden_recall() { + let scenario = QualityScenario { + name: "recall gate".to_string(), + category: "constraint_recall".to_string(), + seed_memories: vec![ + memory( + "mem_required", + "Do not add a background writer.", + "heartwood", + ), + memory("mem_forbidden", "Stale instruction.", "heartwood"), + ], + query: Some("background writer".to_string()), + workflow_prompt: None, + expected_recall: vec![RecallExpectation { + memory_id: Some("mem_required".to_string()), + reason: "required constraint".to_string(), + ..Default::default() + }], + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_forbidden".to_string()), + reason: "stale memory".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + scenario.validate().unwrap(); + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory( + "mem_required", + "Do not add a background writer.", + "heartwood", + ), + score: 0.91, + }], + ) + .unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.constraint_recall_rate, Some(1.0)); + assert_eq!(report.forbidden_recall_rate, Some(0.0)); + assert!(report.expected_recall[0].passed); + assert!(report.forbidden_recall[0].passed); + } + + #[test] + fn validates_forbidden_recall_against_seed_memories() { + let scenario = QualityScenario { + name: "stale recall seed coverage".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: vec![memory("mem_stale", "Old CLI contract.", "heartwood")], + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + ring: Some("heartwood".to_string()), + reason: "seeded stale memory".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + scenario.validate().unwrap(); + } + + #[test] + fn rejects_forbidden_recall_with_missing_seed_selector() { + let scenario = QualityScenario { + name: "stale recall typo".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: vec![memory("mem_stale", "Old CLI contract.", "heartwood")], + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_typo".to_string()), + reason: "typo selector".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let error = scenario.validate().unwrap_err().to_string(); + + assert!(error.contains("forbidden_recall[0]")); + assert!(error.contains("does not match a seed_memory")); + } + + #[test] + fn fails_when_forbidden_memory_is_recalled() { + let scenario = QualityScenario { + name: "stale recall".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: vec![memory("mem_stale", "Old CLI contract.", "heartwood")], + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_stale".to_string()), + reason: "stale CLI contract".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory("mem_stale", "Old CLI contract.", "heartwood"), + score: 0.88, + }], + ) + .unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.forbidden_recall_rate, Some(1.0)); + assert!(!report.forbidden_recall[0].passed); + } + + #[test] + fn classifies_write_candidates_against_expected_decisions() { + let mut spam = memory( + "mem_spam", + "Thinking about options for a moment.", + "heartwood", + ); + spam.tags = vec!["transient".to_string()]; + let mut promoted_without_evidence = memory( + "mem_missing_evidence", + "Promote evaluated proof.", + "heartwood", + ); + promoted_without_evidence.event_type = "evaluation_promotion".to_string(); + promoted_without_evidence.source.ref_.clear(); + let mut broad_heartwood = memory( + "mem_needs_confirmation", + "All projects should prefer this.", + "heartwood", + ); + broad_heartwood.source.ref_.clear(); + + let scenario = QualityScenario { + name: "write gates".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![spam, promoted_without_evidence, broad_heartwood], + expected_write_decisions: vec![ + WriteDecisionExpectation { + memory_id: "mem_spam".to_string(), + decision: "reject".to_string(), + reason: "transient planning chatter".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_missing_evidence".to_string(), + decision: "require_evidence".to_string(), + reason: "promotion needs evidence".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_needs_confirmation".to_string(), + decision: "require_user_confirmation".to_string(), + reason: "broad heartwood needs confirmation".to_string(), + }, + ], + evidence_refs: vec!["evals/run-001".to_string()], + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.spam_rejection_rate, Some(1.0)); + assert_eq!(report.evidence_required_rate, Some(1.0)); + assert_eq!(report.write_decisions.len(), 3); + } + + #[test] + fn quality_pass_fails_when_expected_reject_is_accepted() { + let scenario = QualityScenario { + name: "reject mismatch".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory( + "mem_accept", + "Durable evidence-backed note.", + "cambium", + )], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_accept".to_string(), + decision: "reject".to_string(), + reason: "should have been rejected".to_string(), + }], + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.write_decisions[0].actual, "accept"); + assert!(!report.write_decisions[0].passed); + } + + #[test] + fn quality_pass_fails_when_expected_require_evidence_is_accepted() { + let mut candidate = memory("mem_accept", "Durable evidence-backed note.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-002".to_string(); + let scenario = QualityScenario { + name: "require evidence mismatch".to_string(), + category: "evidence_preservation".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![candidate], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_accept".to_string(), + decision: "require_evidence".to_string(), + reason: "should need evidence".to_string(), + }], + evidence_refs: vec!["evals/run-002".to_string()], + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.write_decisions[0].actual, "accept"); + assert!(!report.write_decisions[0].passed); + } + + #[test] + fn quality_pass_fails_when_confirmation_mismatch_occurs() { + let mut candidate = memory("mem_accept", "Durable evidence-backed note.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-003".to_string(); + let scenario = QualityScenario { + name: "confirmation mismatch".to_string(), + category: "evidence_preservation".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![candidate], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_accept".to_string(), + decision: "require_user_confirmation".to_string(), + reason: "should require confirmation".to_string(), + }], + evidence_refs: vec!["evals/run-003".to_string()], + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.write_decisions[0].actual, "accept"); + assert!(!report.write_decisions[0].passed); + } + + #[test] + fn summarizes_quality_run() { + let passing = QualityScenarioReport { + name: "pass".to_string(), + category: "constraint_recall".to_string(), + constraint_recall_rate: Some(1.0), + forbidden_recall_rate: Some(0.0), + spam_rejection_rate: Some(1.0), + evidence_required_rate: Some(1.0), + behavior_proof_pass: Some(true), + behavior_expectation: None, + quality_pass: true, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_decisions: Vec::new(), + }; + + let run = summarize_quality_run(vec![passing.clone(), passing]); + + assert!(run.quality_pass); + assert_eq!(run.scenario_count, 2); + assert_eq!(run.behavior_proof_pass_rate, Some(1.0)); + } +} diff --git a/crates/tree-ring-memory-sqlite/src/lib.rs b/crates/tree-ring-memory-sqlite/src/lib.rs index f640bdb..45bd8e5 100644 --- a/crates/tree-ring-memory-sqlite/src/lib.rs +++ b/crates/tree-ring-memory-sqlite/src/lib.rs @@ -311,7 +311,32 @@ impl SQLiteMemoryStore { let Some(fts_query) = format_plain_text_fts_query(query) else { return Ok(Vec::new()); }; + self.search_fts_filtered_limited( + &fts_query, + project, + agent_profile, + scope, + rings, + event_types, + include_sensitive, + include_superseded, + limit, + ) + } + #[allow(clippy::too_many_arguments)] + fn search_fts_filtered_limited( + &self, + fts_query: &str, + project: Option<&str>, + agent_profile: Option<&str>, + scope: Option<&str>, + rings: Option<&[String]>, + event_types: Option<&[String]>, + include_sensitive: bool, + include_superseded: bool, + limit: Option, + ) -> TreeRingResult> { let mut sql = String::from( r#" SELECT memories.raw_json @@ -320,7 +345,7 @@ impl SQLiteMemoryStore { WHERE memory_fts MATCH ? "#, ); - let mut parameters = vec![Value::Text(fts_query)]; + let mut parameters = vec![Value::Text(fts_query.to_string())]; if !include_superseded { sql.push_str(" AND memories.superseded_by IS NULL"); @@ -803,11 +828,11 @@ impl<'a> MemoryRetriever<'a> { let mut candidates = Vec::new(); let mut seen_queries = HashSet::new(); + let candidate_limit = Some(limit.saturating_mul(128).clamp(256, 2048)); for search_query in search_queries(query) { if !seen_queries.insert(search_query.clone()) { continue; } - let candidate_limit = Some(limit.saturating_mul(128).clamp(256, 2048)); candidates = self.store.search_text_filtered_limited( &search_query, project, @@ -823,6 +848,21 @@ impl<'a> MemoryRetriever<'a> { break; } } + if candidates.is_empty() { + if let Some(fts_query) = format_plain_text_fts_or_query(query) { + candidates = self.store.search_fts_filtered_limited( + &fts_query, + project, + agent_profile, + scope, + rings, + event_types, + include_sensitive, + include_superseded, + candidate_limit, + )?; + } + } let mut results: Vec = candidates .into_iter() @@ -1199,6 +1239,14 @@ where } fn format_plain_text_fts_query(query: &str) -> Option { + format_plain_text_fts_query_with_operator(query, " AND ") +} + +fn format_plain_text_fts_or_query(query: &str) -> Option { + format_plain_text_fts_query_with_operator(query, " OR ") +} + +fn format_plain_text_fts_query_with_operator(query: &str, operator: &str) -> Option { let terms: Vec = tree_ring_memory_core::recall::terms(query) .into_iter() .filter(|term| !SEARCH_FILLER_TERMS.contains(&term.as_str())) @@ -1207,7 +1255,7 @@ fn format_plain_text_fts_query(query: &str) -> Option { if terms.is_empty() { return None; } - Some(terms.join(" AND ")) + Some(terms.join(operator)) } const SEARCH_FILLER_TERMS: &[&str] = &[ @@ -1405,6 +1453,123 @@ mod tests { assert_eq!(results[0].memory.id, target_id); } + #[test] + fn recall_strict_all_term_match_wins_before_fallback() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut partial = + MemoryEvent::new("Cache invalidation rollout reminder.", "lesson").unwrap(); + partial.salience = 1.0; + partial.confidence = 1.0; + let partial_id = partial.id.clone(); + let mut strict = + MemoryEvent::new("Cache invalidation canary rollout reminder.", "lesson").unwrap(); + strict.salience = 0.1; + strict.confidence = 0.1; + let strict_id = strict.id.clone(); + store.put(&partial).unwrap(); + store.put(&strict).unwrap(); + + let results = MemoryRetriever::new(&store) + .recall( + "cache invalidation canary", + None, + None, + None, + None, + None, + false, + false, + 8, + false, + ) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory.id, strict_id); + assert!(!results.iter().any(|result| result.memory.id == partial_id)); + } + + #[test] + fn recall_falls_back_to_partial_term_match_after_strict_misses() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let target = MemoryEvent::new( + "Quality adapter recalled approved fixture evidence.", + "lesson", + ) + .unwrap(); + let target_id = target.id.clone(); + store.put(&target).unwrap(); + + let results = MemoryRetriever::new(&store) + .recall( + "quality adapter missing transcript", + None, + None, + None, + None, + None, + false, + false, + 8, + false, + ) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory.id, target_id); + } + + #[test] + fn recall_fallback_keeps_default_filters_before_candidate_limit() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut events = Vec::new(); + for index in 0..300 { + let mut event = MemoryEvent::new( + format!("Workflow fallback recall distractor {index}"), + "lesson", + ) + .unwrap(); + event.project = Some("other-project".to_string()); + events.push(event); + } + let mut sensitive = + MemoryEvent::new("Workflow fallback recall sensitive target", "lesson").unwrap(); + sensitive.project = Some("target-project".to_string()); + sensitive.sensitivity = "private".to_string(); + events.push(sensitive); + let mut superseded = + MemoryEvent::new("Workflow fallback recall superseded target", "lesson").unwrap(); + superseded.project = Some("target-project".to_string()); + superseded.superseded_by = Some("replacement-memory".to_string()); + events.push(superseded); + let mut target = MemoryEvent::new("Workflow fallback recall target", "lesson").unwrap(); + target.project = Some("target-project".to_string()); + let target_id = target.id.clone(); + events.push(target); + store.put_many(&events).unwrap(); + + let results = MemoryRetriever::new(&store) + .recall( + "workflow fallback recall impossible", + Some("target-project"), + None, + None, + None, + None, + false, + false, + 1, + false, + ) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory.id, target_id); + } + #[test] fn redact_clears_fts_and_raw_payload() { let dir = tempdir().unwrap(); diff --git a/docs/integrations/agent-skill.md b/docs/integrations/agent-skill.md index 291624a..7598338 100644 --- a/docs/integrations/agent-skill.md +++ b/docs/integrations/agent-skill.md @@ -83,6 +83,20 @@ Outcome mapping: Plain `remember` is still appropriate for user preferences, explicit decisions, and project lessons that do not come from a formal evaluated outcome. +## Memory Quality Gates + +Tree Ring guidance is meant to improve agent behavior, not increase memory volume. +Use these gates when wiring Tree Ring into an agent harness: + +- Recall gates: before substantial or risky work, recall constraints, scars, preferences, and unresolved seeds. +- Trust gates: prefer source-linked, non-superseded, high-confidence memories and re-read authoritative sources when memory conflicts with source files or user instructions. +- Write gates: reject transient planning chatter, duplicate wording, tool noise, and unsupported claims; require evidence refs for promoted or rejected evaluated outcomes. + +The certification suite includes quality scenarios that exercise missed constraints, memory spam, stale truth suppression, and behavior proof. +Quality artifacts are written to +`target/tree-ring-certification/quality/quality-report.json` and +`target/tree-ring-certification/quality/quality-summary.md`. + ## Source Adapter Flow Use DOX and Revolve adapters when the source artifacts already exist locally: diff --git a/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md new file mode 100644 index 0000000..e24328c --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md @@ -0,0 +1,2233 @@ +# Tree Ring Memory Quality Proof Loop Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a certification-backed memory quality loop that proves Tree Ring helps agents recall constraints, reject memory spam, suppress stale truth, and preserve evidence in complex workflows. + +**Architecture:** Add a deterministic quality scenario model and evaluator to `tree-ring-memory-core`, then run it through a certification-only CLI example that loads reviewable fixtures into temporary SQLite stores. Feed the resulting gates into generated agent guidance, and add first-pass TUI fullness styling from the current ring distribution without creating a daemon, hidden writer, or public eval subcommand. + +**Tech Stack:** Rust 2021, serde, serde_json, rusqlite-backed `SQLiteMemoryStore`, existing `MemoryRetriever`, Ratatui, shell certification script, JSON fixtures. + +## Global Constraints + +- Do not add a daemon, sidecar, hosted service, telemetry pipeline, or hidden recorder. +- Do not scrape transcripts or turn event-stream pulses into durable memory. +- Do not add autonomous durable writes outside explicit user, agent, adapter, import, TUI, consolidation, or maintenance actions. +- Do not make Tree Ring memory more authoritative than source files, tests, explicit user instructions, root agent contracts, DOX contracts, or Revolve evidence. +- Do not replace the existing SQLite store, JSONL import/export shape, recall model, or certification workflow. +- Do not add a public `tree-ring eval` subcommand in this pass; keep the quality runner certification-owned until the fixture format settles. +- Keep scenario runs isolated in temporary memory roots. +- Keep TUI fullness separate from pulse and warning state. +- Preserve existing untracked local files; stage only files changed by each task. +- Run `cargo fmt --check`, `cargo test --locked`, `cargo clippy --locked --all-targets`, `git diff --check`, and `sh scripts/certify-tree-ring.sh` before final handoff. + +## Final Hardening Amendment + +This implementation plan now carries the final hardening semantics that +supersede earlier draft snippets: + +- Category validation must enforce a primary observation contract: + `constraint_recall` requires `expected_recall`, + `spam_prevention` requires at least one expected `reject`, + `stale_truth_suppression` requires `forbidden_recall`, + `behavior_proof` requires `behavior_expectation`, and + `evidence_preservation` requires at least one `evaluation_` write candidate. +- `QualityThresholds` are presence-aware optional fields. Omitted thresholds + inherit strict defaults only when their metric has observations. Configuring + a threshold for an inapplicable metric is a validation failure, not a silent + no-op. +- Runner failures must be sanitized before they are written to + `quality-report.json`, `quality-summary.md`, or bubbled back through the + example entrypoint. Reports keep stage plus stable error class, but never raw + fixture payload values. +- `QualityScenarioReport.behavior_expectation` must deserialize as `None` when + older artifacts omit the field. + +--- + +## Scope Check + +This plan covers one subsystem: a memory quality proof loop and its first consumer surfaces. It produces working, testable software in layers: + +1. Core scenario parsing. +2. Core quality evaluation. +3. Default quality fixtures. +4. Certification-only runner. +5. Certification report integration. +6. Generated guidance gates. +7. TUI fullness styling. + +No task creates a background process, network service, public CLI command, or alternate storage backend. + +## File Structure + +- Create `crates/tree-ring-memory-core/src/quality.rs`: scenario structs, parser, deterministic evaluator, quality run aggregation. +- Modify `crates/tree-ring-memory-core/src/lib.rs`: expose the quality module and public quality types. +- Create `fixtures/quality/no-background-writer-constraint.json`: real workflow constraint recall scenario. +- Create `fixtures/quality/transient-planning-spam.json`: real workflow spam-prevention scenario. +- Create `fixtures/quality/stale-cli-contract.json`: real workflow stale-truth suppression scenario. +- Create `fixtures/quality/sensitive-memory-hidden.json`: synthetic sensitive recall scenario. +- Create `fixtures/quality/superseded-heartwood.json`: synthetic supersession scenario. +- Create `fixtures/quality/scar-failure-recall.json`: synthetic scar recall scenario. +- Create `crates/tree-ring-memory-cli/examples/quality_scenarios.rs`: certification-only runner that loads fixtures, runs recall, evaluates reports, and writes JSON/Markdown artifacts. +- Modify `scripts/certify-tree-ring.sh`: run the quality example and include quality metrics in certification artifacts. +- Modify `crates/tree-ring-memory-cli/src/agent_awareness.rs`: generated AGENTS and CLI guidance gates. +- Modify `skills/tree-ring-memory/SKILL.md`: portable recall, trust, and write gates. +- Modify `docs/integrations/agent-skill.md`: document quality-gate usage for agent harnesses. +- Modify `README.md`: document certification quality artifacts. +- Modify `crates/tree-ring-memory-cli/src/tui/model.rs`: add first-pass relative fullness. +- Modify `crates/tree-ring-memory-cli/src/tui/rings.rs`: map fullness to ambient intensity without changing pulse/warning semantics. + +--- + +### Task 1: Add Quality Scenario Model + +**Files:** +- Create: `crates/tree-ring-memory-core/src/quality.rs` +- Modify: `crates/tree-ring-memory-core/src/lib.rs` + +**Interfaces:** +- Produces: `quality::QualityScenario` +- Produces: `quality::QualityThresholds` +- Produces: `quality::RecallExpectation` +- Produces: `quality::WriteDecisionExpectation` +- Produces: `quality::parse_quality_scenario(input: &str) -> TreeRingResult` +- Consumes: `models::MemoryEvent` +- Consumes: `models::TreeRingError` + +- [ ] **Step 1: Write the failing scenario parser tests** + +Add this complete file at `crates/tree-ring-memory-core/src/quality.rs`: + +```rust +use serde::{Deserialize, Serialize}; + +use crate::models::{MemoryEvent, TreeRingError, TreeRingResult}; + +pub const QUALITY_CATEGORIES: &[&str] = &[ + "constraint_recall", + "spam_prevention", + "stale_truth_suppression", + "behavior_proof", +]; + +pub const WRITE_DECISIONS: &[&str] = &[ + "accept", + "reject", + "require_evidence", + "require_user_confirmation", +]; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityScenario { + pub name: String, + pub category: String, + #[serde(default)] + pub seed_memories: Vec, + #[serde(default)] + pub query: Option, + #[serde(default)] + pub workflow_prompt: Option, + #[serde(default)] + pub expected_recall: Vec, + #[serde(default)] + pub forbidden_recall: Vec, + #[serde(default)] + pub write_candidates: Vec, + #[serde(default)] + pub expected_write_decisions: Vec, + #[serde(default)] + pub evidence_refs: Vec, + #[serde(default)] + pub thresholds: QualityThresholds, +} + +impl QualityScenario { + pub fn prompt(&self) -> Option<&str> { + self.query + .as_deref() + .or_else(|| self.workflow_prompt.as_deref()) + } + + pub fn validate(&self) -> TreeRingResult<()> { + if self.name.trim().is_empty() { + return Err(TreeRingError::Validation( + "quality scenario name is required".to_string(), + )); + } + validate_member("quality category", &self.category, QUALITY_CATEGORIES)?; + if self + .prompt() + .map(|value| value.trim().is_empty()) + .unwrap_or(true) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} requires query or workflow_prompt", + self.name + ))); + } + for memory in self.seed_memories.iter().chain(self.write_candidates.iter()) { + memory.validate()?; + } + for expectation in &self.expected_recall { + expectation.validate("expected_recall", &self.name)?; + } + for expectation in &self.forbidden_recall { + expectation.validate("forbidden_recall", &self.name)?; + } + for decision in &self.expected_write_decisions { + decision.validate(&self.name)?; + } + self.thresholds.validate(&self.name)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecallExpectation { + #[serde(default)] + pub memory_id: Option, + #[serde(default)] + pub ring: Option, + #[serde(default)] + pub tag: Option, + #[serde(default)] + pub source_ref: Option, + #[serde(default)] + pub reason: String, +} + +impl RecallExpectation { + fn validate(&self, field: &str, scenario_name: &str) -> TreeRingResult<()> { + if self.memory_id.is_none() + && self.ring.is_none() + && self.tag.is_none() + && self.source_ref.is_none() + { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} {field} entry needs memory_id, ring, tag, or source_ref" + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct WriteDecisionExpectation { + pub memory_id: String, + pub decision: String, + #[serde(default)] + pub reason: String, +} + +impl WriteDecisionExpectation { + fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + if self.memory_id.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} write decision requires memory_id" + ))); + } + validate_member("write decision", &self.decision, WRITE_DECISIONS) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityThresholds { + #[serde(default = "one")] + pub min_constraint_recall_rate: f64, + #[serde(default)] + pub max_forbidden_recall_rate: f64, + #[serde(default = "one")] + pub min_spam_rejection_rate: f64, + #[serde(default = "one")] + pub min_evidence_required_rate: f64, + #[serde(default = "one")] + pub min_behavior_proof_pass_rate: f64, +} + +impl Default for QualityThresholds { + fn default() -> Self { + Self { + min_constraint_recall_rate: 1.0, + max_forbidden_recall_rate: 0.0, + min_spam_rejection_rate: 1.0, + min_evidence_required_rate: 1.0, + min_behavior_proof_pass_rate: 1.0, + } + } +} + +impl QualityThresholds { + fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + for (field, value) in [ + ( + "min_constraint_recall_rate", + self.min_constraint_recall_rate, + ), + ("max_forbidden_recall_rate", self.max_forbidden_recall_rate), + ("min_spam_rejection_rate", self.min_spam_rejection_rate), + ("min_evidence_required_rate", self.min_evidence_required_rate), + ( + "min_behavior_proof_pass_rate", + self.min_behavior_proof_pass_rate, + ), + ] { + if !value.is_finite() || !(0.0..=1.0).contains(&value) { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} threshold {field} must be between 0 and 1" + ))); + } + } + Ok(()) + } +} + +pub fn parse_quality_scenario(input: &str) -> TreeRingResult { + let scenario: QualityScenario = serde_json::from_str(input)?; + scenario.validate()?; + Ok(scenario) +} + +fn validate_member(field: &str, value: &str, allowed: &[&str]) -> TreeRingResult<()> { + if allowed.contains(&value) { + Ok(()) + } else { + Err(TreeRingError::Validation(format!( + "invalid {field}: {value}" + ))) + } +} + +fn one() -> f64 { + 1.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_valid_quality_scenario() { + let input = r#"{ + "name": "constraint recall", + "category": "constraint_recall", + "query": "proof loop background writer", + "seed_memories": [ + { + "id": "mem_quality_constraint", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background writer to proof loop work.", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": ["proof-loop"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + } + ], + "expected_recall": [ + {"memory_id": "mem_quality_constraint", "reason": "must recall proof-loop constraint"} + ] + }"#; + + let scenario = parse_quality_scenario(input).unwrap(); + + assert_eq!(scenario.name, "constraint recall"); + assert_eq!(scenario.category, "constraint_recall"); + assert_eq!(scenario.seed_memories.len(), 1); + assert_eq!(scenario.expected_recall[0].memory_id.as_deref(), Some("mem_quality_constraint")); + } + + #[test] + fn rejects_scenario_without_prompt() { + let input = r#"{"name":"bad","category":"constraint_recall"}"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("requires query or workflow_prompt")); + } + + #[test] + fn rejects_unknown_write_decision() { + let input = r#"{ + "name": "bad decision", + "category": "spam_prevention", + "query": "planning chatter", + "expected_write_decisions": [ + {"memory_id": "mem_candidate", "decision": "maybe"} + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("invalid write decision")); + } +} +``` + +- [ ] **Step 2: Export the quality module** + +Modify `crates/tree-ring-memory-core/src/lib.rs` by adding the module declaration with the other modules: + +```rust +pub mod quality; +``` + +Then add this public export block after the existing `pub use models::{...};` block: + +```rust +pub use quality::{ + parse_quality_scenario, QualityScenario, QualityThresholds, RecallExpectation, + WriteDecisionExpectation, QUALITY_CATEGORIES, WRITE_DECISIONS, +}; +``` + +- [ ] **Step 3: Run the focused parser tests** + +Run: + +```bash +cargo test -p tree-ring-memory-core quality:: --locked +``` + +Expected: PASS with the three quality parser tests passing. + +- [ ] **Step 4: Commit Task 1** + +Run: + +```bash +git add crates/tree-ring-memory-core/src/lib.rs crates/tree-ring-memory-core/src/quality.rs +git commit -m "Add quality scenario model" +``` + +--- + +### Task 2: Add Deterministic Quality Evaluator + +**Files:** +- Modify: `crates/tree-ring-memory-core/src/quality.rs` +- Modify: `crates/tree-ring-memory-core/src/lib.rs` + +**Interfaces:** +- Consumes: `QualityScenario` +- Produces: `quality::QualityRecall` +- Produces: `quality::QualityScenarioReport` +- Produces: `quality::QualityRunReport` +- Produces: `quality::evaluate_quality_scenario(scenario: &QualityScenario, recalls: &[QualityRecall]) -> TreeRingResult` +- Produces: `quality::summarize_quality_run(reports: Vec) -> QualityRunReport` + +- [ ] **Step 1: Add failing evaluator tests** + +Append these tests inside the existing `#[cfg(test)] mod tests` in `crates/tree-ring-memory-core/src/quality.rs`: + +```rust + fn memory(id: &str, summary: &str, ring: &str) -> MemoryEvent { + let mut event = MemoryEvent::new(summary, "lesson").unwrap(); + event.id = id.to_string(); + event.created_at = "2026-07-09T00:00:00Z".to_string(); + event.updated_at = "2026-07-09T00:00:00Z".to_string(); + event.scope = "project".to_string(); + event.project = Some("tree-ring".to_string()); + event.ring = ring.to_string(); + event.source.source_type = "evidence".to_string(); + event.source.ref_ = "docs/spec.md".to_string(); + event.salience = 0.8; + event.confidence = 0.8; + event + } + + #[test] + fn evaluates_required_and_forbidden_recall() { + let mut scenario = QualityScenario { + name: "recall gate".to_string(), + category: "constraint_recall".to_string(), + seed_memories: Vec::new(), + query: Some("background writer".to_string()), + workflow_prompt: None, + expected_recall: vec![RecallExpectation { + memory_id: Some("mem_required".to_string()), + reason: "required constraint".to_string(), + ..Default::default() + }], + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_forbidden".to_string()), + reason: "stale memory".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + scenario.validate().unwrap(); + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory("mem_required", "Do not add a background writer.", "heartwood"), + score: 0.91, + }], + ) + .unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.constraint_recall_rate, 1.0); + assert_eq!(report.forbidden_recall_rate, 0.0); + assert!(report.expected_recall[0].passed); + assert!(report.forbidden_recall[0].passed); + } + + #[test] + fn fails_when_forbidden_memory_is_recalled() { + let scenario = QualityScenario { + name: "stale recall".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: Vec::new(), + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_stale".to_string()), + reason: "stale CLI contract".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory("mem_stale", "Old CLI contract.", "heartwood"), + score: 0.88, + }], + ) + .unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.forbidden_recall_rate, 1.0); + assert!(!report.forbidden_recall[0].passed); + } + + #[test] + fn classifies_write_candidates_against_expected_decisions() { + let mut spam = memory("mem_spam", "Thinking about options for a moment.", "heartwood"); + spam.tags = vec!["transient".to_string()]; + let mut promoted_without_evidence = + memory("mem_missing_evidence", "Promote evaluated proof.", "heartwood"); + promoted_without_evidence.event_type = "evaluation_promotion".to_string(); + promoted_without_evidence.source.ref_.clear(); + let mut broad_heartwood = + memory("mem_needs_confirmation", "All projects should prefer this.", "heartwood"); + broad_heartwood.source.ref_.clear(); + + let scenario = QualityScenario { + name: "write gates".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![spam, promoted_without_evidence, broad_heartwood], + expected_write_decisions: vec![ + WriteDecisionExpectation { + memory_id: "mem_spam".to_string(), + decision: "reject".to_string(), + reason: "transient planning chatter".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_missing_evidence".to_string(), + decision: "require_evidence".to_string(), + reason: "promotion needs evidence".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_needs_confirmation".to_string(), + decision: "require_user_confirmation".to_string(), + reason: "broad heartwood needs confirmation".to_string(), + }, + ], + evidence_refs: vec!["evals/run-001".to_string()], + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.spam_rejection_rate, 1.0); + assert_eq!(report.evidence_required_rate, 1.0); + assert_eq!(report.write_decisions.len(), 3); + } + + #[test] + fn summarizes_quality_run() { + let passing = QualityScenarioReport { + name: "pass".to_string(), + category: "constraint_recall".to_string(), + constraint_recall_rate: 1.0, + forbidden_recall_rate: 0.0, + spam_rejection_rate: 1.0, + evidence_required_rate: 1.0, + behavior_proof_pass: true, + quality_pass: true, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_decisions: Vec::new(), + }; + + let run = summarize_quality_run(vec![passing.clone(), passing]); + + assert!(run.quality_pass); + assert_eq!(run.scenario_count, 2); + assert_eq!(run.behavior_proof_pass_rate, 1.0); + } +``` + +- [ ] **Step 2: Run the focused evaluator tests and verify missing types** + +Run: + +```bash +cargo test -p tree-ring-memory-core quality:: --locked +``` + +Expected: FAIL because `QualityRecall`, `evaluate_quality_scenario`, `QualityScenarioReport`, and `summarize_quality_run` are not defined. + +- [ ] **Step 3: Add evaluator types and functions** + +Insert this code in `crates/tree-ring-memory-core/src/quality.rs` above `pub fn parse_quality_scenario`: + +```rust +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRecall { + pub memory: MemoryEvent, + pub score: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallExpectationReport { + pub expectation: RecallExpectation, + pub matched_memory_ids: Vec, + pub passed: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WriteDecisionReport { + pub memory_id: String, + pub expected: String, + pub actual: String, + pub passed: bool, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityScenarioReport { + pub name: String, + pub category: String, + pub constraint_recall_rate: f64, + pub forbidden_recall_rate: f64, + pub spam_rejection_rate: f64, + pub evidence_required_rate: f64, + pub behavior_proof_pass: bool, + pub quality_pass: bool, + pub expected_recall: Vec, + pub forbidden_recall: Vec, + pub write_decisions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRunReport { + pub ok: bool, + pub scenario_count: usize, + pub constraint_recall_rate: f64, + pub forbidden_recall_rate: f64, + pub spam_rejection_rate: f64, + pub evidence_required_rate: f64, + pub behavior_proof_pass_rate: f64, + pub quality_pass: bool, + pub scenarios: Vec, +} + +pub fn evaluate_quality_scenario( + scenario: &QualityScenario, + recalls: &[QualityRecall], +) -> TreeRingResult { + scenario.validate()?; + + let expected_recall = scenario + .expected_recall + .iter() + .map(|expectation| expected_recall_report(expectation, recalls)) + .collect::>(); + let forbidden_recall = scenario + .forbidden_recall + .iter() + .map(|expectation| forbidden_recall_report(expectation, recalls)) + .collect::>(); + let write_decisions = scenario + .expected_write_decisions + .iter() + .map(|expectation| write_decision_report(expectation, scenario)) + .collect::>(); + + let constraint_recall_rate = pass_rate(&expected_recall); + let forbidden_recall_rate = failure_rate(&forbidden_recall); + let spam_rejection_rate = decision_pass_rate(&write_decisions, "reject"); + let evidence_required_rate = decision_pass_rate(&write_decisions, "require_evidence"); + let behavior_proof_pass = + constraint_recall_rate >= scenario.thresholds.min_constraint_recall_rate + && forbidden_recall_rate <= scenario.thresholds.max_forbidden_recall_rate; + let quality_pass = behavior_proof_pass + && spam_rejection_rate >= scenario.thresholds.min_spam_rejection_rate + && evidence_required_rate >= scenario.thresholds.min_evidence_required_rate; + + Ok(QualityScenarioReport { + name: scenario.name.clone(), + category: scenario.category.clone(), + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass, + quality_pass, + expected_recall, + forbidden_recall, + write_decisions, + }) +} + +pub fn summarize_quality_run(reports: Vec) -> QualityRunReport { + let scenario_count = reports.len(); + let denominator = scenario_count.max(1) as f64; + let constraint_recall_rate = + reports.iter().map(|report| report.constraint_recall_rate).sum::() / denominator; + let forbidden_recall_rate = + reports.iter().map(|report| report.forbidden_recall_rate).sum::() / denominator; + let spam_rejection_rate = + reports.iter().map(|report| report.spam_rejection_rate).sum::() / denominator; + let evidence_required_rate = + reports.iter().map(|report| report.evidence_required_rate).sum::() / denominator; + let behavior_proof_pass_rate = reports + .iter() + .filter(|report| report.behavior_proof_pass) + .count() as f64 + / denominator; + let quality_pass = reports.iter().all(|report| report.quality_pass); + + QualityRunReport { + ok: quality_pass, + scenario_count, + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass_rate, + quality_pass, + scenarios: reports, + } +} + +fn expected_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: !matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn forbidden_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn matching_recall_ids(expectation: &RecallExpectation, recalls: &[QualityRecall]) -> Vec { + recalls + .iter() + .filter(|recall| expectation_matches_memory(expectation, &recall.memory)) + .map(|recall| recall.memory.id.clone()) + .collect() +} + +fn expectation_matches_memory(expectation: &RecallExpectation, memory: &MemoryEvent) -> bool { + if expectation + .memory_id + .as_deref() + .is_some_and(|id| memory.id != id) + { + return false; + } + if expectation + .ring + .as_deref() + .is_some_and(|ring| memory.ring != ring) + { + return false; + } + if expectation + .tag + .as_deref() + .is_some_and(|tag| !memory.tags.iter().any(|candidate| candidate == tag)) + { + return false; + } + if expectation + .source_ref + .as_deref() + .is_some_and(|source_ref| memory.source.ref_ != source_ref) + { + return false; + } + true +} + +fn write_decision_report( + expectation: &WriteDecisionExpectation, + scenario: &QualityScenario, +) -> WriteDecisionReport { + let actual = scenario + .write_candidates + .iter() + .find(|candidate| candidate.id == expectation.memory_id) + .map(|candidate| classify_write_candidate(candidate, &scenario.evidence_refs)) + .unwrap_or_else(|| "reject".to_string()); + WriteDecisionReport { + memory_id: expectation.memory_id.clone(), + expected: expectation.decision.clone(), + passed: actual == expectation.decision, + actual, + reason: expectation.reason.clone(), + } +} + +fn classify_write_candidate(candidate: &MemoryEvent, required_evidence_refs: &[String]) -> String { + if candidate.sensitivity == "secret" { + return "reject".to_string(); + } + if candidate.retention == "ephemeral" + || candidate.tags.iter().any(|tag| { + matches!( + tag.as_str(), + "transient" | "planning-chatter" | "scratchpad" | "tool-noise" + ) + }) + { + return "reject".to_string(); + } + if candidate.event_type.starts_with("evaluation_") && candidate.source.ref_.trim().is_empty() { + return "require_evidence".to_string(); + } + if !required_evidence_refs.is_empty() + && candidate.event_type.starts_with("evaluation_") + && !required_evidence_refs + .iter() + .any(|required| required == &candidate.source.ref_) + { + return "require_evidence".to_string(); + } + if candidate.ring == "heartwood" && candidate.source.ref_.trim().is_empty() { + return "require_user_confirmation".to_string(); + } + "accept".to_string() +} + +fn pass_rate(reports: &[RecallExpectationReport]) -> f64 { + if reports.is_empty() { + return 1.0; + } + reports.iter().filter(|report| report.passed).count() as f64 / reports.len() as f64 +} + +fn failure_rate(reports: &[RecallExpectationReport]) -> f64 { + if reports.is_empty() { + return 0.0; + } + reports.iter().filter(|report| !report.passed).count() as f64 / reports.len() as f64 +} + +fn decision_pass_rate(reports: &[WriteDecisionReport], decision: &str) -> f64 { + let relevant = reports + .iter() + .filter(|report| report.expected == decision) + .collect::>(); + if relevant.is_empty() { + return 1.0; + } + relevant.iter().filter(|report| report.passed).count() as f64 / relevant.len() as f64 +} +``` + +- [ ] **Step 4: Export evaluator types** + +Modify the `pub use quality::{...};` block in `crates/tree-ring-memory-core/src/lib.rs` so it includes: + +```rust + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, + QualityRunReport, QualityScenario, QualityScenarioReport, QualityThresholds, + RecallExpectation, RecallExpectationReport, WriteDecisionExpectation, WriteDecisionReport, + QUALITY_CATEGORIES, WRITE_DECISIONS, +``` + +- [ ] **Step 5: Run the focused evaluator tests** + +Run: + +```bash +cargo test -p tree-ring-memory-core quality:: --locked +``` + +Expected: PASS. + +- [ ] **Step 6: Commit Task 2** + +Run: + +```bash +git add crates/tree-ring-memory-core/src/lib.rs crates/tree-ring-memory-core/src/quality.rs +git commit -m "Add memory quality evaluator" +``` + +--- + +### Task 3: Add Default Quality Fixture Pack + +**Files:** +- Create: `fixtures/quality/no-background-writer-constraint.json` +- Create: `fixtures/quality/transient-planning-spam.json` +- Create: `fixtures/quality/stale-cli-contract.json` +- Create: `fixtures/quality/sensitive-memory-hidden.json` +- Create: `fixtures/quality/superseded-heartwood.json` +- Create: `fixtures/quality/scar-failure-recall.json` + +**Interfaces:** +- Consumes: `quality::parse_quality_scenario` +- Produces: six valid quality scenarios for certification. + +- [ ] **Step 1: Add the real workflow constraint scenario** + +Create `fixtures/quality/no-background-writer-constraint.json`: + +```json +{ + "name": "no-background-writer-constraint", + "category": "constraint_recall", + "query": "proof loop background writer hidden durable writes", + "seed_memories": [ + { + "id": "mem_quality_no_background_writer", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background proof runner, hidden recorder, or autonomous durable writer to Tree Ring proof-loop work.", + "details": "Quality proof must remain explicit, visible, and certification-owned.", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#non-goals", + "quote": "" + }, + "tags": ["proof-loop", "agent-mediated", "no-daemon"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_no_background_writer", + "reason": "proof-loop work must recall the explicit no-background-writer constraint" + } + ], + "thresholds": { + "min_constraint_recall_rate": 1.0, + "max_forbidden_recall_rate": 0.0, + "min_spam_rejection_rate": 1.0, + "min_evidence_required_rate": 1.0, + "min_behavior_proof_pass_rate": 1.0 + } +} +``` + +- [ ] **Step 2: Add the real workflow spam-prevention scenario** + +Create `fixtures/quality/transient-planning-spam.json`: + +```json +{ + "name": "transient-planning-spam", + "category": "spam_prevention", + "query": "memory spam transient planning durable heartwood", + "seed_memories": [ + { + "id": "mem_quality_store_only_durable", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Store only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#agent-guidance-gates", + "quote": "" + }, + "tags": ["write-gate", "memory-quality"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_store_only_durable", + "reason": "the write gate should be recalled before evaluating candidate memories" + } + ], + "write_candidates": [ + { + "id": "mem_candidate_planning_chatter", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "We discussed maybe doing the quality proof loop next.", + "details": "Transient planning chatter should not become durable heartwood.", + "source": { + "type": "manual", + "ref": "", + "quote": "" + }, + "tags": ["planning-chatter", "transient"], + "salience": 0.2, + "confidence": 0.4, + "sensitivity": "normal", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_write_decisions": [ + { + "memory_id": "mem_candidate_planning_chatter", + "decision": "reject", + "reason": "transient planning chatter is memory spam" + } + ] +} +``` + +- [ ] **Step 3: Add the real workflow stale CLI-contract scenario** + +Create `fixtures/quality/stale-cli-contract.json`: + +```json +{ + "name": "stale-cli-contract", + "category": "stale_truth_suppression", + "query": "tree ring helper script cli contract remember recall forget", + "seed_memories": [ + { + "id": "mem_stale_cli_contract", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Old helper scripts can call remember recall and forget without required arguments.", + "details": "This stale CLI contract must not guide current helper-script reviews.", + "source": { + "type": "manual", + "ref": "old-review-note", + "quote": "" + }, + "tags": ["cli-contract", "stale"], + "salience": 0.8, + "confidence": 0.6, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_current_cli_contract", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_current_cli_contract", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "warning", + "summary": "Review Tree Ring helper scripts against the current CLI contract: remember needs --event-type, recall needs a query, and forget needs a memory id.", + "details": "Current source in crates/tree-ring-memory-cli/src/main.rs remains authoritative.", + "source": { + "type": "file", + "ref": "crates/tree-ring-memory-cli/src/main.rs", + "quote": "" + }, + "tags": ["cli-contract", "current"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_stale_cli_contract"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_current_cli_contract", + "reason": "current CLI contract should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_stale_cli_contract", + "reason": "superseded stale CLI contract must be suppressed" + } + ] +} +``` + +- [ ] **Step 4: Add the sensitive-memory hidden scenario** + +Create `fixtures/quality/sensitive-memory-hidden.json`: + +```json +{ + "name": "sensitive-memory-hidden", + "category": "stale_truth_suppression", + "query": "private diagnosis source metadata", + "seed_memories": [ + { + "id": "mem_sensitive_private_diagnosis", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "outer", + "event_type": "lesson", + "summary": "Private diagnosis source metadata should stay hidden by default.", + "details": "", + "source": { + "type": "manual", + "ref": "private-diagnosis-note", + "quote": "" + }, + "tags": ["privacy"], + "salience": 0.7, + "confidence": 0.8, + "sensitivity": "private", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_sensitive_private_diagnosis", + "reason": "private memory is hidden by default" + } + ] +} +``` + +- [ ] **Step 5: Add the superseded-heartwood scenario** + +Create `fixtures/quality/superseded-heartwood.json`: + +```json +{ + "name": "superseded-heartwood", + "category": "stale_truth_suppression", + "query": "durable rule project bridge command", + "seed_memories": [ + { + "id": "mem_old_bridge_rule", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should happen automatically during init.", + "details": "", + "source": { + "type": "manual", + "ref": "old-bridge-note", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.9, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_new_bridge_rule", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_new_bridge_rule", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should be an explicit integrations link command, not automatic init behavior.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-06-tree-ring-agent-mediated-bridges-design.md#command-surface", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_old_bridge_rule"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_new_bridge_rule", + "reason": "replacement heartwood should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_old_bridge_rule", + "reason": "superseded heartwood should not outrank the replacement" + } + ] +} +``` + +- [ ] **Step 6: Add the scar failure recall scenario** + +Create `fixtures/quality/scar-failure-recall.json`: + +```json +{ + "name": "scar-failure-recall", + "category": "behavior_proof", + "query": "failure stale cache rollback", + "seed_memories": [ + { + "id": "mem_scar_stale_cache", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "scar", + "event_type": "warning", + "summary": "Aggressive caching caused a stale rollback failure; recall this scar before cache-related workflow changes.", + "details": "", + "source": { + "type": "evidence", + "ref": "evals/cache-branch/run-013", + "quote": "" + }, + "tags": ["failure", "cache", "rollback"], + "salience": 0.95, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_scar_stale_cache", + "ring": "scar", + "reason": "failure-like queries should surface scar memory" + } + ] +} +``` + +- [ ] **Step 7: Add a fixture parser test** + +Append this test inside `crates/tree-ring-memory-core/src/quality.rs` tests: + +```rust + #[test] + fn parses_default_quality_fixtures() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join("fixtures") + .join("quality"); + let mut parsed = 0usize; + for entry in std::fs::read_dir(&root).unwrap() { + let path = entry.unwrap().path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let input = std::fs::read_to_string(&path).unwrap(); + parse_quality_scenario(&input) + .unwrap_or_else(|err| panic!("{}: {err}", path.display())); + parsed += 1; + } + + assert_eq!(parsed, 6); + } +``` + +- [ ] **Step 8: Run fixture validation tests** + +Run: + +```bash +cargo test -p tree-ring-memory-core parses_default_quality_fixtures --locked +``` + +Expected: PASS. + +- [ ] **Step 9: Commit Task 3** + +Run: + +```bash +git add crates/tree-ring-memory-core/src/quality.rs fixtures/quality +git commit -m "Add default memory quality fixtures" +``` + +--- + +### Task 4: Add Certification-Only Quality Runner + +**Files:** +- Create: `crates/tree-ring-memory-cli/examples/quality_scenarios.rs` + +**Interfaces:** +- Consumes: `tree_ring_memory_core::parse_quality_scenario` +- Consumes: `tree_ring_memory_core::evaluate_quality_scenario` +- Consumes: `tree_ring_memory_core::summarize_quality_run` +- Consumes: `tree_ring_memory_sqlite::{SQLiteMemoryStore, MemoryRetriever}` +- Produces: `quality-report.json` +- Produces: `quality-summary.md` + +- [ ] **Step 1: Add the failing example smoke test** + +Run: + +```bash +cargo run -p tree-ring-memory-cli --example quality_scenarios -- fixtures/quality target/tree-ring-quality-test +``` + +Expected: FAIL because the `quality_scenarios` example does not exist. + +- [ ] **Step 2: Add the quality runner example** + +Create `crates/tree-ring-memory-cli/examples/quality_scenarios.rs`: + +```rust +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use tree_ring_memory_core::{ + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, + QualityRunReport, QualityScenario, QualityScenarioReport, +}; +use tree_ring_memory_sqlite::{MemoryRetriever, SQLiteMemoryStore}; + +fn main() { + if let Err(err) = run() { + eprintln!("quality scenarios failed: {err}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let mut args = std::env::args_os().skip(1); + let fixture_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| "usage: quality_scenarios ".to_string())?; + let output_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| "usage: quality_scenarios ".to_string())?; + if args.next().is_some() { + return Err("usage: quality_scenarios ".to_string()); + } + + fs::create_dir_all(&output_dir).map_err(|err| err.to_string())?; + let scenarios = load_scenarios(&fixture_dir)?; + let mut reports = Vec::new(); + for scenario in scenarios { + reports.push(run_scenario(&scenario)?); + } + + let run_report = summarize_quality_run(reports); + write_reports(&output_dir, &run_report)?; + if !run_report.quality_pass { + return Err(format!( + "quality run failed: {} scenarios evaluated", + run_report.scenario_count + )); + } + println!( + "quality scenarios passed: {} scenario(s)", + run_report.scenario_count + ); + Ok(()) +} + +fn load_scenarios(fixture_dir: &Path) -> Result, String> { + let mut paths = fs::read_dir(fixture_dir) + .map_err(|err| err.to_string())? + .map(|entry| entry.map(|entry| entry.path()).map_err(|err| err.to_string())) + .collect::, _>>()?; + paths.sort(); + + let mut scenarios = Vec::new(); + for path in paths { + if path.extension() != Some(OsStr::new("json")) { + continue; + } + let input = fs::read_to_string(&path).map_err(|err| err.to_string())?; + let scenario = parse_quality_scenario(&input) + .map_err(|err| format!("{}: {err}", path.display()))?; + scenarios.push(scenario); + } + if scenarios.is_empty() { + return Err(format!("no quality scenario json files in {}", fixture_dir.display())); + } + Ok(scenarios) +} + +fn run_scenario(scenario: &QualityScenario) -> Result { + let root = temporary_root(&scenario.name)?; + let db_path = root.join("memory.sqlite"); + let mut store = SQLiteMemoryStore::open(&db_path).map_err(|err| err.to_string())?; + store + .put_many(&scenario.seed_memories) + .map_err(|err| err.to_string())?; + + let prompt = scenario + .prompt() + .ok_or_else(|| format!("quality scenario {} has no prompt", scenario.name))?; + let recalls = MemoryRetriever::new(&store) + .recall( + prompt, + None, + None, + None, + None, + None, + false, + false, + 8, + true, + ) + .map_err(|err| err.to_string())? + .into_iter() + .map(|result| QualityRecall { + memory: result.memory, + score: result.score, + }) + .collect::>(); + + let report = evaluate_quality_scenario(scenario, &recalls).map_err(|err| err.to_string())?; + fs::remove_dir_all(&root).map_err(|err| err.to_string())?; + Ok(report) +} + +fn temporary_root(name: &str) -> Result { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| err.to_string())? + .as_millis(); + let safe_name = name + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) + .collect::(); + let root = std::env::temp_dir().join(format!( + "tree-ring-quality-{}-{}-{safe_name}", + std::process::id(), + millis + )); + if root.exists() { + fs::remove_dir_all(&root).map_err(|err| err.to_string())?; + } + fs::create_dir_all(&root).map_err(|err| err.to_string())?; + Ok(root) +} + +fn write_reports(output_dir: &Path, report: &QualityRunReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report).map_err(|err| err.to_string())?; + fs::write(output_dir.join("quality-report.json"), json).map_err(|err| err.to_string())?; + fs::write(output_dir.join("quality-summary.md"), markdown_summary(report)) + .map_err(|err| err.to_string())?; + Ok(()) +} + +fn markdown_summary(report: &QualityRunReport) -> String { + let mut lines = vec![ + "# Tree Ring Memory Quality Summary".to_string(), + String::new(), + format!("- quality pass: {}", report.quality_pass), + format!("- scenarios: {}", report.scenario_count), + format!( + "- constraint recall rate: {:.3}", + report.constraint_recall_rate + ), + format!( + "- forbidden recall rate: {:.3}", + report.forbidden_recall_rate + ), + format!("- spam rejection rate: {:.3}", report.spam_rejection_rate), + format!( + "- evidence required rate: {:.3}", + report.evidence_required_rate + ), + format!( + "- behavior proof pass rate: {:.3}", + report.behavior_proof_pass_rate + ), + String::new(), + "## Scenarios".to_string(), + String::new(), + ]; + for scenario in &report.scenarios { + lines.push(format!( + "- `{}` [{}]: pass={}", + scenario.name, scenario.category, scenario.quality_pass + )); + } + lines.push(String::new()); + lines.join("\n") +} +``` + +- [ ] **Step 3: Run the quality runner** + +Run: + +```bash +rm -rf target/tree-ring-quality-test +cargo run -p tree-ring-memory-cli --example quality_scenarios -- fixtures/quality target/tree-ring-quality-test +``` + +Expected: PASS and output contains: + +```text +quality scenarios passed: 6 scenario(s) +``` + +- [ ] **Step 4: Inspect generated artifacts** + +Run: + +```bash +test -f target/tree-ring-quality-test/quality-report.json +test -f target/tree-ring-quality-test/quality-summary.md +grep -F '"quality_pass": true' target/tree-ring-quality-test/quality-report.json +grep -F 'Tree Ring Memory Quality Summary' target/tree-ring-quality-test/quality-summary.md +``` + +Expected: all commands pass. + +- [ ] **Step 5: Commit Task 4** + +Run: + +```bash +git add crates/tree-ring-memory-cli/examples/quality_scenarios.rs +git commit -m "Add memory quality scenario runner" +``` + +--- + +### Task 5: Include Quality Metrics In Certification + +**Files:** +- Modify: `scripts/certify-tree-ring.sh` + +**Interfaces:** +- Consumes: `fixtures/quality/*.json` +- Consumes: `cargo run --release -p tree-ring-memory-cli --example quality_scenarios` +- Produces: `target/tree-ring-certification/quality/quality-report.json` +- Produces: `target/tree-ring-certification/quality/quality-summary.md` +- Produces: top-level `metrics.json` field `quality` + +- [ ] **Step 1: Add the failing certification expectation** + +Run: + +```bash +rm -rf target/tree-ring-certification/quality +sh scripts/certify-tree-ring.sh +test -f target/tree-ring-certification/quality/quality-report.json +``` + +Expected: FAIL at the `test -f` command because certification does not run quality scenarios yet. + +- [ ] **Step 2: Run the quality example from certification** + +In `scripts/certify-tree-ring.sh`, add this block after the performance metric extraction block and before Agent Zero handling: + +```sh +quality_out="$OUT_DIR/quality" +mkdir -p "$quality_out" +run cargo run --release -p tree-ring-memory-cli --example quality_scenarios -- "$ROOT/fixtures/quality" "$quality_out" \ + > "$OUT_DIR/quality-run.out" 2>&1 +require_file "$quality_out/quality-report.json" +require_file "$quality_out/quality-summary.md" +grep -F '"quality_pass": true' "$quality_out/quality-report.json" > /dev/null \ + || fail "memory quality scenarios did not pass" +quality_json=$(cat "$quality_out/quality-report.json") +``` + +- [ ] **Step 3: Add quality JSON to metrics** + +In the `cat > "$METRICS" < u16 { + match color { + Color::Rgb(red, green, blue) => red as u16 + green as u16 + blue as u16, + _ => 0, + } + } + + fn stats_with_fullness(ring: &str, total: usize, fullness_level: f64) -> RingStats { + let mut stats = RingStats::empty(ring); + stats.total = total; + stats.fullness_level = fullness_level; + stats + } + + #[test] + fn fullness_brightens_ambient_ring_without_pulse() { + let low = stats_with_fullness("cambium", 1, 0.10); + let high = stats_with_fullness("cambium", 8, 0.80); + + let low_color = animated_color("cambium", Some(&low), 10, 0); + let high_color = animated_color("cambium", Some(&high), 10, 0); + + assert!(brightness_sum(high_color) > brightness_sum(low_color)); + } +``` + +- [ ] **Step 6: Run focused ambient style test and verify it fails** + +Run: + +```bash +cargo test -p tree-ring-memory-cli fullness_brightens_ambient_ring_without_pulse --locked +``` + +Expected: FAIL because fullness does not affect `animated_color`. + +- [ ] **Step 7: Map fullness to ambient intensity** + +Modify `animated_color` in `crates/tree-ring-memory-cli/src/tui/rings.rs` to this full function: + +```rust +fn animated_color(ring: &str, stats: Option<&RingStats>, tick: u64, offset: u64) -> Color { + let warning_level = stats.map(|stats| stats.warning_level).unwrap_or_default(); + let base = theme::ring_color(ring, warning_level); + let Some(stats) = stats else { + return dim_color(base, 0.36); + }; + if stats.total == 0 { + return dim_color(base, 0.34); + } + let fullness = stats.fullness_level.clamp(0.0, 1.0); + if stats.pulse_level > 0.05 && (tick + offset) % 6 < 3 { + return brighten_color(base, 0.28 + fullness * 0.16 + stats.pulse_level * 0.18); + } + if (tick + offset) % 18 < 3 { + return brighten_color(base, 0.08 + fullness * 0.16); + } + if fullness < 0.18 { + return dim_color(base, 0.42 + fullness * 1.35); + } + brighten_color(base, fullness * 0.20) +} +``` + +- [ ] **Step 8: Run TUI focused tests** + +Run: + +```bash +cargo test -p tree-ring-memory-cli fullness --locked +``` + +Expected: PASS. + +- [ ] **Step 9: Commit Task 7** + +Run: + +```bash +git add crates/tree-ring-memory-cli/src/tui/model.rs crates/tree-ring-memory-cli/src/tui/rings.rs +git commit -m "Add ambient ring fullness styling" +``` + +--- + +### Task 8: Final Verification And Documentation Consistency + +**Files:** +- Verify all changed files from Tasks 1-7. + +**Interfaces:** +- Consumes: complete quality proof loop implementation. +- Produces: final verified branch state. + +- [ ] **Step 1: Run formatting** + +Run: + +```bash +cargo fmt --check +``` + +Expected: PASS. + +- [ ] **Step 2: Run locked tests** + +Run: + +```bash +cargo test --locked +``` + +Expected: PASS. + +- [ ] **Step 3: Run Clippy** + +Run: + +```bash +cargo clippy --locked --all-targets +``` + +Expected: PASS with no warnings. + +- [ ] **Step 4: Run certification** + +Run: + +```bash +sh scripts/certify-tree-ring.sh +``` + +Expected: PASS and output includes: + +```text +certification passed +``` + +- [ ] **Step 5: Inspect certification quality artifacts** + +Run: + +```bash +test -f target/tree-ring-certification/quality/quality-report.json +test -f target/tree-ring-certification/quality/quality-summary.md +grep -F '"quality_pass": true' target/tree-ring-certification/quality/quality-report.json +grep -F '"quality"' target/tree-ring-certification/metrics.json +grep -F 'memory quality scenarios: passed' target/tree-ring-certification/summary.md +``` + +Expected: PASS. + +- [ ] **Step 6: Run whitespace check** + +Run: + +```bash +git diff --check +``` + +Expected: PASS with no output. + +- [ ] **Step 7: Review final changed files** + +Run: + +```bash +git status --short +git log --oneline -8 +``` + +Expected: only expected tracked changes are present; pre-existing untracked local files may still appear and must not be staged. + +- [ ] **Step 8: Commit final verification note only if files changed during verification** + +If formatting or documentation consistency changed files, run: + +```bash +git add +git commit -m "Verify memory quality proof loop" +``` + +Expected: create a commit only when verification produced tracked file changes. + +--- + +## Final-Review Hardening Addendum + +This addendum supersedes the historical six-fixture exact count and related +non-null default assumptions in the task evidence above. The final default pack +contains seven fixtures, including `evidence-preservation.json`. Fixture-facing +types reject unknown fields; behavior proof requires an explicit recalled-memory +and decision-change expectation; scenario and run metrics are nullable and +aggregate applicable raw observations; and runner errors preserve completed +scenarios in JSON and markdown with one privacy-safe structured terminal error. + +The final default run must contain seven scenarios, zero errors, numeric values +for all five aggregate metrics, and top-level `"quality_pass": true`. The +certification shell's dependency-free exact line gate remains unchanged. diff --git a/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md new file mode 100644 index 0000000..51b97ef --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md @@ -0,0 +1,335 @@ +# Tree Ring Memory Quality Proof Loop Design + +## Status + +Approved brainstorming direction: build a Memory Quality Proof Loop. + +This spec covers the design only. Implementation planning should follow after +user review. + +## Intent + +Tree Ring Memory already has the right memory lifecycle primitives: scoped +memory events, evidence records, DOX and Revolve adapters, audit, +consolidation, import/export, TUI views, and certification artifacts. + +The next improvement should prove and improve whether Tree Ring actually helps +AI agents with long, dynamic, complex workflows. The product should not merely +store more memory. It should show that agents recall the right constraints, +avoid low-value writes, suppress stale or weak memory, and make better +decisions because memory was present. + +## Goals + +- Reduce missed constraints in complex agent workflows. +- Reduce memory spam from transient planning chatter, duplicate lessons, and + low-value observations. +- Reduce stale-truth risk when old, superseded, sensitive, or weak memories + compete with current source files, tests, user instructions, DOX contracts, + or Revolve evidence. +- Add repeatable proof scenarios that show memory changed the expected agent + decision path. +- Feed proven recall and write gates back into generated agent guidance. +- Keep the first proof surface CLI/CI-friendly before adding richer TUI + quality views. +- Specify ambient ring fullness now so the visual surface can later reflect + real quality and distribution signals. + +## Non-Goals + +- Do not add a daemon, sidecar, hosted service, telemetry pipeline, or hidden + recorder. +- Do not scrape transcripts or turn event-stream pulses into durable memory. +- Do not add autonomous durable writes outside explicit user, agent, adapter, + import, TUI, consolidation, or maintenance actions. +- Do not make Tree Ring memory more authoritative than source files, tests, + explicit user instructions, root agent contracts, DOX contracts, or Revolve + evidence. +- Do not replace the existing SQLite store, JSONL import/export shape, recall + model, or certification workflow. +- Do not implement the full TUI quality cockpit in the first proof-loop pass. + +## Selected Approach + +Use an eval and certification-first proof loop, with guidance updates in the +same lane and TUI visualization as the downstream surface. + +Considered approaches: + +1. Guidance-only hardening + - Tighten generated skills and bridges so agents recall before risky work, + avoid low-value writes, and treat stale memory carefully. + - Fast and useful, but weak as proof that behavior improved. + +2. Eval and certification-first proof loop + - Add repeatable workflow scenarios that test missed constraints, memory + spam, stale recall, and behavior improvement. + - Strongest credibility because Tree Ring can claim improvement from + measurable scenarios rather than product copy. + +3. Operator-first TUI quality cockpit + - Surface fullness, stale-risk, low-confidence, and spam signals visually. + - Valuable for human inspection, but it should follow trustworthy metrics + instead of inventing meaning visually. + +Recommended path: implement approach 2 first, take the practical generated +guidance pieces from approach 1, and make approach 3 consume the resulting +quality metrics. + +## Architecture + +Add a quality layer around existing Tree Ring primitives. The quality layer +consumes memory events, recall results, and proposed write candidates. It does +not own storage and does not need a background process. + +Final hardening semantics for this lane: + +- Every category must carry a primary observation contract during validation. + `constraint_recall` needs at least one `expected_recall`, + `spam_prevention` needs at least one expected `reject` decision, + `stale_truth_suppression` needs at least one `forbidden_recall`, + `behavior_proof` needs `behavior_expectation`, and + `evidence_preservation` needs at least one `evaluation_` write candidate. +- Threshold configuration is presence-aware. Threshold fields are optional in + fixtures and default to strict behavior only when the corresponding metric + has observations. An explicitly configured threshold for a metric with no + observations is invalid and must fail scenario validation. +- Runner failures must emit stable stage plus error-class output in JSON, + markdown, and terminal failure paths. Fixture values, invalid memory field + values, and other raw payload fragments must not be echoed back in failure + messages. + +Target components: + +- `fixtures/quality/`: reviewable quality scenario fixtures. +- Quality scenario parser: validates scenario files and normalizes expected + recall and write rules. +- Quality evaluator: runs deterministic checks over recall results and proposed + memory writes. +- Certification integration: runs the default quality pack and writes summary + artifacts beside current certification output. +- Generated guidance updates: teaches agents concrete recall, trust, and write + gates that are backed by scenarios. +- TUI quality contract: defines how future visual views consume quality and + fullness signals. + +The existing SQLite store remains the durable memory backend. The existing +recall path remains the recall engine under test. The quality loop should test +current behavior before changing behavior. + +## Scenario Model + +Quality scenarios should be readable, deterministic, and code-reviewable. +JSON is the preferred first fixture format because the repo already uses JSON +schemas and fixture files for memory protocol coverage. + +Each scenario should include: + +- `name`: stable scenario id. +- `category`: `constraint_recall`, `spam_prevention`, + `stale_truth_suppression`, `behavior_proof`, or `evidence_preservation`. +- `seed_memories`: memory events loaded into a temporary store. +- `query` or `workflow_prompt`: simulated agent task context. +- `expected_recall`: memory ids, rings, tags, or source refs that should appear. +- `forbidden_recall`: stale, superseded, sensitive, or weak memories that must + not appear by default. +- `write_candidates`: optional proposed memories to evaluate before storage. +- `expected_write_decisions`: `accept`, `reject`, `require_evidence`, or + `require_user_confirmation`. +- `evidence_refs`: source refs required for accepted outcome memories. +- `behavior_expectation`: required recalled memory id, baseline decision, + memory-informed decision, expected decision, and an optional reason. It is + required for `behavior_proof` scenarios. +- `thresholds`: per-scenario minimums or tolerances. + Threshold entries are optional and only legal for metrics the scenario + actually observes. + +First scenario pack: + +- Real Tree Ring/Codex workflow: recall the no-background-writer constraint + before proof-loop work. +- Real Tree Ring/Codex workflow: avoid storing transient planning chatter as + durable heartwood. +- Real Tree Ring/Codex workflow: suppress stale PR or CLI-contract memory when + current source has changed. +- Synthetic edge case: sensitive memory is blocked or hidden by default. +- Synthetic edge case: superseded heartwood does not outrank its replacement. +- Synthetic edge case: failed approach surfaces as a scar for a failure-like + query and changes a baseline retry into the expected rollback-safe decision. +- Synthetic edge case: evaluated outcomes without evidence are held while an + outcome preserving the required evidence ref is accepted. + +## Data Flow + +Quality runs should be explicit and reproducible: + +1. Load a scenario fixture into a temporary `.tree-ring` root. +2. Run existing recall behavior against the scenario query or workflow prompt. +3. Evaluate returned memories against expected and forbidden recall rules. +4. Evaluate optional write candidates before storage. +5. Classify write candidates as useful, spam, stale, evidence-required, or + user-confirmation-required. +6. Emit `quality-report.json` plus a short markdown summary. +7. Include the quality summary in certification output. +8. Feed durable lessons into generated agent guidance and later TUI metrics. + +The quality loop should not mutate the user's real memory root. Scenario runs +use temporary stores and fixture-owned memory events. + +## Agent Guidance Gates + +Generated Tree Ring guidance should make agents do the right thing at the +moments where memory matters. + +Recall gates: + +- Before substantial project work, recall project constraints, scars, user + preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict + already-stored lessons. + +Trust gates: + +- Prefer source-linked, non-superseded, high-confidence memories. +- Treat heartwood as durable only when source evidence or user confirmation + supports it. +- Re-read source files, tests, user instructions, DOX contracts, or Revolve + evidence when memory conflicts with current sources. +- Do not treat sensitive or hidden-by-default memory as ordinary recall context. + +Write gates: + +- Remember only durable decisions, validated lessons, reusable warnings, + corrections, future seeds, and evidence-backed outcomes. +- Reject transient planning chatter, duplicate wording, tool noise, and + unsupported claims as durable memory. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project + heartwood. + +## Metrics + +The quality report should include: + +- `constraint_recall_rate`: required constraints recalled before the workflow. +- `forbidden_recall_rate`: stale, sensitive, superseded, or weak memories + returned when they should not be. +- `spam_rejection_rate`: low-value write candidates rejected. +- `evidence_required_rate`: evaluated candidates correctly held when evidence + is missing or accepted when the required evidence ref is preserved. +- `behavior_proof_pass_rate`: scenarios where memory changed the expected + decision path. +- `quality_pass`: boolean certification gate. + +Rates are observation-weighted across applicable expectations, not averaged +from per-scenario defaults. A scenario or run emits JSON `null` for a dimension +with no observations. Constraint and forbidden recall use their respective +expectation counts; spam uses expected `reject` decisions; evidence uses every +`evaluation_` write candidate; behavior uses scenarios with an explicit +behavior expectation. The default seven-scenario certification pack observes +all five dimensions, so its aggregate metrics remain numeric. + +`quality_pass` should fail when required constraints are missed, forbidden +recall exceeds tolerance, spam candidates are accepted, or required evidence +refs are lost. Scenario thresholds apply only to dimensions with observations, +every write-decision report must match, and an empty run cannot pass. + +## TUI Contract + +The TUI quality cockpit is downstream of CLI/CI quality proof. The first spec +still defines the visual contract so implementation work does not drift. + +Ambient ring fullness uses a hybrid model: + +- First pass: fullness is relative share of memories in the current store. +- Later pass: per-ring thresholds are calibrated from real usage and + certification data. +- Empty rings stay dull or dim. +- Low/full rings are light but not visually dominant. +- High/full rings become brighter and more legible. +- Warning-heavy rings can override the palette, but fullness and warning are + separate signals. +- Pulse remains activity, not fullness. + +Current TUI internals already expose `RingStats.total`, `pulse_level`, warning +state, and per-cell brightness. Fullness should become part of the style +contract that maps ring distribution to intensity. It should not replace pulse +or warning behavior. + +## Error Handling + +- Fixture-facing objects reject unknown fields. Parse and validation failures + should report the fixture path plus a stable error class, without extracting + a scenario name or echoing invalid fields and other raw fixture values. +- Missing expected recall should produce a precise failed expectation. +- Forbidden recall should include the returned memory id and reason. +- Write-decision mismatches should show the candidate id or summary and the + expected decision. +- Certification should preserve completed scenarios in partial JSON and + markdown artifacts on failure. Reports include one terminal structured error + with scenario and path when known, a stable stage, and a privacy-safe message. +- Markdown renders null rates as `n/a` and adds an Errors section when errors + are present. +- Scenario runs should clean up temporary stores on success and avoid touching + real user memory roots. + +## Testing + +Add tests in layers: + +- Unit tests for scenario parsing and validation. +- Unit tests for deterministic recall expectation evaluation. +- Unit tests for write-candidate decision evaluation. +- Integration tests that load fixture stores and run recall checks. +- Certification tests that verify the default quality pack emits JSON and + markdown artifacts. +- Generated guidance tests that check recall, trust, and write gates appear in + `.tree-ring` awareness files and skill docs. +- Later TUI tests that verify fullness changes style intensity without changing + layout or confusing pulse state. + +## Acceptance Criteria + +1. A default quality scenario pack exists under `fixtures/quality/`. +2. Quality runs use temporary memory stores and do not mutate real user roots. +3. Required recall expectations and forbidden recall expectations are + deterministic. +4. Write candidates are classified as accept, reject, require evidence, or + require user confirmation. +5. The default quality run emits `quality-report.json` and a markdown summary. +6. Certification includes quality metrics and fails on quality regressions. +7. Generated guidance includes explicit recall, trust, and write gates. +8. TUI fullness semantics are documented as hybrid relative-share first, + calibrated thresholds later. +9. The seven-scenario default pack includes evidence preservation and produces + numeric passing aggregate values for every quality metric. +10. Runner failures preserve structured partial artifacts without embedding + fixture contents or memory payloads in error messages. + +## Final-Review Hardening + +The final review replaced inferred behavior proof with an explicit decision +change contract, made non-applicable metrics nullable and observation-weighted, +added strict fixture fields and the `evidence_preservation` category, and made +runner failures artifact-producing. This section and the seven-scenario pack +supersede the original six-fixture count while preserving the certification +gate's exact top-level `"quality_pass": true` JSON line. + +## Rollout + +Phase 1: Add fixture format, evaluator, default quality pack, reports, and +certification integration. + +Phase 2: Update generated guidance and skill docs so agents use the proven +recall, trust, and write gates. + +Phase 3: Add TUI quality and fullness visualization after CLI/CI metrics are +stable. + +## Open Follow-Up + +After the first quality pack is running, choose whether the stable surface +should be a dedicated `tree-ring eval` subcommand or remain certification-only +until the fixture format settles. diff --git a/fixtures/quality/evidence-preservation.json b/fixtures/quality/evidence-preservation.json new file mode 100644 index 0000000..4f2dca7 --- /dev/null +++ b/fixtures/quality/evidence-preservation.json @@ -0,0 +1,84 @@ +{ + "name": "evidence-preservation", + "category": "evidence_preservation", + "query": "preserve evidence for evaluated outcomes", + "write_candidates": [ + { + "id": "mem_evaluation_missing_source", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "evaluation_outcome", + "summary": "An evaluated outcome without its source reference.", + "details": "The quality gate must require evidence before accepting it.", + "source": { + "type": "evidence", + "ref": "", + "quote": "" + }, + "tags": ["evaluation", "evidence"], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_evaluation_preserved_source", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "evaluation_outcome", + "summary": "An evaluated outcome that preserves its required source reference.", + "details": "The matching evidence reference makes this candidate acceptable.", + "source": { + "type": "evidence", + "ref": "evals/evidence/run-021", + "quote": "" + }, + "tags": ["evaluation", "evidence"], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_write_decisions": [ + { + "memory_id": "mem_evaluation_missing_source", + "decision": "require_evidence", + "reason": "evaluated outcomes without a source ref require evidence" + }, + { + "memory_id": "mem_evaluation_preserved_source", + "decision": "accept", + "reason": "the evaluated outcome preserves the required evidence ref" + } + ], + "evidence_refs": ["evals/evidence/run-021"] +} diff --git a/fixtures/quality/no-background-writer-constraint.json b/fixtures/quality/no-background-writer-constraint.json new file mode 100644 index 0000000..c5489f5 --- /dev/null +++ b/fixtures/quality/no-background-writer-constraint.json @@ -0,0 +1,45 @@ +{ + "name": "no-background-writer-constraint", + "category": "constraint_recall", + "query": "proof loop background writer hidden durable writes", + "seed_memories": [ + { + "id": "mem_quality_no_background_writer", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background proof runner, hidden recorder, or autonomous durable writer to Tree Ring proof-loop work.", + "details": "Quality proof must remain explicit, visible, and certification-owned.", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#non-goals", + "quote": "" + }, + "tags": ["proof-loop", "agent-mediated", "no-daemon"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_no_background_writer", + "reason": "proof-loop work must recall the explicit no-background-writer constraint" + } + ] +} diff --git a/fixtures/quality/scar-failure-recall.json b/fixtures/quality/scar-failure-recall.json new file mode 100644 index 0000000..788971d --- /dev/null +++ b/fixtures/quality/scar-failure-recall.json @@ -0,0 +1,53 @@ +{ + "name": "scar-failure-recall", + "category": "behavior_proof", + "query": "failure stale cache rollback", + "seed_memories": [ + { + "id": "mem_scar_stale_cache", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "scar", + "event_type": "warning", + "summary": "Aggressive caching caused a stale rollback failure; recall this scar before cache-related workflow changes.", + "details": "", + "source": { + "type": "evidence", + "ref": "evals/cache-branch/run-013", + "quote": "" + }, + "tags": ["failure", "cache", "rollback"], + "salience": 0.95, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_scar_stale_cache", + "ring": "scar", + "reason": "failure-like queries should surface scar memory" + } + ], + "behavior_expectation": { + "required_memory_id": "mem_scar_stale_cache", + "baseline_decision": "retry the cache migration unchanged", + "memory_informed_decision": "roll back the cache migration and inspect stale state before retrying", + "expected_decision": "roll back the cache migration and inspect stale state before retrying", + "reason": "the recalled failure scar changes the workflow to rollback-safe recovery" + } +} diff --git a/fixtures/quality/sensitive-memory-hidden.json b/fixtures/quality/sensitive-memory-hidden.json new file mode 100644 index 0000000..1dbeab4 --- /dev/null +++ b/fixtures/quality/sensitive-memory-hidden.json @@ -0,0 +1,45 @@ +{ + "name": "sensitive-memory-hidden", + "category": "stale_truth_suppression", + "query": "private diagnosis source metadata", + "seed_memories": [ + { + "id": "mem_sensitive_private_diagnosis", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "outer", + "event_type": "lesson", + "summary": "Private diagnosis source metadata should stay hidden by default.", + "details": "", + "source": { + "type": "manual", + "ref": "private-diagnosis-note", + "quote": "" + }, + "tags": ["privacy"], + "salience": 0.7, + "confidence": 0.8, + "sensitivity": "private", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_sensitive_private_diagnosis", + "reason": "private memory is hidden by default" + } + ] +} diff --git a/fixtures/quality/stale-cli-contract.json b/fixtures/quality/stale-cli-contract.json new file mode 100644 index 0000000..8472d27 --- /dev/null +++ b/fixtures/quality/stale-cli-contract.json @@ -0,0 +1,83 @@ +{ + "name": "stale-cli-contract", + "category": "stale_truth_suppression", + "query": "tree ring helper script cli contract remember recall forget", + "seed_memories": [ + { + "id": "mem_stale_cli_contract", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Old helper scripts can call remember recall and forget without required arguments.", + "details": "This stale CLI contract must not guide current helper-script reviews.", + "source": { + "type": "manual", + "ref": "old-review-note", + "quote": "" + }, + "tags": ["cli-contract", "stale"], + "salience": 0.8, + "confidence": 0.6, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_current_cli_contract", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_current_cli_contract", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "warning", + "summary": "Review Tree Ring helper scripts against the current CLI contract: remember needs --event-type, recall needs a query, and forget needs a memory id.", + "details": "Current source in crates/tree-ring-memory-cli/src/main.rs remains authoritative.", + "source": { + "type": "file", + "ref": "crates/tree-ring-memory-cli/src/main.rs", + "quote": "" + }, + "tags": ["cli-contract", "current"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_stale_cli_contract"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_current_cli_contract", + "reason": "current CLI contract should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_stale_cli_contract", + "reason": "superseded stale CLI contract must be suppressed" + } + ] +} diff --git a/fixtures/quality/superseded-heartwood.json b/fixtures/quality/superseded-heartwood.json new file mode 100644 index 0000000..f6a1259 --- /dev/null +++ b/fixtures/quality/superseded-heartwood.json @@ -0,0 +1,83 @@ +{ + "name": "superseded-heartwood", + "category": "stale_truth_suppression", + "query": "durable rule project bridge command", + "seed_memories": [ + { + "id": "mem_old_bridge_rule", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should happen automatically during init.", + "details": "", + "source": { + "type": "manual", + "ref": "old-bridge-note", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.9, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_new_bridge_rule", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_new_bridge_rule", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should be an explicit integrations link command, not automatic init behavior.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-06-tree-ring-agent-mediated-bridges-design.md#command-surface", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_old_bridge_rule"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_new_bridge_rule", + "reason": "replacement heartwood should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_old_bridge_rule", + "reason": "superseded heartwood should not outrank the replacement" + } + ] +} diff --git a/fixtures/quality/transient-planning-spam.json b/fixtures/quality/transient-planning-spam.json new file mode 100644 index 0000000..a2182ab --- /dev/null +++ b/fixtures/quality/transient-planning-spam.json @@ -0,0 +1,86 @@ +{ + "name": "transient-planning-spam", + "category": "spam_prevention", + "query": "memory spam transient planning durable heartwood", + "seed_memories": [ + { + "id": "mem_quality_store_only_durable", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Store only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#agent-guidance-gates", + "quote": "" + }, + "tags": ["write-gate", "memory-quality"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_store_only_durable", + "reason": "the write gate should be recalled before evaluating candidate memories" + } + ], + "write_candidates": [ + { + "id": "mem_candidate_planning_chatter", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "We discussed maybe doing the quality proof loop next.", + "details": "Transient planning chatter should not become durable heartwood.", + "source": { + "type": "manual", + "ref": "", + "quote": "" + }, + "tags": ["planning-chatter", "transient"], + "salience": 0.2, + "confidence": 0.4, + "sensitivity": "normal", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_write_decisions": [ + { + "memory_id": "mem_candidate_planning_chatter", + "decision": "reject", + "reason": "transient planning chatter is memory spam" + } + ] +} diff --git a/scripts/certify-tree-ring.sh b/scripts/certify-tree-ring.sh index 2527263..4b28f4c 100755 --- a/scripts/certify-tree-ring.sh +++ b/scripts/certify-tree-ring.sh @@ -86,6 +86,10 @@ mkdir -p "$OUT_DIR" SUMMARY="$OUT_DIR/summary.md" METRICS="$OUT_DIR/metrics.json" LOG="$OUT_DIR/certification.log" +QUALITY_OUT="$OUT_DIR/quality" +QUALITY_RUN_OUT="$OUT_DIR/quality-run.out" +rm -f "$SUMMARY" "$METRICS" "$QUALITY_RUN_OUT" +rm -rf "$QUALITY_OUT" : > "$LOG" log "certification output: $OUT_DIR" | tee -a "$LOG" @@ -200,6 +204,15 @@ if [ "$EXTENDED" = "1" ]; then [ -n "$perf_50k_json" ] || fail "missing 50k performance metrics" fi +mkdir -p "$QUALITY_OUT" +run cargo run --release -p tree-ring-memory-cli --example quality_scenarios -- "$ROOT/fixtures/quality" "$QUALITY_OUT" \ + > "$QUALITY_RUN_OUT" 2>&1 +require_file "$QUALITY_OUT/quality-report.json" +require_file "$QUALITY_OUT/quality-summary.md" +# The runner re-parses this persisted report and exits nonzero unless its +# top-level status matches the in-memory passing report. +quality_json=$(cat "$QUALITY_OUT/quality-report.json") + agent_zero_status='"skipped"' agent_zero_note='"TREE_RING_AGENT_ZERO_ROOT not set"' if [ -n "$AGENT_ZERO_ROOT" ]; then @@ -247,6 +260,7 @@ cat > "$METRICS" <