From c9fef36b4634bec7dc039ab29501c0291d17a5f7 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 13:26:55 -0400 Subject: [PATCH 01/19] Design memory quality proof loop --- ...e-ring-memory-quality-proof-loop-design.md | 284 ++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md diff --git a/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md new file mode 100644 index 0000000..c6ae687 --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md @@ -0,0 +1,284 @@ +# Tree Ring Memory Quality Proof Loop Design + +## Status + +Approved brainstorming direction: build a Memory Quality Proof Loop. + +This spec covers the design only. Implementation planning should follow after +user review. + +## Intent + +Tree Ring Memory already has the right memory lifecycle primitives: scoped +memory events, evidence records, DOX and Revolve adapters, audit, +consolidation, import/export, TUI views, and certification artifacts. + +The next improvement should prove and improve whether Tree Ring actually helps +AI agents with long, dynamic, complex workflows. The product should not merely +store more memory. It should show that agents recall the right constraints, +avoid low-value writes, suppress stale or weak memory, and make better +decisions because memory was present. + +## Goals + +- Reduce missed constraints in complex agent workflows. +- Reduce memory spam from transient planning chatter, duplicate lessons, and + low-value observations. +- Reduce stale-truth risk when old, superseded, sensitive, or weak memories + compete with current source files, tests, user instructions, DOX contracts, + or Revolve evidence. +- Add repeatable proof scenarios that show memory changed the expected agent + decision path. +- Feed proven recall and write gates back into generated agent guidance. +- Keep the first proof surface CLI/CI-friendly before adding richer TUI + quality views. +- Specify ambient ring fullness now so the visual surface can later reflect + real quality and distribution signals. + +## Non-Goals + +- Do not add a daemon, sidecar, hosted service, telemetry pipeline, or hidden + recorder. +- Do not scrape transcripts or turn event-stream pulses into durable memory. +- Do not add autonomous durable writes outside explicit user, agent, adapter, + import, TUI, consolidation, or maintenance actions. +- Do not make Tree Ring memory more authoritative than source files, tests, + explicit user instructions, root agent contracts, DOX contracts, or Revolve + evidence. +- Do not replace the existing SQLite store, JSONL import/export shape, recall + model, or certification workflow. +- Do not implement the full TUI quality cockpit in the first proof-loop pass. + +## Selected Approach + +Use an eval and certification-first proof loop, with guidance updates in the +same lane and TUI visualization as the downstream surface. + +Considered approaches: + +1. Guidance-only hardening + - Tighten generated skills and bridges so agents recall before risky work, + avoid low-value writes, and treat stale memory carefully. + - Fast and useful, but weak as proof that behavior improved. + +2. Eval and certification-first proof loop + - Add repeatable workflow scenarios that test missed constraints, memory + spam, stale recall, and behavior improvement. + - Strongest credibility because Tree Ring can claim improvement from + measurable scenarios rather than product copy. + +3. Operator-first TUI quality cockpit + - Surface fullness, stale-risk, low-confidence, and spam signals visually. + - Valuable for human inspection, but it should follow trustworthy metrics + instead of inventing meaning visually. + +Recommended path: implement approach 2 first, take the practical generated +guidance pieces from approach 1, and make approach 3 consume the resulting +quality metrics. + +## Architecture + +Add a quality layer around existing Tree Ring primitives. The quality layer +consumes memory events, recall results, and proposed write candidates. It does +not own storage and does not need a background process. + +Target components: + +- `fixtures/quality/`: reviewable quality scenario fixtures. +- Quality scenario parser: validates scenario files and normalizes expected + recall and write rules. +- Quality evaluator: runs deterministic checks over recall results and proposed + memory writes. +- Certification integration: runs the default quality pack and writes summary + artifacts beside current certification output. +- Generated guidance updates: teaches agents concrete recall, trust, and write + gates that are backed by scenarios. +- TUI quality contract: defines how future visual views consume quality and + fullness signals. + +The existing SQLite store remains the durable memory backend. The existing +recall path remains the recall engine under test. The quality loop should test +current behavior before changing behavior. + +## Scenario Model + +Quality scenarios should be readable, deterministic, and code-reviewable. +JSON is the preferred first fixture format because the repo already uses JSON +schemas and fixture files for memory protocol coverage. + +Each scenario should include: + +- `name`: stable scenario id. +- `category`: `constraint_recall`, `spam_prevention`, + `stale_truth_suppression`, or `behavior_proof`. +- `seed_memories`: memory events loaded into a temporary store. +- `query` or `workflow_prompt`: simulated agent task context. +- `expected_recall`: memory ids, rings, tags, or source refs that should appear. +- `forbidden_recall`: stale, superseded, sensitive, or weak memories that must + not appear by default. +- `write_candidates`: optional proposed memories to evaluate before storage. +- `expected_write_decisions`: `accept`, `reject`, `require_evidence`, or + `require_user_confirmation`. +- `evidence_refs`: source refs required for accepted outcome memories. +- `thresholds`: per-scenario minimums or tolerances. + +First scenario pack: + +- Real Tree Ring/Codex workflow: recall the no-background-writer constraint + before proof-loop work. +- Real Tree Ring/Codex workflow: avoid storing transient planning chatter as + durable heartwood. +- Real Tree Ring/Codex workflow: suppress stale PR or CLI-contract memory when + current source has changed. +- Synthetic edge case: sensitive memory is blocked or hidden by default. +- Synthetic edge case: superseded heartwood does not outrank its replacement. +- Synthetic edge case: failed approach surfaces as a scar for a failure-like + query. + +## Data Flow + +Quality runs should be explicit and reproducible: + +1. Load a scenario fixture into a temporary `.tree-ring` root. +2. Run existing recall behavior against the scenario query or workflow prompt. +3. Evaluate returned memories against expected and forbidden recall rules. +4. Evaluate optional write candidates before storage. +5. Classify write candidates as useful, spam, stale, evidence-required, or + user-confirmation-required. +6. Emit `quality-report.json` plus a short markdown summary. +7. Include the quality summary in certification output. +8. Feed durable lessons into generated agent guidance and later TUI metrics. + +The quality loop should not mutate the user's real memory root. Scenario runs +use temporary stores and fixture-owned memory events. + +## Agent Guidance Gates + +Generated Tree Ring guidance should make agents do the right thing at the +moments where memory matters. + +Recall gates: + +- Before substantial project work, recall project constraints, scars, user + preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict + already-stored lessons. + +Trust gates: + +- Prefer source-linked, non-superseded, high-confidence memories. +- Treat heartwood as durable only when source evidence or user confirmation + supports it. +- Re-read source files, tests, user instructions, DOX contracts, or Revolve + evidence when memory conflicts with current sources. +- Do not treat sensitive or hidden-by-default memory as ordinary recall context. + +Write gates: + +- Remember only durable decisions, validated lessons, reusable warnings, + corrections, future seeds, and evidence-backed outcomes. +- Reject transient planning chatter, duplicate wording, tool noise, and + unsupported claims as durable memory. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project + heartwood. + +## Metrics + +The quality report should include: + +- `constraint_recall_rate`: required constraints recalled before the workflow. +- `forbidden_recall_rate`: stale, sensitive, superseded, or weak memories + returned when they should not be. +- `spam_rejection_rate`: low-value write candidates rejected. +- `evidence_required_rate`: accepted outcome memories that preserve required + evidence refs. +- `behavior_proof_pass_rate`: scenarios where memory changed the expected + decision path. +- `quality_pass`: boolean certification gate. + +`quality_pass` should fail when required constraints are missed, forbidden +recall exceeds tolerance, spam candidates are accepted, or required evidence +refs are lost. + +## TUI Contract + +The TUI quality cockpit is downstream of CLI/CI quality proof. The first spec +still defines the visual contract so implementation work does not drift. + +Ambient ring fullness uses a hybrid model: + +- First pass: fullness is relative share of memories in the current store. +- Later pass: per-ring thresholds are calibrated from real usage and + certification data. +- Empty rings stay dull or dim. +- Low/full rings are light but not visually dominant. +- High/full rings become brighter and more legible. +- Warning-heavy rings can override the palette, but fullness and warning are + separate signals. +- Pulse remains activity, not fullness. + +Current TUI internals already expose `RingStats.total`, `pulse_level`, warning +state, and per-cell brightness. Fullness should become part of the style +contract that maps ring distribution to intensity. It should not replace pulse +or warning behavior. + +## Error Handling + +- Invalid scenario fixtures should report the scenario name, file path, and + invalid field. +- Missing expected recall should produce a precise failed expectation. +- Forbidden recall should include the returned memory id and reason. +- Write-decision mismatches should show the candidate id or summary and the + expected decision. +- Certification should preserve partial quality artifacts on failure. +- Scenario runs should clean up temporary stores on success and avoid touching + real user memory roots. + +## Testing + +Add tests in layers: + +- Unit tests for scenario parsing and validation. +- Unit tests for deterministic recall expectation evaluation. +- Unit tests for write-candidate decision evaluation. +- Integration tests that load fixture stores and run recall checks. +- Certification tests that verify the default quality pack emits JSON and + markdown artifacts. +- Generated guidance tests that check recall, trust, and write gates appear in + `.tree-ring` awareness files and skill docs. +- Later TUI tests that verify fullness changes style intensity without changing + layout or confusing pulse state. + +## Acceptance Criteria + +1. A default quality scenario pack exists under `fixtures/quality/`. +2. Quality runs use temporary memory stores and do not mutate real user roots. +3. Required recall expectations and forbidden recall expectations are + deterministic. +4. Write candidates are classified as accept, reject, require evidence, or + require user confirmation. +5. The default quality run emits `quality-report.json` and a markdown summary. +6. Certification includes quality metrics and fails on quality regressions. +7. Generated guidance includes explicit recall, trust, and write gates. +8. TUI fullness semantics are documented as hybrid relative-share first, + calibrated thresholds later. + +## Rollout + +Phase 1: Add fixture format, evaluator, default quality pack, reports, and +certification integration. + +Phase 2: Update generated guidance and skill docs so agents use the proven +recall, trust, and write gates. + +Phase 3: Add TUI quality and fullness visualization after CLI/CI metrics are +stable. + +## Open Follow-Up + +After the first quality pack is running, choose whether the stable surface +should be a dedicated `tree-ring eval` subcommand or remain certification-only +until the fixture format settles. From 00f571a21c3c4691de4c1a4ba2fb8065d6f6f1b8 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 13:39:53 -0400 Subject: [PATCH 02/19] Plan memory quality proof loop --- ...-quality-proof-loop-implementation-plan.md | 2195 +++++++++++++++++ 1 file changed, 2195 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md diff --git a/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md new file mode 100644 index 0000000..def0a9d --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md @@ -0,0 +1,2195 @@ +# Tree Ring Memory Quality Proof Loop Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a certification-backed memory quality loop that proves Tree Ring helps agents recall constraints, reject memory spam, suppress stale truth, and preserve evidence in complex workflows. + +**Architecture:** Add a deterministic quality scenario model and evaluator to `tree-ring-memory-core`, then run it through a certification-only CLI example that loads reviewable fixtures into temporary SQLite stores. Feed the resulting gates into generated agent guidance, and add first-pass TUI fullness styling from the current ring distribution without creating a daemon, hidden writer, or public eval subcommand. + +**Tech Stack:** Rust 2021, serde, serde_json, rusqlite-backed `SQLiteMemoryStore`, existing `MemoryRetriever`, Ratatui, shell certification script, JSON fixtures. + +## Global Constraints + +- Do not add a daemon, sidecar, hosted service, telemetry pipeline, or hidden recorder. +- Do not scrape transcripts or turn event-stream pulses into durable memory. +- Do not add autonomous durable writes outside explicit user, agent, adapter, import, TUI, consolidation, or maintenance actions. +- Do not make Tree Ring memory more authoritative than source files, tests, explicit user instructions, root agent contracts, DOX contracts, or Revolve evidence. +- Do not replace the existing SQLite store, JSONL import/export shape, recall model, or certification workflow. +- Do not add a public `tree-ring eval` subcommand in this pass; keep the quality runner certification-owned until the fixture format settles. +- Keep scenario runs isolated in temporary memory roots. +- Keep TUI fullness separate from pulse and warning state. +- Preserve existing untracked local files; stage only files changed by each task. +- Run `cargo fmt --check`, `cargo test --locked`, `cargo clippy --locked --all-targets`, `git diff --check`, and `sh scripts/certify-tree-ring.sh` before final handoff. + +--- + +## Scope Check + +This plan covers one subsystem: a memory quality proof loop and its first consumer surfaces. It produces working, testable software in layers: + +1. Core scenario parsing. +2. Core quality evaluation. +3. Default quality fixtures. +4. Certification-only runner. +5. Certification report integration. +6. Generated guidance gates. +7. TUI fullness styling. + +No task creates a background process, network service, public CLI command, or alternate storage backend. + +## File Structure + +- Create `crates/tree-ring-memory-core/src/quality.rs`: scenario structs, parser, deterministic evaluator, quality run aggregation. +- Modify `crates/tree-ring-memory-core/src/lib.rs`: expose the quality module and public quality types. +- Create `fixtures/quality/no-background-writer-constraint.json`: real workflow constraint recall scenario. +- Create `fixtures/quality/transient-planning-spam.json`: real workflow spam-prevention scenario. +- Create `fixtures/quality/stale-cli-contract.json`: real workflow stale-truth suppression scenario. +- Create `fixtures/quality/sensitive-memory-hidden.json`: synthetic sensitive recall scenario. +- Create `fixtures/quality/superseded-heartwood.json`: synthetic supersession scenario. +- Create `fixtures/quality/scar-failure-recall.json`: synthetic scar recall scenario. +- Create `crates/tree-ring-memory-cli/examples/quality_scenarios.rs`: certification-only runner that loads fixtures, runs recall, evaluates reports, and writes JSON/Markdown artifacts. +- Modify `scripts/certify-tree-ring.sh`: run the quality example and include quality metrics in certification artifacts. +- Modify `crates/tree-ring-memory-cli/src/agent_awareness.rs`: generated AGENTS and CLI guidance gates. +- Modify `skills/tree-ring-memory/SKILL.md`: portable recall, trust, and write gates. +- Modify `docs/integrations/agent-skill.md`: document quality-gate usage for agent harnesses. +- Modify `README.md`: document certification quality artifacts. +- Modify `crates/tree-ring-memory-cli/src/tui/model.rs`: add first-pass relative fullness. +- Modify `crates/tree-ring-memory-cli/src/tui/rings.rs`: map fullness to ambient intensity without changing pulse/warning semantics. + +--- + +### Task 1: Add Quality Scenario Model + +**Files:** +- Create: `crates/tree-ring-memory-core/src/quality.rs` +- Modify: `crates/tree-ring-memory-core/src/lib.rs` + +**Interfaces:** +- Produces: `quality::QualityScenario` +- Produces: `quality::QualityThresholds` +- Produces: `quality::RecallExpectation` +- Produces: `quality::WriteDecisionExpectation` +- Produces: `quality::parse_quality_scenario(input: &str) -> TreeRingResult` +- Consumes: `models::MemoryEvent` +- Consumes: `models::TreeRingError` + +- [ ] **Step 1: Write the failing scenario parser tests** + +Add this complete file at `crates/tree-ring-memory-core/src/quality.rs`: + +```rust +use serde::{Deserialize, Serialize}; + +use crate::models::{MemoryEvent, TreeRingError, TreeRingResult}; + +pub const QUALITY_CATEGORIES: &[&str] = &[ + "constraint_recall", + "spam_prevention", + "stale_truth_suppression", + "behavior_proof", +]; + +pub const WRITE_DECISIONS: &[&str] = &[ + "accept", + "reject", + "require_evidence", + "require_user_confirmation", +]; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityScenario { + pub name: String, + pub category: String, + #[serde(default)] + pub seed_memories: Vec, + #[serde(default)] + pub query: Option, + #[serde(default)] + pub workflow_prompt: Option, + #[serde(default)] + pub expected_recall: Vec, + #[serde(default)] + pub forbidden_recall: Vec, + #[serde(default)] + pub write_candidates: Vec, + #[serde(default)] + pub expected_write_decisions: Vec, + #[serde(default)] + pub evidence_refs: Vec, + #[serde(default)] + pub thresholds: QualityThresholds, +} + +impl QualityScenario { + pub fn prompt(&self) -> Option<&str> { + self.query + .as_deref() + .or_else(|| self.workflow_prompt.as_deref()) + } + + pub fn validate(&self) -> TreeRingResult<()> { + if self.name.trim().is_empty() { + return Err(TreeRingError::Validation( + "quality scenario name is required".to_string(), + )); + } + validate_member("quality category", &self.category, QUALITY_CATEGORIES)?; + if self + .prompt() + .map(|value| value.trim().is_empty()) + .unwrap_or(true) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} requires query or workflow_prompt", + self.name + ))); + } + for memory in self.seed_memories.iter().chain(self.write_candidates.iter()) { + memory.validate()?; + } + for expectation in &self.expected_recall { + expectation.validate("expected_recall", &self.name)?; + } + for expectation in &self.forbidden_recall { + expectation.validate("forbidden_recall", &self.name)?; + } + for decision in &self.expected_write_decisions { + decision.validate(&self.name)?; + } + self.thresholds.validate(&self.name)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecallExpectation { + #[serde(default)] + pub memory_id: Option, + #[serde(default)] + pub ring: Option, + #[serde(default)] + pub tag: Option, + #[serde(default)] + pub source_ref: Option, + #[serde(default)] + pub reason: String, +} + +impl RecallExpectation { + fn validate(&self, field: &str, scenario_name: &str) -> TreeRingResult<()> { + if self.memory_id.is_none() + && self.ring.is_none() + && self.tag.is_none() + && self.source_ref.is_none() + { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} {field} entry needs memory_id, ring, tag, or source_ref" + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct WriteDecisionExpectation { + pub memory_id: String, + pub decision: String, + #[serde(default)] + pub reason: String, +} + +impl WriteDecisionExpectation { + fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + if self.memory_id.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} write decision requires memory_id" + ))); + } + validate_member("write decision", &self.decision, WRITE_DECISIONS) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityThresholds { + #[serde(default = "one")] + pub min_constraint_recall_rate: f64, + #[serde(default)] + pub max_forbidden_recall_rate: f64, + #[serde(default = "one")] + pub min_spam_rejection_rate: f64, + #[serde(default = "one")] + pub min_evidence_required_rate: f64, + #[serde(default = "one")] + pub min_behavior_proof_pass_rate: f64, +} + +impl Default for QualityThresholds { + fn default() -> Self { + Self { + min_constraint_recall_rate: 1.0, + max_forbidden_recall_rate: 0.0, + min_spam_rejection_rate: 1.0, + min_evidence_required_rate: 1.0, + min_behavior_proof_pass_rate: 1.0, + } + } +} + +impl QualityThresholds { + fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + for (field, value) in [ + ( + "min_constraint_recall_rate", + self.min_constraint_recall_rate, + ), + ("max_forbidden_recall_rate", self.max_forbidden_recall_rate), + ("min_spam_rejection_rate", self.min_spam_rejection_rate), + ("min_evidence_required_rate", self.min_evidence_required_rate), + ( + "min_behavior_proof_pass_rate", + self.min_behavior_proof_pass_rate, + ), + ] { + if !value.is_finite() || !(0.0..=1.0).contains(&value) { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} threshold {field} must be between 0 and 1" + ))); + } + } + Ok(()) + } +} + +pub fn parse_quality_scenario(input: &str) -> TreeRingResult { + let scenario: QualityScenario = serde_json::from_str(input)?; + scenario.validate()?; + Ok(scenario) +} + +fn validate_member(field: &str, value: &str, allowed: &[&str]) -> TreeRingResult<()> { + if allowed.contains(&value) { + Ok(()) + } else { + Err(TreeRingError::Validation(format!( + "invalid {field}: {value}" + ))) + } +} + +fn one() -> f64 { + 1.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_valid_quality_scenario() { + let input = r#"{ + "name": "constraint recall", + "category": "constraint_recall", + "query": "proof loop background writer", + "seed_memories": [ + { + "id": "mem_quality_constraint", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background writer to proof loop work.", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": ["proof-loop"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + } + ], + "expected_recall": [ + {"memory_id": "mem_quality_constraint", "reason": "must recall proof-loop constraint"} + ] + }"#; + + let scenario = parse_quality_scenario(input).unwrap(); + + assert_eq!(scenario.name, "constraint recall"); + assert_eq!(scenario.category, "constraint_recall"); + assert_eq!(scenario.seed_memories.len(), 1); + assert_eq!(scenario.expected_recall[0].memory_id.as_deref(), Some("mem_quality_constraint")); + } + + #[test] + fn rejects_scenario_without_prompt() { + let input = r#"{"name":"bad","category":"constraint_recall"}"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("requires query or workflow_prompt")); + } + + #[test] + fn rejects_unknown_write_decision() { + let input = r#"{ + "name": "bad decision", + "category": "spam_prevention", + "query": "planning chatter", + "expected_write_decisions": [ + {"memory_id": "mem_candidate", "decision": "maybe"} + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("invalid write decision")); + } +} +``` + +- [ ] **Step 2: Export the quality module** + +Modify `crates/tree-ring-memory-core/src/lib.rs` by adding the module declaration with the other modules: + +```rust +pub mod quality; +``` + +Then add this public export block after the existing `pub use models::{...};` block: + +```rust +pub use quality::{ + parse_quality_scenario, QualityScenario, QualityThresholds, RecallExpectation, + WriteDecisionExpectation, QUALITY_CATEGORIES, WRITE_DECISIONS, +}; +``` + +- [ ] **Step 3: Run the focused parser tests** + +Run: + +```bash +cargo test -p tree-ring-memory-core quality:: --locked +``` + +Expected: PASS with the three quality parser tests passing. + +- [ ] **Step 4: Commit Task 1** + +Run: + +```bash +git add crates/tree-ring-memory-core/src/lib.rs crates/tree-ring-memory-core/src/quality.rs +git commit -m "Add quality scenario model" +``` + +--- + +### Task 2: Add Deterministic Quality Evaluator + +**Files:** +- Modify: `crates/tree-ring-memory-core/src/quality.rs` +- Modify: `crates/tree-ring-memory-core/src/lib.rs` + +**Interfaces:** +- Consumes: `QualityScenario` +- Produces: `quality::QualityRecall` +- Produces: `quality::QualityScenarioReport` +- Produces: `quality::QualityRunReport` +- Produces: `quality::evaluate_quality_scenario(scenario: &QualityScenario, recalls: &[QualityRecall]) -> TreeRingResult` +- Produces: `quality::summarize_quality_run(reports: Vec) -> QualityRunReport` + +- [ ] **Step 1: Add failing evaluator tests** + +Append these tests inside the existing `#[cfg(test)] mod tests` in `crates/tree-ring-memory-core/src/quality.rs`: + +```rust + fn memory(id: &str, summary: &str, ring: &str) -> MemoryEvent { + let mut event = MemoryEvent::new(summary, "lesson").unwrap(); + event.id = id.to_string(); + event.created_at = "2026-07-09T00:00:00Z".to_string(); + event.updated_at = "2026-07-09T00:00:00Z".to_string(); + event.scope = "project".to_string(); + event.project = Some("tree-ring".to_string()); + event.ring = ring.to_string(); + event.source.source_type = "evidence".to_string(); + event.source.ref_ = "docs/spec.md".to_string(); + event.salience = 0.8; + event.confidence = 0.8; + event + } + + #[test] + fn evaluates_required_and_forbidden_recall() { + let mut scenario = QualityScenario { + name: "recall gate".to_string(), + category: "constraint_recall".to_string(), + seed_memories: Vec::new(), + query: Some("background writer".to_string()), + workflow_prompt: None, + expected_recall: vec![RecallExpectation { + memory_id: Some("mem_required".to_string()), + reason: "required constraint".to_string(), + ..Default::default() + }], + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_forbidden".to_string()), + reason: "stale memory".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + scenario.validate().unwrap(); + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory("mem_required", "Do not add a background writer.", "heartwood"), + score: 0.91, + }], + ) + .unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.constraint_recall_rate, 1.0); + assert_eq!(report.forbidden_recall_rate, 0.0); + assert!(report.expected_recall[0].passed); + assert!(report.forbidden_recall[0].passed); + } + + #[test] + fn fails_when_forbidden_memory_is_recalled() { + let scenario = QualityScenario { + name: "stale recall".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: Vec::new(), + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_stale".to_string()), + reason: "stale CLI contract".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory("mem_stale", "Old CLI contract.", "heartwood"), + score: 0.88, + }], + ) + .unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.forbidden_recall_rate, 1.0); + assert!(!report.forbidden_recall[0].passed); + } + + #[test] + fn classifies_write_candidates_against_expected_decisions() { + let mut spam = memory("mem_spam", "Thinking about options for a moment.", "heartwood"); + spam.tags = vec!["transient".to_string()]; + let mut promoted_without_evidence = + memory("mem_missing_evidence", "Promote evaluated proof.", "heartwood"); + promoted_without_evidence.event_type = "evaluation_promotion".to_string(); + promoted_without_evidence.source.ref_.clear(); + let mut broad_heartwood = + memory("mem_needs_confirmation", "All projects should prefer this.", "heartwood"); + broad_heartwood.source.ref_.clear(); + + let scenario = QualityScenario { + name: "write gates".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![spam, promoted_without_evidence, broad_heartwood], + expected_write_decisions: vec![ + WriteDecisionExpectation { + memory_id: "mem_spam".to_string(), + decision: "reject".to_string(), + reason: "transient planning chatter".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_missing_evidence".to_string(), + decision: "require_evidence".to_string(), + reason: "promotion needs evidence".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_needs_confirmation".to_string(), + decision: "require_user_confirmation".to_string(), + reason: "broad heartwood needs confirmation".to_string(), + }, + ], + evidence_refs: vec!["evals/run-001".to_string()], + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.spam_rejection_rate, 1.0); + assert_eq!(report.evidence_required_rate, 1.0); + assert_eq!(report.write_decisions.len(), 3); + } + + #[test] + fn summarizes_quality_run() { + let passing = QualityScenarioReport { + name: "pass".to_string(), + category: "constraint_recall".to_string(), + constraint_recall_rate: 1.0, + forbidden_recall_rate: 0.0, + spam_rejection_rate: 1.0, + evidence_required_rate: 1.0, + behavior_proof_pass: true, + quality_pass: true, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_decisions: Vec::new(), + }; + + let run = summarize_quality_run(vec![passing.clone(), passing]); + + assert!(run.quality_pass); + assert_eq!(run.scenario_count, 2); + assert_eq!(run.behavior_proof_pass_rate, 1.0); + } +``` + +- [ ] **Step 2: Run the focused evaluator tests and verify missing types** + +Run: + +```bash +cargo test -p tree-ring-memory-core quality:: --locked +``` + +Expected: FAIL because `QualityRecall`, `evaluate_quality_scenario`, `QualityScenarioReport`, and `summarize_quality_run` are not defined. + +- [ ] **Step 3: Add evaluator types and functions** + +Insert this code in `crates/tree-ring-memory-core/src/quality.rs` above `pub fn parse_quality_scenario`: + +```rust +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRecall { + pub memory: MemoryEvent, + pub score: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallExpectationReport { + pub expectation: RecallExpectation, + pub matched_memory_ids: Vec, + pub passed: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WriteDecisionReport { + pub memory_id: String, + pub expected: String, + pub actual: String, + pub passed: bool, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityScenarioReport { + pub name: String, + pub category: String, + pub constraint_recall_rate: f64, + pub forbidden_recall_rate: f64, + pub spam_rejection_rate: f64, + pub evidence_required_rate: f64, + pub behavior_proof_pass: bool, + pub quality_pass: bool, + pub expected_recall: Vec, + pub forbidden_recall: Vec, + pub write_decisions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRunReport { + pub ok: bool, + pub scenario_count: usize, + pub constraint_recall_rate: f64, + pub forbidden_recall_rate: f64, + pub spam_rejection_rate: f64, + pub evidence_required_rate: f64, + pub behavior_proof_pass_rate: f64, + pub quality_pass: bool, + pub scenarios: Vec, +} + +pub fn evaluate_quality_scenario( + scenario: &QualityScenario, + recalls: &[QualityRecall], +) -> TreeRingResult { + scenario.validate()?; + + let expected_recall = scenario + .expected_recall + .iter() + .map(|expectation| expected_recall_report(expectation, recalls)) + .collect::>(); + let forbidden_recall = scenario + .forbidden_recall + .iter() + .map(|expectation| forbidden_recall_report(expectation, recalls)) + .collect::>(); + let write_decisions = scenario + .expected_write_decisions + .iter() + .map(|expectation| write_decision_report(expectation, scenario)) + .collect::>(); + + let constraint_recall_rate = pass_rate(&expected_recall); + let forbidden_recall_rate = failure_rate(&forbidden_recall); + let spam_rejection_rate = decision_pass_rate(&write_decisions, "reject"); + let evidence_required_rate = decision_pass_rate(&write_decisions, "require_evidence"); + let behavior_proof_pass = + constraint_recall_rate >= scenario.thresholds.min_constraint_recall_rate + && forbidden_recall_rate <= scenario.thresholds.max_forbidden_recall_rate; + let quality_pass = behavior_proof_pass + && spam_rejection_rate >= scenario.thresholds.min_spam_rejection_rate + && evidence_required_rate >= scenario.thresholds.min_evidence_required_rate; + + Ok(QualityScenarioReport { + name: scenario.name.clone(), + category: scenario.category.clone(), + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass, + quality_pass, + expected_recall, + forbidden_recall, + write_decisions, + }) +} + +pub fn summarize_quality_run(reports: Vec) -> QualityRunReport { + let scenario_count = reports.len(); + let denominator = scenario_count.max(1) as f64; + let constraint_recall_rate = + reports.iter().map(|report| report.constraint_recall_rate).sum::() / denominator; + let forbidden_recall_rate = + reports.iter().map(|report| report.forbidden_recall_rate).sum::() / denominator; + let spam_rejection_rate = + reports.iter().map(|report| report.spam_rejection_rate).sum::() / denominator; + let evidence_required_rate = + reports.iter().map(|report| report.evidence_required_rate).sum::() / denominator; + let behavior_proof_pass_rate = reports + .iter() + .filter(|report| report.behavior_proof_pass) + .count() as f64 + / denominator; + let quality_pass = reports.iter().all(|report| report.quality_pass); + + QualityRunReport { + ok: quality_pass, + scenario_count, + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass_rate, + quality_pass, + scenarios: reports, + } +} + +fn expected_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: !matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn forbidden_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn matching_recall_ids(expectation: &RecallExpectation, recalls: &[QualityRecall]) -> Vec { + recalls + .iter() + .filter(|recall| expectation_matches_memory(expectation, &recall.memory)) + .map(|recall| recall.memory.id.clone()) + .collect() +} + +fn expectation_matches_memory(expectation: &RecallExpectation, memory: &MemoryEvent) -> bool { + if expectation + .memory_id + .as_deref() + .is_some_and(|id| memory.id != id) + { + return false; + } + if expectation + .ring + .as_deref() + .is_some_and(|ring| memory.ring != ring) + { + return false; + } + if expectation + .tag + .as_deref() + .is_some_and(|tag| !memory.tags.iter().any(|candidate| candidate == tag)) + { + return false; + } + if expectation + .source_ref + .as_deref() + .is_some_and(|source_ref| memory.source.ref_ != source_ref) + { + return false; + } + true +} + +fn write_decision_report( + expectation: &WriteDecisionExpectation, + scenario: &QualityScenario, +) -> WriteDecisionReport { + let actual = scenario + .write_candidates + .iter() + .find(|candidate| candidate.id == expectation.memory_id) + .map(|candidate| classify_write_candidate(candidate, &scenario.evidence_refs)) + .unwrap_or_else(|| "reject".to_string()); + WriteDecisionReport { + memory_id: expectation.memory_id.clone(), + expected: expectation.decision.clone(), + passed: actual == expectation.decision, + actual, + reason: expectation.reason.clone(), + } +} + +fn classify_write_candidate(candidate: &MemoryEvent, required_evidence_refs: &[String]) -> String { + if candidate.sensitivity == "secret" { + return "reject".to_string(); + } + if candidate.retention == "ephemeral" + || candidate.tags.iter().any(|tag| { + matches!( + tag.as_str(), + "transient" | "planning-chatter" | "scratchpad" | "tool-noise" + ) + }) + { + return "reject".to_string(); + } + if candidate.event_type.starts_with("evaluation_") && candidate.source.ref_.trim().is_empty() { + return "require_evidence".to_string(); + } + if !required_evidence_refs.is_empty() + && candidate.event_type.starts_with("evaluation_") + && !required_evidence_refs + .iter() + .any(|required| required == &candidate.source.ref_) + { + return "require_evidence".to_string(); + } + if candidate.ring == "heartwood" && candidate.source.ref_.trim().is_empty() { + return "require_user_confirmation".to_string(); + } + "accept".to_string() +} + +fn pass_rate(reports: &[RecallExpectationReport]) -> f64 { + if reports.is_empty() { + return 1.0; + } + reports.iter().filter(|report| report.passed).count() as f64 / reports.len() as f64 +} + +fn failure_rate(reports: &[RecallExpectationReport]) -> f64 { + if reports.is_empty() { + return 0.0; + } + reports.iter().filter(|report| !report.passed).count() as f64 / reports.len() as f64 +} + +fn decision_pass_rate(reports: &[WriteDecisionReport], decision: &str) -> f64 { + let relevant = reports + .iter() + .filter(|report| report.expected == decision) + .collect::>(); + if relevant.is_empty() { + return 1.0; + } + relevant.iter().filter(|report| report.passed).count() as f64 / relevant.len() as f64 +} +``` + +- [ ] **Step 4: Export evaluator types** + +Modify the `pub use quality::{...};` block in `crates/tree-ring-memory-core/src/lib.rs` so it includes: + +```rust + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, + QualityRunReport, QualityScenario, QualityScenarioReport, QualityThresholds, + RecallExpectation, RecallExpectationReport, WriteDecisionExpectation, WriteDecisionReport, + QUALITY_CATEGORIES, WRITE_DECISIONS, +``` + +- [ ] **Step 5: Run the focused evaluator tests** + +Run: + +```bash +cargo test -p tree-ring-memory-core quality:: --locked +``` + +Expected: PASS. + +- [ ] **Step 6: Commit Task 2** + +Run: + +```bash +git add crates/tree-ring-memory-core/src/lib.rs crates/tree-ring-memory-core/src/quality.rs +git commit -m "Add memory quality evaluator" +``` + +--- + +### Task 3: Add Default Quality Fixture Pack + +**Files:** +- Create: `fixtures/quality/no-background-writer-constraint.json` +- Create: `fixtures/quality/transient-planning-spam.json` +- Create: `fixtures/quality/stale-cli-contract.json` +- Create: `fixtures/quality/sensitive-memory-hidden.json` +- Create: `fixtures/quality/superseded-heartwood.json` +- Create: `fixtures/quality/scar-failure-recall.json` + +**Interfaces:** +- Consumes: `quality::parse_quality_scenario` +- Produces: six valid quality scenarios for certification. + +- [ ] **Step 1: Add the real workflow constraint scenario** + +Create `fixtures/quality/no-background-writer-constraint.json`: + +```json +{ + "name": "no-background-writer-constraint", + "category": "constraint_recall", + "query": "proof loop background writer hidden durable writes", + "seed_memories": [ + { + "id": "mem_quality_no_background_writer", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background proof runner, hidden recorder, or autonomous durable writer to Tree Ring proof-loop work.", + "details": "Quality proof must remain explicit, visible, and certification-owned.", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#non-goals", + "quote": "" + }, + "tags": ["proof-loop", "agent-mediated", "no-daemon"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_no_background_writer", + "reason": "proof-loop work must recall the explicit no-background-writer constraint" + } + ], + "thresholds": { + "min_constraint_recall_rate": 1.0, + "max_forbidden_recall_rate": 0.0, + "min_spam_rejection_rate": 1.0, + "min_evidence_required_rate": 1.0, + "min_behavior_proof_pass_rate": 1.0 + } +} +``` + +- [ ] **Step 2: Add the real workflow spam-prevention scenario** + +Create `fixtures/quality/transient-planning-spam.json`: + +```json +{ + "name": "transient-planning-spam", + "category": "spam_prevention", + "query": "memory spam transient planning durable heartwood", + "seed_memories": [ + { + "id": "mem_quality_store_only_durable", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Store only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#agent-guidance-gates", + "quote": "" + }, + "tags": ["write-gate", "memory-quality"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_store_only_durable", + "reason": "the write gate should be recalled before evaluating candidate memories" + } + ], + "write_candidates": [ + { + "id": "mem_candidate_planning_chatter", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "We discussed maybe doing the quality proof loop next.", + "details": "Transient planning chatter should not become durable heartwood.", + "source": { + "type": "manual", + "ref": "", + "quote": "" + }, + "tags": ["planning-chatter", "transient"], + "salience": 0.2, + "confidence": 0.4, + "sensitivity": "normal", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_write_decisions": [ + { + "memory_id": "mem_candidate_planning_chatter", + "decision": "reject", + "reason": "transient planning chatter is memory spam" + } + ] +} +``` + +- [ ] **Step 3: Add the real workflow stale CLI-contract scenario** + +Create `fixtures/quality/stale-cli-contract.json`: + +```json +{ + "name": "stale-cli-contract", + "category": "stale_truth_suppression", + "query": "tree ring helper script cli contract remember recall forget", + "seed_memories": [ + { + "id": "mem_stale_cli_contract", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Old helper scripts can call remember recall and forget without required arguments.", + "details": "This stale CLI contract must not guide current helper-script reviews.", + "source": { + "type": "manual", + "ref": "old-review-note", + "quote": "" + }, + "tags": ["cli-contract", "stale"], + "salience": 0.8, + "confidence": 0.6, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_current_cli_contract", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_current_cli_contract", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "warning", + "summary": "Review Tree Ring helper scripts against the current CLI contract: remember needs --event-type, recall needs a query, and forget needs a memory id.", + "details": "Current source in crates/tree-ring-memory-cli/src/main.rs remains authoritative.", + "source": { + "type": "file", + "ref": "crates/tree-ring-memory-cli/src/main.rs", + "quote": "" + }, + "tags": ["cli-contract", "current"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_stale_cli_contract"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_current_cli_contract", + "reason": "current CLI contract should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_stale_cli_contract", + "reason": "superseded stale CLI contract must be suppressed" + } + ] +} +``` + +- [ ] **Step 4: Add the sensitive-memory hidden scenario** + +Create `fixtures/quality/sensitive-memory-hidden.json`: + +```json +{ + "name": "sensitive-memory-hidden", + "category": "stale_truth_suppression", + "query": "private diagnosis source metadata", + "seed_memories": [ + { + "id": "mem_sensitive_private_diagnosis", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "outer", + "event_type": "lesson", + "summary": "Private diagnosis source metadata should stay hidden by default.", + "details": "", + "source": { + "type": "manual", + "ref": "private-diagnosis-note", + "quote": "" + }, + "tags": ["privacy"], + "salience": 0.7, + "confidence": 0.8, + "sensitivity": "private", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_sensitive_private_diagnosis", + "reason": "private memory is hidden by default" + } + ] +} +``` + +- [ ] **Step 5: Add the superseded-heartwood scenario** + +Create `fixtures/quality/superseded-heartwood.json`: + +```json +{ + "name": "superseded-heartwood", + "category": "stale_truth_suppression", + "query": "durable rule project bridge command", + "seed_memories": [ + { + "id": "mem_old_bridge_rule", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should happen automatically during init.", + "details": "", + "source": { + "type": "manual", + "ref": "old-bridge-note", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.9, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_new_bridge_rule", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_new_bridge_rule", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should be an explicit integrations link command, not automatic init behavior.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-06-tree-ring-agent-mediated-bridges-design.md#command-surface", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_old_bridge_rule"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_new_bridge_rule", + "reason": "replacement heartwood should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_old_bridge_rule", + "reason": "superseded heartwood should not outrank the replacement" + } + ] +} +``` + +- [ ] **Step 6: Add the scar failure recall scenario** + +Create `fixtures/quality/scar-failure-recall.json`: + +```json +{ + "name": "scar-failure-recall", + "category": "behavior_proof", + "query": "failure stale cache rollback", + "seed_memories": [ + { + "id": "mem_scar_stale_cache", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "scar", + "event_type": "warning", + "summary": "Aggressive caching caused a stale rollback failure; recall this scar before cache-related workflow changes.", + "details": "", + "source": { + "type": "evidence", + "ref": "evals/cache-branch/run-013", + "quote": "" + }, + "tags": ["failure", "cache", "rollback"], + "salience": 0.95, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_scar_stale_cache", + "ring": "scar", + "reason": "failure-like queries should surface scar memory" + } + ] +} +``` + +- [ ] **Step 7: Add a fixture parser test** + +Append this test inside `crates/tree-ring-memory-core/src/quality.rs` tests: + +```rust + #[test] + fn parses_default_quality_fixtures() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join("fixtures") + .join("quality"); + let mut parsed = 0usize; + for entry in std::fs::read_dir(&root).unwrap() { + let path = entry.unwrap().path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let input = std::fs::read_to_string(&path).unwrap(); + parse_quality_scenario(&input) + .unwrap_or_else(|err| panic!("{}: {err}", path.display())); + parsed += 1; + } + + assert_eq!(parsed, 6); + } +``` + +- [ ] **Step 8: Run fixture validation tests** + +Run: + +```bash +cargo test -p tree-ring-memory-core parses_default_quality_fixtures --locked +``` + +Expected: PASS. + +- [ ] **Step 9: Commit Task 3** + +Run: + +```bash +git add crates/tree-ring-memory-core/src/quality.rs fixtures/quality +git commit -m "Add default memory quality fixtures" +``` + +--- + +### Task 4: Add Certification-Only Quality Runner + +**Files:** +- Create: `crates/tree-ring-memory-cli/examples/quality_scenarios.rs` + +**Interfaces:** +- Consumes: `tree_ring_memory_core::parse_quality_scenario` +- Consumes: `tree_ring_memory_core::evaluate_quality_scenario` +- Consumes: `tree_ring_memory_core::summarize_quality_run` +- Consumes: `tree_ring_memory_sqlite::{SQLiteMemoryStore, MemoryRetriever}` +- Produces: `quality-report.json` +- Produces: `quality-summary.md` + +- [ ] **Step 1: Add the failing example smoke test** + +Run: + +```bash +cargo run -p tree-ring-memory-cli --example quality_scenarios -- fixtures/quality target/tree-ring-quality-test +``` + +Expected: FAIL because the `quality_scenarios` example does not exist. + +- [ ] **Step 2: Add the quality runner example** + +Create `crates/tree-ring-memory-cli/examples/quality_scenarios.rs`: + +```rust +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use tree_ring_memory_core::{ + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, + QualityRunReport, QualityScenario, QualityScenarioReport, +}; +use tree_ring_memory_sqlite::{MemoryRetriever, SQLiteMemoryStore}; + +fn main() { + if let Err(err) = run() { + eprintln!("quality scenarios failed: {err}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let mut args = std::env::args_os().skip(1); + let fixture_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| "usage: quality_scenarios ".to_string())?; + let output_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| "usage: quality_scenarios ".to_string())?; + if args.next().is_some() { + return Err("usage: quality_scenarios ".to_string()); + } + + fs::create_dir_all(&output_dir).map_err(|err| err.to_string())?; + let scenarios = load_scenarios(&fixture_dir)?; + let mut reports = Vec::new(); + for scenario in scenarios { + reports.push(run_scenario(&scenario)?); + } + + let run_report = summarize_quality_run(reports); + write_reports(&output_dir, &run_report)?; + if !run_report.quality_pass { + return Err(format!( + "quality run failed: {} scenarios evaluated", + run_report.scenario_count + )); + } + println!( + "quality scenarios passed: {} scenario(s)", + run_report.scenario_count + ); + Ok(()) +} + +fn load_scenarios(fixture_dir: &Path) -> Result, String> { + let mut paths = fs::read_dir(fixture_dir) + .map_err(|err| err.to_string())? + .map(|entry| entry.map(|entry| entry.path()).map_err(|err| err.to_string())) + .collect::, _>>()?; + paths.sort(); + + let mut scenarios = Vec::new(); + for path in paths { + if path.extension() != Some(OsStr::new("json")) { + continue; + } + let input = fs::read_to_string(&path).map_err(|err| err.to_string())?; + let scenario = parse_quality_scenario(&input) + .map_err(|err| format!("{}: {err}", path.display()))?; + scenarios.push(scenario); + } + if scenarios.is_empty() { + return Err(format!("no quality scenario json files in {}", fixture_dir.display())); + } + Ok(scenarios) +} + +fn run_scenario(scenario: &QualityScenario) -> Result { + let root = temporary_root(&scenario.name)?; + let db_path = root.join("memory.sqlite"); + let mut store = SQLiteMemoryStore::open(&db_path).map_err(|err| err.to_string())?; + store + .put_many(&scenario.seed_memories) + .map_err(|err| err.to_string())?; + + let prompt = scenario + .prompt() + .ok_or_else(|| format!("quality scenario {} has no prompt", scenario.name))?; + let recalls = MemoryRetriever::new(&store) + .recall( + prompt, + None, + None, + None, + None, + None, + false, + false, + 8, + true, + ) + .map_err(|err| err.to_string())? + .into_iter() + .map(|result| QualityRecall { + memory: result.memory, + score: result.score, + }) + .collect::>(); + + let report = evaluate_quality_scenario(scenario, &recalls).map_err(|err| err.to_string())?; + fs::remove_dir_all(&root).map_err(|err| err.to_string())?; + Ok(report) +} + +fn temporary_root(name: &str) -> Result { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| err.to_string())? + .as_millis(); + let safe_name = name + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) + .collect::(); + let root = std::env::temp_dir().join(format!( + "tree-ring-quality-{}-{}-{safe_name}", + std::process::id(), + millis + )); + if root.exists() { + fs::remove_dir_all(&root).map_err(|err| err.to_string())?; + } + fs::create_dir_all(&root).map_err(|err| err.to_string())?; + Ok(root) +} + +fn write_reports(output_dir: &Path, report: &QualityRunReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report).map_err(|err| err.to_string())?; + fs::write(output_dir.join("quality-report.json"), json).map_err(|err| err.to_string())?; + fs::write(output_dir.join("quality-summary.md"), markdown_summary(report)) + .map_err(|err| err.to_string())?; + Ok(()) +} + +fn markdown_summary(report: &QualityRunReport) -> String { + let mut lines = vec![ + "# Tree Ring Memory Quality Summary".to_string(), + String::new(), + format!("- quality pass: {}", report.quality_pass), + format!("- scenarios: {}", report.scenario_count), + format!( + "- constraint recall rate: {:.3}", + report.constraint_recall_rate + ), + format!( + "- forbidden recall rate: {:.3}", + report.forbidden_recall_rate + ), + format!("- spam rejection rate: {:.3}", report.spam_rejection_rate), + format!( + "- evidence required rate: {:.3}", + report.evidence_required_rate + ), + format!( + "- behavior proof pass rate: {:.3}", + report.behavior_proof_pass_rate + ), + String::new(), + "## Scenarios".to_string(), + String::new(), + ]; + for scenario in &report.scenarios { + lines.push(format!( + "- `{}` [{}]: pass={}", + scenario.name, scenario.category, scenario.quality_pass + )); + } + lines.push(String::new()); + lines.join("\n") +} +``` + +- [ ] **Step 3: Run the quality runner** + +Run: + +```bash +rm -rf target/tree-ring-quality-test +cargo run -p tree-ring-memory-cli --example quality_scenarios -- fixtures/quality target/tree-ring-quality-test +``` + +Expected: PASS and output contains: + +```text +quality scenarios passed: 6 scenario(s) +``` + +- [ ] **Step 4: Inspect generated artifacts** + +Run: + +```bash +test -f target/tree-ring-quality-test/quality-report.json +test -f target/tree-ring-quality-test/quality-summary.md +grep -F '"quality_pass": true' target/tree-ring-quality-test/quality-report.json +grep -F 'Tree Ring Memory Quality Summary' target/tree-ring-quality-test/quality-summary.md +``` + +Expected: all commands pass. + +- [ ] **Step 5: Commit Task 4** + +Run: + +```bash +git add crates/tree-ring-memory-cli/examples/quality_scenarios.rs +git commit -m "Add memory quality scenario runner" +``` + +--- + +### Task 5: Include Quality Metrics In Certification + +**Files:** +- Modify: `scripts/certify-tree-ring.sh` + +**Interfaces:** +- Consumes: `fixtures/quality/*.json` +- Consumes: `cargo run --release -p tree-ring-memory-cli --example quality_scenarios` +- Produces: `target/tree-ring-certification/quality/quality-report.json` +- Produces: `target/tree-ring-certification/quality/quality-summary.md` +- Produces: top-level `metrics.json` field `quality` + +- [ ] **Step 1: Add the failing certification expectation** + +Run: + +```bash +rm -rf target/tree-ring-certification/quality +sh scripts/certify-tree-ring.sh +test -f target/tree-ring-certification/quality/quality-report.json +``` + +Expected: FAIL at the `test -f` command because certification does not run quality scenarios yet. + +- [ ] **Step 2: Run the quality example from certification** + +In `scripts/certify-tree-ring.sh`, add this block after the performance metric extraction block and before Agent Zero handling: + +```sh +quality_out="$OUT_DIR/quality" +mkdir -p "$quality_out" +run cargo run --release -p tree-ring-memory-cli --example quality_scenarios -- "$ROOT/fixtures/quality" "$quality_out" \ + > "$OUT_DIR/quality-run.out" 2>&1 +require_file "$quality_out/quality-report.json" +require_file "$quality_out/quality-summary.md" +grep -F '"quality_pass": true' "$quality_out/quality-report.json" > /dev/null \ + || fail "memory quality scenarios did not pass" +quality_json=$(cat "$quality_out/quality-report.json") +``` + +- [ ] **Step 3: Add quality JSON to metrics** + +In the `cat > "$METRICS" < u16 { + match color { + Color::Rgb(red, green, blue) => red as u16 + green as u16 + blue as u16, + _ => 0, + } + } + + fn stats_with_fullness(ring: &str, total: usize, fullness_level: f64) -> RingStats { + let mut stats = RingStats::empty(ring); + stats.total = total; + stats.fullness_level = fullness_level; + stats + } + + #[test] + fn fullness_brightens_ambient_ring_without_pulse() { + let low = stats_with_fullness("cambium", 1, 0.10); + let high = stats_with_fullness("cambium", 8, 0.80); + + let low_color = animated_color("cambium", Some(&low), 10, 0); + let high_color = animated_color("cambium", Some(&high), 10, 0); + + assert!(brightness_sum(high_color) > brightness_sum(low_color)); + } +``` + +- [ ] **Step 6: Run focused ambient style test and verify it fails** + +Run: + +```bash +cargo test -p tree-ring-memory-cli fullness_brightens_ambient_ring_without_pulse --locked +``` + +Expected: FAIL because fullness does not affect `animated_color`. + +- [ ] **Step 7: Map fullness to ambient intensity** + +Modify `animated_color` in `crates/tree-ring-memory-cli/src/tui/rings.rs` to this full function: + +```rust +fn animated_color(ring: &str, stats: Option<&RingStats>, tick: u64, offset: u64) -> Color { + let warning_level = stats.map(|stats| stats.warning_level).unwrap_or_default(); + let base = theme::ring_color(ring, warning_level); + let Some(stats) = stats else { + return dim_color(base, 0.36); + }; + if stats.total == 0 { + return dim_color(base, 0.34); + } + let fullness = stats.fullness_level.clamp(0.0, 1.0); + if stats.pulse_level > 0.05 && (tick + offset) % 6 < 3 { + return brighten_color(base, 0.28 + fullness * 0.16 + stats.pulse_level * 0.18); + } + if (tick + offset) % 18 < 3 { + return brighten_color(base, 0.08 + fullness * 0.16); + } + if fullness < 0.18 { + return dim_color(base, 0.42 + fullness * 1.35); + } + brighten_color(base, fullness * 0.20) +} +``` + +- [ ] **Step 8: Run TUI focused tests** + +Run: + +```bash +cargo test -p tree-ring-memory-cli fullness --locked +``` + +Expected: PASS. + +- [ ] **Step 9: Commit Task 7** + +Run: + +```bash +git add crates/tree-ring-memory-cli/src/tui/model.rs crates/tree-ring-memory-cli/src/tui/rings.rs +git commit -m "Add ambient ring fullness styling" +``` + +--- + +### Task 8: Final Verification And Documentation Consistency + +**Files:** +- Verify all changed files from Tasks 1-7. + +**Interfaces:** +- Consumes: complete quality proof loop implementation. +- Produces: final verified branch state. + +- [ ] **Step 1: Run formatting** + +Run: + +```bash +cargo fmt --check +``` + +Expected: PASS. + +- [ ] **Step 2: Run locked tests** + +Run: + +```bash +cargo test --locked +``` + +Expected: PASS. + +- [ ] **Step 3: Run Clippy** + +Run: + +```bash +cargo clippy --locked --all-targets +``` + +Expected: PASS with no warnings. + +- [ ] **Step 4: Run certification** + +Run: + +```bash +sh scripts/certify-tree-ring.sh +``` + +Expected: PASS and output includes: + +```text +certification passed +``` + +- [ ] **Step 5: Inspect certification quality artifacts** + +Run: + +```bash +test -f target/tree-ring-certification/quality/quality-report.json +test -f target/tree-ring-certification/quality/quality-summary.md +grep -F '"quality_pass": true' target/tree-ring-certification/quality/quality-report.json +grep -F '"quality"' target/tree-ring-certification/metrics.json +grep -F 'memory quality scenarios: passed' target/tree-ring-certification/summary.md +``` + +Expected: PASS. + +- [ ] **Step 6: Run whitespace check** + +Run: + +```bash +git diff --check +``` + +Expected: PASS with no output. + +- [ ] **Step 7: Review final changed files** + +Run: + +```bash +git status --short +git log --oneline -8 +``` + +Expected: only expected tracked changes are present; pre-existing untracked local files may still appear and must not be staged. + +- [ ] **Step 8: Commit final verification note only if files changed during verification** + +If formatting or documentation consistency changed files, run: + +```bash +git add +git commit -m "Verify memory quality proof loop" +``` + +Expected: create a commit only when verification produced tracked file changes. From c5c0227be291492b21d80eac3af0a7e75fa402bd Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 13:51:12 -0400 Subject: [PATCH 03/19] Add quality scenario model --- crates/tree-ring-memory-core/src/lib.rs | 5 + crates/tree-ring-memory-core/src/quality.rs | 275 ++++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 crates/tree-ring-memory-core/src/quality.rs diff --git a/crates/tree-ring-memory-core/src/lib.rs b/crates/tree-ring-memory-core/src/lib.rs index 10c8759..c0661c3 100644 --- a/crates/tree-ring-memory-core/src/lib.rs +++ b/crates/tree-ring-memory-core/src/lib.rs @@ -4,6 +4,7 @@ pub mod dox; pub mod import_export; pub mod maintenance; pub mod models; +pub mod quality; pub mod recall; pub mod revolve; pub mod sensitivity; @@ -26,6 +27,10 @@ pub use maintenance::{ pub use models::{ now_iso, MemoryEvent, MemoryLink, MemoryReview, MemorySource, TreeRingError, TreeRingResult, }; +pub use quality::{ + parse_quality_scenario, QualityScenario, QualityThresholds, RecallExpectation, + WriteDecisionExpectation, QUALITY_CATEGORIES, WRITE_DECISIONS, +}; pub use recall::{RecallRanking, RecallScore, RecallScorer}; pub use revolve::{collect_revolve_memories, RevolveSyncReport, RevolveSyncRequest}; pub use sensitivity::{SensitivityGuard, SensitivityResult}; diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs new file mode 100644 index 0000000..cdcf8ba --- /dev/null +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -0,0 +1,275 @@ +use serde::{Deserialize, Serialize}; + +use crate::models::{MemoryEvent, TreeRingError, TreeRingResult}; + +pub const QUALITY_CATEGORIES: &[&str] = &[ + "constraint_recall", + "spam_prevention", + "stale_truth_suppression", + "behavior_proof", +]; + +pub const WRITE_DECISIONS: &[&str] = &[ + "accept", + "reject", + "require_evidence", + "require_user_confirmation", +]; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityScenario { + pub name: String, + pub category: String, + #[serde(default)] + pub seed_memories: Vec, + #[serde(default)] + pub query: Option, + #[serde(default)] + pub workflow_prompt: Option, + #[serde(default)] + pub expected_recall: Vec, + #[serde(default)] + pub forbidden_recall: Vec, + #[serde(default)] + pub write_candidates: Vec, + #[serde(default)] + pub expected_write_decisions: Vec, + #[serde(default)] + pub evidence_refs: Vec, + #[serde(default)] + pub thresholds: QualityThresholds, +} + +impl QualityScenario { + pub fn prompt(&self) -> Option<&str> { + self.query + .as_deref() + .or_else(|| self.workflow_prompt.as_deref()) + } + + pub fn validate(&self) -> TreeRingResult<()> { + if self.name.trim().is_empty() { + return Err(TreeRingError::Validation( + "quality scenario name is required".to_string(), + )); + } + validate_member("quality category", &self.category, QUALITY_CATEGORIES)?; + if self + .prompt() + .map(|value| value.trim().is_empty()) + .unwrap_or(true) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} requires query or workflow_prompt", + self.name + ))); + } + for memory in self.seed_memories.iter().chain(self.write_candidates.iter()) { + memory.validate()?; + } + for expectation in &self.expected_recall { + expectation.validate("expected_recall", &self.name)?; + } + for expectation in &self.forbidden_recall { + expectation.validate("forbidden_recall", &self.name)?; + } + for decision in &self.expected_write_decisions { + decision.validate(&self.name)?; + } + self.thresholds.validate(&self.name)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecallExpectation { + #[serde(default)] + pub memory_id: Option, + #[serde(default)] + pub ring: Option, + #[serde(default)] + pub tag: Option, + #[serde(default)] + pub source_ref: Option, + #[serde(default)] + pub reason: String, +} + +impl RecallExpectation { + fn validate(&self, field: &str, scenario_name: &str) -> TreeRingResult<()> { + if self.memory_id.is_none() + && self.ring.is_none() + && self.tag.is_none() + && self.source_ref.is_none() + { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} {field} entry needs memory_id, ring, tag, or source_ref" + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct WriteDecisionExpectation { + pub memory_id: String, + pub decision: String, + #[serde(default)] + pub reason: String, +} + +impl WriteDecisionExpectation { + fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + if self.memory_id.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} write decision requires memory_id" + ))); + } + validate_member("write decision", &self.decision, WRITE_DECISIONS) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityThresholds { + #[serde(default = "one")] + pub min_constraint_recall_rate: f64, + #[serde(default)] + pub max_forbidden_recall_rate: f64, + #[serde(default = "one")] + pub min_spam_rejection_rate: f64, + #[serde(default = "one")] + pub min_evidence_required_rate: f64, + #[serde(default = "one")] + pub min_behavior_proof_pass_rate: f64, +} + +impl Default for QualityThresholds { + fn default() -> Self { + Self { + min_constraint_recall_rate: 1.0, + max_forbidden_recall_rate: 0.0, + min_spam_rejection_rate: 1.0, + min_evidence_required_rate: 1.0, + min_behavior_proof_pass_rate: 1.0, + } + } +} + +impl QualityThresholds { + fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + for (field, value) in [ + ( + "min_constraint_recall_rate", + self.min_constraint_recall_rate, + ), + ("max_forbidden_recall_rate", self.max_forbidden_recall_rate), + ("min_spam_rejection_rate", self.min_spam_rejection_rate), + ("min_evidence_required_rate", self.min_evidence_required_rate), + ( + "min_behavior_proof_pass_rate", + self.min_behavior_proof_pass_rate, + ), + ] { + if !value.is_finite() || !(0.0..=1.0).contains(&value) { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} threshold {field} must be between 0 and 1" + ))); + } + } + Ok(()) + } +} + +pub fn parse_quality_scenario(input: &str) -> TreeRingResult { + let scenario: QualityScenario = serde_json::from_str(input)?; + scenario.validate()?; + Ok(scenario) +} + +fn validate_member(field: &str, value: &str, allowed: &[&str]) -> TreeRingResult<()> { + if allowed.contains(&value) { + Ok(()) + } else { + Err(TreeRingError::Validation(format!( + "invalid {field}: {value}" + ))) + } +} + +fn one() -> f64 { + 1.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_valid_quality_scenario() { + let input = r#"{ + "name": "constraint recall", + "category": "constraint_recall", + "query": "proof loop background writer", + "seed_memories": [ + { + "id": "mem_quality_constraint", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background writer to proof loop work.", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": ["proof-loop"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + } + ], + "expected_recall": [ + {"memory_id": "mem_quality_constraint", "reason": "must recall proof-loop constraint"} + ] + }"#; + + let scenario = parse_quality_scenario(input).unwrap(); + + assert_eq!(scenario.name, "constraint recall"); + assert_eq!(scenario.category, "constraint_recall"); + assert_eq!(scenario.seed_memories.len(), 1); + assert_eq!(scenario.expected_recall[0].memory_id.as_deref(), Some("mem_quality_constraint")); + } + + #[test] + fn rejects_scenario_without_prompt() { + let input = r#"{"name":"bad","category":"constraint_recall"}"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("requires query or workflow_prompt")); + } + + #[test] + fn rejects_unknown_write_decision() { + let input = r#"{ + "name": "bad decision", + "category": "spam_prevention", + "query": "planning chatter", + "expected_write_decisions": [ + {"memory_id": "mem_candidate", "decision": "maybe"} + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("invalid write decision")); + } +} From 7d41b97d4351fa0faeeb79490f2944745f4ebfe5 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 13:55:14 -0400 Subject: [PATCH 04/19] Tighten quality scenario validation --- crates/tree-ring-memory-core/src/quality.rs | 81 +++++++++++++++++++-- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index cdcf8ba..dc7a904 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -42,9 +42,14 @@ pub struct QualityScenario { impl QualityScenario { pub fn prompt(&self) -> Option<&str> { - self.query + if let Some(query) = self.query.as_deref() { + if !query.trim().is_empty() { + return Some(query); + } + } + self.workflow_prompt .as_deref() - .or_else(|| self.workflow_prompt.as_deref()) + .filter(|value| !value.trim().is_empty()) } pub fn validate(&self) -> TreeRingResult<()> { @@ -76,6 +81,14 @@ impl QualityScenario { for decision in &self.expected_write_decisions { decision.validate(&self.name)?; } + for (index, evidence_ref) in self.evidence_refs.iter().enumerate() { + if evidence_ref.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {} evidence_refs[{}] must not be blank", + self.name, index + ))); + } + } self.thresholds.validate(&self.name)?; Ok(()) } @@ -97,13 +110,19 @@ pub struct RecallExpectation { impl RecallExpectation { fn validate(&self, field: &str, scenario_name: &str) -> TreeRingResult<()> { - if self.memory_id.is_none() - && self.ring.is_none() - && self.tag.is_none() - && self.source_ref.is_none() - { + let has_nonblank_selector = [ + self.memory_id.as_deref(), + self.ring.as_deref(), + self.tag.as_deref(), + self.source_ref.as_deref(), + ] + .into_iter() + .flatten() + .any(|value| !value.trim().is_empty()); + + if !has_nonblank_selector { return Err(TreeRingError::Validation(format!( - "quality scenario {scenario_name} {field} entry needs memory_id, ring, tag, or source_ref" + "quality scenario {scenario_name} {field} entry must include a nonblank memory_id, ring, tag, or source_ref" ))); } Ok(()) @@ -272,4 +291,50 @@ mod tests { assert!(error.contains("invalid write decision")); } + + #[test] + fn rejects_blank_evidence_refs() { + let input = r#"{ + "name": "bad evidence refs", + "category": "behavior_proof", + "query": "prove behavior", + "evidence_refs": ["docs/spec.md", " "] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("evidence_refs")); + assert!(error.contains("must not be blank")); + } + + #[test] + fn rejects_blank_recall_selectors() { + let input = r#"{ + "name": "blank recall selectors", + "category": "constraint_recall", + "query": "proof loop", + "expected_recall": [ + {"memory_id": " ", "ring": "", "tag": " ", "source_ref": "\t"} + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("expected_recall")); + assert!(error.contains("must include a nonblank")); + } + + #[test] + fn uses_workflow_prompt_when_query_is_blank() { + let input = r#"{ + "name": "workflow prompt fallback", + "category": "behavior_proof", + "query": " ", + "workflow_prompt": " validate behavior proof " + }"#; + + let scenario = parse_quality_scenario(input).unwrap(); + + assert_eq!(scenario.prompt(), Some(" validate behavior proof ")); + } } From d0ec95cc587659ad54da96051b80370879db28da Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 14:00:01 -0400 Subject: [PATCH 05/19] Add memory quality evaluator --- crates/tree-ring-memory-core/src/lib.rs | 6 +- crates/tree-ring-memory-core/src/quality.rs | 431 ++++++++++++++++++++ 2 files changed, 435 insertions(+), 2 deletions(-) diff --git a/crates/tree-ring-memory-core/src/lib.rs b/crates/tree-ring-memory-core/src/lib.rs index c0661c3..3a75fe1 100644 --- a/crates/tree-ring-memory-core/src/lib.rs +++ b/crates/tree-ring-memory-core/src/lib.rs @@ -28,8 +28,10 @@ pub use models::{ now_iso, MemoryEvent, MemoryLink, MemoryReview, MemorySource, TreeRingError, TreeRingResult, }; pub use quality::{ - parse_quality_scenario, QualityScenario, QualityThresholds, RecallExpectation, - WriteDecisionExpectation, QUALITY_CATEGORIES, WRITE_DECISIONS, + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, + QualityRunReport, QualityScenario, QualityScenarioReport, QualityThresholds, + RecallExpectation, RecallExpectationReport, WriteDecisionExpectation, WriteDecisionReport, + QUALITY_CATEGORIES, WRITE_DECISIONS, }; pub use recall::{RecallRanking, RecallScore, RecallScorer}; pub use revolve::{collect_revolve_memories, RevolveSyncReport, RevolveSyncRequest}; diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index dc7a904..f620235 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -199,6 +199,135 @@ impl QualityThresholds { } } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRecall { + pub memory: MemoryEvent, + pub score: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RecallExpectationReport { + pub expectation: RecallExpectation, + pub matched_memory_ids: Vec, + pub passed: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WriteDecisionReport { + pub memory_id: String, + pub expected: String, + pub actual: String, + pub passed: bool, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityScenarioReport { + pub name: String, + pub category: String, + pub constraint_recall_rate: f64, + pub forbidden_recall_rate: f64, + pub spam_rejection_rate: f64, + pub evidence_required_rate: f64, + pub behavior_proof_pass: bool, + pub quality_pass: bool, + pub expected_recall: Vec, + pub forbidden_recall: Vec, + pub write_decisions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct QualityRunReport { + pub ok: bool, + pub scenario_count: usize, + pub constraint_recall_rate: f64, + pub forbidden_recall_rate: f64, + pub spam_rejection_rate: f64, + pub evidence_required_rate: f64, + pub behavior_proof_pass_rate: f64, + pub quality_pass: bool, + pub scenarios: Vec, +} + +pub fn evaluate_quality_scenario( + scenario: &QualityScenario, + recalls: &[QualityRecall], +) -> TreeRingResult { + scenario.validate()?; + + let expected_recall = scenario + .expected_recall + .iter() + .map(|expectation| expected_recall_report(expectation, recalls)) + .collect::>(); + let forbidden_recall = scenario + .forbidden_recall + .iter() + .map(|expectation| forbidden_recall_report(expectation, recalls)) + .collect::>(); + let write_decisions = scenario + .expected_write_decisions + .iter() + .map(|expectation| write_decision_report(expectation, scenario)) + .collect::>(); + + let constraint_recall_rate = pass_rate(&expected_recall); + let forbidden_recall_rate = failure_rate(&forbidden_recall); + let spam_rejection_rate = decision_pass_rate(&write_decisions, "reject"); + let evidence_required_rate = decision_pass_rate(&write_decisions, "require_evidence"); + let behavior_proof_pass = + constraint_recall_rate >= scenario.thresholds.min_constraint_recall_rate + && forbidden_recall_rate <= scenario.thresholds.max_forbidden_recall_rate; + let quality_pass = behavior_proof_pass + && spam_rejection_rate >= scenario.thresholds.min_spam_rejection_rate + && evidence_required_rate >= scenario.thresholds.min_evidence_required_rate; + + Ok(QualityScenarioReport { + name: scenario.name.clone(), + category: scenario.category.clone(), + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass, + quality_pass, + expected_recall, + forbidden_recall, + write_decisions, + }) +} + +pub fn summarize_quality_run(reports: Vec) -> QualityRunReport { + let scenario_count = reports.len(); + let denominator = scenario_count.max(1) as f64; + let constraint_recall_rate = + reports.iter().map(|report| report.constraint_recall_rate).sum::() / denominator; + let forbidden_recall_rate = + reports.iter().map(|report| report.forbidden_recall_rate).sum::() / denominator; + let spam_rejection_rate = + reports.iter().map(|report| report.spam_rejection_rate).sum::() / denominator; + let evidence_required_rate = + reports.iter().map(|report| report.evidence_required_rate).sum::() / denominator; + let behavior_proof_pass_rate = reports + .iter() + .filter(|report| report.behavior_proof_pass) + .count() as f64 + / denominator; + let quality_pass = reports.iter().all(|report| report.quality_pass); + + QualityRunReport { + ok: quality_pass, + scenario_count, + constraint_recall_rate, + forbidden_recall_rate, + spam_rejection_rate, + evidence_required_rate, + behavior_proof_pass_rate, + quality_pass, + scenarios: reports, + } +} + pub fn parse_quality_scenario(input: &str) -> TreeRingResult { let scenario: QualityScenario = serde_json::from_str(input)?; scenario.validate()?; @@ -219,10 +348,164 @@ fn one() -> f64 { 1.0 } +fn expected_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: !matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn forbidden_recall_report( + expectation: &RecallExpectation, + recalls: &[QualityRecall], +) -> RecallExpectationReport { + let matched_memory_ids = matching_recall_ids(expectation, recalls); + RecallExpectationReport { + expectation: expectation.clone(), + passed: matched_memory_ids.is_empty(), + matched_memory_ids, + } +} + +fn matching_recall_ids(expectation: &RecallExpectation, recalls: &[QualityRecall]) -> Vec { + recalls + .iter() + .filter(|recall| expectation_matches_memory(expectation, &recall.memory)) + .map(|recall| recall.memory.id.clone()) + .collect() +} + +fn expectation_matches_memory(expectation: &RecallExpectation, memory: &MemoryEvent) -> bool { + if expectation + .memory_id + .as_deref() + .is_some_and(|id| memory.id != id) + { + return false; + } + if expectation + .ring + .as_deref() + .is_some_and(|ring| memory.ring != ring) + { + return false; + } + if expectation + .tag + .as_deref() + .is_some_and(|tag| !memory.tags.iter().any(|candidate| candidate == tag)) + { + return false; + } + if expectation + .source_ref + .as_deref() + .is_some_and(|source_ref| memory.source.ref_ != source_ref) + { + return false; + } + true +} + +fn write_decision_report( + expectation: &WriteDecisionExpectation, + scenario: &QualityScenario, +) -> WriteDecisionReport { + let actual = scenario + .write_candidates + .iter() + .find(|candidate| candidate.id == expectation.memory_id) + .map(|candidate| classify_write_candidate(candidate, &scenario.evidence_refs)) + .unwrap_or_else(|| "reject".to_string()); + WriteDecisionReport { + memory_id: expectation.memory_id.clone(), + expected: expectation.decision.clone(), + passed: actual == expectation.decision, + actual, + reason: expectation.reason.clone(), + } +} + +fn classify_write_candidate(candidate: &MemoryEvent, required_evidence_refs: &[String]) -> String { + if candidate.sensitivity == "secret" { + return "reject".to_string(); + } + if candidate.retention == "ephemeral" + || candidate.tags.iter().any(|tag| { + matches!( + tag.as_str(), + "transient" | "planning-chatter" | "scratchpad" | "tool-noise" + ) + }) + { + return "reject".to_string(); + } + if candidate.event_type.starts_with("evaluation_") && candidate.source.ref_.trim().is_empty() { + return "require_evidence".to_string(); + } + if !required_evidence_refs.is_empty() + && candidate.event_type.starts_with("evaluation_") + && !required_evidence_refs + .iter() + .any(|required| required == &candidate.source.ref_) + { + return "require_evidence".to_string(); + } + if candidate.ring == "heartwood" && candidate.source.ref_.trim().is_empty() { + return "require_user_confirmation".to_string(); + } + "accept".to_string() +} + +fn pass_rate(reports: &[RecallExpectationReport]) -> f64 { + if reports.is_empty() { + return 1.0; + } + reports.iter().filter(|report| report.passed).count() as f64 / reports.len() as f64 +} + +fn failure_rate(reports: &[RecallExpectationReport]) -> f64 { + if reports.is_empty() { + return 0.0; + } + reports.iter().filter(|report| !report.passed).count() as f64 / reports.len() as f64 +} + +fn decision_pass_rate(reports: &[WriteDecisionReport], decision: &str) -> f64 { + let relevant = reports + .iter() + .filter(|report| report.expected == decision) + .collect::>(); + if relevant.is_empty() { + return 1.0; + } + relevant.iter().filter(|report| report.passed).count() as f64 / relevant.len() as f64 +} + #[cfg(test)] mod tests { use super::*; + fn memory(id: &str, summary: &str, ring: &str) -> MemoryEvent { + let mut event = MemoryEvent::new(summary, "lesson").unwrap(); + event.id = id.to_string(); + event.created_at = "2026-07-09T00:00:00Z".to_string(); + event.updated_at = "2026-07-09T00:00:00Z".to_string(); + event.scope = "project".to_string(); + event.project = Some("tree-ring".to_string()); + event.ring = ring.to_string(); + event.source.source_type = "evidence".to_string(); + event.source.ref_ = "docs/spec.md".to_string(); + event.salience = 0.8; + event.confidence = 0.8; + event + } + #[test] fn parses_valid_quality_scenario() { let input = r#"{ @@ -337,4 +620,152 @@ mod tests { assert_eq!(scenario.prompt(), Some(" validate behavior proof ")); } + + #[test] + fn evaluates_required_and_forbidden_recall() { + let scenario = QualityScenario { + name: "recall gate".to_string(), + category: "constraint_recall".to_string(), + seed_memories: Vec::new(), + query: Some("background writer".to_string()), + workflow_prompt: None, + expected_recall: vec![RecallExpectation { + memory_id: Some("mem_required".to_string()), + reason: "required constraint".to_string(), + ..Default::default() + }], + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_forbidden".to_string()), + reason: "stale memory".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + scenario.validate().unwrap(); + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory("mem_required", "Do not add a background writer.", "heartwood"), + score: 0.91, + }], + ) + .unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.constraint_recall_rate, 1.0); + assert_eq!(report.forbidden_recall_rate, 0.0); + assert!(report.expected_recall[0].passed); + assert!(report.forbidden_recall[0].passed); + } + + #[test] + fn fails_when_forbidden_memory_is_recalled() { + let scenario = QualityScenario { + name: "stale recall".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: Vec::new(), + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_stale".to_string()), + reason: "stale CLI contract".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario( + &scenario, + &[QualityRecall { + memory: memory("mem_stale", "Old CLI contract.", "heartwood"), + score: 0.88, + }], + ) + .unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.forbidden_recall_rate, 1.0); + assert!(!report.forbidden_recall[0].passed); + } + + #[test] + fn classifies_write_candidates_against_expected_decisions() { + let mut spam = memory("mem_spam", "Thinking about options for a moment.", "heartwood"); + spam.tags = vec!["transient".to_string()]; + let mut promoted_without_evidence = + memory("mem_missing_evidence", "Promote evaluated proof.", "heartwood"); + promoted_without_evidence.event_type = "evaluation_promotion".to_string(); + promoted_without_evidence.source.ref_.clear(); + let mut broad_heartwood = + memory("mem_needs_confirmation", "All projects should prefer this.", "heartwood"); + broad_heartwood.source.ref_.clear(); + + let scenario = QualityScenario { + name: "write gates".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![spam, promoted_without_evidence, broad_heartwood], + expected_write_decisions: vec![ + WriteDecisionExpectation { + memory_id: "mem_spam".to_string(), + decision: "reject".to_string(), + reason: "transient planning chatter".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_missing_evidence".to_string(), + decision: "require_evidence".to_string(), + reason: "promotion needs evidence".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_needs_confirmation".to_string(), + decision: "require_user_confirmation".to_string(), + reason: "broad heartwood needs confirmation".to_string(), + }, + ], + evidence_refs: vec!["evals/run-001".to_string()], + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(report.quality_pass); + assert_eq!(report.spam_rejection_rate, 1.0); + assert_eq!(report.evidence_required_rate, 1.0); + assert_eq!(report.write_decisions.len(), 3); + } + + #[test] + fn summarizes_quality_run() { + let passing = QualityScenarioReport { + name: "pass".to_string(), + category: "constraint_recall".to_string(), + constraint_recall_rate: 1.0, + forbidden_recall_rate: 0.0, + spam_rejection_rate: 1.0, + evidence_required_rate: 1.0, + behavior_proof_pass: true, + quality_pass: true, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_decisions: Vec::new(), + }; + + let run = summarize_quality_run(vec![passing.clone(), passing]); + + assert!(run.quality_pass); + assert_eq!(run.scenario_count, 2); + assert_eq!(run.behavior_proof_pass_rate, 1.0); + } } From 8c72dec4dfb943e196a0928bfd293b0570f331d0 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 14:06:32 -0400 Subject: [PATCH 06/19] Close quality evaluator gate holes --- crates/tree-ring-memory-core/src/quality.rs | 247 +++++++++++++++++++- 1 file changed, 240 insertions(+), 7 deletions(-) diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index f620235..ba3bf6e 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; use crate::models::{MemoryEvent, TreeRingError, TreeRingResult}; @@ -81,6 +82,7 @@ impl QualityScenario { for decision in &self.expected_write_decisions { decision.validate(&self.name)?; } + validate_write_decision_coverage(self)?; for (index, evidence_ref) in self.evidence_refs.iter().enumerate() { if evidence_ref.trim().is_empty() { return Err(TreeRingError::Validation(format!( @@ -269,7 +271,7 @@ pub fn evaluate_quality_scenario( .expected_write_decisions .iter() .map(|expectation| write_decision_report(expectation, scenario)) - .collect::>(); + .collect::>>()?; let constraint_recall_rate = pass_rate(&expected_recall); let forbidden_recall_rate = failure_rate(&forbidden_recall); @@ -279,6 +281,7 @@ pub fn evaluate_quality_scenario( constraint_recall_rate >= scenario.thresholds.min_constraint_recall_rate && forbidden_recall_rate <= scenario.thresholds.max_forbidden_recall_rate; let quality_pass = behavior_proof_pass + && write_decisions.iter().all(|report| report.passed) && spam_rejection_rate >= scenario.thresholds.min_spam_rejection_rate && evidence_required_rate >= scenario.thresholds.min_evidence_required_rate; @@ -348,6 +351,52 @@ fn one() -> f64 { 1.0 } +fn validate_write_decision_coverage(scenario: &QualityScenario) -> TreeRingResult<()> { + let mut candidate_ids = HashSet::new(); + for (index, candidate) in scenario.write_candidates.iter().enumerate() { + if !candidate_ids.insert(candidate.id.as_str()) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} write_candidates[{index}] duplicates memory_id {}", + scenario.name, candidate.id + ))); + } + } + + let candidate_index_by_id = scenario + .write_candidates + .iter() + .enumerate() + .map(|(index, candidate)| (candidate.id.as_str(), index)) + .collect::>(); + let mut decision_index_by_id = HashMap::new(); + + for (index, decision) in scenario.expected_write_decisions.iter().enumerate() { + if let Some(previous_index) = decision_index_by_id.insert(decision.memory_id.as_str(), index) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} expected_write_decisions[{index}] duplicates memory_id {} from expected_write_decisions[{previous_index}]", + scenario.name, decision.memory_id + ))); + } + if !candidate_index_by_id.contains_key(decision.memory_id.as_str()) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} expected_write_decisions[{index}] memory_id {} does not match any write_candidate", + scenario.name, decision.memory_id + ))); + } + } + + for (index, candidate) in scenario.write_candidates.iter().enumerate() { + if !decision_index_by_id.contains_key(candidate.id.as_str()) { + return Err(TreeRingError::Validation(format!( + "quality scenario {} write_candidates[{index}] id {} is missing expected_write_decision coverage", + scenario.name, candidate.id + ))); + } + } + + Ok(()) +} + fn expected_recall_report( expectation: &RecallExpectation, recalls: &[QualityRecall], @@ -415,20 +464,25 @@ fn expectation_matches_memory(expectation: &RecallExpectation, memory: &MemoryEv fn write_decision_report( expectation: &WriteDecisionExpectation, scenario: &QualityScenario, -) -> WriteDecisionReport { - let actual = scenario +) -> TreeRingResult { + let candidate = scenario .write_candidates .iter() .find(|candidate| candidate.id == expectation.memory_id) - .map(|candidate| classify_write_candidate(candidate, &scenario.evidence_refs)) - .unwrap_or_else(|| "reject".to_string()); - WriteDecisionReport { + .ok_or_else(|| { + TreeRingError::Validation(format!( + "quality scenario {} expected write decision for memory_id {} could not resolve a write_candidate", + scenario.name, expectation.memory_id + )) + })?; + let actual = classify_write_candidate(candidate, &scenario.evidence_refs); + Ok(WriteDecisionReport { memory_id: expectation.memory_id.clone(), expected: expectation.decision.clone(), passed: actual == expectation.decision, actual, reason: expectation.reason.clone(), - } + }) } fn classify_write_candidate(candidate: &MemoryEvent, required_evidence_refs: &[String]) -> String { @@ -621,6 +675,104 @@ mod tests { assert_eq!(scenario.prompt(), Some(" validate behavior proof ")); } + #[test] + fn rejects_uncovered_write_candidate() { + let input = r#"{ + "name": "missing decision", + "category": "spam_prevention", + "query": "write gates", + "write_candidates": [ + { + "id": "mem_candidate", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Candidate without expected decision.", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + } + ] + }"#; + + let error = parse_quality_scenario(input).unwrap_err().to_string(); + + assert!(error.contains("write_candidates[0]")); + assert!(error.contains("missing expected_write_decision")); + } + + #[test] + fn rejects_orphan_write_decision_mapping() { + let scenario = QualityScenario { + name: "orphan decision".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory("mem_real", "Real candidate.", "heartwood")], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_missing".to_string(), + decision: "reject".to_string(), + reason: "orphan mapping".to_string(), + }], + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let error = scenario.validate().unwrap_err().to_string(); + + assert!(error.contains("expected_write_decisions[0]")); + assert!(error.contains("does not match any write_candidate")); + } + + #[test] + fn rejects_duplicate_write_decision_mapping() { + let scenario = QualityScenario { + name: "duplicate decision".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory("mem_real", "Real candidate.", "heartwood")], + expected_write_decisions: vec![ + WriteDecisionExpectation { + memory_id: "mem_real".to_string(), + decision: "reject".to_string(), + reason: "first mapping".to_string(), + }, + WriteDecisionExpectation { + memory_id: "mem_real".to_string(), + decision: "accept".to_string(), + reason: "duplicate mapping".to_string(), + }, + ], + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let error = scenario.validate().unwrap_err().to_string(); + + assert!(error.contains("expected_write_decisions[1]")); + assert!(error.contains("duplicate")); + } + #[test] fn evaluates_required_and_forbidden_recall() { let scenario = QualityScenario { @@ -746,6 +898,87 @@ mod tests { assert_eq!(report.write_decisions.len(), 3); } + #[test] + fn quality_pass_fails_when_expected_reject_is_accepted() { + let scenario = QualityScenario { + name: "reject mismatch".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory("mem_accept", "Durable evidence-backed note.", "cambium")], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_accept".to_string(), + decision: "reject".to_string(), + reason: "should have been rejected".to_string(), + }], + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.write_decisions[0].actual, "accept"); + assert!(!report.write_decisions[0].passed); + } + + #[test] + fn quality_pass_fails_when_expected_require_evidence_is_accepted() { + let scenario = QualityScenario { + name: "require evidence mismatch".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory("mem_accept", "Durable evidence-backed note.", "cambium")], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_accept".to_string(), + decision: "require_evidence".to_string(), + reason: "should need evidence".to_string(), + }], + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.write_decisions[0].actual, "accept"); + assert!(!report.write_decisions[0].passed); + } + + #[test] + fn quality_pass_fails_when_confirmation_mismatch_occurs() { + let scenario = QualityScenario { + name: "confirmation mismatch".to_string(), + category: "spam_prevention".to_string(), + seed_memories: Vec::new(), + query: Some("write gates".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: vec![memory("mem_accept", "Durable evidence-backed note.", "cambium")], + expected_write_decisions: vec![WriteDecisionExpectation { + memory_id: "mem_accept".to_string(), + decision: "require_user_confirmation".to_string(), + reason: "should require confirmation".to_string(), + }], + evidence_refs: Vec::new(), + thresholds: QualityThresholds::default(), + }; + + let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); + + assert!(!report.quality_pass); + assert_eq!(report.write_decisions[0].actual, "accept"); + assert!(!report.write_decisions[0].passed); + } + #[test] fn summarizes_quality_run() { let passing = QualityScenarioReport { From 19051b14305407478ba03b79accffe7c26ca2b7f Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 14:11:46 -0400 Subject: [PATCH 07/19] Add default memory quality fixtures --- crates/tree-ring-memory-core/src/quality.rs | 21 +++++ .../no-background-writer-constraint.json | 52 +++++++++++ fixtures/quality/scar-failure-recall.json | 46 ++++++++++ fixtures/quality/sensitive-memory-hidden.json | 45 ++++++++++ fixtures/quality/stale-cli-contract.json | 83 ++++++++++++++++++ fixtures/quality/superseded-heartwood.json | 83 ++++++++++++++++++ fixtures/quality/transient-planning-spam.json | 86 +++++++++++++++++++ 7 files changed, 416 insertions(+) create mode 100644 fixtures/quality/no-background-writer-constraint.json create mode 100644 fixtures/quality/scar-failure-recall.json create mode 100644 fixtures/quality/sensitive-memory-hidden.json create mode 100644 fixtures/quality/stale-cli-contract.json create mode 100644 fixtures/quality/superseded-heartwood.json create mode 100644 fixtures/quality/transient-planning-spam.json diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index ba3bf6e..bf87039 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -740,6 +740,27 @@ mod tests { assert!(error.contains("does not match any write_candidate")); } + #[test] + fn parses_default_quality_fixtures() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join("fixtures") + .join("quality"); + let mut parsed = 0usize; + for entry in std::fs::read_dir(&root).unwrap() { + let path = entry.unwrap().path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let input = std::fs::read_to_string(&path).unwrap(); + parse_quality_scenario(&input) + .unwrap_or_else(|err| panic!("{}: {err}", path.display())); + parsed += 1; + } + + assert_eq!(parsed, 6); + } + #[test] fn rejects_duplicate_write_decision_mapping() { let scenario = QualityScenario { diff --git a/fixtures/quality/no-background-writer-constraint.json b/fixtures/quality/no-background-writer-constraint.json new file mode 100644 index 0000000..88a0aba --- /dev/null +++ b/fixtures/quality/no-background-writer-constraint.json @@ -0,0 +1,52 @@ +{ + "name": "no-background-writer-constraint", + "category": "constraint_recall", + "query": "proof loop background writer hidden durable writes", + "seed_memories": [ + { + "id": "mem_quality_no_background_writer", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Do not add a background proof runner, hidden recorder, or autonomous durable writer to Tree Ring proof-loop work.", + "details": "Quality proof must remain explicit, visible, and certification-owned.", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#non-goals", + "quote": "" + }, + "tags": ["proof-loop", "agent-mediated", "no-daemon"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_no_background_writer", + "reason": "proof-loop work must recall the explicit no-background-writer constraint" + } + ], + "thresholds": { + "min_constraint_recall_rate": 1.0, + "max_forbidden_recall_rate": 0.0, + "min_spam_rejection_rate": 1.0, + "min_evidence_required_rate": 1.0, + "min_behavior_proof_pass_rate": 1.0 + } +} diff --git a/fixtures/quality/scar-failure-recall.json b/fixtures/quality/scar-failure-recall.json new file mode 100644 index 0000000..2d6668c --- /dev/null +++ b/fixtures/quality/scar-failure-recall.json @@ -0,0 +1,46 @@ +{ + "name": "scar-failure-recall", + "category": "behavior_proof", + "query": "failure stale cache rollback", + "seed_memories": [ + { + "id": "mem_scar_stale_cache", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "scar", + "event_type": "warning", + "summary": "Aggressive caching caused a stale rollback failure; recall this scar before cache-related workflow changes.", + "details": "", + "source": { + "type": "evidence", + "ref": "evals/cache-branch/run-013", + "quote": "" + }, + "tags": ["failure", "cache", "rollback"], + "salience": 0.95, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_scar_stale_cache", + "ring": "scar", + "reason": "failure-like queries should surface scar memory" + } + ] +} diff --git a/fixtures/quality/sensitive-memory-hidden.json b/fixtures/quality/sensitive-memory-hidden.json new file mode 100644 index 0000000..1dbeab4 --- /dev/null +++ b/fixtures/quality/sensitive-memory-hidden.json @@ -0,0 +1,45 @@ +{ + "name": "sensitive-memory-hidden", + "category": "stale_truth_suppression", + "query": "private diagnosis source metadata", + "seed_memories": [ + { + "id": "mem_sensitive_private_diagnosis", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "outer", + "event_type": "lesson", + "summary": "Private diagnosis source metadata should stay hidden by default.", + "details": "", + "source": { + "type": "manual", + "ref": "private-diagnosis-note", + "quote": "" + }, + "tags": ["privacy"], + "salience": 0.7, + "confidence": 0.8, + "sensitivity": "private", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_sensitive_private_diagnosis", + "reason": "private memory is hidden by default" + } + ] +} diff --git a/fixtures/quality/stale-cli-contract.json b/fixtures/quality/stale-cli-contract.json new file mode 100644 index 0000000..8472d27 --- /dev/null +++ b/fixtures/quality/stale-cli-contract.json @@ -0,0 +1,83 @@ +{ + "name": "stale-cli-contract", + "category": "stale_truth_suppression", + "query": "tree ring helper script cli contract remember recall forget", + "seed_memories": [ + { + "id": "mem_stale_cli_contract", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Old helper scripts can call remember recall and forget without required arguments.", + "details": "This stale CLI contract must not guide current helper-script reviews.", + "source": { + "type": "manual", + "ref": "old-review-note", + "quote": "" + }, + "tags": ["cli-contract", "stale"], + "salience": 0.8, + "confidence": 0.6, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_current_cli_contract", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_current_cli_contract", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "warning", + "summary": "Review Tree Ring helper scripts against the current CLI contract: remember needs --event-type, recall needs a query, and forget needs a memory id.", + "details": "Current source in crates/tree-ring-memory-cli/src/main.rs remains authoritative.", + "source": { + "type": "file", + "ref": "crates/tree-ring-memory-cli/src/main.rs", + "quote": "" + }, + "tags": ["cli-contract", "current"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_stale_cli_contract"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_current_cli_contract", + "reason": "current CLI contract should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_stale_cli_contract", + "reason": "superseded stale CLI contract must be suppressed" + } + ] +} diff --git a/fixtures/quality/superseded-heartwood.json b/fixtures/quality/superseded-heartwood.json new file mode 100644 index 0000000..f6a1259 --- /dev/null +++ b/fixtures/quality/superseded-heartwood.json @@ -0,0 +1,83 @@ +{ + "name": "superseded-heartwood", + "category": "stale_truth_suppression", + "query": "durable rule project bridge command", + "seed_memories": [ + { + "id": "mem_old_bridge_rule", + "created_at": "2026-07-08T00:00:00Z", + "updated_at": "2026-07-08T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should happen automatically during init.", + "details": "", + "source": { + "type": "manual", + "ref": "old-bridge-note", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.9, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": "mem_new_bridge_rule", + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_new_bridge_rule", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Durable rule: project bridge setup should be an explicit integrations link command, not automatic init behavior.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-06-tree-ring-agent-mediated-bridges-design.md#command-surface", + "quote": "" + }, + "tags": ["bridge", "durable-rule"], + "salience": 0.95, + "confidence": 0.95, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": ["mem_old_bridge_rule"], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_new_bridge_rule", + "reason": "replacement heartwood should be recalled" + } + ], + "forbidden_recall": [ + { + "memory_id": "mem_old_bridge_rule", + "reason": "superseded heartwood should not outrank the replacement" + } + ] +} diff --git a/fixtures/quality/transient-planning-spam.json b/fixtures/quality/transient-planning-spam.json new file mode 100644 index 0000000..a2182ab --- /dev/null +++ b/fixtures/quality/transient-planning-spam.json @@ -0,0 +1,86 @@ +{ + "name": "transient-planning-spam", + "category": "spam_prevention", + "query": "memory spam transient planning durable heartwood", + "seed_memories": [ + { + "id": "mem_quality_store_only_durable", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "decision", + "summary": "Store only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes.", + "details": "", + "source": { + "type": "evidence", + "ref": "docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md#agent-guidance-gates", + "quote": "" + }, + "tags": ["write-gate", "memory-quality"], + "salience": 0.9, + "confidence": 0.9, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_recall": [ + { + "memory_id": "mem_quality_store_only_durable", + "reason": "the write gate should be recalled before evaluating candidate memories" + } + ], + "write_candidates": [ + { + "id": "mem_candidate_planning_chatter", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "We discussed maybe doing the quality proof loop next.", + "details": "Transient planning chatter should not become durable heartwood.", + "source": { + "type": "manual", + "ref": "", + "quote": "" + }, + "tags": ["planning-chatter", "transient"], + "salience": 0.2, + "confidence": 0.4, + "sensitivity": "normal", + "retention": "normal", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_write_decisions": [ + { + "memory_id": "mem_candidate_planning_chatter", + "decision": "reject", + "reason": "transient planning chatter is memory spam" + } + ] +} From c17640504cfd98e45effafca5d1a27329138bf9f Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 14:30:05 -0400 Subject: [PATCH 08/19] Add resilient recall fallback --- crates/tree-ring-memory-sqlite/src/lib.rs | 171 +++++++++++++++++++++- 1 file changed, 168 insertions(+), 3 deletions(-) diff --git a/crates/tree-ring-memory-sqlite/src/lib.rs b/crates/tree-ring-memory-sqlite/src/lib.rs index f640bdb..45bd8e5 100644 --- a/crates/tree-ring-memory-sqlite/src/lib.rs +++ b/crates/tree-ring-memory-sqlite/src/lib.rs @@ -311,7 +311,32 @@ impl SQLiteMemoryStore { let Some(fts_query) = format_plain_text_fts_query(query) else { return Ok(Vec::new()); }; + self.search_fts_filtered_limited( + &fts_query, + project, + agent_profile, + scope, + rings, + event_types, + include_sensitive, + include_superseded, + limit, + ) + } + #[allow(clippy::too_many_arguments)] + fn search_fts_filtered_limited( + &self, + fts_query: &str, + project: Option<&str>, + agent_profile: Option<&str>, + scope: Option<&str>, + rings: Option<&[String]>, + event_types: Option<&[String]>, + include_sensitive: bool, + include_superseded: bool, + limit: Option, + ) -> TreeRingResult> { let mut sql = String::from( r#" SELECT memories.raw_json @@ -320,7 +345,7 @@ impl SQLiteMemoryStore { WHERE memory_fts MATCH ? "#, ); - let mut parameters = vec![Value::Text(fts_query)]; + let mut parameters = vec![Value::Text(fts_query.to_string())]; if !include_superseded { sql.push_str(" AND memories.superseded_by IS NULL"); @@ -803,11 +828,11 @@ impl<'a> MemoryRetriever<'a> { let mut candidates = Vec::new(); let mut seen_queries = HashSet::new(); + let candidate_limit = Some(limit.saturating_mul(128).clamp(256, 2048)); for search_query in search_queries(query) { if !seen_queries.insert(search_query.clone()) { continue; } - let candidate_limit = Some(limit.saturating_mul(128).clamp(256, 2048)); candidates = self.store.search_text_filtered_limited( &search_query, project, @@ -823,6 +848,21 @@ impl<'a> MemoryRetriever<'a> { break; } } + if candidates.is_empty() { + if let Some(fts_query) = format_plain_text_fts_or_query(query) { + candidates = self.store.search_fts_filtered_limited( + &fts_query, + project, + agent_profile, + scope, + rings, + event_types, + include_sensitive, + include_superseded, + candidate_limit, + )?; + } + } let mut results: Vec = candidates .into_iter() @@ -1199,6 +1239,14 @@ where } fn format_plain_text_fts_query(query: &str) -> Option { + format_plain_text_fts_query_with_operator(query, " AND ") +} + +fn format_plain_text_fts_or_query(query: &str) -> Option { + format_plain_text_fts_query_with_operator(query, " OR ") +} + +fn format_plain_text_fts_query_with_operator(query: &str, operator: &str) -> Option { let terms: Vec = tree_ring_memory_core::recall::terms(query) .into_iter() .filter(|term| !SEARCH_FILLER_TERMS.contains(&term.as_str())) @@ -1207,7 +1255,7 @@ fn format_plain_text_fts_query(query: &str) -> Option { if terms.is_empty() { return None; } - Some(terms.join(" AND ")) + Some(terms.join(operator)) } const SEARCH_FILLER_TERMS: &[&str] = &[ @@ -1405,6 +1453,123 @@ mod tests { assert_eq!(results[0].memory.id, target_id); } + #[test] + fn recall_strict_all_term_match_wins_before_fallback() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut partial = + MemoryEvent::new("Cache invalidation rollout reminder.", "lesson").unwrap(); + partial.salience = 1.0; + partial.confidence = 1.0; + let partial_id = partial.id.clone(); + let mut strict = + MemoryEvent::new("Cache invalidation canary rollout reminder.", "lesson").unwrap(); + strict.salience = 0.1; + strict.confidence = 0.1; + let strict_id = strict.id.clone(); + store.put(&partial).unwrap(); + store.put(&strict).unwrap(); + + let results = MemoryRetriever::new(&store) + .recall( + "cache invalidation canary", + None, + None, + None, + None, + None, + false, + false, + 8, + false, + ) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory.id, strict_id); + assert!(!results.iter().any(|result| result.memory.id == partial_id)); + } + + #[test] + fn recall_falls_back_to_partial_term_match_after_strict_misses() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let target = MemoryEvent::new( + "Quality adapter recalled approved fixture evidence.", + "lesson", + ) + .unwrap(); + let target_id = target.id.clone(); + store.put(&target).unwrap(); + + let results = MemoryRetriever::new(&store) + .recall( + "quality adapter missing transcript", + None, + None, + None, + None, + None, + false, + false, + 8, + false, + ) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory.id, target_id); + } + + #[test] + fn recall_fallback_keeps_default_filters_before_candidate_limit() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut events = Vec::new(); + for index in 0..300 { + let mut event = MemoryEvent::new( + format!("Workflow fallback recall distractor {index}"), + "lesson", + ) + .unwrap(); + event.project = Some("other-project".to_string()); + events.push(event); + } + let mut sensitive = + MemoryEvent::new("Workflow fallback recall sensitive target", "lesson").unwrap(); + sensitive.project = Some("target-project".to_string()); + sensitive.sensitivity = "private".to_string(); + events.push(sensitive); + let mut superseded = + MemoryEvent::new("Workflow fallback recall superseded target", "lesson").unwrap(); + superseded.project = Some("target-project".to_string()); + superseded.superseded_by = Some("replacement-memory".to_string()); + events.push(superseded); + let mut target = MemoryEvent::new("Workflow fallback recall target", "lesson").unwrap(); + target.project = Some("target-project".to_string()); + let target_id = target.id.clone(); + events.push(target); + store.put_many(&events).unwrap(); + + let results = MemoryRetriever::new(&store) + .recall( + "workflow fallback recall impossible", + Some("target-project"), + None, + None, + None, + None, + false, + false, + 1, + false, + ) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory.id, target_id); + } + #[test] fn redact_clears_fts_and_raw_payload() { let dir = tempdir().unwrap(); From c42488b666ed51a3431def1b251fa7de8bfbd078 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 14:34:58 -0400 Subject: [PATCH 09/19] Add memory quality scenario runner --- .../examples/quality_scenarios.rs | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 crates/tree-ring-memory-cli/examples/quality_scenarios.rs diff --git a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs new file mode 100644 index 0000000..c739609 --- /dev/null +++ b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs @@ -0,0 +1,221 @@ +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use tree_ring_memory_core::{ + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, + QualityRunReport, QualityScenario, QualityScenarioReport, +}; +use tree_ring_memory_sqlite::{MemoryRetriever, SQLiteMemoryStore}; + +const USAGE: &str = "usage: quality_scenarios "; +const RECALL_LIMIT: usize = 8; + +fn main() { + if let Err(err) = run() { + eprintln!("quality scenarios failed: {err}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let mut args = std::env::args_os().skip(1); + let fixture_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| USAGE.to_string())?; + let output_dir = args + .next() + .map(PathBuf::from) + .ok_or_else(|| USAGE.to_string())?; + if args.next().is_some() { + return Err(USAGE.to_string()); + } + + fs::create_dir_all(&output_dir) + .map_err(|err| format!("create output dir {}: {err}", output_dir.display()))?; + + let scenarios = load_scenarios(&fixture_dir)?; + let mut reports = Vec::with_capacity(scenarios.len()); + for scenario in &scenarios { + reports.push(run_scenario(scenario)?); + } + + let run_report = summarize_quality_run(reports); + write_reports(&output_dir, &run_report)?; + if !run_report.quality_pass { + return Err(format!( + "quality run failed: {} scenarios evaluated", + run_report.scenario_count + )); + } + + println!( + "quality scenarios passed: {} scenario(s)", + run_report.scenario_count + ); + Ok(()) +} + +fn load_scenarios(fixture_dir: &Path) -> Result, String> { + let mut paths = fs::read_dir(fixture_dir) + .map_err(|err| format!("read fixture dir {}: {err}", fixture_dir.display()))? + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(|err| err.to_string()) + }) + .collect::, _>>()?; + paths.sort(); + + let mut scenarios = Vec::new(); + for path in paths { + if path.extension() != Some(OsStr::new("json")) { + continue; + } + let input = fs::read_to_string(&path) + .map_err(|err| format!("read fixture {}: {err}", path.display()))?; + let scenario = + parse_quality_scenario(&input).map_err(|err| format!("{}: {err}", path.display()))?; + scenarios.push(scenario); + } + + if scenarios.is_empty() { + return Err(format!( + "no quality scenario json files in {}", + fixture_dir.display() + )); + } + + Ok(scenarios) +} + +fn run_scenario(scenario: &QualityScenario) -> Result { + let root = TemporaryRoot::new(&scenario.name)?; + let db_path = root.path().join("memory.sqlite"); + let mut store = SQLiteMemoryStore::open(&db_path) + .map_err(|err| format!("open scenario store {}: {err}", scenario.name))?; + store + .put_many(&scenario.seed_memories) + .map_err(|err| format!("seed scenario store {}: {err}", scenario.name))?; + + let prompt = scenario + .prompt() + .ok_or_else(|| format!("quality scenario {} has no prompt", scenario.name))?; + let recalls = MemoryRetriever::new(&store) + .recall( + prompt, + None, + None, + None, + None, + None, + false, + false, + RECALL_LIMIT, + true, + ) + .map_err(|err| format!("recall scenario {}: {err}", scenario.name))? + .into_iter() + .map(|result| QualityRecall { + memory: result.memory, + score: result.score, + }) + .collect::>(); + + evaluate_quality_scenario(scenario, &recalls) + .map_err(|err| format!("evaluate scenario {}: {err}", scenario.name)) +} + +fn write_reports(output_dir: &Path, report: &QualityRunReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report).map_err(|err| err.to_string())?; + fs::write(output_dir.join("quality-report.json"), json) + .map_err(|err| format!("write quality-report.json: {err}"))?; + fs::write( + output_dir.join("quality-summary.md"), + markdown_summary(report), + ) + .map_err(|err| format!("write quality-summary.md: {err}"))?; + Ok(()) +} + +fn markdown_summary(report: &QualityRunReport) -> String { + let mut lines = vec![ + "# Tree Ring Memory Quality Summary".to_string(), + String::new(), + format!("- quality pass: {}", report.quality_pass), + format!("- scenarios: {}", report.scenario_count), + format!( + "- constraint recall rate: {:.3}", + report.constraint_recall_rate + ), + format!( + "- forbidden recall rate: {:.3}", + report.forbidden_recall_rate + ), + format!("- spam rejection rate: {:.3}", report.spam_rejection_rate), + format!( + "- evidence required rate: {:.3}", + report.evidence_required_rate + ), + format!( + "- behavior proof pass rate: {:.3}", + report.behavior_proof_pass_rate + ), + String::new(), + "## Scenarios".to_string(), + String::new(), + ]; + + for scenario in &report.scenarios { + lines.push(format!( + "- `{}` [{}]: pass={}", + scenario.name, scenario.category, scenario.quality_pass + )); + } + + lines.push(String::new()); + lines.join("\n") +} + +struct TemporaryRoot { + path: PathBuf, +} + +impl TemporaryRoot { + fn new(name: &str) -> Result { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| err.to_string())? + .as_millis(); + let safe_name = name + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) + .collect::(); + let path = std::env::temp_dir().join(format!( + "tree-ring-quality-{}-{}-{safe_name}", + std::process::id(), + millis + )); + if path.exists() { + fs::remove_dir_all(&path) + .map_err(|err| format!("remove existing temp root {}: {err}", path.display()))?; + } + fs::create_dir_all(&path) + .map_err(|err| format!("create temp root {}: {err}", path.display()))?; + Ok(Self { path }) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TemporaryRoot { + fn drop(&mut self) { + if self.path.exists() { + let _ = fs::remove_dir_all(&self.path); + } + } +} From d3a8e50b912cda69382511afd3b30993055e9c2a Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 14:40:55 -0400 Subject: [PATCH 10/19] Format memory quality core --- crates/tree-ring-memory-core/src/lib.rs | 6 +- crates/tree-ring-memory-core/src/quality.rs | 98 ++++++++++++++++----- 2 files changed, 77 insertions(+), 27 deletions(-) diff --git a/crates/tree-ring-memory-core/src/lib.rs b/crates/tree-ring-memory-core/src/lib.rs index 3a75fe1..0688288 100644 --- a/crates/tree-ring-memory-core/src/lib.rs +++ b/crates/tree-ring-memory-core/src/lib.rs @@ -29,9 +29,9 @@ pub use models::{ }; pub use quality::{ evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, - QualityRunReport, QualityScenario, QualityScenarioReport, QualityThresholds, - RecallExpectation, RecallExpectationReport, WriteDecisionExpectation, WriteDecisionReport, - QUALITY_CATEGORIES, WRITE_DECISIONS, + QualityRunReport, QualityScenario, QualityScenarioReport, QualityThresholds, RecallExpectation, + RecallExpectationReport, WriteDecisionExpectation, WriteDecisionReport, QUALITY_CATEGORIES, + WRITE_DECISIONS, }; pub use recall::{RecallRanking, RecallScore, RecallScorer}; pub use revolve::{collect_revolve_memories, RevolveSyncReport, RevolveSyncRequest}; diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index bf87039..a2992e4 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -70,7 +70,11 @@ impl QualityScenario { self.name ))); } - for memory in self.seed_memories.iter().chain(self.write_candidates.iter()) { + for memory in self + .seed_memories + .iter() + .chain(self.write_candidates.iter()) + { memory.validate()?; } for expectation in &self.expected_recall { @@ -185,7 +189,10 @@ impl QualityThresholds { ), ("max_forbidden_recall_rate", self.max_forbidden_recall_rate), ("min_spam_rejection_rate", self.min_spam_rejection_rate), - ("min_evidence_required_rate", self.min_evidence_required_rate), + ( + "min_evidence_required_rate", + self.min_evidence_required_rate, + ), ( "min_behavior_proof_pass_rate", self.min_behavior_proof_pass_rate, @@ -277,9 +284,9 @@ pub fn evaluate_quality_scenario( let forbidden_recall_rate = failure_rate(&forbidden_recall); let spam_rejection_rate = decision_pass_rate(&write_decisions, "reject"); let evidence_required_rate = decision_pass_rate(&write_decisions, "require_evidence"); - let behavior_proof_pass = - constraint_recall_rate >= scenario.thresholds.min_constraint_recall_rate - && forbidden_recall_rate <= scenario.thresholds.max_forbidden_recall_rate; + let behavior_proof_pass = constraint_recall_rate + >= scenario.thresholds.min_constraint_recall_rate + && forbidden_recall_rate <= scenario.thresholds.max_forbidden_recall_rate; let quality_pass = behavior_proof_pass && write_decisions.iter().all(|report| report.passed) && spam_rejection_rate >= scenario.thresholds.min_spam_rejection_rate @@ -303,14 +310,26 @@ pub fn evaluate_quality_scenario( pub fn summarize_quality_run(reports: Vec) -> QualityRunReport { let scenario_count = reports.len(); let denominator = scenario_count.max(1) as f64; - let constraint_recall_rate = - reports.iter().map(|report| report.constraint_recall_rate).sum::() / denominator; - let forbidden_recall_rate = - reports.iter().map(|report| report.forbidden_recall_rate).sum::() / denominator; - let spam_rejection_rate = - reports.iter().map(|report| report.spam_rejection_rate).sum::() / denominator; - let evidence_required_rate = - reports.iter().map(|report| report.evidence_required_rate).sum::() / denominator; + let constraint_recall_rate = reports + .iter() + .map(|report| report.constraint_recall_rate) + .sum::() + / denominator; + let forbidden_recall_rate = reports + .iter() + .map(|report| report.forbidden_recall_rate) + .sum::() + / denominator; + let spam_rejection_rate = reports + .iter() + .map(|report| report.spam_rejection_rate) + .sum::() + / denominator; + let evidence_required_rate = reports + .iter() + .map(|report| report.evidence_required_rate) + .sum::() + / denominator; let behavior_proof_pass_rate = reports .iter() .filter(|report| report.behavior_proof_pass) @@ -371,7 +390,9 @@ fn validate_write_decision_coverage(scenario: &QualityScenario) -> TreeRingResul let mut decision_index_by_id = HashMap::new(); for (index, decision) in scenario.expected_write_decisions.iter().enumerate() { - if let Some(previous_index) = decision_index_by_id.insert(decision.memory_id.as_str(), index) { + if let Some(previous_index) = + decision_index_by_id.insert(decision.memory_id.as_str(), index) + { return Err(TreeRingError::Validation(format!( "quality scenario {} expected_write_decisions[{index}] duplicates memory_id {} from expected_write_decisions[{previous_index}]", scenario.name, decision.memory_id @@ -601,7 +622,10 @@ mod tests { assert_eq!(scenario.name, "constraint recall"); assert_eq!(scenario.category, "constraint_recall"); assert_eq!(scenario.seed_memories.len(), 1); - assert_eq!(scenario.expected_recall[0].memory_id.as_deref(), Some("mem_quality_constraint")); + assert_eq!( + scenario.expected_recall[0].memory_id.as_deref(), + Some("mem_quality_constraint") + ); } #[test] @@ -822,7 +846,11 @@ mod tests { let report = evaluate_quality_scenario( &scenario, &[QualityRecall { - memory: memory("mem_required", "Do not add a background writer.", "heartwood"), + memory: memory( + "mem_required", + "Do not add a background writer.", + "heartwood", + ), score: 0.91, }], ) @@ -871,14 +899,24 @@ mod tests { #[test] fn classifies_write_candidates_against_expected_decisions() { - let mut spam = memory("mem_spam", "Thinking about options for a moment.", "heartwood"); + let mut spam = memory( + "mem_spam", + "Thinking about options for a moment.", + "heartwood", + ); spam.tags = vec!["transient".to_string()]; - let mut promoted_without_evidence = - memory("mem_missing_evidence", "Promote evaluated proof.", "heartwood"); + let mut promoted_without_evidence = memory( + "mem_missing_evidence", + "Promote evaluated proof.", + "heartwood", + ); promoted_without_evidence.event_type = "evaluation_promotion".to_string(); promoted_without_evidence.source.ref_.clear(); - let mut broad_heartwood = - memory("mem_needs_confirmation", "All projects should prefer this.", "heartwood"); + let mut broad_heartwood = memory( + "mem_needs_confirmation", + "All projects should prefer this.", + "heartwood", + ); broad_heartwood.source.ref_.clear(); let scenario = QualityScenario { @@ -929,7 +967,11 @@ mod tests { workflow_prompt: None, expected_recall: Vec::new(), forbidden_recall: Vec::new(), - write_candidates: vec![memory("mem_accept", "Durable evidence-backed note.", "cambium")], + write_candidates: vec![memory( + "mem_accept", + "Durable evidence-backed note.", + "cambium", + )], expected_write_decisions: vec![WriteDecisionExpectation { memory_id: "mem_accept".to_string(), decision: "reject".to_string(), @@ -956,7 +998,11 @@ mod tests { workflow_prompt: None, expected_recall: Vec::new(), forbidden_recall: Vec::new(), - write_candidates: vec![memory("mem_accept", "Durable evidence-backed note.", "cambium")], + write_candidates: vec![memory( + "mem_accept", + "Durable evidence-backed note.", + "cambium", + )], expected_write_decisions: vec![WriteDecisionExpectation { memory_id: "mem_accept".to_string(), decision: "require_evidence".to_string(), @@ -983,7 +1029,11 @@ mod tests { workflow_prompt: None, expected_recall: Vec::new(), forbidden_recall: Vec::new(), - write_candidates: vec![memory("mem_accept", "Durable evidence-backed note.", "cambium")], + write_candidates: vec![memory( + "mem_accept", + "Durable evidence-backed note.", + "cambium", + )], expected_write_decisions: vec![WriteDecisionExpectation { memory_id: "mem_accept".to_string(), decision: "require_user_confirmation".to_string(), From 1e9afbafd9e6931e56fcc3877fac1d5fc0391469 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 14:51:18 -0400 Subject: [PATCH 11/19] Include memory quality in certification --- scripts/certify-tree-ring.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/certify-tree-ring.sh b/scripts/certify-tree-ring.sh index 2527263..e24063b 100755 --- a/scripts/certify-tree-ring.sh +++ b/scripts/certify-tree-ring.sh @@ -200,6 +200,16 @@ if [ "$EXTENDED" = "1" ]; then [ -n "$perf_50k_json" ] || fail "missing 50k performance metrics" fi +quality_out="$OUT_DIR/quality" +mkdir -p "$quality_out" +run cargo run --release -p tree-ring-memory-cli --example quality_scenarios -- "$ROOT/fixtures/quality" "$quality_out" \ + > "$OUT_DIR/quality-run.out" 2>&1 +require_file "$quality_out/quality-report.json" +require_file "$quality_out/quality-summary.md" +grep -F '"quality_pass": true' "$quality_out/quality-report.json" > /dev/null \ + || fail "memory quality scenarios did not pass" +quality_json=$(cat "$quality_out/quality-report.json") + agent_zero_status='"skipped"' agent_zero_note='"TREE_RING_AGENT_ZERO_ROOT not set"' if [ -n "$AGENT_ZERO_ROOT" ]; then @@ -247,6 +257,7 @@ cat > "$METRICS" < Date: Thu, 9 Jul 2026 15:01:00 -0400 Subject: [PATCH 12/19] Tighten certification quality artifact gating --- scripts/certify-tree-ring.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/certify-tree-ring.sh b/scripts/certify-tree-ring.sh index e24063b..ab157ee 100755 --- a/scripts/certify-tree-ring.sh +++ b/scripts/certify-tree-ring.sh @@ -86,6 +86,10 @@ mkdir -p "$OUT_DIR" SUMMARY="$OUT_DIR/summary.md" METRICS="$OUT_DIR/metrics.json" LOG="$OUT_DIR/certification.log" +QUALITY_OUT="$OUT_DIR/quality" +QUALITY_RUN_OUT="$OUT_DIR/quality-run.out" +rm -f "$SUMMARY" "$METRICS" "$QUALITY_RUN_OUT" +rm -rf "$QUALITY_OUT" : > "$LOG" log "certification output: $OUT_DIR" | tee -a "$LOG" @@ -200,15 +204,14 @@ if [ "$EXTENDED" = "1" ]; then [ -n "$perf_50k_json" ] || fail "missing 50k performance metrics" fi -quality_out="$OUT_DIR/quality" -mkdir -p "$quality_out" -run cargo run --release -p tree-ring-memory-cli --example quality_scenarios -- "$ROOT/fixtures/quality" "$quality_out" \ - > "$OUT_DIR/quality-run.out" 2>&1 -require_file "$quality_out/quality-report.json" -require_file "$quality_out/quality-summary.md" -grep -F '"quality_pass": true' "$quality_out/quality-report.json" > /dev/null \ +mkdir -p "$QUALITY_OUT" +run cargo run --release -p tree-ring-memory-cli --example quality_scenarios -- "$ROOT/fixtures/quality" "$QUALITY_OUT" \ + > "$QUALITY_RUN_OUT" 2>&1 +require_file "$QUALITY_OUT/quality-report.json" +require_file "$QUALITY_OUT/quality-summary.md" +grep -Fx ' "quality_pass": true,' "$QUALITY_OUT/quality-report.json" > /dev/null \ || fail "memory quality scenarios did not pass" -quality_json=$(cat "$quality_out/quality-report.json") +quality_json=$(cat "$QUALITY_OUT/quality-report.json") agent_zero_status='"skipped"' agent_zero_note='"TREE_RING_AGENT_ZERO_ROOT not set"' From a452cdc495f2d10c8f5c3c7f3595064d9a869699 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 15:09:24 -0400 Subject: [PATCH 13/19] Add memory quality guidance gates --- README.md | 7 +++ .../src/agent_awareness.rs | 60 +++++++++++++++++++ docs/integrations/agent-skill.md | 11 ++++ skills/tree-ring-memory/SKILL.md | 25 ++++++++ 4 files changed, 103 insertions(+) diff --git a/README.md b/README.md index 98a8b22..fd08cc7 100644 --- a/README.md +++ b/README.md @@ -390,6 +390,13 @@ Agent Zero plugin smoke enabled: - 30k recall max: 14.705 ms. - Agent Zero plugin smoke: passed. +Certification also runs the default memory quality scenario pack under +`fixtures/quality/`. Those scenarios prove recall gates, spam rejection, +stale-truth suppression, evidence requirements, and behavior-proof outcomes. +The quality report is written to +`target/tree-ring-certification/quality/quality-report.json` with a readable +summary at `target/tree-ring-certification/quality/quality-summary.md`. + `scripts/package-release.sh` builds the Rust CLI in release mode, creates a platform tarball under `dist/`, and writes a SHA-256 checksum file. Tag pushes run the release artifact workflow for Linux and macOS. diff --git a/crates/tree-ring-memory-cli/src/agent_awareness.rs b/crates/tree-ring-memory-cli/src/agent_awareness.rs index 66d1fd7..61aef89 100644 --- a/crates/tree-ring-memory-cli/src/agent_awareness.rs +++ b/crates/tree-ring-memory-cli/src/agent_awareness.rs @@ -51,6 +51,15 @@ Adapter rules: - Run adapter commands with `--dry-run` before writing memory. - `tree-ring integrations scan` is read-only; add harness bridge references manually until a link command is available. +Memory quality gates: + +- Before risky work, recall constraints, scars, user preferences, and unresolved seeds. +- Before trusting memory, prefer source-linked, non-superseded, high-confidence results. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Before writing memory, reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + Safety rules: - Do not store secrets, credentials, private keys, or raw chain-of-thought. @@ -154,6 +163,28 @@ call `tree-ring recall`, `tree-ring remember`, `tree-ring evidence`, `tree-ring forget`, `tree-ring consolidate --dry-run`, or `tree-ring maintain`. They do not authorize hidden transcript scraping or autonomous durable writes. +## Memory Quality Gates + +Recall gates: + +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. + +Trust gates: + +- Prefer source-linked, non-superseded, high-confidence memories. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Do not treat sensitive or hidden-by-default memory as ordinary recall context. + +Write gates: + +- Remember only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes. +- Reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + ## DOX Integration If this project uses DOX-style `AGENTS.md` traversal, merge the relevant @@ -234,6 +265,35 @@ mod tests { assert!(agents.contains("Tree Ring Memory Project Contract")); } + #[test] + fn generated_agents_file_mentions_quality_gates() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + + ensure_agent_awareness(&root).unwrap(); + let agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + + assert!(agents.contains("Memory Quality Gates")); + assert!(agents.contains("Recall gates")); + assert!(agents.contains("Trust gates")); + assert!(agents.contains("Write gates")); + assert!(agents.contains("Reject transient planning chatter")); + } + + #[test] + fn generated_cli_reference_mentions_quality_gates() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + + ensure_agent_awareness(&root).unwrap(); + let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + + assert!(cli.contains("Memory quality gates")); + assert!(cli.contains("Before risky work, recall constraints")); + assert!(cli.contains("Before trusting memory, prefer source-linked")); + assert!(cli.contains("Before writing memory, reject transient planning chatter")); + } + #[test] fn generated_agents_file_quotes_roots_with_spaces() { let dir = tempdir().unwrap(); diff --git a/docs/integrations/agent-skill.md b/docs/integrations/agent-skill.md index 291624a..6d80f44 100644 --- a/docs/integrations/agent-skill.md +++ b/docs/integrations/agent-skill.md @@ -83,6 +83,17 @@ Outcome mapping: Plain `remember` is still appropriate for user preferences, explicit decisions, and project lessons that do not come from a formal evaluated outcome. +## Memory Quality Gates + +Tree Ring guidance is meant to improve agent behavior, not increase memory volume. +Use these gates when wiring Tree Ring into an agent harness: + +- Recall gates: before substantial or risky work, recall constraints, scars, preferences, and unresolved seeds. +- Trust gates: prefer source-linked, non-superseded, high-confidence memories and re-read authoritative sources when memory conflicts with source files or user instructions. +- Write gates: reject transient planning chatter, duplicate wording, tool noise, and unsupported claims; require evidence refs for promoted or rejected evaluated outcomes. + +The certification suite includes quality scenarios that exercise missed constraints, memory spam, stale truth suppression, and behavior proof. + ## Source Adapter Flow Use DOX and Revolve adapters when the source artifacts already exist locally: diff --git a/skills/tree-ring-memory/SKILL.md b/skills/tree-ring-memory/SKILL.md index ea1855c..8c746ec 100644 --- a/skills/tree-ring-memory/SKILL.md +++ b/skills/tree-ring-memory/SKILL.md @@ -100,6 +100,31 @@ Evidence outcome mapping: - `deferred`: seed for promising unresolved options - `observed`: outer-ring evaluation result +## Memory Quality Gates + +Use these gates before relying on or writing memory. + +Recall gates: + +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. + +Trust gates: + +- Prefer source-linked, non-superseded, high-confidence memories. +- Treat heartwood as durable only when source evidence or user confirmation supports it. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Do not treat sensitive or hidden-by-default memory as ordinary recall context. + +Write gates: + +- Remember only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes. +- Reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + ## Ring Selection Use these rings: From 88811469d3d5b605944b843c040e5f76792b1bba Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 15:18:02 -0400 Subject: [PATCH 14/19] Backfill generated quality guidance gates --- .../src/agent_awareness.rs | 378 +++++++++++++++++- docs/integrations/agent-skill.md | 3 + 2 files changed, 374 insertions(+), 7 deletions(-) diff --git a/crates/tree-ring-memory-cli/src/agent_awareness.rs b/crates/tree-ring-memory-cli/src/agent_awareness.rs index 61aef89..fd5aada 100644 --- a/crates/tree-ring-memory-cli/src/agent_awareness.rs +++ b/crates/tree-ring-memory-cli/src/agent_awareness.rs @@ -4,6 +4,15 @@ use std::path::{Path, PathBuf}; const SKILL_TEMPLATE: &str = include_str!("../../../skills/tree-ring-memory/SKILL.md"); const DOX_TEMPLATE: &str = include_str!("../../../templates/dox/AGENTS.md"); +const AGENT_HEADER: &str = "# Tree Ring Memory Agent Instructions"; +const CLI_HEADER: &str = "# Tree Ring Memory CLI Quick Reference"; +const SKILL_FRONT_MATTER_MARKER: &str = "name: tree-ring-memory"; +const AGENT_QUALITY_GATES_HEADING: &str = "## Memory Quality Gates"; +const AGENT_QUALITY_GATES_ANCHOR: &str = "## DOX Integration"; +const CLI_QUALITY_GATES_HEADING: &str = "Memory quality gates:"; +const CLI_QUALITY_GATES_ANCHOR: &str = "Safety rules:"; +const SKILL_QUALITY_GATES_HEADING: &str = "## Memory Quality Gates"; +const SKILL_QUALITY_GATES_ANCHOR: &str = "## Ring Selection"; const CLI_REFERENCE: &str = r#"# Tree Ring Memory CLI Quick Reference Tree Ring Memory is a local-first memory lifecycle layer for AI agents. @@ -53,7 +62,10 @@ Adapter rules: Memory quality gates: -- Before risky work, recall constraints, scars, user preferences, and unresolved seeds. +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. - Before trusting memory, prefer source-linked, non-superseded, high-confidence results. - Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. - Before writing memory, reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. @@ -82,19 +94,61 @@ pub fn ensure_agent_awareness(root: &Path) -> Result bool, + section_heading: &str, + anchor: &str, + section: Option<&str>, ) -> Result<(), String> { if path.exists() { + maybe_backfill_generated_file(path, recognizer, section_heading, anchor, section)?; report.existing.push(path.to_path_buf()); return Ok(()); } @@ -106,9 +160,61 @@ fn write_if_missing( Ok(()) } +fn maybe_backfill_generated_file( + path: &Path, + recognizer: fn(&str) -> bool, + section_heading: &str, + anchor: &str, + section: Option<&str>, +) -> Result<(), String> { + let Some(section) = section else { + return Ok(()); + }; + + let existing = fs::read_to_string(path).map_err(|err| err.to_string())?; + if !recognizer(&existing) || existing.contains(section_heading) { + return Ok(()); + } + + let Some(updated) = insert_section_before_anchor(&existing, section, anchor) else { + return Ok(()); + }; + if updated != existing { + fs::write(path, updated).map_err(|err| err.to_string())?; + } + Ok(()) +} + +fn extract_section<'a>(content: &'a str, heading: &str, anchor: &str) -> Option<&'a str> { + let start = content.find(heading)?; + let end = content[start..].find(anchor)? + start; + Some(&content[start..end]) +} + +fn insert_section_before_anchor(content: &str, section: &str, anchor: &str) -> Option { + let anchor_index = content.find(anchor)?; + let mut updated = String::with_capacity(content.len() + section.len()); + updated.push_str(&content[..anchor_index]); + updated.push_str(section); + updated.push_str(&content[anchor_index..]); + Some(updated) +} + +fn is_generated_agents_file(content: &str) -> bool { + content.starts_with(AGENT_HEADER) +} + +fn is_generated_cli_file(content: &str) -> bool { + content.starts_with(CLI_HEADER) +} + +fn is_generated_skill_file(content: &str) -> bool { + content.starts_with("---\n") && content.contains(SKILL_FRONT_MATTER_MARKER) +} + fn agent_contract(root: &Path) -> String { format!( - r#"# Tree Ring Memory Agent Instructions + r#"{AGENT_HEADER} This directory contains Tree Ring Memory's local project memory store. @@ -212,6 +318,7 @@ outcomes become seeds, and observed outcomes become outer-ring evidence. {DOX_TEMPLATE} "#, + AGENT_HEADER = AGENT_HEADER, root = shell_path(root) ) } @@ -230,6 +337,75 @@ mod tests { use super::*; use tempfile::tempdir; + const TEST_AGENT_QUALITY_GATES_SECTION: &str = r#"## Memory Quality Gates + +Recall gates: + +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. + +Trust gates: + +- Prefer source-linked, non-superseded, high-confidence memories. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Do not treat sensitive or hidden-by-default memory as ordinary recall context. + +Write gates: + +- Remember only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes. +- Reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + +"#; + + const TEST_CLI_QUALITY_GATES_SECTION: &str = r#"Memory quality gates: + +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. +- Before trusting memory, prefer source-linked, non-superseded, high-confidence results. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Before writing memory, reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + +"#; + + const TEST_SKILL_QUALITY_GATES_SECTION: &str = r#"## Memory Quality Gates + +Use these gates before relying on or writing memory. + +Recall gates: + +- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. +- Before risky changes, recall warnings and evidence-linked prior failures. +- Before repeating a workflow, recall prior errors and accepted procedures. +- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. + +Trust gates: + +- Prefer source-linked, non-superseded, high-confidence memories. +- Treat heartwood as durable only when source evidence or user confirmation supports it. +- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. +- Do not treat sensitive or hidden-by-default memory as ordinary recall context. + +Write gates: + +- Remember only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes. +- Reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. +- Require evidence refs for promoted or rejected evaluated outcomes. +- Require user confirmation before creating or promoting broad cross-project heartwood. + +"#; + + const TEST_AGENT_QUALITY_GATES_ANCHOR: &str = "## DOX Integration"; + const TEST_CLI_QUALITY_GATES_ANCHOR: &str = "Safety rules:"; + const TEST_SKILL_QUALITY_GATES_ANCHOR: &str = "## Ring Selection"; + #[test] fn init_writes_agent_awareness_files_without_overwriting() { let dir = tempdir().unwrap(); @@ -289,11 +465,199 @@ mod tests { let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); assert!(cli.contains("Memory quality gates")); - assert!(cli.contains("Before risky work, recall constraints")); + assert!(cli.contains("Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds.")); + assert!(cli + .contains("Before risky changes, recall warnings and evidence-linked prior failures.")); + assert!(cli + .contains("Before repeating a workflow, recall prior errors and accepted procedures.")); + assert!(cli.contains("Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons.")); assert!(cli.contains("Before trusting memory, prefer source-linked")); assert!(cli.contains("Before writing memory, reject transient planning chatter")); } + #[test] + fn generated_backfills_quality_gates_into_recognized_stale_generated_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + + fs::write( + root.join("AGENTS.md"), + agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + + let report = ensure_agent_awareness(&root).unwrap(); + assert_eq!(report.created.len(), 0); + assert_eq!(report.existing.len(), 3); + + let agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + assert!(agents.contains(TEST_AGENT_QUALITY_GATES_SECTION.trim())); + assert!(cli.contains(TEST_CLI_QUALITY_GATES_SECTION.trim())); + assert!(skill.contains(TEST_SKILL_QUALITY_GATES_SECTION.trim())); + } + + #[test] + fn generated_backfill_inserts_quality_gates_at_canonical_anchors() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + + fs::write( + root.join("AGENTS.md"), + agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + let agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + assert!(agents.contains(&format!( + "{}{}", + TEST_AGENT_QUALITY_GATES_SECTION, TEST_AGENT_QUALITY_GATES_ANCHOR + ))); + assert!(cli.contains(&format!( + "{}{}", + TEST_CLI_QUALITY_GATES_SECTION, TEST_CLI_QUALITY_GATES_ANCHOR + ))); + assert!(skill.contains(&format!( + "{}{}", + TEST_SKILL_QUALITY_GATES_SECTION, TEST_SKILL_QUALITY_GATES_ANCHOR + ))); + } + + #[test] + fn generated_backfill_preserves_custom_content_in_recognized_generated_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + + let agents_custom = "\nCustom note: keep me.\n"; + let cli_custom = "\nCustom alias note: keep me.\n"; + let skill_custom = "\nCustom workflow note: keep me.\n"; + + fs::write( + root.join("AGENTS.md"), + agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, "") + agents_custom, + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, "") + cli_custom, + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, "") + skill_custom, + ) + .unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + assert!(fs::read_to_string(root.join("AGENTS.md")) + .unwrap() + .ends_with(agents_custom)); + assert!(fs::read_to_string(root.join("CLI.md")) + .unwrap() + .ends_with(cli_custom)); + assert!(fs::read_to_string(root.join("SKILL.md")) + .unwrap() + .ends_with(skill_custom)); + } + + #[test] + fn generated_backfill_leaves_arbitrary_custom_files_untouched() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + + fs::write(root.join("AGENTS.md"), "# Custom project contract\n").unwrap(); + fs::write(root.join("CLI.md"), "# Custom CLI guide\n").unwrap(); + fs::write(root.join("SKILL.md"), "# Custom skill\n").unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + assert_eq!( + fs::read_to_string(root.join("AGENTS.md")).unwrap(), + "# Custom project contract\n" + ); + assert_eq!( + fs::read_to_string(root.join("CLI.md")).unwrap(), + "# Custom CLI guide\n" + ); + assert_eq!( + fs::read_to_string(root.join("SKILL.md")).unwrap(), + "# Custom skill\n" + ); + } + + #[test] + fn generated_backfill_is_idempotent_for_recognized_stale_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + + fs::write( + root.join("AGENTS.md"), + agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + fs::write( + root.join("CLI.md"), + CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + fs::write( + root.join("SKILL.md"), + SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, ""), + ) + .unwrap(); + + ensure_agent_awareness(&root).unwrap(); + let first_agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let first_cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let first_skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + let second_agents = fs::read_to_string(root.join("AGENTS.md")).unwrap(); + let second_cli = fs::read_to_string(root.join("CLI.md")).unwrap(); + let second_skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + + assert_eq!(first_agents, second_agents); + assert_eq!(first_cli, second_cli); + assert_eq!(first_skill, second_skill); + assert_eq!(second_agents.matches("## Memory Quality Gates").count(), 1); + assert_eq!(second_cli.matches("Memory quality gates:").count(), 1); + assert_eq!(second_skill.matches("## Memory Quality Gates").count(), 1); + } + #[test] fn generated_agents_file_quotes_roots_with_spaces() { let dir = tempdir().unwrap(); diff --git a/docs/integrations/agent-skill.md b/docs/integrations/agent-skill.md index 6d80f44..7598338 100644 --- a/docs/integrations/agent-skill.md +++ b/docs/integrations/agent-skill.md @@ -93,6 +93,9 @@ Use these gates when wiring Tree Ring into an agent harness: - Write gates: reject transient planning chatter, duplicate wording, tool noise, and unsupported claims; require evidence refs for promoted or rejected evaluated outcomes. The certification suite includes quality scenarios that exercise missed constraints, memory spam, stale truth suppression, and behavior proof. +Quality artifacts are written to +`target/tree-ring-certification/quality/quality-report.json` and +`target/tree-ring-certification/quality/quality-summary.md`. ## Source Adapter Flow From 47cd238fd4c15ec6dc808556b6552e055e537f97 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 15:23:39 -0400 Subject: [PATCH 15/19] Add ambient ring fullness styling --- crates/tree-ring-memory-cli/src/tui/model.rs | 26 +++++++++++++++ crates/tree-ring-memory-cli/src/tui/rings.rs | 35 ++++++++++++++++++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/crates/tree-ring-memory-cli/src/tui/model.rs b/crates/tree-ring-memory-cli/src/tui/model.rs index 578905a..fbc1a25 100644 --- a/crates/tree-ring-memory-cli/src/tui/model.rs +++ b/crates/tree-ring-memory-cli/src/tui/model.rs @@ -14,6 +14,7 @@ pub struct RingStats { pub average_confidence: f64, pub newest_at: Option, pub oldest_at: Option, + pub fullness_level: f64, pub pulse_level: f64, pub warning_level: f64, } @@ -30,6 +31,7 @@ impl RingStats { average_confidence: 0.0, newest_at: None, oldest_at: None, + fullness_level: 0.0, pulse_level: 0.0, warning_level: 0.0, } @@ -106,6 +108,11 @@ impl DashboardStats { stats.average_salience /= stats.total as f64; stats.average_confidence /= stats.total as f64; } + stats.fullness_level = if dashboard.total == 0 { + 0.0 + } else { + stats.total as f64 / dashboard.total as f64 + }; stats.warning_level = ring_warning_level(stats); if let Some(previous_stats) = previous.and_then(|previous| previous.ring(&stats.ring)) { if stats.total != previous_stats.total { @@ -213,4 +220,23 @@ mod tests { assert_eq!(second.ring("cambium").unwrap().pulse_level, 1.0); } + + #[test] + fn derives_relative_ring_fullness() { + let dashboard = DashboardStats::from_memories( + &[ + event("Fresh one", "cambium", "lesson"), + event("Fresh two", "cambium", "lesson"), + event("Durable", "heartwood", "decision"), + ], + None, + ); + + assert_eq!(dashboard.ring("cambium").unwrap().fullness_level, 2.0 / 3.0); + assert_eq!( + dashboard.ring("heartwood").unwrap().fullness_level, + 1.0 / 3.0 + ); + assert_eq!(dashboard.ring("outer").unwrap().fullness_level, 0.0); + } } diff --git a/crates/tree-ring-memory-cli/src/tui/rings.rs b/crates/tree-ring-memory-cli/src/tui/rings.rs index 90a68eb..d2fc91a 100644 --- a/crates/tree-ring-memory-cli/src/tui/rings.rs +++ b/crates/tree-ring-memory-cli/src/tui/rings.rs @@ -321,13 +321,17 @@ fn animated_color(ring: &str, stats: Option<&RingStats>, tick: u64, offset: u64) if stats.total == 0 { return dim_color(base, 0.34); } + let fullness = stats.fullness_level.clamp(0.0, 1.0); if stats.pulse_level > 0.05 && (tick + offset) % 6 < 3 { - return brighten_color(base, 0.34 + (stats.pulse_level * 0.18)); + return brighten_color(base, 0.28 + fullness * 0.16 + stats.pulse_level * 0.18); } if (tick + offset) % 18 < 3 { - return brighten_color(base, 0.18); + return brighten_color(base, 0.08 + fullness * 0.16); } - base + if fullness < 0.18 { + return dim_color(base, 0.42 + fullness * 1.35); + } + brighten_color(base, fullness * 0.20) } fn brighten_color(color: Color, amount: f64) -> Color { @@ -422,6 +426,20 @@ mod tests { use super::*; use tree_ring_memory_core::MemoryEvent; + fn brightness_sum(color: Color) -> u16 { + match color { + Color::Rgb(red, green, blue) => red as u16 + green as u16 + blue as u16, + _ => 0, + } + } + + fn stats_with_fullness(ring: &str, total: usize, fullness_level: f64) -> RingStats { + let mut stats = RingStats::empty(ring); + stats.total = total; + stats.fullness_level = fullness_level; + stats + } + #[test] fn ambient_lines_include_all_core_ring_labels() { let mut memory = MemoryEvent::new("Durable", "lesson").unwrap(); @@ -485,4 +503,15 @@ mod tests { assert_ne!(early, later); } + + #[test] + fn fullness_brightens_ambient_ring_without_pulse() { + let low = stats_with_fullness("cambium", 1, 0.10); + let high = stats_with_fullness("cambium", 8, 0.80); + + let low_color = animated_color("cambium", Some(&low), 10, 0); + let high_color = animated_color("cambium", Some(&high), 10, 0); + + assert!(brightness_sum(high_color) > brightness_sum(low_color)); + } } From ff5393519b838e86a61e72b2f9560ee4d591e4e9 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 15:59:17 -0400 Subject: [PATCH 16/19] Harden memory quality proof semantics --- .../examples/quality_scenarios.rs | 298 +++++++-- .../src/agent_awareness.rs | 237 ++++--- crates/tree-ring-memory-core/src/lib.rs | 8 +- crates/tree-ring-memory-core/src/quality.rs | 596 +++++++++++++++--- ...-quality-proof-loop-implementation-plan.md | 16 + ...e-ring-memory-quality-proof-loop-design.md | 47 +- fixtures/quality/evidence-preservation.json | 84 +++ fixtures/quality/scar-failure-recall.json | 9 +- 8 files changed, 1056 insertions(+), 239 deletions(-) create mode 100644 fixtures/quality/evidence-preservation.json diff --git a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs index c739609..6c63b5f 100644 --- a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs +++ b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs @@ -5,7 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tree_ring_memory_core::{ evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, - QualityRunReport, QualityScenario, QualityScenarioReport, + QualityRunError, QualityRunReport, QualityScenario, QualityScenarioReport, }; use tree_ring_memory_sqlite::{MemoryRetriever, SQLiteMemoryStore}; @@ -33,17 +33,7 @@ fn run() -> Result<(), String> { return Err(USAGE.to_string()); } - fs::create_dir_all(&output_dir) - .map_err(|err| format!("create output dir {}: {err}", output_dir.display()))?; - - let scenarios = load_scenarios(&fixture_dir)?; - let mut reports = Vec::with_capacity(scenarios.len()); - for scenario in &scenarios { - reports.push(run_scenario(scenario)?); - } - - let run_report = summarize_quality_run(reports); - write_reports(&output_dir, &run_report)?; + let run_report = run_quality_scenarios(&fixture_dir, &output_dir)?; if !run_report.quality_pass { return Err(format!( "quality run failed: {} scenarios evaluated", @@ -58,51 +48,156 @@ fn run() -> Result<(), String> { Ok(()) } -fn load_scenarios(fixture_dir: &Path) -> Result, String> { - let mut paths = fs::read_dir(fixture_dir) - .map_err(|err| format!("read fixture dir {}: {err}", fixture_dir.display()))? +fn run_quality_scenarios( + fixture_dir: &Path, + output_dir: &Path, +) -> Result { + fs::create_dir_all(output_dir) + .map_err(|err| format!("create output dir {}: {err}", output_dir.display()))?; + + let paths = match fixture_paths(fixture_dir) { + Ok(paths) if !paths.is_empty() => paths, + Ok(_) => { + return write_failed_report( + output_dir, + Vec::new(), + QualityRunError { + scenario: None, + path: Some(fixture_dir.display().to_string()), + stage: "fixture_directory".to_string(), + message: "no quality scenario JSON files found".to_string(), + }, + ); + } + Err(error) => return write_failed_report(output_dir, Vec::new(), error), + }; + + let mut reports = Vec::with_capacity(paths.len()); + for path in paths { + let input = match fs::read_to_string(&path) { + Ok(input) => input, + Err(err) => { + return write_failed_report( + output_dir, + reports, + QualityRunError { + scenario: None, + path: Some(path.display().to_string()), + stage: "file_read".to_string(), + message: err.to_string(), + }, + ); + } + }; + let scenario_name = scenario_name_hint(&input); + let scenario = match parse_quality_scenario(&input) { + Ok(scenario) => scenario, + Err(err) => { + return write_failed_report( + output_dir, + reports, + QualityRunError { + scenario: scenario_name, + path: Some(path.display().to_string()), + stage: "parse".to_string(), + message: err.to_string(), + }, + ); + } + }; + match run_scenario(&scenario) { + Ok(report) => reports.push(report), + Err(failure) => { + return write_failed_report( + output_dir, + reports, + QualityRunError { + scenario: Some(scenario.name), + path: Some(path.display().to_string()), + stage: failure.stage.to_string(), + message: failure.message, + }, + ); + } + } + } + + let report = summarize_quality_run(reports); + write_reports(output_dir, &report)?; + Ok(report) +} + +fn fixture_paths(fixture_dir: &Path) -> Result, QualityRunError> { + let entries = fs::read_dir(fixture_dir).map_err(|err| QualityRunError { + scenario: None, + path: Some(fixture_dir.display().to_string()), + stage: "fixture_directory".to_string(), + message: err.to_string(), + })?; + let mut paths = entries .map(|entry| { entry .map(|entry| entry.path()) - .map_err(|err| err.to_string()) + .map_err(|err| QualityRunError { + scenario: None, + path: Some(fixture_dir.display().to_string()), + stage: "fixture_directory".to_string(), + message: err.to_string(), + }) }) .collect::, _>>()?; + paths.retain(|path| path.extension() == Some(OsStr::new("json"))); paths.sort(); + Ok(paths) +} - let mut scenarios = Vec::new(); - for path in paths { - if path.extension() != Some(OsStr::new("json")) { - continue; - } - let input = fs::read_to_string(&path) - .map_err(|err| format!("read fixture {}: {err}", path.display()))?; - let scenario = - parse_quality_scenario(&input).map_err(|err| format!("{}: {err}", path.display()))?; - scenarios.push(scenario); - } +fn scenario_name_hint(input: &str) -> Option { + serde_json::from_str::(input) + .ok()? + .get("name")? + .as_str() + .map(str::to_string) +} - if scenarios.is_empty() { - return Err(format!( - "no quality scenario json files in {}", - fixture_dir.display() - )); - } +fn write_failed_report( + output_dir: &Path, + reports: Vec, + error: QualityRunError, +) -> Result { + let mut report = summarize_quality_run(reports); + report.ok = false; + report.quality_pass = false; + report.errors.push(error); + write_reports(output_dir, &report)?; + Ok(report) +} - Ok(scenarios) +struct ScenarioRunFailure { + stage: &'static str, + message: String, } -fn run_scenario(scenario: &QualityScenario) -> Result { - let root = TemporaryRoot::new(&scenario.name)?; +fn run_scenario(scenario: &QualityScenario) -> Result { + let root = TemporaryRoot::new(&scenario.name).map_err(|message| ScenarioRunFailure { + stage: "store_open", + message, + })?; let db_path = root.path().join("memory.sqlite"); - let mut store = SQLiteMemoryStore::open(&db_path) - .map_err(|err| format!("open scenario store {}: {err}", scenario.name))?; + let mut store = SQLiteMemoryStore::open(&db_path).map_err(|err| ScenarioRunFailure { + stage: "store_open", + message: err.to_string(), + })?; store .put_many(&scenario.seed_memories) - .map_err(|err| format!("seed scenario store {}: {err}", scenario.name))?; + .map_err(|err| ScenarioRunFailure { + stage: "seed", + message: err.to_string(), + })?; - let prompt = scenario - .prompt() - .ok_or_else(|| format!("quality scenario {} has no prompt", scenario.name))?; + let prompt = scenario.prompt().ok_or_else(|| ScenarioRunFailure { + stage: "evaluation", + message: "validated scenario has no prompt".to_string(), + })?; let recalls = MemoryRetriever::new(&store) .recall( prompt, @@ -116,7 +211,10 @@ fn run_scenario(scenario: &QualityScenario) -> Result Result>(); - evaluate_quality_scenario(scenario, &recalls) - .map_err(|err| format!("evaluate scenario {}: {err}", scenario.name)) + evaluate_quality_scenario(scenario, &recalls).map_err(|err| ScenarioRunFailure { + stage: "evaluation", + message: err.to_string(), + }) } fn write_reports(output_dir: &Path, report: &QualityRunReport) -> Result<(), String> { @@ -147,21 +247,24 @@ fn markdown_summary(report: &QualityRunReport) -> String { format!("- quality pass: {}", report.quality_pass), format!("- scenarios: {}", report.scenario_count), format!( - "- constraint recall rate: {:.3}", - report.constraint_recall_rate + "- constraint recall rate: {}", + format_rate(report.constraint_recall_rate) + ), + format!( + "- forbidden recall rate: {}", + format_rate(report.forbidden_recall_rate) ), format!( - "- forbidden recall rate: {:.3}", - report.forbidden_recall_rate + "- spam rejection rate: {}", + format_rate(report.spam_rejection_rate) ), - format!("- spam rejection rate: {:.3}", report.spam_rejection_rate), format!( - "- evidence required rate: {:.3}", - report.evidence_required_rate + "- evidence required rate: {}", + format_rate(report.evidence_required_rate) ), format!( - "- behavior proof pass rate: {:.3}", - report.behavior_proof_pass_rate + "- behavior proof pass rate: {}", + format_rate(report.behavior_proof_pass_rate) ), String::new(), "## Scenarios".to_string(), @@ -175,10 +278,26 @@ fn markdown_summary(report: &QualityRunReport) -> String { )); } + if !report.errors.is_empty() { + lines.extend([String::new(), "## Errors".to_string(), String::new()]); + for error in &report.errors { + let scenario = error.scenario.as_deref().unwrap_or("unknown"); + let path = error.path.as_deref().unwrap_or("unknown"); + lines.push(format!( + "- stage={} scenario=`{scenario}` path=`{path}`: {}", + error.stage, error.message + )); + } + } + lines.push(String::new()); lines.join("\n") } +fn format_rate(rate: Option) -> String { + rate.map_or_else(|| "n/a".to_string(), |rate| format!("{rate:.3}")) +} + struct TemporaryRoot { path: PathBuf, } @@ -219,3 +338,74 @@ impl Drop for TemporaryRoot { } } } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn copy_valid_fixture(fixture_dir: &Path, name: &str) { + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join("fixtures/quality/no-background-writer-constraint.json"); + fs::copy(source, fixture_dir.join(name)).unwrap(); + } + + #[test] + fn writes_partial_artifacts_when_a_later_fixture_fails_to_parse() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + copy_valid_fixture(fixtures.path(), "01-valid.json"); + fs::write( + fixtures.path().join("02-private-invalid.json"), + r#"{ + "name": "invalid-private-fixture", + "category": "constraint_recall", + "query": "private-memory-payload", + "expected_recal": [] + }"#, + ) + .unwrap(); + + let report = run_quality_scenarios(fixtures.path(), output.path()).unwrap(); + + assert!(!report.ok); + assert!(!report.quality_pass); + assert_eq!(report.scenario_count, 1); + assert_eq!(report.errors.len(), 1); + assert_eq!(report.errors[0].stage, "parse"); + assert_eq!( + report.errors[0].scenario.as_deref(), + Some("invalid-private-fixture") + ); + assert!(report.errors[0] + .path + .as_deref() + .is_some_and(|path| path.ends_with("02-private-invalid.json"))); + + let json = fs::read_to_string(output.path().join("quality-report.json")).unwrap(); + let markdown = fs::read_to_string(output.path().join("quality-summary.md")).unwrap(); + assert!(!json.contains("private-memory-payload")); + assert!(!markdown.contains("private-memory-payload")); + assert!(markdown.contains("## Errors")); + assert!(markdown.contains("stage=parse")); + } + + #[test] + fn writes_failed_artifacts_for_an_empty_fixture_directory() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + + let report = run_quality_scenarios(fixtures.path(), output.path()).unwrap(); + + assert!(!report.ok); + assert!(!report.quality_pass); + assert_eq!(report.scenario_count, 0); + assert_eq!(report.errors.len(), 1); + assert_eq!(report.errors[0].stage, "fixture_directory"); + assert!(output.path().join("quality-report.json").is_file()); + let markdown = fs::read_to_string(output.path().join("quality-summary.md")).unwrap(); + assert!(markdown.contains("constraint recall rate: n/a")); + assert!(markdown.contains("## Errors")); + } +} diff --git a/crates/tree-ring-memory-cli/src/agent_awareness.rs b/crates/tree-ring-memory-cli/src/agent_awareness.rs index fd5aada..8334aaa 100644 --- a/crates/tree-ring-memory-cli/src/agent_awareness.rs +++ b/crates/tree-ring-memory-cli/src/agent_awareness.rs @@ -186,9 +186,23 @@ fn maybe_backfill_generated_file( } fn extract_section<'a>(content: &'a str, heading: &str, anchor: &str) -> Option<&'a str> { + let (start, end) = managed_section_range(content, heading, anchor)?; + Some(&content[start..end]) +} + +#[cfg(test)] +fn remove_managed_section(content: &str, heading: &str, anchor: &str) -> Option { + let (start, end) = managed_section_range(content, heading, anchor)?; + let mut stale = String::with_capacity(content.len() - (end - start)); + stale.push_str(&content[..start]); + stale.push_str(&content[end..]); + Some(stale) +} + +fn managed_section_range(content: &str, heading: &str, anchor: &str) -> Option<(usize, usize)> { let start = content.find(heading)?; let end = content[start..].find(anchor)? + start; - Some(&content[start..end]) + Some((start, end)) } fn insert_section_before_anchor(content: &str, section: &str, anchor: &str) -> Option { @@ -337,74 +351,32 @@ mod tests { use super::*; use tempfile::tempdir; - const TEST_AGENT_QUALITY_GATES_SECTION: &str = r#"## Memory Quality Gates - -Recall gates: - -- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. -- Before risky changes, recall warnings and evidence-linked prior failures. -- Before repeating a workflow, recall prior errors and accepted procedures. -- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. - -Trust gates: - -- Prefer source-linked, non-superseded, high-confidence memories. -- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. -- Do not treat sensitive or hidden-by-default memory as ordinary recall context. - -Write gates: - -- Remember only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes. -- Reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. -- Require evidence refs for promoted or rejected evaluated outcomes. -- Require user confirmation before creating or promoting broad cross-project heartwood. - -"#; - - const TEST_CLI_QUALITY_GATES_SECTION: &str = r#"Memory quality gates: - -- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. -- Before risky changes, recall warnings and evidence-linked prior failures. -- Before repeating a workflow, recall prior errors and accepted procedures. -- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. -- Before trusting memory, prefer source-linked, non-superseded, high-confidence results. -- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. -- Before writing memory, reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. -- Require evidence refs for promoted or rejected evaluated outcomes. -- Require user confirmation before creating or promoting broad cross-project heartwood. - -"#; - - const TEST_SKILL_QUALITY_GATES_SECTION: &str = r#"## Memory Quality Gates - -Use these gates before relying on or writing memory. - -Recall gates: - -- Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. -- Before risky changes, recall warnings and evidence-linked prior failures. -- Before repeating a workflow, recall prior errors and accepted procedures. -- Before closeout, recall recent decisions so memory updates do not contradict already-stored lessons. - -Trust gates: - -- Prefer source-linked, non-superseded, high-confidence memories. -- Treat heartwood as durable only when source evidence or user confirmation supports it. -- Re-read source files, tests, explicit user instructions, DOX contracts, or Revolve evidence when memory conflicts with current sources. -- Do not treat sensitive or hidden-by-default memory as ordinary recall context. - -Write gates: + #[test] + fn generated_canonical_managed_section_can_be_removed_and_reinserted() { + let root = Path::new("/tmp/tree-ring-quality-fixture"); + let canonical = agent_contract(root); + let section = extract_section( + &canonical, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); -- Remember only durable decisions, validated lessons, reusable warnings, corrections, future seeds, and evidence-backed outcomes. -- Reject transient planning chatter, duplicate wording, tool noise, and unsupported claims. -- Require evidence refs for promoted or rejected evaluated outcomes. -- Require user confirmation before creating or promoting broad cross-project heartwood. + let stale = remove_managed_section( + &canonical, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let restored = + insert_section_before_anchor(&stale, section, AGENT_QUALITY_GATES_ANCHOR).unwrap(); -"#; + assert_eq!(restored, canonical); + } - const TEST_AGENT_QUALITY_GATES_ANCHOR: &str = "## DOX Integration"; - const TEST_CLI_QUALITY_GATES_ANCHOR: &str = "Safety rules:"; - const TEST_SKILL_QUALITY_GATES_ANCHOR: &str = "## Ring Selection"; + fn stale_fixture(content: &str, heading: &str, anchor: &str) -> String { + remove_managed_section(content, heading, anchor).unwrap() + } #[test] fn init_writes_agent_awareness_files_without_overwriting() { @@ -480,20 +452,51 @@ Write gates: let dir = tempdir().unwrap(); let root = dir.path().join(".tree-ring"); fs::create_dir_all(&root).unwrap(); + let canonical_agents = agent_contract(&root); + let agent_section = extract_section( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let cli_section = extract_section( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let skill_section = extract_section( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + .unwrap(); fs::write( root.join("AGENTS.md"), - agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, ""), + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); fs::write( root.join("CLI.md"), - CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, ""), + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); fs::write( root.join("SKILL.md"), - SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, ""), + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); @@ -505,9 +508,9 @@ Write gates: let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); - assert!(agents.contains(TEST_AGENT_QUALITY_GATES_SECTION.trim())); - assert!(cli.contains(TEST_CLI_QUALITY_GATES_SECTION.trim())); - assert!(skill.contains(TEST_SKILL_QUALITY_GATES_SECTION.trim())); + assert!(agents.contains(agent_section.trim())); + assert!(cli.contains(cli_section.trim())); + assert!(skill.contains(skill_section.trim())); } #[test] @@ -515,20 +518,51 @@ Write gates: let dir = tempdir().unwrap(); let root = dir.path().join(".tree-ring"); fs::create_dir_all(&root).unwrap(); + let canonical_agents = agent_contract(&root); + let agent_section = extract_section( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let cli_section = extract_section( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ) + .unwrap(); + let skill_section = extract_section( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + .unwrap(); fs::write( root.join("AGENTS.md"), - agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, ""), + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); fs::write( root.join("CLI.md"), - CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, ""), + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); fs::write( root.join("SKILL.md"), - SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, ""), + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); @@ -538,18 +572,9 @@ Write gates: let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); - assert!(agents.contains(&format!( - "{}{}", - TEST_AGENT_QUALITY_GATES_SECTION, TEST_AGENT_QUALITY_GATES_ANCHOR - ))); - assert!(cli.contains(&format!( - "{}{}", - TEST_CLI_QUALITY_GATES_SECTION, TEST_CLI_QUALITY_GATES_ANCHOR - ))); - assert!(skill.contains(&format!( - "{}{}", - TEST_SKILL_QUALITY_GATES_SECTION, TEST_SKILL_QUALITY_GATES_ANCHOR - ))); + assert!(agents.contains(&format!("{}{}", agent_section, AGENT_QUALITY_GATES_ANCHOR))); + assert!(cli.contains(&format!("{}{}", cli_section, CLI_QUALITY_GATES_ANCHOR))); + assert!(skill.contains(&format!("{}{}", skill_section, SKILL_QUALITY_GATES_ANCHOR))); } #[test] @@ -561,20 +586,33 @@ Write gates: let agents_custom = "\nCustom note: keep me.\n"; let cli_custom = "\nCustom alias note: keep me.\n"; let skill_custom = "\nCustom workflow note: keep me.\n"; + let canonical_agents = agent_contract(&root); fs::write( root.join("AGENTS.md"), - agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, "") + agents_custom, + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ) + agents_custom, ) .unwrap(); fs::write( root.join("CLI.md"), - CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, "") + cli_custom, + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ) + cli_custom, ) .unwrap(); fs::write( root.join("SKILL.md"), - SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, "") + skill_custom, + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + skill_custom, ) .unwrap(); @@ -622,20 +660,33 @@ Write gates: let dir = tempdir().unwrap(); let root = dir.path().join(".tree-ring"); fs::create_dir_all(&root).unwrap(); + let canonical_agents = agent_contract(&root); fs::write( root.join("AGENTS.md"), - agent_contract(&root).replace(TEST_AGENT_QUALITY_GATES_SECTION, ""), + stale_fixture( + &canonical_agents, + AGENT_QUALITY_GATES_HEADING, + AGENT_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); fs::write( root.join("CLI.md"), - CLI_REFERENCE.replace(TEST_CLI_QUALITY_GATES_SECTION, ""), + stale_fixture( + CLI_REFERENCE, + CLI_QUALITY_GATES_HEADING, + CLI_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); fs::write( root.join("SKILL.md"), - SKILL_TEMPLATE.replace(TEST_SKILL_QUALITY_GATES_SECTION, ""), + stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ), ) .unwrap(); diff --git a/crates/tree-ring-memory-core/src/lib.rs b/crates/tree-ring-memory-core/src/lib.rs index 0688288..f92f60b 100644 --- a/crates/tree-ring-memory-core/src/lib.rs +++ b/crates/tree-ring-memory-core/src/lib.rs @@ -28,10 +28,10 @@ pub use models::{ now_iso, MemoryEvent, MemoryLink, MemoryReview, MemorySource, TreeRingError, TreeRingResult, }; pub use quality::{ - evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, - QualityRunReport, QualityScenario, QualityScenarioReport, QualityThresholds, RecallExpectation, - RecallExpectationReport, WriteDecisionExpectation, WriteDecisionReport, QUALITY_CATEGORIES, - WRITE_DECISIONS, + evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, BehaviorExpectation, + BehaviorExpectationReport, QualityRecall, QualityRunError, QualityRunReport, QualityScenario, + QualityScenarioReport, QualityThresholds, RecallExpectation, RecallExpectationReport, + WriteDecisionExpectation, WriteDecisionReport, QUALITY_CATEGORIES, WRITE_DECISIONS, }; pub use recall::{RecallRanking, RecallScore, RecallScorer}; pub use revolve::{collect_revolve_memories, RevolveSyncReport, RevolveSyncRequest}; diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index a2992e4..4b1dda9 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -8,6 +8,7 @@ pub const QUALITY_CATEGORIES: &[&str] = &[ "spam_prevention", "stale_truth_suppression", "behavior_proof", + "evidence_preservation", ]; pub const WRITE_DECISIONS: &[&str] = &[ @@ -18,6 +19,7 @@ pub const WRITE_DECISIONS: &[&str] = &[ ]; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct QualityScenario { pub name: String, pub category: String, @@ -38,6 +40,8 @@ pub struct QualityScenario { #[serde(default)] pub evidence_refs: Vec, #[serde(default)] + pub behavior_expectation: Option, + #[serde(default)] pub thresholds: QualityThresholds, } @@ -95,12 +99,22 @@ impl QualityScenario { ))); } } + if self.category == "behavior_proof" && self.behavior_expectation.is_none() { + return Err(TreeRingError::Validation(format!( + "quality scenario {} category behavior_proof requires behavior_expectation", + self.name + ))); + } + if let Some(expectation) = &self.behavior_expectation { + expectation.validate(self)?; + } self.thresholds.validate(&self.name)?; Ok(()) } } #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct RecallExpectation { #[serde(default)] pub memory_id: Option, @@ -136,6 +150,7 @@ impl RecallExpectation { } #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct WriteDecisionExpectation { pub memory_id: String, pub decision: String, @@ -154,7 +169,61 @@ impl WriteDecisionExpectation { } } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BehaviorExpectation { + pub required_memory_id: String, + pub baseline_decision: String, + pub memory_informed_decision: String, + pub expected_decision: String, + #[serde(default)] + pub reason: Option, +} + +impl BehaviorExpectation { + fn validate(&self, scenario: &QualityScenario) -> TreeRingResult<()> { + for (field, value) in [ + ("required_memory_id", self.required_memory_id.as_str()), + ("baseline_decision", self.baseline_decision.as_str()), + ( + "memory_informed_decision", + self.memory_informed_decision.as_str(), + ), + ("expected_decision", self.expected_decision.as_str()), + ] { + if value.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {} behavior_expectation {field} must not be blank", + scenario.name + ))); + } + } + if self + .reason + .as_deref() + .is_some_and(|reason| reason.trim().is_empty()) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} behavior_expectation reason must not be blank when provided", + scenario.name + ))); + } + if !scenario + .seed_memories + .iter() + .any(|memory| memory.id == self.required_memory_id) + { + return Err(TreeRingError::Validation(format!( + "quality scenario {} behavior_expectation required_memory_id {} does not match a seed_memory", + scenario.name, self.required_memory_id + ))); + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct QualityThresholds { #[serde(default = "one")] pub min_constraint_recall_rate: f64, @@ -228,34 +297,58 @@ pub struct WriteDecisionReport { pub actual: String, pub passed: bool, pub reason: String, + #[serde(default)] + pub evidence_applicable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BehaviorExpectationReport { + pub expectation: BehaviorExpectation, + pub required_memory_recalled: bool, + pub decision_changed: bool, + pub expected_decision_reached: bool, + pub passed: bool, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct QualityScenarioReport { pub name: String, pub category: String, - pub constraint_recall_rate: f64, - pub forbidden_recall_rate: f64, - pub spam_rejection_rate: f64, - pub evidence_required_rate: f64, - pub behavior_proof_pass: bool, + pub constraint_recall_rate: Option, + pub forbidden_recall_rate: Option, + pub spam_rejection_rate: Option, + pub evidence_required_rate: Option, + pub behavior_proof_pass: Option, + pub behavior_expectation: Option, pub quality_pass: bool, pub expected_recall: Vec, pub forbidden_recall: Vec, pub write_decisions: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct QualityRunError { + #[serde(default)] + pub scenario: Option, + #[serde(default)] + pub path: Option, + pub stage: String, + pub message: String, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct QualityRunReport { pub ok: bool, pub scenario_count: usize, - pub constraint_recall_rate: f64, - pub forbidden_recall_rate: f64, - pub spam_rejection_rate: f64, - pub evidence_required_rate: f64, - pub behavior_proof_pass_rate: f64, + pub constraint_recall_rate: Option, + pub forbidden_recall_rate: Option, + pub spam_rejection_rate: Option, + pub evidence_required_rate: Option, + pub behavior_proof_pass_rate: Option, pub quality_pass: bool, pub scenarios: Vec, + #[serde(default)] + pub errors: Vec, } pub fn evaluate_quality_scenario( @@ -282,15 +375,35 @@ pub fn evaluate_quality_scenario( let constraint_recall_rate = pass_rate(&expected_recall); let forbidden_recall_rate = failure_rate(&forbidden_recall); - let spam_rejection_rate = decision_pass_rate(&write_decisions, "reject"); - let evidence_required_rate = decision_pass_rate(&write_decisions, "require_evidence"); - let behavior_proof_pass = constraint_recall_rate - >= scenario.thresholds.min_constraint_recall_rate - && forbidden_recall_rate <= scenario.thresholds.max_forbidden_recall_rate; - let quality_pass = behavior_proof_pass - && write_decisions.iter().all(|report| report.passed) - && spam_rejection_rate >= scenario.thresholds.min_spam_rejection_rate - && evidence_required_rate >= scenario.thresholds.min_evidence_required_rate; + let spam_rejection_rate = + decision_pass_rate(&write_decisions, |report| report.expected == "reject"); + let evidence_required_rate = + decision_pass_rate(&write_decisions, |report| report.evidence_applicable); + let behavior_expectation = scenario + .behavior_expectation + .as_ref() + .map(|expectation| behavior_expectation_report(expectation, recalls)); + let behavior_proof_pass = behavior_expectation.as_ref().map(|report| report.passed); + let behavior_rate = behavior_proof_pass.map(|passed| if passed { 1.0 } else { 0.0 }); + let quality_pass = minimum_met( + constraint_recall_rate, + scenario.thresholds.min_constraint_recall_rate, + ) && maximum_met( + forbidden_recall_rate, + scenario.thresholds.max_forbidden_recall_rate, + ) && write_decisions.iter().all(|report| report.passed) + && minimum_met( + spam_rejection_rate, + scenario.thresholds.min_spam_rejection_rate, + ) + && minimum_met( + evidence_required_rate, + scenario.thresholds.min_evidence_required_rate, + ) + && minimum_met( + behavior_rate, + scenario.thresholds.min_behavior_proof_pass_rate, + ); Ok(QualityScenarioReport { name: scenario.name.clone(), @@ -300,6 +413,7 @@ pub fn evaluate_quality_scenario( spam_rejection_rate, evidence_required_rate, behavior_proof_pass, + behavior_expectation, quality_pass, expected_recall, forbidden_recall, @@ -309,33 +423,36 @@ pub fn evaluate_quality_scenario( pub fn summarize_quality_run(reports: Vec) -> QualityRunReport { let scenario_count = reports.len(); - let denominator = scenario_count.max(1) as f64; - let constraint_recall_rate = reports - .iter() - .map(|report| report.constraint_recall_rate) - .sum::() - / denominator; - let forbidden_recall_rate = reports - .iter() - .map(|report| report.forbidden_recall_rate) - .sum::() - / denominator; - let spam_rejection_rate = reports - .iter() - .map(|report| report.spam_rejection_rate) - .sum::() - / denominator; - let evidence_required_rate = reports - .iter() - .map(|report| report.evidence_required_rate) - .sum::() - / denominator; - let behavior_proof_pass_rate = reports - .iter() - .filter(|report| report.behavior_proof_pass) - .count() as f64 - / denominator; - let quality_pass = reports.iter().all(|report| report.quality_pass); + let constraint_recall_rate = aggregate_rate( + reports + .iter() + .flat_map(|report| report.expected_recall.iter().map(|item| item.passed)), + ); + let forbidden_recall_rate = aggregate_rate( + reports + .iter() + .flat_map(|report| report.forbidden_recall.iter().map(|item| !item.passed)), + ); + let spam_rejection_rate = aggregate_rate(reports.iter().flat_map(|report| { + report + .write_decisions + .iter() + .filter(|item| item.expected == "reject") + .map(|item| item.passed) + })); + let evidence_required_rate = aggregate_rate(reports.iter().flat_map(|report| { + report + .write_decisions + .iter() + .filter(|item| item.evidence_applicable) + .map(|item| item.passed) + })); + let behavior_proof_pass_rate = aggregate_rate( + reports + .iter() + .filter_map(|report| report.behavior_proof_pass), + ); + let quality_pass = scenario_count > 0 && reports.iter().all(|report| report.quality_pass); QualityRunReport { ok: quality_pass, @@ -347,6 +464,7 @@ pub fn summarize_quality_run(reports: Vec) -> QualityRunR behavior_proof_pass_rate, quality_pass, scenarios: reports, + errors: Vec::new(), } } @@ -442,6 +560,25 @@ fn forbidden_recall_report( } } +fn behavior_expectation_report( + expectation: &BehaviorExpectation, + recalls: &[QualityRecall], +) -> BehaviorExpectationReport { + let required_memory_recalled = recalls + .iter() + .any(|recall| recall.memory.id == expectation.required_memory_id); + let decision_changed = expectation.baseline_decision != expectation.memory_informed_decision; + let expected_decision_reached = + expectation.memory_informed_decision == expectation.expected_decision; + BehaviorExpectationReport { + expectation: expectation.clone(), + required_memory_recalled, + decision_changed, + expected_decision_reached, + passed: required_memory_recalled && decision_changed && expected_decision_reached, + } +} + fn matching_recall_ids(expectation: &RecallExpectation, recalls: &[QualityRecall]) -> Vec { recalls .iter() @@ -503,6 +640,7 @@ fn write_decision_report( passed: actual == expectation.decision, actual, reason: expectation.reason.clone(), + evidence_applicable: candidate.event_type.starts_with("evaluation_"), }) } @@ -537,29 +675,39 @@ fn classify_write_candidate(candidate: &MemoryEvent, required_evidence_refs: &[S "accept".to_string() } -fn pass_rate(reports: &[RecallExpectationReport]) -> f64 { - if reports.is_empty() { - return 1.0; - } - reports.iter().filter(|report| report.passed).count() as f64 / reports.len() as f64 +fn pass_rate(reports: &[RecallExpectationReport]) -> Option { + aggregate_rate(reports.iter().map(|report| report.passed)) } -fn failure_rate(reports: &[RecallExpectationReport]) -> f64 { - if reports.is_empty() { - return 0.0; - } - reports.iter().filter(|report| !report.passed).count() as f64 / reports.len() as f64 +fn failure_rate(reports: &[RecallExpectationReport]) -> Option { + aggregate_rate(reports.iter().map(|report| !report.passed)) } -fn decision_pass_rate(reports: &[WriteDecisionReport], decision: &str) -> f64 { - let relevant = reports - .iter() - .filter(|report| report.expected == decision) - .collect::>(); - if relevant.is_empty() { - return 1.0; - } - relevant.iter().filter(|report| report.passed).count() as f64 / relevant.len() as f64 +fn decision_pass_rate( + reports: &[WriteDecisionReport], + applicable: impl Fn(&WriteDecisionReport) -> bool, +) -> Option { + aggregate_rate( + reports + .iter() + .filter(|report| applicable(report)) + .map(|report| report.passed), + ) +} + +fn aggregate_rate(observations: impl Iterator) -> Option { + let (passed, total) = observations.fold((0usize, 0usize), |(passed, total), observation| { + (passed + usize::from(observation), total + 1) + }); + (total > 0).then_some(passed as f64 / total as f64) +} + +fn minimum_met(rate: Option, threshold: f64) -> bool { + rate.is_none_or(|rate| rate >= threshold) +} + +fn maximum_met(rate: Option, threshold: f64) -> bool { + rate.is_none_or(|rate| rate <= threshold) } #[cfg(test)] @@ -581,6 +729,287 @@ mod tests { event } + fn scenario(name: &str, category: &str) -> QualityScenario { + QualityScenario { + name: name.to_string(), + category: category.to_string(), + seed_memories: Vec::new(), + query: Some("quality proof".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: Vec::new(), + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + } + } + + fn report_json(scenario: &QualityScenario, recalls: &[QualityRecall]) -> serde_json::Value { + serde_json::to_value(evaluate_quality_scenario(scenario, recalls).unwrap()).unwrap() + } + + #[test] + fn rejects_unknown_top_level_fixture_field() { + let error = parse_quality_scenario( + r#"{ + "name": "misspelled recall", + "category": "constraint_recall", + "query": "quality proof", + "expected_recal": [] + }"#, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("unknown field `expected_recal`"), "{error}"); + } + + #[test] + fn rejects_unknown_threshold_field() { + let error = parse_quality_scenario( + r#"{ + "name": "misspelled threshold", + "category": "constraint_recall", + "query": "quality proof", + "thresholds": {"min_constraint_recal_rate": 1.0} + }"#, + ) + .unwrap_err() + .to_string(); + + assert!( + error.contains("unknown field `min_constraint_recal_rate`"), + "{error}" + ); + } + + #[test] + fn rejects_unknown_nested_expectation_field() { + let error = parse_quality_scenario( + r#"{ + "name": "misspelled nested field", + "category": "constraint_recall", + "query": "quality proof", + "expected_recall": [{"memoryid": "mem_required"}] + }"#, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("unknown field `memoryid`"), "{error}"); + } + + #[test] + fn behavior_proof_requires_explicit_expectation() { + let error = parse_quality_scenario( + r#"{ + "name": "implicit behavior proof", + "category": "behavior_proof", + "query": "quality proof" + }"#, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("requires behavior_expectation"), "{error}"); + } + + #[test] + fn behavior_proof_reports_an_observed_decision_change() { + let required = memory( + "mem_required_behavior", + "Rollback instead of retrying the stale cache migration.", + "scar", + ); + let input = serde_json::json!({ + "name": "explicit behavior proof", + "category": "behavior_proof", + "query": "stale cache migration failed", + "seed_memories": [required], + "behavior_expectation": { + "required_memory_id": "mem_required_behavior", + "baseline_decision": "retry the migration unchanged", + "memory_informed_decision": "rollback and inspect cache state", + "expected_decision": "rollback and inspect cache state", + "reason": "the scar changes the recovery decision" + } + }); + let scenario = parse_quality_scenario(&input.to_string()).unwrap(); + let recalls = [QualityRecall { + memory: scenario.seed_memories[0].clone(), + score: 0.9, + }]; + + let report = report_json(&scenario, &recalls); + + assert_eq!(report["behavior_proof_pass"], true); + assert_eq!( + report["behavior_expectation"]["required_memory_recalled"], + true + ); + assert_eq!(report["behavior_expectation"]["decision_changed"], true); + assert_eq!( + report["behavior_expectation"]["expected_decision_reached"], + true + ); + assert_eq!(report["behavior_expectation"]["passed"], true); + } + + #[test] + fn behavior_threshold_applies_to_an_applicable_failure() { + let required = memory("mem_behavior_threshold", "Use rollback recovery.", "scar"); + let input = serde_json::json!({ + "name": "behavior threshold", + "category": "behavior_proof", + "query": "failed recovery", + "seed_memories": [required], + "behavior_expectation": { + "required_memory_id": "mem_behavior_threshold", + "baseline_decision": "retry unchanged", + "memory_informed_decision": "retry unchanged", + "expected_decision": "rollback" + }, + "thresholds": {"min_behavior_proof_pass_rate": 0.0} + }); + let mut scenario = parse_quality_scenario(&input.to_string()).unwrap(); + let recalls = [QualityRecall { + memory: scenario.seed_memories[0].clone(), + score: 0.9, + }]; + + assert_eq!(report_json(&scenario, &recalls)["quality_pass"], true); + scenario.thresholds.min_behavior_proof_pass_rate = 0.5; + assert_eq!(report_json(&scenario, &recalls)["quality_pass"], false); + } + + #[test] + fn metrics_are_null_without_applicable_observations() { + let report = report_json(&scenario("no observations", "constraint_recall"), &[]); + + assert!(report["constraint_recall_rate"].is_null()); + assert!(report["forbidden_recall_rate"].is_null()); + assert!(report["spam_rejection_rate"].is_null()); + assert!(report["evidence_required_rate"].is_null()); + assert!(report["behavior_proof_pass"].is_null()); + } + + #[test] + fn run_metrics_weight_only_applicable_expectations() { + let mut failing = scenario("missed constraint", "constraint_recall"); + failing.expected_recall = vec![RecallExpectation { + memory_id: Some("mem_missing".to_string()), + ..Default::default() + }]; + let unrelated = scenario("unrelated", "spam_prevention"); + let reports = vec![ + evaluate_quality_scenario(&failing, &[]).unwrap(), + evaluate_quality_scenario(&unrelated, &[]).unwrap(), + ]; + + let run = serde_json::to_value(summarize_quality_run(reports)).unwrap(); + + assert_eq!(run["constraint_recall_rate"], 0.0); + assert!(run["spam_rejection_rate"].is_null()); + } + + #[test] + fn forbidden_run_rate_uses_failures_over_applicable_expectations() { + let mut mixed = scenario("mixed forbidden recall", "stale_truth_suppression"); + mixed.forbidden_recall = vec![ + RecallExpectation { + memory_id: Some("mem_forbidden_recalled".to_string()), + ..Default::default() + }, + RecallExpectation { + memory_id: Some("mem_forbidden_hidden".to_string()), + ..Default::default() + }, + ]; + let recalls = [QualityRecall { + memory: memory("mem_forbidden_recalled", "Stale instruction.", "heartwood"), + score: 0.8, + }]; + let reports = vec![ + evaluate_quality_scenario(&mixed, &recalls).unwrap(), + evaluate_quality_scenario(&scenario("unrelated", "spam_prevention"), &[]).unwrap(), + ]; + + let run = serde_json::to_value(summarize_quality_run(reports)).unwrap(); + + assert_eq!(run["forbidden_recall_rate"], 0.5); + } + + #[test] + fn applicable_thresholds_gate_scenario_quality() { + let required = memory("mem_recalled", "Required constraint.", "heartwood"); + let mut recall = scenario("threshold gates", "constraint_recall"); + recall.expected_recall = vec![ + RecallExpectation { + memory_id: Some("mem_recalled".to_string()), + ..Default::default() + }, + RecallExpectation { + memory_id: Some("mem_missing".to_string()), + ..Default::default() + }, + ]; + recall.thresholds.min_constraint_recall_rate = 0.5; + let recalls = [QualityRecall { + memory: required, + score: 0.9, + }]; + assert_eq!(report_json(&recall, &recalls)["quality_pass"], true); + recall.thresholds.min_constraint_recall_rate = 0.6; + assert_eq!(report_json(&recall, &recalls)["quality_pass"], false); + + let mut forbidden = scenario("forbidden threshold", "stale_truth_suppression"); + forbidden.forbidden_recall = vec![RecallExpectation { + memory_id: Some("mem_stale".to_string()), + ..Default::default() + }]; + forbidden.thresholds.max_forbidden_recall_rate = 0.0; + let stale = [QualityRecall { + memory: memory("mem_stale", "Stale instruction.", "heartwood"), + score: 0.9, + }]; + assert_eq!(report_json(&forbidden, &stale)["quality_pass"], false); + } + + #[test] + fn evaluation_write_decisions_are_evidence_applicable() { + let mut candidate = memory( + "mem_evaluation", + "Preserve the evaluated outcome with its evidence.", + "cambium", + ); + candidate.event_type = "evaluation_result".to_string(); + candidate.source.ref_.clear(); + let mut scenario = scenario("evidence applicability", "evidence_preservation"); + scenario.write_candidates = vec![candidate]; + scenario.expected_write_decisions = vec![WriteDecisionExpectation { + memory_id: "mem_evaluation".to_string(), + decision: "accept".to_string(), + reason: "accepted outcomes preserve evidence".to_string(), + }]; + + let report = report_json(&scenario, &[]); + + assert_eq!(report["evidence_required_rate"], 0.0); + assert_eq!(report["write_decisions"][0]["evidence_applicable"], true); + } + + #[test] + fn zero_scenario_run_is_failed_and_has_null_metrics() { + let report = serde_json::to_value(summarize_quality_run(Vec::new())).unwrap(); + + assert_eq!(report["ok"], false); + assert_eq!(report["quality_pass"], false); + assert!(report["constraint_recall_rate"].is_null()); + assert!(report["behavior_proof_pass_rate"].is_null()); + } + #[test] fn parses_valid_quality_scenario() { let input = r#"{ @@ -657,7 +1086,7 @@ mod tests { fn rejects_blank_evidence_refs() { let input = r#"{ "name": "bad evidence refs", - "category": "behavior_proof", + "category": "constraint_recall", "query": "prove behavior", "evidence_refs": ["docs/spec.md", " "] }"#; @@ -689,7 +1118,7 @@ mod tests { fn uses_workflow_prompt_when_query_is_blank() { let input = r#"{ "name": "workflow prompt fallback", - "category": "behavior_proof", + "category": "constraint_recall", "query": " ", "workflow_prompt": " validate behavior proof " }"#; @@ -755,6 +1184,7 @@ mod tests { reason: "orphan mapping".to_string(), }], evidence_refs: Vec::new(), + behavior_expectation: None, thresholds: QualityThresholds::default(), }; @@ -782,7 +1212,7 @@ mod tests { parsed += 1; } - assert_eq!(parsed, 6); + assert_eq!(parsed, 7); } #[test] @@ -809,6 +1239,7 @@ mod tests { }, ], evidence_refs: Vec::new(), + behavior_expectation: None, thresholds: QualityThresholds::default(), }; @@ -839,6 +1270,7 @@ mod tests { write_candidates: Vec::new(), expected_write_decisions: Vec::new(), evidence_refs: Vec::new(), + behavior_expectation: None, thresholds: QualityThresholds::default(), }; scenario.validate().unwrap(); @@ -857,8 +1289,8 @@ mod tests { .unwrap(); assert!(report.quality_pass); - assert_eq!(report.constraint_recall_rate, 1.0); - assert_eq!(report.forbidden_recall_rate, 0.0); + assert_eq!(report.constraint_recall_rate, Some(1.0)); + assert_eq!(report.forbidden_recall_rate, Some(0.0)); assert!(report.expected_recall[0].passed); assert!(report.forbidden_recall[0].passed); } @@ -880,6 +1312,7 @@ mod tests { write_candidates: Vec::new(), expected_write_decisions: Vec::new(), evidence_refs: Vec::new(), + behavior_expectation: None, thresholds: QualityThresholds::default(), }; @@ -893,7 +1326,7 @@ mod tests { .unwrap(); assert!(!report.quality_pass); - assert_eq!(report.forbidden_recall_rate, 1.0); + assert_eq!(report.forbidden_recall_rate, Some(1.0)); assert!(!report.forbidden_recall[0].passed); } @@ -946,14 +1379,15 @@ mod tests { }, ], evidence_refs: vec!["evals/run-001".to_string()], + behavior_expectation: None, thresholds: QualityThresholds::default(), }; let report = evaluate_quality_scenario(&scenario, &[]).unwrap(); assert!(report.quality_pass); - assert_eq!(report.spam_rejection_rate, 1.0); - assert_eq!(report.evidence_required_rate, 1.0); + assert_eq!(report.spam_rejection_rate, Some(1.0)); + assert_eq!(report.evidence_required_rate, Some(1.0)); assert_eq!(report.write_decisions.len(), 3); } @@ -978,6 +1412,7 @@ mod tests { reason: "should have been rejected".to_string(), }], evidence_refs: Vec::new(), + behavior_expectation: None, thresholds: QualityThresholds::default(), }; @@ -1009,6 +1444,7 @@ mod tests { reason: "should need evidence".to_string(), }], evidence_refs: Vec::new(), + behavior_expectation: None, thresholds: QualityThresholds::default(), }; @@ -1040,6 +1476,7 @@ mod tests { reason: "should require confirmation".to_string(), }], evidence_refs: Vec::new(), + behavior_expectation: None, thresholds: QualityThresholds::default(), }; @@ -1055,11 +1492,12 @@ mod tests { let passing = QualityScenarioReport { name: "pass".to_string(), category: "constraint_recall".to_string(), - constraint_recall_rate: 1.0, - forbidden_recall_rate: 0.0, - spam_rejection_rate: 1.0, - evidence_required_rate: 1.0, - behavior_proof_pass: true, + constraint_recall_rate: Some(1.0), + forbidden_recall_rate: Some(0.0), + spam_rejection_rate: Some(1.0), + evidence_required_rate: Some(1.0), + behavior_proof_pass: Some(true), + behavior_expectation: None, quality_pass: true, expected_recall: Vec::new(), forbidden_recall: Vec::new(), @@ -1070,6 +1508,6 @@ mod tests { assert!(run.quality_pass); assert_eq!(run.scenario_count, 2); - assert_eq!(run.behavior_proof_pass_rate, 1.0); + assert_eq!(run.behavior_proof_pass_rate, Some(1.0)); } } diff --git a/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md index def0a9d..db85382 100644 --- a/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md +++ b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md @@ -2193,3 +2193,19 @@ git commit -m "Verify memory quality proof loop" ``` Expected: create a commit only when verification produced tracked file changes. + +--- + +## Final-Review Hardening Addendum + +This addendum supersedes the historical six-fixture exact count and related +non-null default assumptions in the task evidence above. The final default pack +contains seven fixtures, including `evidence-preservation.json`. Fixture-facing +types reject unknown fields; behavior proof requires an explicit recalled-memory +and decision-change expectation; scenario and run metrics are nullable and +aggregate applicable raw observations; and runner errors preserve completed +scenarios in JSON and markdown with one privacy-safe structured terminal error. + +The final default run must contain seven scenarios, zero errors, numeric values +for all five aggregate metrics, and top-level `"quality_pass": true`. The +certification shell's dependency-free exact line gate remains unchanged. diff --git a/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md index c6ae687..409cbb5 100644 --- a/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md +++ b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md @@ -110,7 +110,7 @@ Each scenario should include: - `name`: stable scenario id. - `category`: `constraint_recall`, `spam_prevention`, - `stale_truth_suppression`, or `behavior_proof`. + `stale_truth_suppression`, `behavior_proof`, or `evidence_preservation`. - `seed_memories`: memory events loaded into a temporary store. - `query` or `workflow_prompt`: simulated agent task context. - `expected_recall`: memory ids, rings, tags, or source refs that should appear. @@ -120,6 +120,9 @@ Each scenario should include: - `expected_write_decisions`: `accept`, `reject`, `require_evidence`, or `require_user_confirmation`. - `evidence_refs`: source refs required for accepted outcome memories. +- `behavior_expectation`: required recalled memory id, baseline decision, + memory-informed decision, expected decision, and an optional reason. It is + required for `behavior_proof` scenarios. - `thresholds`: per-scenario minimums or tolerances. First scenario pack: @@ -133,7 +136,9 @@ First scenario pack: - Synthetic edge case: sensitive memory is blocked or hidden by default. - Synthetic edge case: superseded heartwood does not outrank its replacement. - Synthetic edge case: failed approach surfaces as a scar for a failure-like - query. + query and changes a baseline retry into the expected rollback-safe decision. +- Synthetic edge case: evaluated outcomes without evidence are held while an + outcome preserving the required evidence ref is accepted. ## Data Flow @@ -193,15 +198,24 @@ The quality report should include: - `forbidden_recall_rate`: stale, sensitive, superseded, or weak memories returned when they should not be. - `spam_rejection_rate`: low-value write candidates rejected. -- `evidence_required_rate`: accepted outcome memories that preserve required - evidence refs. +- `evidence_required_rate`: evaluated candidates correctly held when evidence + is missing or accepted when the required evidence ref is preserved. - `behavior_proof_pass_rate`: scenarios where memory changed the expected decision path. - `quality_pass`: boolean certification gate. +Rates are observation-weighted across applicable expectations, not averaged +from per-scenario defaults. A scenario or run emits JSON `null` for a dimension +with no observations. Constraint and forbidden recall use their respective +expectation counts; spam uses expected `reject` decisions; evidence uses every +`evaluation_` write candidate; behavior uses scenarios with an explicit +behavior expectation. The default seven-scenario certification pack observes +all five dimensions, so its aggregate metrics remain numeric. + `quality_pass` should fail when required constraints are missed, forbidden recall exceeds tolerance, spam candidates are accepted, or required evidence -refs are lost. +refs are lost. Scenario thresholds apply only to dimensions with observations, +every write-decision report must match, and an empty run cannot pass. ## TUI Contract @@ -227,13 +241,17 @@ or warning behavior. ## Error Handling -- Invalid scenario fixtures should report the scenario name, file path, and - invalid field. +- Fixture-facing objects reject unknown fields. Invalid scenario fixtures + should report the scenario name, file path, and invalid field. - Missing expected recall should produce a precise failed expectation. - Forbidden recall should include the returned memory id and reason. - Write-decision mismatches should show the candidate id or summary and the expected decision. -- Certification should preserve partial quality artifacts on failure. +- Certification should preserve completed scenarios in partial JSON and + markdown artifacts on failure. Reports include one terminal structured error + with scenario and path when known, a stable stage, and a privacy-safe message. +- Markdown renders null rates as `n/a` and adds an Errors section when errors + are present. - Scenario runs should clean up temporary stores on success and avoid touching real user memory roots. @@ -265,6 +283,19 @@ Add tests in layers: 7. Generated guidance includes explicit recall, trust, and write gates. 8. TUI fullness semantics are documented as hybrid relative-share first, calibrated thresholds later. +9. The seven-scenario default pack includes evidence preservation and produces + numeric passing aggregate values for every quality metric. +10. Runner failures preserve structured partial artifacts without embedding + fixture contents or memory payloads in error messages. + +## Final-Review Hardening + +The final review replaced inferred behavior proof with an explicit decision +change contract, made non-applicable metrics nullable and observation-weighted, +added strict fixture fields and the `evidence_preservation` category, and made +runner failures artifact-producing. This section and the seven-scenario pack +supersede the original six-fixture count while preserving the certification +gate's exact top-level `"quality_pass": true` JSON line. ## Rollout diff --git a/fixtures/quality/evidence-preservation.json b/fixtures/quality/evidence-preservation.json new file mode 100644 index 0000000..4f2dca7 --- /dev/null +++ b/fixtures/quality/evidence-preservation.json @@ -0,0 +1,84 @@ +{ + "name": "evidence-preservation", + "category": "evidence_preservation", + "query": "preserve evidence for evaluated outcomes", + "write_candidates": [ + { + "id": "mem_evaluation_missing_source", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "evaluation_outcome", + "summary": "An evaluated outcome without its source reference.", + "details": "The quality gate must require evidence before accepting it.", + "source": { + "type": "evidence", + "ref": "", + "quote": "" + }, + "tags": ["evaluation", "evidence"], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + }, + { + "id": "mem_evaluation_preserved_source", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "Tree_Ring_Memory", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "evaluation_outcome", + "summary": "An evaluated outcome that preserves its required source reference.", + "details": "The matching evidence reference makes this candidate acceptable.", + "source": { + "type": "evidence", + "ref": "evals/evidence/run-021", + "quote": "" + }, + "tags": ["evaluation", "evidence"], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": { + "needs_review": false, + "review_reason": null, + "reviewed_at": null, + "reviewed_by": null + } + } + ], + "expected_write_decisions": [ + { + "memory_id": "mem_evaluation_missing_source", + "decision": "require_evidence", + "reason": "evaluated outcomes without a source ref require evidence" + }, + { + "memory_id": "mem_evaluation_preserved_source", + "decision": "accept", + "reason": "the evaluated outcome preserves the required evidence ref" + } + ], + "evidence_refs": ["evals/evidence/run-021"] +} diff --git a/fixtures/quality/scar-failure-recall.json b/fixtures/quality/scar-failure-recall.json index 2d6668c..788971d 100644 --- a/fixtures/quality/scar-failure-recall.json +++ b/fixtures/quality/scar-failure-recall.json @@ -42,5 +42,12 @@ "ring": "scar", "reason": "failure-like queries should surface scar memory" } - ] + ], + "behavior_expectation": { + "required_memory_id": "mem_scar_stale_cache", + "baseline_decision": "retry the cache migration unchanged", + "memory_informed_decision": "roll back the cache migration and inspect stale state before retrying", + "expected_decision": "roll back the cache migration and inspect stale state before retrying", + "reason": "the recalled failure scar changes the workflow to rollback-safe recovery" + } } From 6e213105102ef67de7d9c1d22aaddab67ffb9ab3 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 17:01:08 -0400 Subject: [PATCH 17/19] Close quality proof edge cases --- .../examples/quality_scenarios.rs | 191 ++++-- crates/tree-ring-memory-core/src/quality.rs | 562 +++++++++++++++--- ...-quality-proof-loop-implementation-plan.md | 22 + ...e-ring-memory-quality-proof-loop-design.md | 24 +- .../no-background-writer-constraint.json | 9 +- 5 files changed, 671 insertions(+), 137 deletions(-) diff --git a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs index 6c63b5f..c84ec82 100644 --- a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs +++ b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs @@ -1,16 +1,18 @@ use std::ffi::OsStr; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use tree_ring_memory_core::{ evaluate_quality_scenario, parse_quality_scenario, summarize_quality_run, QualityRecall, - QualityRunError, QualityRunReport, QualityScenario, QualityScenarioReport, + QualityRunError, QualityRunReport, QualityScenario, QualityScenarioReport, TreeRingError, }; use tree_ring_memory_sqlite::{MemoryRetriever, SQLiteMemoryStore}; const USAGE: &str = "usage: quality_scenarios "; const RECALL_LIMIT: usize = 8; +static TEMP_ROOT_COUNTER: AtomicU64 = AtomicU64::new(0); fn main() { if let Err(err) = run() { @@ -33,13 +35,7 @@ fn run() -> Result<(), String> { return Err(USAGE.to_string()); } - let run_report = run_quality_scenarios(&fixture_dir, &output_dir)?; - if !run_report.quality_pass { - return Err(format!( - "quality run failed: {} scenarios evaluated", - run_report.scenario_count - )); - } + let run_report = run_quality_command(&fixture_dir, &output_dir)?; println!( "quality scenarios passed: {} scenario(s)", @@ -48,12 +44,22 @@ fn run() -> Result<(), String> { Ok(()) } +fn run_quality_command(fixture_dir: &Path, output_dir: &Path) -> Result { + let run_report = run_quality_scenarios(fixture_dir, output_dir)?; + if !run_report.quality_pass { + return Err(format!( + "quality run failed: {} scenarios evaluated", + run_report.scenario_count + )); + } + Ok(run_report) +} + fn run_quality_scenarios( fixture_dir: &Path, output_dir: &Path, ) -> Result { - fs::create_dir_all(output_dir) - .map_err(|err| format!("create output dir {}: {err}", output_dir.display()))?; + fs::create_dir_all(output_dir).map_err(|_| "output_directory_create_error".to_string())?; let paths = match fixture_paths(fixture_dir) { Ok(paths) if !paths.is_empty() => paths, @@ -76,7 +82,7 @@ fn run_quality_scenarios( for path in paths { let input = match fs::read_to_string(&path) { Ok(input) => input, - Err(err) => { + Err(_) => { return write_failed_report( output_dir, reports, @@ -84,12 +90,11 @@ fn run_quality_scenarios( scenario: None, path: Some(path.display().to_string()), stage: "file_read".to_string(), - message: err.to_string(), + message: "fixture_file_read_error".to_string(), }, ); } }; - let scenario_name = scenario_name_hint(&input); let scenario = match parse_quality_scenario(&input) { Ok(scenario) => scenario, Err(err) => { @@ -97,10 +102,10 @@ fn run_quality_scenarios( output_dir, reports, QualityRunError { - scenario: scenario_name, + scenario: None, path: Some(path.display().to_string()), stage: "parse".to_string(), - message: err.to_string(), + message: classify_parse_error(&err).to_string(), }, ); } @@ -128,21 +133,21 @@ fn run_quality_scenarios( } fn fixture_paths(fixture_dir: &Path) -> Result, QualityRunError> { - let entries = fs::read_dir(fixture_dir).map_err(|err| QualityRunError { + let entries = fs::read_dir(fixture_dir).map_err(|_| QualityRunError { scenario: None, path: Some(fixture_dir.display().to_string()), stage: "fixture_directory".to_string(), - message: err.to_string(), + message: "fixture_directory_read_error".to_string(), })?; let mut paths = entries .map(|entry| { entry .map(|entry| entry.path()) - .map_err(|err| QualityRunError { + .map_err(|_| QualityRunError { scenario: None, path: Some(fixture_dir.display().to_string()), stage: "fixture_directory".to_string(), - message: err.to_string(), + message: "fixture_directory_read_error".to_string(), }) }) .collect::, _>>()?; @@ -151,14 +156,6 @@ fn fixture_paths(fixture_dir: &Path) -> Result, QualityRunError> { Ok(paths) } -fn scenario_name_hint(input: &str) -> Option { - serde_json::from_str::(input) - .ok()? - .get("name")? - .as_str() - .map(str::to_string) -} - fn write_failed_report( output_dir: &Path, reports: Vec, @@ -183,20 +180,20 @@ fn run_scenario(scenario: &QualityScenario) -> Result Result Result Result<(), String> { - let json = serde_json::to_string_pretty(report).map_err(|err| err.to_string())?; + let json = + serde_json::to_string_pretty(report).map_err(|_| "report_json_encode_error".to_string())?; fs::write(output_dir.join("quality-report.json"), json) - .map_err(|err| format!("write quality-report.json: {err}"))?; + .map_err(|_| "report_json_write_error".to_string())?; fs::write( output_dir.join("quality-summary.md"), markdown_summary(report), ) - .map_err(|err| format!("write quality-summary.md: {err}"))?; + .map_err(|_| "report_markdown_write_error".to_string())?; Ok(()) } +fn classify_parse_error(error: &TreeRingError) -> &'static str { + match error { + TreeRingError::Json(_) => "scenario_parse_error", + TreeRingError::Validation(_) => "scenario_validation_error", + _ => "scenario_parse_error", + } +} + +fn classify_evaluation_error(error: &TreeRingError) -> &'static str { + match error { + TreeRingError::Validation(_) => "scenario_validation_error", + _ => "scenario_evaluation_error", + } +} + fn markdown_summary(report: &QualityRunReport) -> String { let mut lines = vec![ "# Tree Ring Memory Quality Summary".to_string(), @@ -304,25 +317,25 @@ struct TemporaryRoot { impl TemporaryRoot { fn new(name: &str) -> Result { - let millis = SystemTime::now() + let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) - .map_err(|err| err.to_string())? - .as_millis(); + .map_err(|_| "temporary_root_clock_error".to_string())? + .as_nanos(); + let unique = TEMP_ROOT_COUNTER.fetch_add(1, Ordering::Relaxed); let safe_name = name .chars() .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) .collect::(); let path = std::env::temp_dir().join(format!( - "tree-ring-quality-{}-{}-{safe_name}", + "tree-ring-quality-{}-{}-{}-{safe_name}", std::process::id(), - millis + nanos, + unique )); if path.exists() { - fs::remove_dir_all(&path) - .map_err(|err| format!("remove existing temp root {}: {err}", path.display()))?; + fs::remove_dir_all(&path).map_err(|_| "temporary_root_cleanup_error".to_string())?; } - fs::create_dir_all(&path) - .map_err(|err| format!("create temp root {}: {err}", path.display()))?; + fs::create_dir_all(&path).map_err(|_| "temporary_root_create_error".to_string())?; Ok(Self { path }) } @@ -344,6 +357,8 @@ mod tests { use super::*; use tempfile::tempdir; + const SECRET_MARKER: &str = "EDGE-SECRET-MARKER-7f1c2e"; + fn copy_valid_fixture(fixture_dir: &Path, name: &str) { let source = Path::new(env!("CARGO_MANIFEST_DIR")) .join("../..") @@ -351,6 +366,48 @@ mod tests { fs::copy(source, fixture_dir.join(name)).unwrap(); } + fn write_invalid_memory_fixture(fixture_dir: &Path) { + fs::write( + fixture_dir.join("invalid-memory.json"), + format!( + r#"{{ + "name": "invalid-memory-{SECRET_MARKER}", + "category": "constraint_recall", + "query": "quality proof", + "seed_memories": [ + {{ + "id": "mem_invalid_secret", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "{SECRET_MARKER}", + "event_type": "decision", + "summary": "Invalid memory", + "details": "", + "source": {{"type": "evidence", "ref": "docs/spec.md", "quote": ""}}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {{"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null}} + }} + ], + "expected_recall": [ + {{"memory_id": "mem_invalid_secret"}} + ] + }}"# + ), + ) + .unwrap(); + } + #[test] fn writes_partial_artifacts_when_a_later_fixture_fails_to_parse() { let fixtures = tempdir().unwrap(); @@ -374,10 +431,8 @@ mod tests { assert_eq!(report.scenario_count, 1); assert_eq!(report.errors.len(), 1); assert_eq!(report.errors[0].stage, "parse"); - assert_eq!( - report.errors[0].scenario.as_deref(), - Some("invalid-private-fixture") - ); + assert_eq!(report.errors[0].message, "scenario_parse_error"); + assert!(report.errors[0].scenario.is_none()); assert!(report.errors[0] .path .as_deref() @@ -391,6 +446,40 @@ mod tests { assert!(markdown.contains("stage=parse")); } + #[test] + fn sanitizes_validation_failures_before_writing_reports() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + copy_valid_fixture(fixtures.path(), "01-valid.json"); + write_invalid_memory_fixture(fixtures.path()); + + let report = run_quality_scenarios(fixtures.path(), output.path()).unwrap(); + let json = fs::read_to_string(output.path().join("quality-report.json")).unwrap(); + let markdown = fs::read_to_string(output.path().join("quality-summary.md")).unwrap(); + + assert!(!report.ok); + assert!(!report.quality_pass); + assert_eq!(report.errors.len(), 1); + assert_eq!(report.errors[0].stage, "parse"); + assert_eq!(report.errors[0].message, "scenario_validation_error"); + assert!(report.errors[0].scenario.is_none()); + assert!(!report.errors[0].message.contains(SECRET_MARKER)); + assert!(!json.contains(SECRET_MARKER)); + assert!(!markdown.contains(SECRET_MARKER)); + } + + #[test] + fn sanitizes_validation_failures_in_returned_command_error() { + let fixtures = tempdir().unwrap(); + let output = tempdir().unwrap(); + write_invalid_memory_fixture(fixtures.path()); + + let error = run_quality_command(fixtures.path(), output.path()).unwrap_err(); + + assert_eq!(error, "quality run failed: 0 scenarios evaluated"); + assert!(!error.contains(SECRET_MARKER)); + } + #[test] fn writes_failed_artifacts_for_an_empty_fixture_directory() { let fixtures = tempdir().unwrap(); diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index 4b1dda9..625c9c9 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -108,9 +108,74 @@ impl QualityScenario { if let Some(expectation) = &self.behavior_expectation { expectation.validate(self)?; } - self.thresholds.validate(&self.name)?; + self.validate_primary_observation_contract()?; + self.thresholds + .validate(&self.name, self.metric_applicability())?; Ok(()) } + + fn validate_primary_observation_contract(&self) -> TreeRingResult<()> { + match self.category.as_str() { + "constraint_recall" if self.expected_recall.is_empty() => { + Err(TreeRingError::Validation(format!( + "quality scenario {} category constraint_recall requires at least one expected_recall", + self.name + ))) + } + "spam_prevention" + if !self + .expected_write_decisions + .iter() + .any(|decision| decision.decision == "reject") => + { + Err(TreeRingError::Validation(format!( + "quality scenario {} category spam_prevention requires at least one expected_write_decision of reject", + self.name + ))) + } + "stale_truth_suppression" if self.forbidden_recall.is_empty() => { + Err(TreeRingError::Validation(format!( + "quality scenario {} category stale_truth_suppression requires at least one forbidden_recall", + self.name + ))) + } + "evidence_preservation" if !self.has_evaluation_write_candidate() => { + Err(TreeRingError::Validation(format!( + "quality scenario {} category evidence_preservation requires at least one evaluation write_candidate", + self.name + ))) + } + _ => Ok(()), + } + } + + fn metric_applicability(&self) -> MetricApplicability { + MetricApplicability { + constraint_recall: !self.expected_recall.is_empty(), + forbidden_recall: !self.forbidden_recall.is_empty(), + spam_rejection: self + .expected_write_decisions + .iter() + .any(|decision| decision.decision == "reject"), + evidence_required: self.has_evaluation_write_candidate(), + behavior_proof: self.behavior_expectation.is_some(), + } + } + + fn has_evaluation_write_candidate(&self) -> bool { + self.write_candidates + .iter() + .any(|candidate| candidate.event_type.starts_with("evaluation_")) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct MetricApplicability { + constraint_recall: bool, + forbidden_recall: bool, + spam_rejection: bool, + evidence_required: bool, + behavior_proof: bool, } #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -222,35 +287,27 @@ impl BehaviorExpectation { } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct QualityThresholds { - #[serde(default = "one")] - pub min_constraint_recall_rate: f64, #[serde(default)] - pub max_forbidden_recall_rate: f64, - #[serde(default = "one")] - pub min_spam_rejection_rate: f64, - #[serde(default = "one")] - pub min_evidence_required_rate: f64, - #[serde(default = "one")] - pub min_behavior_proof_pass_rate: f64, -} - -impl Default for QualityThresholds { - fn default() -> Self { - Self { - min_constraint_recall_rate: 1.0, - max_forbidden_recall_rate: 0.0, - min_spam_rejection_rate: 1.0, - min_evidence_required_rate: 1.0, - min_behavior_proof_pass_rate: 1.0, - } - } + pub min_constraint_recall_rate: Option, + #[serde(default)] + pub max_forbidden_recall_rate: Option, + #[serde(default)] + pub min_spam_rejection_rate: Option, + #[serde(default)] + pub min_evidence_required_rate: Option, + #[serde(default)] + pub min_behavior_proof_pass_rate: Option, } impl QualityThresholds { - fn validate(&self, scenario_name: &str) -> TreeRingResult<()> { + fn validate( + &self, + scenario_name: &str, + applicability: MetricApplicability, + ) -> TreeRingResult<()> { for (field, value) in [ ( "min_constraint_recall_rate", @@ -267,12 +324,50 @@ impl QualityThresholds { self.min_behavior_proof_pass_rate, ), ] { - if !value.is_finite() || !(0.0..=1.0).contains(&value) { + if value.is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) { return Err(TreeRingError::Validation(format!( "quality scenario {scenario_name} threshold {field} must be between 0 and 1" ))); } } + for (field, configured, applicable, required_observation) in [ + ( + "min_constraint_recall_rate", + self.min_constraint_recall_rate.is_some(), + applicability.constraint_recall, + "expected_recall observations", + ), + ( + "max_forbidden_recall_rate", + self.max_forbidden_recall_rate.is_some(), + applicability.forbidden_recall, + "forbidden_recall observations", + ), + ( + "min_spam_rejection_rate", + self.min_spam_rejection_rate.is_some(), + applicability.spam_rejection, + "expected_write_decision reject observations", + ), + ( + "min_evidence_required_rate", + self.min_evidence_required_rate.is_some(), + applicability.evidence_required, + "evaluation write_candidate observations", + ), + ( + "min_behavior_proof_pass_rate", + self.min_behavior_proof_pass_rate.is_some(), + applicability.behavior_proof, + "behavior_expectation observations", + ), + ] { + if configured && !applicable { + return Err(TreeRingError::Validation(format!( + "quality scenario {scenario_name} threshold {field} is inapplicable without {required_observation}" + ))); + } + } Ok(()) } } @@ -319,6 +414,7 @@ pub struct QualityScenarioReport { pub spam_rejection_rate: Option, pub evidence_required_rate: Option, pub behavior_proof_pass: Option, + #[serde(default)] pub behavior_expectation: Option, pub quality_pass: bool, pub expected_recall: Vec, @@ -385,25 +481,21 @@ pub fn evaluate_quality_scenario( .map(|expectation| behavior_expectation_report(expectation, recalls)); let behavior_proof_pass = behavior_expectation.as_ref().map(|report| report.passed); let behavior_rate = behavior_proof_pass.map(|passed| if passed { 1.0 } else { 0.0 }); - let quality_pass = minimum_met( - constraint_recall_rate, - scenario.thresholds.min_constraint_recall_rate, - ) && maximum_met( - forbidden_recall_rate, - scenario.thresholds.max_forbidden_recall_rate, - ) && write_decisions.iter().all(|report| report.passed) - && minimum_met( - spam_rejection_rate, - scenario.thresholds.min_spam_rejection_rate, - ) - && minimum_met( - evidence_required_rate, - scenario.thresholds.min_evidence_required_rate, - ) - && minimum_met( - behavior_rate, - scenario.thresholds.min_behavior_proof_pass_rate, - ); + let thresholds = scenario + .metric_applicability() + .thresholds(&scenario.thresholds); + let quality_pass = + minimum_met( + constraint_recall_rate, + thresholds.min_constraint_recall_rate, + ) && maximum_met(forbidden_recall_rate, thresholds.max_forbidden_recall_rate) + && write_decisions.iter().all(|report| report.passed) + && minimum_met(spam_rejection_rate, thresholds.min_spam_rejection_rate) + && minimum_met( + evidence_required_rate, + thresholds.min_evidence_required_rate, + ) + && minimum_met(behavior_rate, thresholds.min_behavior_proof_pass_rate); Ok(QualityScenarioReport { name: scenario.name.clone(), @@ -484,10 +576,6 @@ fn validate_member(field: &str, value: &str, allowed: &[&str]) -> TreeRingResult } } -fn one() -> f64 { - 1.0 -} - fn validate_write_decision_coverage(scenario: &QualityScenario) -> TreeRingResult<()> { let mut candidate_ids = HashSet::new(); for (index, candidate) in scenario.write_candidates.iter().enumerate() { @@ -702,12 +790,43 @@ fn aggregate_rate(observations: impl Iterator) -> Option { (total > 0).then_some(passed as f64 / total as f64) } -fn minimum_met(rate: Option, threshold: f64) -> bool { - rate.is_none_or(|rate| rate >= threshold) +#[derive(Debug, Clone, Copy, PartialEq)] +struct EffectiveThresholds { + min_constraint_recall_rate: Option, + max_forbidden_recall_rate: Option, + min_spam_rejection_rate: Option, + min_evidence_required_rate: Option, + min_behavior_proof_pass_rate: Option, +} + +impl MetricApplicability { + fn thresholds(&self, configured: &QualityThresholds) -> EffectiveThresholds { + EffectiveThresholds { + min_constraint_recall_rate: configured + .min_constraint_recall_rate + .or(self.constraint_recall.then_some(1.0)), + max_forbidden_recall_rate: configured + .max_forbidden_recall_rate + .or(self.forbidden_recall.then_some(0.0)), + min_spam_rejection_rate: configured + .min_spam_rejection_rate + .or(self.spam_rejection.then_some(1.0)), + min_evidence_required_rate: configured + .min_evidence_required_rate + .or(self.evidence_required.then_some(1.0)), + min_behavior_proof_pass_rate: configured + .min_behavior_proof_pass_rate + .or(self.behavior_proof.then_some(1.0)), + } + } +} + +fn minimum_met(rate: Option, threshold: Option) -> bool { + threshold.is_none_or(|threshold| rate.is_some_and(|rate| rate >= threshold)) } -fn maximum_met(rate: Option, threshold: f64) -> bool { - rate.is_none_or(|rate| rate <= threshold) +fn maximum_met(rate: Option, threshold: Option) -> bool { + threshold.is_none_or(|threshold| rate.is_some_and(|rate| rate <= threshold)) } #[cfg(test)] @@ -816,6 +935,271 @@ mod tests { assert!(error.contains("requires behavior_expectation"), "{error}"); } + #[test] + fn constraint_recall_requires_expected_recall_observations() { + let error = parse_quality_scenario( + r#"{ + "name": "empty constraint recall", + "category": "constraint_recall", + "query": "proof loop" + }"#, + ) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one expected_recall"), + "{error}" + ); + } + + #[test] + fn spam_prevention_requires_reject_observations() { + let input = serde_json::json!({ + "name": "empty spam prevention", + "category": "spam_prevention", + "query": "memory spam", + "write_candidates": [{ + "id": "mem_non_reject_candidate", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "lesson", + "summary": "Durable note", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_non_reject_candidate", + "decision": "accept" + }] + }); + + let error = parse_quality_scenario(&input.to_string()) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one expected_write_decision of reject"), + "{error}" + ); + } + + #[test] + fn stale_truth_suppression_requires_forbidden_recall_observations() { + let error = parse_quality_scenario( + r#"{ + "name": "empty stale truth suppression", + "category": "stale_truth_suppression", + "query": "stale rule" + }"#, + ) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one forbidden_recall"), + "{error}" + ); + } + + #[test] + fn evidence_preservation_requires_evaluation_write_candidates() { + let input = serde_json::json!({ + "name": "empty evidence preservation", + "category": "evidence_preservation", + "query": "preserve evidence", + "write_candidates": [{ + "id": "mem_non_evaluation_candidate", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "lesson", + "summary": "Regular lesson", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_non_evaluation_candidate", + "decision": "accept" + }] + }); + + let error = parse_quality_scenario(&input.to_string()) + .unwrap_err() + .to_string(); + + assert!( + error.contains("requires at least one evaluation write_candidate"), + "{error}" + ); + } + + #[test] + fn rejects_explicit_thresholds_for_inapplicable_metrics() { + let cases = [ + ( + serde_json::json!({ + "name": "irrelevant constraint threshold", + "category": "stale_truth_suppression", + "query": "stale rule", + "forbidden_recall": [{"memory_id": "mem_stale"}], + "thresholds": {"min_constraint_recall_rate": 1.0} + }), + "min_constraint_recall_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant forbidden threshold", + "category": "constraint_recall", + "query": "proof loop", + "expected_recall": [{"memory_id": "mem_required"}], + "thresholds": {"max_forbidden_recall_rate": 0.0} + }), + "max_forbidden_recall_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant spam threshold", + "category": "evidence_preservation", + "query": "preserve evidence", + "write_candidates": [{ + "id": "mem_eval", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "cambium", + "event_type": "evaluation_outcome", + "summary": "Evidence-backed evaluation", + "details": "", + "source": {"type": "evidence", "ref": "evals/run-001", "quote": ""}, + "tags": ["evaluation"], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_eval", + "decision": "accept" + }], + "evidence_refs": ["evals/run-001"], + "thresholds": {"min_spam_rejection_rate": 1.0} + }), + "min_spam_rejection_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant evidence threshold", + "category": "spam_prevention", + "query": "memory spam", + "write_candidates": [{ + "id": "mem_spam", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "Transient planning chatter", + "details": "", + "source": {"type": "manual", "ref": "", "quote": ""}, + "tags": ["transient"], + "salience": 0.2, + "confidence": 0.2, + "sensitivity": "normal", + "retention": "ephemeral", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], + "expected_write_decisions": [{ + "memory_id": "mem_spam", + "decision": "reject" + }], + "thresholds": {"min_evidence_required_rate": 1.0} + }), + "min_evidence_required_rate", + ), + ( + serde_json::json!({ + "name": "irrelevant behavior threshold", + "category": "constraint_recall", + "query": "proof loop", + "expected_recall": [{"memory_id": "mem_required"}], + "thresholds": {"min_behavior_proof_pass_rate": 1.0} + }), + "min_behavior_proof_pass_rate", + ), + ]; + + for (input, field) in cases { + let error = parse_quality_scenario(&input.to_string()) + .unwrap_err() + .to_string(); + assert!(error.contains(field), "{field}: {error}"); + assert!(error.contains("inapplicable"), "{field}: {error}"); + } + } + + #[test] + fn older_quality_reports_deserialize_without_behavior_expectation() { + let report = serde_json::from_str::( + r#"{ + "name": "legacy report", + "category": "constraint_recall", + "constraint_recall_rate": 1.0, + "forbidden_recall_rate": null, + "spam_rejection_rate": null, + "evidence_required_rate": null, + "behavior_proof_pass": null, + "quality_pass": true, + "expected_recall": [], + "forbidden_recall": [], + "write_decisions": [] + }"#, + ) + .unwrap(); + + assert!(report.behavior_expectation.is_none()); + } + #[test] fn behavior_proof_reports_an_observed_decision_change() { let required = memory( @@ -880,15 +1264,21 @@ mod tests { }]; assert_eq!(report_json(&scenario, &recalls)["quality_pass"], true); - scenario.thresholds.min_behavior_proof_pass_rate = 0.5; + scenario.thresholds.min_behavior_proof_pass_rate = Some(0.5); assert_eq!(report_json(&scenario, &recalls)["quality_pass"], false); } #[test] - fn metrics_are_null_without_applicable_observations() { - let report = report_json(&scenario("no observations", "constraint_recall"), &[]); + fn inapplicable_metrics_are_null_when_a_primary_observation_exists() { + let mut scenario = scenario("constraint only", "constraint_recall"); + scenario.expected_recall = vec![RecallExpectation { + memory_id: Some("mem_missing".to_string()), + ..Default::default() + }]; + let report = report_json(&scenario, &[]); - assert!(report["constraint_recall_rate"].is_null()); + assert_eq!(report["constraint_recall_rate"], 0.0); + assert_eq!(report["quality_pass"], false); assert!(report["forbidden_recall_rate"].is_null()); assert!(report["spam_rejection_rate"].is_null()); assert!(report["evidence_required_rate"].is_null()); @@ -902,7 +1292,17 @@ mod tests { memory_id: Some("mem_missing".to_string()), ..Default::default() }]; - let unrelated = scenario("unrelated", "spam_prevention"); + let mut unrelated = scenario("unrelated", "evidence_preservation"); + let mut candidate = memory("mem_eval", "Evidence-backed evaluation.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-001".to_string(); + unrelated.write_candidates = vec![candidate]; + unrelated.expected_write_decisions = vec![WriteDecisionExpectation { + memory_id: "mem_eval".to_string(), + decision: "accept".to_string(), + reason: "valid unrelated evidence scenario".to_string(), + }]; + unrelated.evidence_refs = vec!["evals/run-001".to_string()]; let reports = vec![ evaluate_quality_scenario(&failing, &[]).unwrap(), evaluate_quality_scenario(&unrelated, &[]).unwrap(), @@ -931,9 +1331,20 @@ mod tests { memory: memory("mem_forbidden_recalled", "Stale instruction.", "heartwood"), score: 0.8, }]; + let mut unrelated = scenario("unrelated", "evidence_preservation"); + let mut candidate = memory("mem_eval", "Evidence-backed evaluation.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-001".to_string(); + unrelated.write_candidates = vec![candidate]; + unrelated.expected_write_decisions = vec![WriteDecisionExpectation { + memory_id: "mem_eval".to_string(), + decision: "accept".to_string(), + reason: "valid unrelated evidence scenario".to_string(), + }]; + unrelated.evidence_refs = vec!["evals/run-001".to_string()]; let reports = vec![ evaluate_quality_scenario(&mixed, &recalls).unwrap(), - evaluate_quality_scenario(&scenario("unrelated", "spam_prevention"), &[]).unwrap(), + evaluate_quality_scenario(&unrelated, &[]).unwrap(), ]; let run = serde_json::to_value(summarize_quality_run(reports)).unwrap(); @@ -955,13 +1366,13 @@ mod tests { ..Default::default() }, ]; - recall.thresholds.min_constraint_recall_rate = 0.5; + recall.thresholds.min_constraint_recall_rate = Some(0.5); let recalls = [QualityRecall { memory: required, score: 0.9, }]; assert_eq!(report_json(&recall, &recalls)["quality_pass"], true); - recall.thresholds.min_constraint_recall_rate = 0.6; + recall.thresholds.min_constraint_recall_rate = Some(0.6); assert_eq!(report_json(&recall, &recalls)["quality_pass"], false); let mut forbidden = scenario("forbidden threshold", "stale_truth_suppression"); @@ -969,7 +1380,7 @@ mod tests { memory_id: Some("mem_stale".to_string()), ..Default::default() }]; - forbidden.thresholds.max_forbidden_recall_rate = 0.0; + forbidden.thresholds.max_forbidden_recall_rate = Some(0.0); let stale = [QualityRecall { memory: memory("mem_stale", "Stale instruction.", "heartwood"), score: 0.9, @@ -1120,7 +1531,8 @@ mod tests { "name": "workflow prompt fallback", "category": "constraint_recall", "query": " ", - "workflow_prompt": " validate behavior proof " + "workflow_prompt": " validate behavior proof ", + "expected_recall": [{"memory_id": "mem_required"}] }"#; let scenario = parse_quality_scenario(input).unwrap(); @@ -1425,25 +1837,24 @@ mod tests { #[test] fn quality_pass_fails_when_expected_require_evidence_is_accepted() { + let mut candidate = memory("mem_accept", "Durable evidence-backed note.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-002".to_string(); let scenario = QualityScenario { name: "require evidence mismatch".to_string(), - category: "spam_prevention".to_string(), + category: "evidence_preservation".to_string(), seed_memories: Vec::new(), query: Some("write gates".to_string()), workflow_prompt: None, expected_recall: Vec::new(), forbidden_recall: Vec::new(), - write_candidates: vec![memory( - "mem_accept", - "Durable evidence-backed note.", - "cambium", - )], + write_candidates: vec![candidate], expected_write_decisions: vec![WriteDecisionExpectation { memory_id: "mem_accept".to_string(), decision: "require_evidence".to_string(), reason: "should need evidence".to_string(), }], - evidence_refs: Vec::new(), + evidence_refs: vec!["evals/run-002".to_string()], behavior_expectation: None, thresholds: QualityThresholds::default(), }; @@ -1457,25 +1868,24 @@ mod tests { #[test] fn quality_pass_fails_when_confirmation_mismatch_occurs() { + let mut candidate = memory("mem_accept", "Durable evidence-backed note.", "cambium"); + candidate.event_type = "evaluation_outcome".to_string(); + candidate.source.ref_ = "evals/run-003".to_string(); let scenario = QualityScenario { name: "confirmation mismatch".to_string(), - category: "spam_prevention".to_string(), + category: "evidence_preservation".to_string(), seed_memories: Vec::new(), query: Some("write gates".to_string()), workflow_prompt: None, expected_recall: Vec::new(), forbidden_recall: Vec::new(), - write_candidates: vec![memory( - "mem_accept", - "Durable evidence-backed note.", - "cambium", - )], + write_candidates: vec![candidate], expected_write_decisions: vec![WriteDecisionExpectation { memory_id: "mem_accept".to_string(), decision: "require_user_confirmation".to_string(), reason: "should require confirmation".to_string(), }], - evidence_refs: Vec::new(), + evidence_refs: vec!["evals/run-003".to_string()], behavior_expectation: None, thresholds: QualityThresholds::default(), }; diff --git a/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md index db85382..e24328c 100644 --- a/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md +++ b/docs/superpowers/plans/2026-07-09-tree-ring-memory-quality-proof-loop-implementation-plan.md @@ -21,6 +21,28 @@ - Preserve existing untracked local files; stage only files changed by each task. - Run `cargo fmt --check`, `cargo test --locked`, `cargo clippy --locked --all-targets`, `git diff --check`, and `sh scripts/certify-tree-ring.sh` before final handoff. +## Final Hardening Amendment + +This implementation plan now carries the final hardening semantics that +supersede earlier draft snippets: + +- Category validation must enforce a primary observation contract: + `constraint_recall` requires `expected_recall`, + `spam_prevention` requires at least one expected `reject`, + `stale_truth_suppression` requires `forbidden_recall`, + `behavior_proof` requires `behavior_expectation`, and + `evidence_preservation` requires at least one `evaluation_` write candidate. +- `QualityThresholds` are presence-aware optional fields. Omitted thresholds + inherit strict defaults only when their metric has observations. Configuring + a threshold for an inapplicable metric is a validation failure, not a silent + no-op. +- Runner failures must be sanitized before they are written to + `quality-report.json`, `quality-summary.md`, or bubbled back through the + example entrypoint. Reports keep stage plus stable error class, but never raw + fixture payload values. +- `QualityScenarioReport.behavior_expectation` must deserialize as `None` when + older artifacts omit the field. + --- ## Scope Check diff --git a/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md index 409cbb5..51b97ef 100644 --- a/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md +++ b/docs/superpowers/specs/2026-07-09-tree-ring-memory-quality-proof-loop-design.md @@ -82,6 +82,23 @@ Add a quality layer around existing Tree Ring primitives. The quality layer consumes memory events, recall results, and proposed write candidates. It does not own storage and does not need a background process. +Final hardening semantics for this lane: + +- Every category must carry a primary observation contract during validation. + `constraint_recall` needs at least one `expected_recall`, + `spam_prevention` needs at least one expected `reject` decision, + `stale_truth_suppression` needs at least one `forbidden_recall`, + `behavior_proof` needs `behavior_expectation`, and + `evidence_preservation` needs at least one `evaluation_` write candidate. +- Threshold configuration is presence-aware. Threshold fields are optional in + fixtures and default to strict behavior only when the corresponding metric + has observations. An explicitly configured threshold for a metric with no + observations is invalid and must fail scenario validation. +- Runner failures must emit stable stage plus error-class output in JSON, + markdown, and terminal failure paths. Fixture values, invalid memory field + values, and other raw payload fragments must not be echoed back in failure + messages. + Target components: - `fixtures/quality/`: reviewable quality scenario fixtures. @@ -124,6 +141,8 @@ Each scenario should include: memory-informed decision, expected decision, and an optional reason. It is required for `behavior_proof` scenarios. - `thresholds`: per-scenario minimums or tolerances. + Threshold entries are optional and only legal for metrics the scenario + actually observes. First scenario pack: @@ -241,8 +260,9 @@ or warning behavior. ## Error Handling -- Fixture-facing objects reject unknown fields. Invalid scenario fixtures - should report the scenario name, file path, and invalid field. +- Fixture-facing objects reject unknown fields. Parse and validation failures + should report the fixture path plus a stable error class, without extracting + a scenario name or echoing invalid fields and other raw fixture values. - Missing expected recall should produce a precise failed expectation. - Forbidden recall should include the returned memory id and reason. - Write-decision mismatches should show the candidate id or summary and the diff --git a/fixtures/quality/no-background-writer-constraint.json b/fixtures/quality/no-background-writer-constraint.json index 88a0aba..c5489f5 100644 --- a/fixtures/quality/no-background-writer-constraint.json +++ b/fixtures/quality/no-background-writer-constraint.json @@ -41,12 +41,5 @@ "memory_id": "mem_quality_no_background_writer", "reason": "proof-loop work must recall the explicit no-background-writer constraint" } - ], - "thresholds": { - "min_constraint_recall_rate": 1.0, - "max_forbidden_recall_rate": 0.0, - "min_spam_rejection_rate": 1.0, - "min_evidence_required_rate": 1.0, - "min_behavior_proof_pass_rate": 1.0 - } + ] } From 06f3808a5504b35106ad16c64adf1baee99b7a84 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 17:07:34 -0400 Subject: [PATCH 18/19] Validate forbidden recall seeds --- crates/tree-ring-memory-core/src/quality.rs | 113 +++++++++++++++++++- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index 625c9c9..8ad8c95 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -84,8 +84,14 @@ impl QualityScenario { for expectation in &self.expected_recall { expectation.validate("expected_recall", &self.name)?; } - for expectation in &self.forbidden_recall { + for (index, expectation) in self.forbidden_recall.iter().enumerate() { expectation.validate("forbidden_recall", &self.name)?; + if matching_seed_memory_ids(expectation, &self.seed_memories).is_empty() { + return Err(TreeRingError::Validation(format!( + "quality scenario {} forbidden_recall[{}] does not match a seed_memory", + self.name, index + ))); + } } for decision in &self.expected_write_decisions { decision.validate(&self.name)?; @@ -675,6 +681,17 @@ fn matching_recall_ids(expectation: &RecallExpectation, recalls: &[QualityRecall .collect() } +fn matching_seed_memory_ids( + expectation: &RecallExpectation, + seed_memories: &[MemoryEvent], +) -> Vec { + seed_memories + .iter() + .filter(|memory| expectation_matches_memory(expectation, memory)) + .map(|memory| memory.id.clone()) + .collect() +} + fn expectation_matches_memory(expectation: &RecallExpectation, memory: &MemoryEvent) -> bool { if expectation .memory_id @@ -1069,6 +1086,29 @@ mod tests { "name": "irrelevant constraint threshold", "category": "stale_truth_suppression", "query": "stale rule", + "seed_memories": [{ + "id": "mem_stale", + "created_at": "2026-07-09T00:00:00Z", + "updated_at": "2026-07-09T00:00:00Z", + "project": "tree-ring", + "agent_profile": null, + "scope": "project", + "ring": "heartwood", + "event_type": "lesson", + "summary": "Stale instruction", + "details": "", + "source": {"type": "evidence", "ref": "docs/spec.md", "quote": ""}, + "tags": [], + "salience": 0.8, + "confidence": 0.8, + "sensitivity": "normal", + "retention": "durable", + "expires_at": null, + "supersedes": [], + "superseded_by": null, + "links": [], + "review": {"needs_review": false, "review_reason": null, "reviewed_at": null, "reviewed_by": null} + }], "forbidden_recall": [{"memory_id": "mem_stale"}], "thresholds": {"min_constraint_recall_rate": 1.0} }), @@ -1317,6 +1357,14 @@ mod tests { #[test] fn forbidden_run_rate_uses_failures_over_applicable_expectations() { let mut mixed = scenario("mixed forbidden recall", "stale_truth_suppression"); + mixed.seed_memories = vec![ + memory("mem_forbidden_recalled", "Stale instruction.", "heartwood"), + memory( + "mem_forbidden_hidden", + "Hidden stale instruction.", + "heartwood", + ), + ]; mixed.forbidden_recall = vec![ RecallExpectation { memory_id: Some("mem_forbidden_recalled".to_string()), @@ -1376,6 +1424,7 @@ mod tests { assert_eq!(report_json(&recall, &recalls)["quality_pass"], false); let mut forbidden = scenario("forbidden threshold", "stale_truth_suppression"); + forbidden.seed_memories = vec![memory("mem_stale", "Stale instruction.", "heartwood")]; forbidden.forbidden_recall = vec![RecallExpectation { memory_id: Some("mem_stale".to_string()), ..Default::default() @@ -1666,7 +1715,14 @@ mod tests { let scenario = QualityScenario { name: "recall gate".to_string(), category: "constraint_recall".to_string(), - seed_memories: Vec::new(), + seed_memories: vec![ + memory( + "mem_required", + "Do not add a background writer.", + "heartwood", + ), + memory("mem_forbidden", "Stale instruction.", "heartwood"), + ], query: Some("background writer".to_string()), workflow_prompt: None, expected_recall: vec![RecallExpectation { @@ -1707,12 +1763,63 @@ mod tests { assert!(report.forbidden_recall[0].passed); } + #[test] + fn validates_forbidden_recall_against_seed_memories() { + let scenario = QualityScenario { + name: "stale recall seed coverage".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: vec![memory("mem_stale", "Old CLI contract.", "heartwood")], + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + ring: Some("heartwood".to_string()), + reason: "seeded stale memory".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + scenario.validate().unwrap(); + } + + #[test] + fn rejects_forbidden_recall_with_missing_seed_selector() { + let scenario = QualityScenario { + name: "stale recall typo".to_string(), + category: "stale_truth_suppression".to_string(), + seed_memories: vec![memory("mem_stale", "Old CLI contract.", "heartwood")], + query: Some("cli contract".to_string()), + workflow_prompt: None, + expected_recall: Vec::new(), + forbidden_recall: vec![RecallExpectation { + memory_id: Some("mem_typo".to_string()), + reason: "typo selector".to_string(), + ..Default::default() + }], + write_candidates: Vec::new(), + expected_write_decisions: Vec::new(), + evidence_refs: Vec::new(), + behavior_expectation: None, + thresholds: QualityThresholds::default(), + }; + + let error = scenario.validate().unwrap_err().to_string(); + + assert!(error.contains("forbidden_recall[0]")); + assert!(error.contains("does not match a seed_memory")); + } + #[test] fn fails_when_forbidden_memory_is_recalled() { let scenario = QualityScenario { name: "stale recall".to_string(), category: "stale_truth_suppression".to_string(), - seed_memories: Vec::new(), + seed_memories: vec![memory("mem_stale", "Old CLI contract.", "heartwood")], query: Some("cli contract".to_string()), workflow_prompt: None, expected_recall: Vec::new(), From 10c153f28141899153f91f6c00f0bd9db3bcb6c4 Mon Sep 17 00:00:00 2001 From: TerminallyLazy Date: Thu, 9 Jul 2026 17:46:19 -0400 Subject: [PATCH 19/19] Address PR review feedback --- .../examples/quality_scenarios.rs | 50 +++++++++++++++++-- .../src/agent_awareness.rs | 31 +++++++++++- crates/tree-ring-memory-core/src/quality.rs | 22 +++----- scripts/certify-tree-ring.sh | 4 +- 4 files changed, 83 insertions(+), 24 deletions(-) diff --git a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs index c84ec82..1df3fa1 100644 --- a/crates/tree-ring-memory-cli/examples/quality_scenarios.rs +++ b/crates/tree-ring-memory-cli/examples/quality_scenarios.rs @@ -46,13 +46,28 @@ fn run() -> Result<(), String> { fn run_quality_command(fixture_dir: &Path, output_dir: &Path) -> Result { let run_report = run_quality_scenarios(fixture_dir, output_dir)?; - if !run_report.quality_pass { + verify_persisted_quality_report(output_dir, &run_report)?; + Ok(run_report) +} + +fn verify_persisted_quality_report( + output_dir: &Path, + expected: &QualityRunReport, +) -> Result<(), String> { + let json = fs::read_to_string(output_dir.join("quality-report.json")) + .map_err(|_| "quality_report_read_error".to_string())?; + let persisted = serde_json::from_str::(&json) + .map_err(|_| "quality_report_parse_error".to_string())?; + if persisted != *expected { + return Err("quality_report_mismatch_error".to_string()); + } + if !persisted.ok || !persisted.quality_pass { return Err(format!( "quality run failed: {} scenarios evaluated", - run_report.scenario_count + persisted.scenario_count )); } - Ok(run_report) + Ok(()) } fn run_quality_scenarios( @@ -206,7 +221,7 @@ fn run_scenario(scenario: &QualityScenario) -> Result Option<( fn insert_section_before_anchor(content: &str, section: &str, anchor: &str) -> Option { let anchor_index = content.find(anchor)?; + let section = if content.contains("\r\n") { + Cow::Owned(section.replace('\n', "\r\n")) + } else { + Cow::Borrowed(section) + }; let mut updated = String::with_capacity(content.len() + section.len()); updated.push_str(&content[..anchor_index]); - updated.push_str(section); + updated.push_str(§ion); updated.push_str(&content[anchor_index..]); Some(updated) } @@ -223,7 +229,8 @@ fn is_generated_cli_file(content: &str) -> bool { } fn is_generated_skill_file(content: &str) -> bool { - content.starts_with("---\n") && content.contains(SKILL_FRONT_MATTER_MARKER) + (content.starts_with("---\n") || content.starts_with("---\r\n")) + && content.contains(SKILL_FRONT_MATTER_MARKER) } fn agent_contract(root: &Path) -> String { @@ -513,6 +520,26 @@ mod tests { assert!(skill.contains(skill_section.trim())); } + #[test] + fn generated_backfill_recognizes_and_preserves_crlf_skill_files() { + let dir = tempdir().unwrap(); + let root = dir.path().join(".tree-ring"); + fs::create_dir_all(&root).unwrap(); + let stale_skill = stale_fixture( + SKILL_TEMPLATE, + SKILL_QUALITY_GATES_HEADING, + SKILL_QUALITY_GATES_ANCHOR, + ) + .replace('\n', "\r\n"); + fs::write(root.join("SKILL.md"), stale_skill).unwrap(); + + ensure_agent_awareness(&root).unwrap(); + + let skill = fs::read_to_string(root.join("SKILL.md")).unwrap(); + assert!(skill.contains("## Memory Quality Gates\r\n")); + assert!(!skill.replace("\r\n", "").contains('\n')); + } + #[test] fn generated_backfill_inserts_quality_gates_at_canonical_anchors() { let dir = tempdir().unwrap(); diff --git a/crates/tree-ring-memory-core/src/quality.rs b/crates/tree-ring-memory-core/src/quality.rs index 8ad8c95..d553206 100644 --- a/crates/tree-ring-memory-core/src/quality.rs +++ b/crates/tree-ring-memory-core/src/quality.rs @@ -86,7 +86,7 @@ impl QualityScenario { } for (index, expectation) in self.forbidden_recall.iter().enumerate() { expectation.validate("forbidden_recall", &self.name)?; - if matching_seed_memory_ids(expectation, &self.seed_memories).is_empty() { + if !has_matching_seed_memory(expectation, &self.seed_memories) { return Err(TreeRingError::Validation(format!( "quality scenario {} forbidden_recall[{}] does not match a seed_memory", self.name, index @@ -593,12 +593,6 @@ fn validate_write_decision_coverage(scenario: &QualityScenario) -> TreeRingResul } } - let candidate_index_by_id = scenario - .write_candidates - .iter() - .enumerate() - .map(|(index, candidate)| (candidate.id.as_str(), index)) - .collect::>(); let mut decision_index_by_id = HashMap::new(); for (index, decision) in scenario.expected_write_decisions.iter().enumerate() { @@ -610,7 +604,7 @@ fn validate_write_decision_coverage(scenario: &QualityScenario) -> TreeRingResul scenario.name, decision.memory_id ))); } - if !candidate_index_by_id.contains_key(decision.memory_id.as_str()) { + if !candidate_ids.contains(decision.memory_id.as_str()) { return Err(TreeRingError::Validation(format!( "quality scenario {} expected_write_decisions[{index}] memory_id {} does not match any write_candidate", scenario.name, decision.memory_id @@ -681,15 +675,13 @@ fn matching_recall_ids(expectation: &RecallExpectation, recalls: &[QualityRecall .collect() } -fn matching_seed_memory_ids( +fn has_matching_seed_memory( expectation: &RecallExpectation, seed_memories: &[MemoryEvent], -) -> Vec { +) -> bool { seed_memories .iter() - .filter(|memory| expectation_matches_memory(expectation, memory)) - .map(|memory| memory.id.clone()) - .collect() + .any(|memory| expectation_matches_memory(expectation, memory)) } fn expectation_matches_memory(expectation: &RecallExpectation, memory: &MemoryEvent) -> bool { @@ -768,9 +760,7 @@ fn classify_write_candidate(candidate: &MemoryEvent, required_evidence_refs: &[S } if !required_evidence_refs.is_empty() && candidate.event_type.starts_with("evaluation_") - && !required_evidence_refs - .iter() - .any(|required| required == &candidate.source.ref_) + && !required_evidence_refs.contains(&candidate.source.ref_) { return "require_evidence".to_string(); } diff --git a/scripts/certify-tree-ring.sh b/scripts/certify-tree-ring.sh index ab157ee..4b28f4c 100755 --- a/scripts/certify-tree-ring.sh +++ b/scripts/certify-tree-ring.sh @@ -209,8 +209,8 @@ run cargo run --release -p tree-ring-memory-cli --example quality_scenarios -- " > "$QUALITY_RUN_OUT" 2>&1 require_file "$QUALITY_OUT/quality-report.json" require_file "$QUALITY_OUT/quality-summary.md" -grep -Fx ' "quality_pass": true,' "$QUALITY_OUT/quality-report.json" > /dev/null \ - || fail "memory quality scenarios did not pass" +# The runner re-parses this persisted report and exits nonzero unless its +# top-level status matches the in-memory passing report. quality_json=$(cat "$QUALITY_OUT/quality-report.json") agent_zero_status='"skipped"'