diff --git a/docs/branch-review-records/1048a2a62bd4a4306cb90f6c144e94427f6cf21d5a9f449f983b170ddb35a4b5.record.md b/docs/branch-review-records/1048a2a62bd4a4306cb90f6c144e94427f6cf21d5a9f449f983b170ddb35a4b5.record.md new file mode 100644 index 0000000000..afcdeedbf7 --- /dev/null +++ b/docs/branch-review-records/1048a2a62bd4a4306cb90f6c144e94427f6cf21d5a9f449f983b170ddb35a4b5.record.md @@ -0,0 +1 @@ +| 2026-08-17 | claude/g1-rag-document-context-qn9ubx | f59b358d5ddba2480c585196b262afe1f4181632 | packet G1 document_context provenance tag + ledger issueRowFingerprint display-id fix + two queued G1 closures (supersedes the f5f146a record: two further commits landed on this branch) | approved — G1 is a provenance tag only, deriveConfidence and synthetic_similarity_count unchanged and pinned; ledger fix restores issues:done for ULID-backed rows with null-safety preserved; no retrieval behaviour change, no canary per owner decision 2026-08-17 Option B | verify:pr-local all 18 stages green at this head; full unit 642 files / 6879 passed 4 skipped; eval:rag:offline 36 golden 25 suites 603/603; 4 G1 pins + 3 ledger pins mutation-checked | diff --git a/docs/branch-review-records/3f5dad0c7109ddc3f59bd3ce42fa9df1587d3cbcd938a01b8950f1f1e746472a.record.md b/docs/branch-review-records/3f5dad0c7109ddc3f59bd3ce42fa9df1587d3cbcd938a01b8950f1f1e746472a.record.md new file mode 100644 index 0000000000..1fd9a2015d --- /dev/null +++ b/docs/branch-review-records/3f5dad0c7109ddc3f59bd3ce42fa9df1587d3cbcd938a01b8950f1f1e746472a.record.md @@ -0,0 +1 @@ +| 2026-08-17 | claude/g1-rag-document-context-qn9ubx | f5f146a497a4ed04fc1db99b7b8f90a00a1cea30 | packet G1 — document_context similarity_origin provenance tag (types.ts, answer-stream-contract.ts, rag-row-contracts.ts, rag.ts telemetry export) + 4 discriminating pins + H5a/HANDOVER docs | approved — provenance tag only; deriveConfidence and synthetic_similarity_count unchanged and pinned; no retrieval behaviour change, no canary per owner decision 2026-08-17 Option B | vitest 5 touched suites 58/58; full unit 642 files / 6879 passed 4 skipped; eval:rag:offline 36 golden 25 suites 603/603; check:rag:fixtures; verify:pr-local all 18 stages green (lint, typecheck, test, build, eval:rag:offline); 4 pins mutation-checked | diff --git a/docs/clinical-hazard-analysis.md b/docs/clinical-hazard-analysis.md index e3cb7567e8..52ccb3fb08 100644 --- a/docs/clinical-hazard-analysis.md +++ b/docs/clinical-hazard-analysis.md @@ -133,6 +133,31 @@ keep telemetry's `synthetic_similarity_count` from counting it. Option A (tag as downgrade without a measured safety gain. Implementation packet: `docs/rag-improvement/HANDOVER.md` §G1 (no retrieval behaviour change; no canary). +**H5a — Option B implemented (packet G1, 2026-08-17).** `"document_context"` is now a declared +value of the `similarity_origin` union ([`types.ts`](../src/lib/types.ts)), accepted by the +streamed-preview client-source validator +([`answer-stream-contract.ts`](../src/lib/answer-stream-contract.ts)), and stamped on every row +`buildDocumentSummaryResults` builds +([`rag-row-contracts.ts`](../src/lib/rag/rag-row-contracts.ts)). The fabricated `similarity: 1` +is therefore no longer indistinguishable from a perfect cosine at any surface that reads a row. +Per the decision, `deriveConfidence` ([`rag-answer-support.ts`](../src/lib/rag/rag-answer-support.ts)) +is **unchanged** — it still excludes only `"synthetic_text"` from `strongestNonSynthetic`, so a +document summary still reaches `"high"` — and `synthetic_similarity_count` +([`rag.ts`](../src/lib/rag/rag.ts)) still counts only `"synthetic_text"`, keeping the RC9 +imputed-score signal a single population. Four discriminating pins hold the split: the summary +rows carry the tag ([`rag-retrieval-row-contract.test.ts`](../tests/rag-retrieval-row-contract.test.ts)); +two `"document_context"` citations at ≥ 0.82 still yield `"high"` while the identical scores +tagged `"synthetic_text"` still yield `"medium"`; and the telemetry counter ignores the new value +([`rag-score.test.ts`](../tests/rag-score.test.ts)). Each was mutation-checked — folding +`"document_context"` into the `deriveConfidence` exclusion (rejected Option A), widening the +telemetry filter, and removing the stamp each turn the matching pin red. + +**Residual after G1.** This closes the _legibility_ half of the live residual, not the underlying +question of whether a constant 1.0 should ever contribute to a confidence label. The tag now makes +that reachable: any future gate can discriminate the document-summary route without re-deriving +provenance. The fast-path half of H5a (`"synthetic_text"`, already excluded from `"high"`) and the +`scoreValue` gap in the row above are unchanged by this packet. + ### H6 — Adversarial / corrupted document content changes the answer _Pathway: uploaded document text → model context → answer. Detailed in the companion threat model; summarized here as a hazard._ diff --git a/docs/outstanding-issues-inbox/34c0f9bf-22fe-495b-a828-f73bbd4cfddf.json b/docs/outstanding-issues-inbox/34c0f9bf-22fe-495b-a828-f73bbd4cfddf.json new file mode 100644 index 0000000000..739eb4eaa5 --- /dev/null +++ b/docs/outstanding-issues-inbox/34c0f9bf-22fe-495b-a828-f73bbd4cfddf.json @@ -0,0 +1,11 @@ +{ + "version": 2, + "id": "34c0f9bf-22fe-495b-a828-f73bbd4cfddf", + "createdOn": "2026-08-17", + "action": "done", + "payload": { + "id": "#0MSNT8", + "outcome": "Implemented as packet G1 on branch claude/g1-rag-document-context-qn9ubx. Every element of the queued Option B scope landed: document_context added to the similarity_origin union in src/lib/types.ts and to src/lib/answer-stream-contract.ts (as an allow-set, so the union and the stream validator cannot drift), stamped in buildDocumentSummaryResults (src/lib/rag/rag-row-contracts.ts), deriveConfidence left unchanged, rag.ts synthetic_similarity_count left counting only synthetic_text, and docs/clinical-hazard-analysis.md H5a updated to mark the decision implemented. Four discriminating pins, each mutation-checked against the change it exists to catch: summary rows carry the tag (tests/rag-retrieval-row-contract.test.ts); two document_context citations at >= 0.82 still yield high while the identical scores tagged synthetic_text still yield medium, and the telemetry counter ignores the new value (tests/rag-score.test.ts); the stream validator accepts every declared union member with a compile-time exhaustiveness guard (tests/answer-incremental-delivery.test.ts). Gates: vitest 58/58 across the five touched suites, full unit 642 files / 6879 passed, eval:rag:offline 36 golden cases / 603 tests, check:rag:fixtures, verify:pr-local all 18 stages green. No canary per the decision. HANDOVER status row updated. Closes the paired governance question row #J912J9.", + "baseRowFingerprint": "5d3f833d9e969448a55703d05c2423ccd92eb9176410abf285825e88bcc0daec" + } +} diff --git a/docs/outstanding-issues-inbox/fb7d42c0-ee67-426e-907e-f2c306afc38b.json b/docs/outstanding-issues-inbox/fb7d42c0-ee67-426e-907e-f2c306afc38b.json new file mode 100644 index 0000000000..186313e76d --- /dev/null +++ b/docs/outstanding-issues-inbox/fb7d42c0-ee67-426e-907e-f2c306afc38b.json @@ -0,0 +1,11 @@ +{ + "version": 2, + "id": "fb7d42c0-ee67-426e-907e-f2c306afc38b", + "createdOn": "2026-08-17", + "action": "done", + "payload": { + "id": "#J912J9", + "outcome": "Answered and implemented (packet G1, PR for branch claude/g1-rag-document-context-qn9ubx). Owner decided Option B on 2026-08-17: document-summary rows keep the high confidence label, and the fabricated similarity gets its own provenance value rather than being folded into synthetic_text. Landed: document_context added to the similarity_origin union (src/lib/types.ts) and to the streamed-preview client-source validator (src/lib/answer-stream-contract.ts), and stamped in buildDocumentSummaryResults (src/lib/rag/rag-row-contracts.ts). Per the decision deriveConfidence (src/lib/rag/rag-answer-support.ts) is unchanged and still excludes only synthetic_text, and rag.ts synthetic_similarity_count still counts only synthetic_text; both are pinned by discriminating tests that go red on the rejected Option A fold. Option A (tag as synthetic_text so summaries cap at medium) recorded as rejected. docs/clinical-hazard-analysis.md H5a marks the decision implemented and names the residual: the tag closes the legibility gap, not the deeper question of whether a constant 1.0 should contribute to a confidence label, but any future gate can now discriminate the route without re-deriving provenance. No retrieval behaviour change; no canary.", + "baseRowFingerprint": "e072b25f5773f1945471f8081b307f5f74766ef400aaad8c01d27f186066eea6" + } +} diff --git a/docs/rag-improvement/HANDOVER.md b/docs/rag-improvement/HANDOVER.md index 6e79fff7b0..d01fd1c33b 100644 --- a/docs/rag-improvement/HANDOVER.md +++ b/docs/rag-improvement/HANDOVER.md @@ -76,7 +76,7 @@ generation-quality verdict on fallback`), merged 2026-08-13 — structured | S1b | A1 rung 3 (R1): pre-deadline strong routing for dosing class | `claude/s1b-rag-dosing-routing-6u1mik` | #2035 | Merged 2026-08-17 (PR #2035, merge `92f7618`) | canary pair pending: baseline run 32025082010 (`2bd146eed`) -> post-merge dispatch (owner-approved); offline 586/586 + verify:pr-local heavy scope green | | S1c | A1 residuals R2 + R3: claim-support strictness | `claude/s1c-residuals-r2-r3-4pb1at` | #2052 | Merged 2026-08-17 (merge `b8e774bcd`; follow-ups #2063, #2065) | canary pair: baseline run 32049952885 -> post run 32052479537 (`084f63799`): recall 1.0/1.0, zero per-case rr regressions, answer gate 44/44 | | S1d | A1 final-gate gap recovery: hedged cited low-confidence fast answers must recover extractively, not collapse to a citation-free `provider_source_gap` | `claude/s1d-final-gate-gap-recovery-dxgrn2` | #2054 | Merged 2026-08-17 (merge `0bbd64fbc`); canary pending (owner approval) | needs post-merge canary pair (baseline run 32039841070, `e6ad0d5db`); offline: verify:pr-local heavy green, eval:rag:offline 604/604, 6 new discriminating fixtures | -| G1 | Governance: provenance tag for document-summary rows (Option B) | `claude/rag-g1-document-context-origin-` | — | Ready — disjoint; owner decided Option B 2026-08-17 | no canary (no behaviour change) | +| G1 | Governance: provenance tag for document-summary rows (Option B) | `claude/g1-rag-document-context-qn9ubx` | #2053 | Implemented 2026-08-17 - awaiting review/merge | no canary by decision (provenance tag only). Offline: 58/58 across the five touched suites; `eval:rag:offline`, `check:rag:fixtures`, `verify:pr-local` - see PR body | | S2 | A2 (+A3): composition menu + moderate length | `claude/rag-a2-composition-` | — | Blocked on the S1d canary pair (S1c pair green); dispatch when green | canary pair + `eval:answer-quality` + Gate E | | S2b | A3: moderate length (if separate review needed) | `claude/rag-a3-length-` | — | Blocked on S2 | — | | S3 | A4: follow-up suggestion refinement | `claude/rag-a4-follow-ups-` | — | Blocked on S2 + S2b | — | diff --git a/scripts/check-outstanding-issues.mjs b/scripts/check-outstanding-issues.mjs index dfac7ee768..2422a3d74d 100644 --- a/scripts/check-outstanding-issues.mjs +++ b/scripts/check-outstanding-issues.mjs @@ -33,7 +33,7 @@ import { execFileSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { createHash } from "node:crypto"; -import { canonicalLegacyIssueId, issueIdCitations, parseIssueIdCell } from "./issue-id.mjs"; +import { canonicalLegacyIssueId, isIssueDisplayId, issueIdCitations, parseIssueIdCell } from "./issue-id.mjs"; export const ISSUES_PATH = "docs/outstanding-issues.md"; @@ -250,22 +250,34 @@ export function parseIssues(markdown) { }; } +// Two id generations coexist in the ledger: legacy zero-padded sequential ids +// (`#001`) and the Crockford display locators minted from a row's ULID +// (`#J912J9`). Rows of both kinds are addressed by display id everywhere else, +// but this lookup resolved only the numeric form — so for every row created +// after the ULID migration it returned null, and `ledger-inbox.mjs` reads a null +// fingerprint as "no such row" and refuses the request. The visible symptom was +// `npm run issues:done '#J912J9'` failing with "is not in Open items" against a +// row plainly present in Open items, which made the optimistic-concurrency check +// unreachable for exactly the rows that have it available (they carry a ULID). export function issueRowFingerprint(markdown, issueId) { - const trimmed = String(issueId).trim(); - const numericMatch = trimmed.match(/^#(\d+)$/); - const number = numericMatch ? Number(numericMatch[1]) : null; - if (numericMatch && !Number.isFinite(number)) return null; - - // Legacy numeric ids match by number (tolerating zero-padding differences); - // ULID-suffix display ids minted by reconcile match by the exact display id. - // Without the second arm, issues:done could never close a reconciled row. - const row = parseIssues(markdown).rows.find( - (entry) => - entry.table === "open" && - entry.valid && - entry.raw && - (numericMatch ? entry.number === number : entry.id === trimmed), - ); + const id = String(issueId).trim(); + const legacy = id.match(/^#(\d+)$/); + const number = legacy ? Number(legacy[1]) : null; + if (legacy) { + if (!Number.isFinite(number)) return null; + } else if (!isIssueDisplayId(id)) { + return null; + } + + const open = parseIssues(markdown).rows.filter((entry) => entry.table === "open" && entry.valid && entry.raw); + // Exact display id first, and only then the legacy numeric interpretation. + // Crockford's alphabet includes 0-9, so a ULID-derived locator can be entirely + // digits (`#041061`) and is indistinguishable from a legacy id by pattern + // alone — branching on shape would silently miss exactly those rows. The + // fallback preserves the old behaviour of resolving a non-canonical legacy id + // (`#5`) to its zero-padded row, and the exact-id arm preserves + // `issues:done` closing ULID-suffix display ids minted by reconcile. + const row = open.find((entry) => entry.id === id) ?? (legacy ? open.find((entry) => entry.number === number) : null); if (!row) return null; const normalized = `| ${cells(row.raw).join(" | ")} |`; return createHash("sha256").update(normalized).digest("hex"); diff --git a/src/lib/answer-stream-contract.ts b/src/lib/answer-stream-contract.ts index a5d2de32b6..914fe087ed 100644 --- a/src/lib/answer-stream-contract.ts +++ b/src/lib/answer-stream-contract.ts @@ -47,6 +47,10 @@ const answerSectionKinds = new Set([ "verification", ]); const answerSectionSupportLevels = new Set(["direct", "partial", "nearby", "unsupported"]); +// Mirrors the `similarity_origin` union in `types.ts`. Kept as an allow-set rather than a +// chain of `!==` comparisons so adding a provenance value there cannot silently leave the +// streamed preview rejecting a payload the `final` response accepts. +const similarityOriginValues = new Set(["cosine", "synthetic_text", "document_context"]); const citationProvenanceValues = new Set([ "model_selected", "section_selected", @@ -165,8 +169,7 @@ function isClientSource(value: unknown): value is SearchResult { if ( "similarity_origin" in value && value.similarity_origin !== undefined && - value.similarity_origin !== "cosine" && - value.similarity_origin !== "synthetic_text" + !similarityOriginValues.has(value.similarity_origin as string) ) { return false; } diff --git a/src/lib/rag/rag-row-contracts.ts b/src/lib/rag/rag-row-contracts.ts index 8b140732cf..c868d7f82f 100644 --- a/src/lib/rag/rag-row-contracts.ts +++ b/src/lib/rag/rag-row-contracts.ts @@ -183,7 +183,23 @@ export function assertIndexUnitRows(rows: unknown, rpc: string): asserts rows is assertRowsAgainst(z.array(indexUnitRowSchema), rows, rpc); } -/** Build and validate the locally retrieved rows used as document-summary context. */ +/** + * Build and validate the locally retrieved rows used as document-summary context. + * + * `similarity: 1` here is a constant, not a measured cosine: `summarizeDocument` loads every + * committed chunk of one document, so there is no query to score against — the document IS the + * query. The `similarity_origin: "document_context"` tag makes that provenance explicit rather + * than leaving a fabricated 1.0 indistinguishable from a perfect vector match (H5a live + * residual, `docs/clinical-hazard-analysis.md`; owner decision 2026-08-17, Option B). + * + * The tag is deliberately NOT `"synthetic_text"`. That value marks scores imputed from lexical + * or structural match strength on the general answer path, where a title hit can masquerade as + * semantic evidence; `deriveConfidence` (`rag-answer-support.ts`) therefore excludes it from the + * "high" bar. This route has no match strength to inflate and its citations are verified by the + * same grounding pipeline, so the confidence derivation is unchanged and the "high" label stays. + * Reusing `"synthetic_text"` here would silently cap every document summary at "medium" — + * recorded as rejected Option A. `tests/rag-score.test.ts` pins both halves of that split. + */ export function buildDocumentSummaryResults( chunks: unknown[], document: { title: string; file_name: string; metadata?: unknown }, @@ -194,6 +210,7 @@ export function buildDocumentSummaryResults( file_name: document.file_name, source_metadata: normalizeOptionalSourceMetadata(document.metadata), similarity: 1, + similarity_origin: "document_context" as const, images: [], })); assertRetrievalRows(results, "document_summary_context"); diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 3804702622..e45e81abf1 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -537,8 +537,8 @@ function provenanceLayerKeys(result: SearchResult) { return layers; } -/** Record search score telemetry. */ -function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchResult[]) { +/** Record search score telemetry. Exported for `tests/rag-score.test.ts`; not a route surface. */ +export function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchResult[]) { if (!results.length) { telemetry.top_score = 0; telemetry.second_top_score = 0; @@ -570,6 +570,7 @@ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchR telemetry.score_spread = Number(Math.max(0, telemetry.top_score - telemetry.second_top_score).toFixed(4)); telemetry.score_distinct_documents = new Set(results.map((result) => result.document_id)).size; telemetry.retrieval_candidate_count = results.length; + // Strict equality, deliberately: "document_context" rows carry the constant 1 with no match strength to inflate; counting them here would mix two populations and make the RC9 signal unreadable. Pinned by tests/rag-score.test.ts. telemetry.synthetic_similarity_count = results.filter( (result) => result.similarity_origin === "synthetic_text", ).length; diff --git a/src/lib/types.ts b/src/lib/types.ts index 8eb9bf6c11..9b48543742 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -348,7 +348,17 @@ export type SearchResult = { // signals (document-lookup, memory-card, table-facts fast paths) rather than a real cosine. // Coverage/threshold gates are calibrated for cosine values; this tag lets telemetry measure // how often synthetic scores cross those gates before any recalibration. - similarity_origin?: "cosine" | "synthetic_text"; + // + // "document_context" marks the constant `similarity: 1` that `buildDocumentSummaryResults` + // (`rag/rag-row-contracts.ts`) stamps on document-summary rows. It is NOT a measured cosine + // either, but it is also not the same hazard as "synthetic_text": that route's only caller is + // `summarizeDocument`, where the "query" IS the document, so every committed chunk is context + // by construction rather than by a match score. Owner decision 2026-08-17 (Option B, + // `docs/clinical-hazard-analysis.md` H5a) keeps the confidence derivation unchanged and adds + // this value purely so the provenance is legible to telemetry, review, and future gating — + // `deriveConfidence` deliberately excludes only "synthetic_text", pinned by + // `tests/rag-score.test.ts`. + similarity_origin?: "cosine" | "synthetic_text" | "document_context"; text_rank?: number; hybrid_score?: number; // Lexical/keyword relevance (0-1) for text-only fallback rows. This is NOT a diff --git a/tests/answer-incremental-delivery.test.ts b/tests/answer-incremental-delivery.test.ts index 6789921d74..83d62fc4dc 100644 --- a/tests/answer-incremental-delivery.test.ts +++ b/tests/answer-incremental-delivery.test.ts @@ -102,6 +102,36 @@ describe("verified-unit stream contract (#100 Phase 0)", () => { expect(isDeliverableVerifiedUnit({ ...previewUnit(), selectedContextCount: 1.5 })).toBe(false); }); + // G1 (docs/clinical-hazard-analysis.md H5a). The streamed preview must accept exactly the + // `similarity_origin` union in types.ts. If the union grows and this allow-set does not, the + // preview silently rejects a payload `final` accepts — the client then falls back to the + // slow path with no error anyone sees. + it("accepts every declared similarity_origin value and nothing else", () => { + const declaredOrigins = ["cosine", "synthetic_text", "document_context"] as const; + // Compile-time exhaustiveness: adding a member to the union without listing it here makes + // `Unlisted` something other than `never`, and this annotation stops typechecking. Without + // it the runtime loop below would keep passing while silently skipping the new value. + type Unlisted = Exclude, (typeof declaredOrigins)[number]>; + const everyOriginListed: [Unlisted] extends [never] ? true : never = true; + expect(everyOriginListed).toBe(true); + + for (const origin of declaredOrigins) { + expect( + isDeliverableVerifiedUnit({ + ...previewUnit(), + sources: [trimSourceForClient(makeSource({ similarity_origin: origin }))], + }), + ).toBe(true); + } + + expect( + isDeliverableVerifiedUnit({ + ...previewUnit(), + sources: [trimSourceForClient(makeSource({ similarity_origin: "made_up_origin" as never }))], + }), + ).toBe(false); + }); + it("rejects raw server fields at the stream boundary", () => { expect( isDeliverableVerifiedUnit({ diff --git a/tests/outstanding-issues-writer.test.ts b/tests/outstanding-issues-writer.test.ts index c1f5b7bd49..4c71bb0668 100644 --- a/tests/outstanding-issues-writer.test.ts +++ b/tests/outstanding-issues-writer.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { parseIssues } from "../scripts/check-outstanding-issues.mjs"; +import { issueRowFingerprint, parseIssues } from "../scripts/check-outstanding-issues.mjs"; import { displayIdForUlid, issueUlid } from "../scripts/issue-id.mjs"; import { addIssue, escapeCell, resolveIssue, splitCells, updateIssue } from "../scripts/outstanding-issues.mjs"; @@ -52,6 +52,50 @@ describe("outstanding-issues writer", () => { expect(parseIssues(next).nextId).toBe(7); }); + // Regression: `issueRowFingerprint` resolved only the legacy numeric form, so + // for every row minted after the ULID migration it returned null — and + // ledger-inbox.mjs reads null as "no such row" and refuses the request. That + // made `npm run issues:done` unusable for any Crockford-id row, reporting + // "is not in Open items" about a row sitting in Open items. + it("fingerprints open rows by either id generation", () => { + const withCrockfordRow = addIssue(ledger, { summary: "third" }, { date: "2026-02-02", issueUlid: TEST_ULID }); + + const crockford = issueRowFingerprint(withCrockfordRow, TEST_ID); + const legacy = issueRowFingerprint(withCrockfordRow, "#005"); + + expect(crockford).toMatch(/^[0-9a-f]{64}$/); + expect(legacy).toMatch(/^[0-9a-f]{64}$/); + expect(crockford).not.toBe(legacy); + + // The fingerprint is what makes the concurrency check meaningful: it must + // track the row's content, not just its identity. + const edited = updateIssue(withCrockfordRow, TEST_ID, { summary: "third edited" }); + expect(issueRowFingerprint(edited, TEST_ID)).not.toBe(crockford); + }); + + // The subtle half. Crockford's alphabet includes 0-9, so a ULID-derived + // locator can be entirely digits — `TEST_ID` here is `#041061`. Deciding the + // lookup from the id's SHAPE therefore reads such a row as a legacy id and + // hunts for a sequential number that no ULID row has, which is how the first + // attempt at this fix still returned null for a row it could plainly see. + it("resolves an all-digit display id to its ULID row, not a legacy number", () => { + expect(TEST_ID).toMatch(/^#\d+$/); + const next = addIssue(ledger, { summary: "third" }, { date: "2026-02-02", issueUlid: TEST_ULID }); + + expect(rowFor(next, TEST_ID)?.number).toBeNull(); + expect(issueRowFingerprint(next, TEST_ID)).toMatch(/^[0-9a-f]{64}$/); + }); + + it("returns null for ids that are absent, archived, or malformed", () => { + // Still null-safe: the inbox's "no such row" refusal must survive for a row + // that genuinely is not in Open items, or the fix would trade one broken + // command for a silently wrong one. + expect(issueRowFingerprint(ledger, TEST_ID)).toBeNull(); + expect(issueRowFingerprint(ledger, "#001")).toBeNull(); // archived, not open + expect(issueRowFingerprint(ledger, "#nope")).toBeNull(); + expect(issueRowFingerprint(ledger, "not-an-id")).toBeNull(); + }); + it("escapes pipes in prose instead of creating columns", () => { const next = addIssue(ledger, { summary: "a | b", detail: "c | d" }, { date: "2026-02-02", issueUlid: TEST_ULID }); const row = rowFor(next, TEST_ID); diff --git a/tests/rag-retrieval-row-contract.test.ts b/tests/rag-retrieval-row-contract.test.ts index ac1b7a0467..2ebf95c807 100644 --- a/tests/rag-retrieval-row-contract.test.ts +++ b/tests/rag-retrieval-row-contract.test.ts @@ -75,6 +75,28 @@ describe("retrieval row shape contract", () => { }); }); + // G1 governance pin (docs/clinical-hazard-analysis.md H5a; owner decision 2026-08-17). + // The `similarity: 1` above is a constant, not a measured cosine — on this route the + // document IS the query. Leaving it untagged is what let a fabricated 1.0 look identical + // to a perfect vector match; the tag makes the provenance legible without changing the + // confidence label (`deriveConfidence` excludes only "synthetic_text", pinned in + // tests/rag-score.test.ts). + it("tags the constant document-summary similarity with its own provenance value", () => { + const rows = buildDocumentSummaryResults([hybridRow({ similarity: 0.2 }), hybridRow({ id: "second-chunk" })], { + title: "Current title", + file_name: "current.pdf", + metadata: {}, + }); + + expect(rows).toHaveLength(2); + for (const row of rows) { + expect(row.similarity).toBe(1); + expect(row.similarity_origin).toBe("document_context"); + // Not "synthetic_text": that value caps confidence at "medium" (rejected Option A). + expect(row.similarity_origin).not.toBe("synthetic_text"); + } + }); + it("preserves unknown columns so an RPC version difference is not data loss", () => { const rows: unknown = [hybridRow({ document_labels: [{ id: "l1" }], a_future_column: 42 })]; diff --git a/tests/rag-score.test.ts b/tests/rag-score.test.ts index 4a006b28d4..94364d94ee 100644 --- a/tests/rag-score.test.ts +++ b/tests/rag-score.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import type { SearchResult } from "../src/lib/types"; -import { deriveConfidence, scoreValue } from "../src/lib/rag/rag"; +import type { SearchTelemetry } from "../src/lib/rag/rag-contracts"; +import { deriveConfidence, recordSearchScoreTelemetry, scoreValue } from "../src/lib/rag/rag"; describe("scoreValue", () => { const base: SearchResult = { @@ -85,4 +86,79 @@ describe("deriveConfidence (RC9 synthetic similarity)", () => { expect(deriveConfidence([result({ id: "a", similarity: 0.9 })], [])).toBe("unsupported"); expect(deriveConfidence([result({ id: "a", similarity: 0.4 })], [{ chunk_id: "a" }])).toBe("low"); }); + + // G1 governance pin (docs/clinical-hazard-analysis.md H5a; owner decision 2026-08-17, + // Option B). `buildDocumentSummaryResults` now tags its constant `similarity: 1` as + // "document_context". That tag is provenance ONLY — it must not change the confidence a + // clinician reads. The two tests below are a discriminating pair: same shape, same scores, + // only the origin value differs, and they must disagree. If a future edit folds + // "document_context" into the `strongestNonSynthetic` exclusion, the first goes red and + // every document summary silently drops from "high" to "medium" (rejected Option A). + // Conversely, if the "synthetic_text" exclusion is ever dropped, the second goes red. + it('keeps "high" for two document-context citations at or above the 0.82 bar', () => { + const cited = [ + result({ id: "a", similarity: 1, similarity_origin: "document_context" }), + result({ id: "b", similarity: 0.82, similarity_origin: "document_context" }), + ]; + expect(deriveConfidence(cited, [{ chunk_id: "a" }, { chunk_id: "b" }])).toBe("high"); + }); + + it('still refuses "high" for the same scores tagged "synthetic_text"', () => { + const cited = [ + result({ id: "a", similarity: 1, similarity_origin: "synthetic_text" }), + result({ id: "b", similarity: 0.82, similarity_origin: "synthetic_text" }), + ]; + expect(deriveConfidence(cited, [{ chunk_id: "a" }, { chunk_id: "b" }])).toBe("medium"); + }); + + it("still needs two accepted citations before document-context evidence reaches high", () => { + const cited = [result({ id: "a", similarity: 1, similarity_origin: "document_context" })]; + expect(deriveConfidence(cited, [{ chunk_id: "a" }])).toBe("medium"); + }); +}); + +describe("synthetic_similarity_count telemetry (RC9)", () => { + function result(overrides: Partial & { id: string }): SearchResult { + return { + document_id: "doc-1", + title: "Document", + file_name: "document.pdf", + page_number: 1, + chunk_index: 0, + section_heading: null, + content: "test", + image_ids: [], + similarity: 0, + images: [], + ...overrides, + }; + } + + // The RC9 counter measures imputed lexical/structural scores crossing cosine-calibrated + // gates. "document_context" is a different provenance on a route with no match strength to + // inflate; counting it would mix two populations into one unreadable number. + it('counts only "synthetic_text", never "document_context" or untagged cosine rows', () => { + const telemetry = {} as SearchTelemetry; + + recordSearchScoreTelemetry(telemetry, [ + result({ id: "a", similarity: 0.89, similarity_origin: "synthetic_text" }), + result({ id: "b", similarity: 1, similarity_origin: "document_context" }), + result({ id: "c", similarity: 0.91, similarity_origin: "cosine" }), + result({ id: "d", similarity: 0.77 }), + ]); + + expect(telemetry.synthetic_similarity_count).toBe(1); + expect(telemetry.retrieval_candidate_count).toBe(4); + }); + + it("reports zero for a document-summary result set", () => { + const telemetry = {} as SearchTelemetry; + + recordSearchScoreTelemetry(telemetry, [ + result({ id: "a", similarity: 1, similarity_origin: "document_context" }), + result({ id: "b", similarity: 1, similarity_origin: "document_context" }), + ]); + + expect(telemetry.synthetic_similarity_count).toBe(0); + }); });