From ce586565a3069a40409c086781b88930ff3a8f77 Mon Sep 17 00:00:00 2001 From: James Kane Date: Tue, 18 Aug 2026 08:07:52 -0500 Subject: [PATCH 01/33] docs(ste): the checker and the project dictionary for Simplified Technical English MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Navigator's comments are to follow ASD-STE100. Two things have to exist before any of them can be judged compliant, and neither is a rewrite. `documents/STE-dictionary.md` is the project dictionary. STE permits three classes of word: its own ~900-word approved list, Technical Names, and Technical Verbs. A project must declare the second and third itself, because the approved list contains no `biosample`, no `contig`, no `haplogroup`. The file also records the nine rules in short form, the substitutions this codebase needs most, and what STE does *not* reach: identifiers, code inside backticks, and commit messages. `scripts/ste-check.py` enforces the seven rules a script can judge without a part-of-speech tagger — sentence length, active voice, `-ing` forms, noun clusters, vocabulary, paragraph length, and idiom. It reads the Technical Names out of the dictionary, so declaring a term is enough to stop the checker flagging it. It is advisory and always exits 0; the intent is to see the number, not to gate a commit on it. Scope is Rust comments under `crates/`. Markdown is out, and `--all` includes it for information only. The baseline as this lands is 12,023 violations across 272 files. Three bugs found while calibrating it against real files, each of which had inflated the count: every doc block in a file merged into one paragraph, Markdown list items merged into one paragraph, and irregular participles matched as suffixes, so "present" read as `pre` + `sent` and reported plain adjectives as passive voice. Co-Authored-By: Claude Opus 5 (1M context) --- documents/STE-dictionary.md | 111 ++++++++++++++ scripts/ste-check.py | 285 ++++++++++++++++++++++++++++++++++++ 2 files changed, 396 insertions(+) create mode 100644 documents/STE-dictionary.md create mode 100755 scripts/ste-check.py diff --git a/documents/STE-dictionary.md b/documents/STE-dictionary.md new file mode 100644 index 00000000..fd1b2fcf --- /dev/null +++ b/documents/STE-dictionary.md @@ -0,0 +1,111 @@ +# Simplified Technical English — project dictionary + +Navigator's comments and documentation follow **ASD-STE100 Simplified Technical English**. + +STE permits three classes of word: the ~900 words in the STE approved dictionary, **Technical +Names**, and **Technical Verbs**. A project must declare its own Technical Names and Technical +Verbs, because the approved dictionary contains no domain vocabulary. This file is that +declaration. A word that is not in the STE dictionary and not in a table below must not appear in a +comment or a document. + +`scripts/ste-check.py` enforces the mechanical rules. It cannot enforce vocabulary beyond the +curated list it carries, so use this file when you write. + +## The rules, in short + +| Rule | Requirement | +|---|---| +| STE 1 | Use one approved word for one meaning. Do not use a word as more than one part of speech. | +| STE 2 | Do not use an `-ing` form as a verb or an adjective. A Technical Name that ends in `-ing` is permitted. | +| STE 3 | Write in the active voice. Use the simple present, the simple past, or the simple future. | +| STE 4 | Write instructions as commands. Give one instruction in one sentence. | +| STE 5 | Do not use more than three nouns together. | +| STE 6 | Write sentences of 20 words or fewer in a procedure, and 25 words or fewer in a description. | +| STE 7 | Write paragraphs of six sentences or fewer. Write about one topic in one paragraph. | +| STE 8 | Do not use slang, idiom, metaphor, or humour. | +| STE 9 | Keep the articles (`a`, `an`, `the`). Do not remove words to make a sentence short. | + +## Technical Names + +A Technical Name is a noun. It can be a compound noun. It cannot be a verb. + +### Genetics and sequence data + +alignment · allele · ancestry · admixture · autosome · base · biosample · build · call · caller · +chromosome · consensus · contig · coverage · depth · donor · genome · genotype · haplogroup · +haplotype · indel · kit · lineage · marker · panel · pedigree · ploidy · position · read · +read metrics · reference · reference genome · region · segment · sequence · sequence run · sex · +signature · site · subject · variant · Y-STR + +### File and data formats + +BAM · BED · CRAM · FASTA · gVCF · index · JSON · masterVar · sidecar · TSV · VCF + +### Application concepts + +app · artifact · cache · command · event · liftover · migration · outbox · profile · project · +query · realignment · record · row · schema · store · table · workspace · worker + +### Federation + +AppView · attestation · consent · device key · DID · exchange · handle · IBD · PDS · record key · +session · signature · suggestion · token + +## Technical Verbs + +STE permits a project to declare Technical Verbs when no approved verb has the meaning. Use these +only with the meaning given. + +| Verb | Meaning in this project | +|---|---| +| to align | to map reads to a reference genome | +| to cache | to keep a result for later use | +| to call | to find a genotype or a haplogroup from read data | +| to genotype | to find the alleles at a site | +| to import | to read a file into the workspace | +| to index | to make an index file for an alignment | +| to lift over | to change coordinates from one build to another | +| to publish | to send a record to a PDS | +| to realign | to map the reads of an alignment to a different reference genome | +| to sign | to make a cryptographic signature | +| to sync | to send records to a PDS and to read records from a PDS | + +## Words to avoid, and what to write + +The approved dictionary gives one word for one meaning. These replacements occur most frequently in +this codebase. + +| Do not write | Write | +|---|---| +| additionally, furthermore, moreover | also | +| approximately | about | +| cannot | can not | +| due to, owing to | because of | +| ensure | make sure | +| however, nevertheless | but | +| in order to | to | +| indicate, demonstrate | show | +| multiple, numerous | many, more than one | +| perform, execute, conduct | do | +| prior to | before | +| require | need | +| simply, merely, solely | only, or delete | +| subsequent to | after | +| sufficient | enough | +| thus, hence, therefore | so | +| utilize, leverage | use | +| various | different | +| verify, validate | check | +| via | by, through | +| whilst | while | + +Do not use a contraction. Write `do not`, not `don't`. + +## What STE does not change + +- **Identifiers.** Function, type, field, and file names are code. STE does not apply to them. +- **Code in a comment.** Text in backticks is code, not prose. +- **Commit messages and pull request text.** These are a record of a decision, not product + documentation. They are outside the scope of this standard. +- **The rationale itself.** STE controls *how* you write a reason. It does not tell you to delete + the reason. A comment must still say why the code is as it is. diff --git a/scripts/ste-check.py b/scripts/ste-check.py new file mode 100755 index 00000000..8b645a79 --- /dev/null +++ b/scripts/ste-check.py @@ -0,0 +1,285 @@ +"""Check comments against the mechanically-checkable ASD-STE100 Simplified Technical English rules. + +Advisory. It always exits 0 -- it reports, it does not gate. See documents/STE-dictionary.md for +the project Technical Names and Technical Verbs, which this script cannot check. + + python3 scripts/ste-check.py # all Rust comments (the enforced scope) + python3 scripts/ste-check.py --detail # with examples + python3 scripts/ste-check.py path/to/file.rs # one file + python3 scripts/ste-check.py --all # include Markdown (out of scope, for information) + + +Covers the rules a script can judge without a POS tagger: + STE 1.x approved vocabulary (curated subset of common offenders + replacements) + STE 2.x no -ing participles used as verbs/adjectives + STE 3.x active voice; simple tenses + STE 5.x noun clusters of more than three words + STE 6.x sentence length (20 procedural / 25 descriptive) + STE 7.x paragraph length (max 6 sentences) + STE 8.x no slang, idiom, metaphor, jargon +""" +import os, re, sys, json +from collections import Counter, defaultdict + +# --- STE non-approved words -> approved alternative (curated, high-frequency subset) ------- +NOT_APPROVED = { + "utilize": "use", "utilise": "use", "utilizes": "uses", "utilized": "used", + "via": "by / through", "per": "for each", "vice": "instead of", + "prior to": "before", "subsequent to": "after", "in order to": "to", + "due to": "because of", "owing to": "because of", "as well as": "and", + "in the event that": "if", "in case": "if", "provided that": "if", + "additionally": "also", "furthermore": "also", "moreover": "also", + "however": "but", "nevertheless": "but", "nonetheless": "but", + "hence": "so", "thus": "so", "therefore": "so", "whilst": "while", + "amongst": "among", "whereby": "by which", "wherein": "in which", + "aforementioned": "the ... described before", "said": "the", + "obtain": "get", "acquire": "get", "commence": "start", "initiate": "start", + "terminate": "stop", "cease": "stop", "endeavour": "try", "attempt": "try", + "ascertain": "find out", "determine": "find", "require": "need", + "sufficient": "enough", "numerous": "many", "multiple": "more than one", + "approximately": "about", "regarding": "about", "concerning": "about", + "possess": "have", "purchase": "buy", "assist": "help", "permit": "let", + "indicate": "show", "demonstrate": "show", "illustrate": "show", + "facilitate": "help", "leverage": "use", "handle": "control", + "perform": "do", "conduct": "do", "execute": "do", + "ensure": "make sure", "verify": "check", "validate": "check", + "prohibit": "do not let", "eliminate": "remove", "modify": "change", + "accomplish": "do", "encounter": "find", "identify": "find", + "sole": "only", "solely": "only", "merely": "only", "simply": "only", + "essentially": "", "basically": "", "actually": "", "really": "", + "quite": "", "rather": "", "fairly": "", "somewhat": "", + "various": "different", "several": "some", "certain": "some", + "considerable": "large", "substantial": "large", "significant": "large", + "minimal": "small", "optimal": "best", "optimum": "best", + "prior": "earlier", "latter": "second", "former": "first", + "cannot": "can not", "won't": "will not", "don't": "do not", + "doesn't": "does not", "isn't": "is not", "it's": "it is", + "we've": "we have", "they're": "they are", "that's": "that is", + "wouldn't": "would not", "couldn't": "could not", "didn't": "did not", + "hasn't": "has not", "haven't": "have not", "aren't": "are not", + "let's": "let us", "there's": "there is", "what's": "what is", +} + +# --- project Technical Names, read from documents/STE-dictionary.md ------------------------- +def _technical_names(): + """Words the project declares as Technical Names / Technical Verbs. STE permits these.""" + names = set() + try: + here = os.path.dirname(os.path.abspath(globals().get("__file__", "scripts/x"))) + doc = open(os.path.join(here, os.pardir, "documents", "STE-dictionary.md"), + encoding="utf-8").read() + except OSError: + return names + body = doc.split("## Technical Names", 1) + if len(body) < 2: + return names + body = body[1].split("## Words to avoid", 1)[0] + for tok in re.split(r"[\u00b7|\n,()]", body): + tok = tok.strip().strip("*`").lower() + if tok and re.fullmatch(r"[a-z][a-z0-9 -]*", tok): + names.add(tok) + for w in tok.split(): + names.add(w) + return names + + +TECHNICAL = _technical_names() + +# --- idiom / metaphor / informal (STE 8) ---------------------------------------------------- +IDIOMS = [ + "God file", "god file", "under the hood", "out of the box", "rule of thumb", + "hand-rolled", "hand rolled", "hammering", "hammer", "byte for byte", + "byte-for-byte", "the headline", "falls out of", "fell behind", "pays off", + "earns its keep", "earned its keep", "loose ends", "knock-on", "gotcha", + "smell test", "cheap", "expensive", "dead code", "wired in", "baked in", + "boils down", "kick off", "kicks off", "spin up", "tear down", "teardown", + "in flight", "in the wild", "happy path", "sad path", "sanity check", + "belt and braces", "first-class", "second-class", "nuked", "blew up", + "silently", "quietly", "magic", "magical", "clever", "ugly", "nasty", + "painful", "pain", "trivial", "obvious", "simply put", "of course", + "arguably", "unfortunately", "sadly", "luckily", "surprisingly", + "the point is", "worth noting", "note that", "keep in mind", "bear in mind", + "a stone's throw", "on the fly", "at a glance", "hunting for", "chase", + "drift apart", "drift", "stands in the way", "standing in for", +] + +BE = r"(?:is|are|was|were|be|been|being|get|gets|got)" +# Irregular participles must match as whole words. As a suffix, "sent" matched "present" and +# "set" matched "offset", which reported plain adjectives as passive voice. +IRREGULAR = r"built|set|put|read|kept|held|left|made|sent|done|shown|known|thrown|written|driven" +PASSIVE = re.compile(rf"\b{BE}\s+(?:\w+ly\s+)?(\w+(?:ed|en|own|ung)|(?:{IRREGULAR}))\b", re.I) +# Words that end in -en/-ed but are not participles. +NOT_PARTICIPLE = { + "often", "when", "then", "even", "open", "green", "seven", "ten", "children", "women", "men", + "garden", "token", "golden", "sudden", "hidden", "wooden", "kitchen", "screen", "between", + "need", "indeed", "speed", "seed", "feed", "exceed", "red", "bed", "led", "ahead", "instead", +} +ING = re.compile(r"\b(\w{3,}ing)\b", re.I) +ING_OK = { + # technical names / gerund-nouns STE permits as established terms + "string", "strings", "ring", "spring", "during", "bring", "thing", "things", + "nothing", "something", "anything", "everything", "morning", "king", "wing", + "sing", "swing", "engineering", "sequencing", "painting", "matching", + "mapping", "encoding", "decoding", "indexing", "logging", "polling", + "signing", "warning", "warnings", "setting", "settings", "heading", + "listing", "ordering", "padding", "casing", "tracking", "caching", + "processing", "pending", "missing", "remaining", "existing", "following", + "corresponding", "underlying", "according", "including", + # Technical Names from documents/STE-dictionary.md that end in -ing. + "operating", "pacing", "sampling", "scaling", "streaming", "spilling", "phasing", + "binning", "masking", "trimming", "clipping", "calling", "sorting", "merging", + "reading", "writing", "counting", "timing", "build", "backing", +} +SENT_SPLIT = re.compile(r"(?<=[.!?:;])\s+(?=[A-Z`\[(])") + + +def sentences(text): + return [s.strip() for s in SENT_SPLIT.split(text) if s.strip()] + + +def strip_code(text): + """Remove inline code, fenced code, links and paths — STE judges prose, not identifiers.""" + text = re.sub(r"```.*?```", " ", text, flags=re.S) + text = re.sub(r"`[^`]*`", " CODE ", text) + text = re.sub(r"https?://\S+", " URL ", text) + text = re.sub(r"\[\[?[^\]]*\]\]?(\([^)]*\))?", " LINK ", text) + text = re.sub(r"\b[\w/.-]+\.(rs|md|sql|toml|json|yml)\b", " FILE ", text) + return text + + +def extract_rust(path): + """Comment text from a .rs file, as (line_no, text).""" + out = [] + for i, raw in enumerate(open(path, encoding="utf-8", errors="replace"), 1): + s = raw.strip() + m = re.match(r"^(///|//!|//)\s?(.*)$", s) + if m and not s.startswith("////"): + out.append((i, m.group(2))) + return out + + +def extract_md(path): + out = [] + fence = False + for i, raw in enumerate(open(path, encoding="utf-8", errors="replace"), 1): + s = raw.rstrip("\n") + if s.strip().startswith("```"): + fence = not fence + continue + if fence or not s.strip() or s.strip().startswith(("|", ">", "#", "---")): + continue + out.append((i, s.strip())) + return out + + +def analyse(items, kind): + """items: [(line, text)] -> violations by rule.""" + v = defaultdict(list) + # Group consecutive lines into paragraphs for sentence-level rules. + para, start, last = [], None, None + paras = [] + for ln, t in items: + # A blank comment line, a gap in line numbers (code between doc blocks), or the start of a + # Markdown list item ends a paragraph. A list item is its own unit of prose. + if not t.strip() or (last is not None and ln != last + 1) or re.match(r"[-*+]\s|\d+\.\s", t.strip()): + if para: + paras.append((start, " ".join(para))) + para, start = [], None + if t.strip(): + if start is None: + start = ln + para.append(t) + last = ln + if para: + paras.append((start, " ".join(para))) + + for ln, ptext in paras: + clean = strip_code(ptext) + sents = sentences(clean) + if len(sents) > 6: + v["STE7 paragraph >6 sentences"].append((ln, f"{len(sents)} sentences")) + for s in sents: + words = re.findall(r"[A-Za-z][\w'-]*", s) + n = len(words) + if n > 25: + v["STE6 sentence >25 words"].append((ln, f"{n}w: {s[:90]}")) + pm = PASSIVE.search(s) + if pm and pm.group(1).lower() not in NOT_PARTICIPLE: + v["STE3 passive voice"].append((ln, s[:90])) + for w in ING.findall(s): + if w.lower() not in ING_OK: + v["STE2 -ing form"].append((ln, w)) + low = " " + clean.lower() + " " + for bad, good in NOT_APPROVED.items(): + if bad in TECHNICAL: + continue + if re.search(rf"\b{re.escape(bad)}\b", low): + v["STE1 non-approved word"].append((ln, f"{bad} -> {good or 'delete'}")) + for idiom in IDIOMS: + if re.search(rf"\b{re.escape(idiom.lower())}\b", low): + v["STE8 idiom/metaphor/informal"].append((ln, idiom)) + if "—" in ptext or " -- " in ptext: + v["STE6 em-dash aside"].append((ln, "")) + return v + + +def main(): + detail = "--detail" in sys.argv + only = None + for a in sys.argv[1:]: + if not a.startswith("--"): + only = a + totals = Counter() + per_file = Counter() + examples = defaultdict(list) + + include_md = "--all" in sys.argv + targets = [] + # Walk only the requested path, so a single-file or single-crate check is fast. + root = only if only and os.path.isdir(only) else (os.path.dirname(only) or "." if only else ".") + if only and os.path.isfile(only): + root = os.path.dirname(only) or "." + for dp, dn, fn in os.walk(root): + if any(x in dp for x in ("/target", "/.git", "/node_modules", "/.claude/worktrees")): + continue + for f in fn: + p = os.path.join(dp, f) + if only and only not in p: + continue + if f.endswith(".rs"): + targets.append((p, extract_rust, "rust")) + elif f.endswith(".md") and include_md: + targets.append((p, extract_md, "md")) + + for p, fn, kind in targets: + try: + v = analyse(fn(p), kind) + except Exception: + continue + c = sum(len(x) for x in v.values()) + if c: + per_file[p] = c + for rule, hits in v.items(): + totals[rule] += len(hits) + for h in hits[:2]: + examples[rule].append((p, h)) + + print(f"{'RULE':<34} {'VIOLATIONS':>10}") + print("-" * 46) + for rule, n in totals.most_common(): + print(f"{rule:<34} {n:>10,}") + print("-" * 46) + print(f"{'TOTAL':<34} {sum(totals.values()):>10,}") + print(f"\nfiles with >=1 violation: {len(per_file)}") + print("\nworst 15 files:") + for p, n in per_file.most_common(15): + print(f" {n:>5} {p}") + if detail: + print("\n=== examples ===") + for rule in totals: + print(f"\n## {rule}") + for p, (ln, txt) in examples[rule][:6]: + print(f" {p}:{ln} {txt}") + + +main() From 22fc1299fc09fa6c6f86317d7797fc4cd9076480 Mon Sep 17 00:00:00 2001 From: James Kane Date: Tue, 18 Aug 2026 10:08:39 -0500 Subject: [PATCH 02/33] docs(ste): the mechanical vocabulary substitutions, across every crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unambiguous half of STE rule 1: a word with exactly one approved replacement regardless of context. `utilize` to `use`, `prior to` to `before`, `whilst` to `while`, `approximately` to `about`, `however` to `but` at the start of a sentence, and every contraction expanded. A script did this, and it is deliberately conservative. It never enters a backtick span, and it leaves every judgement call for the hand pass — `via`, `due to`, `determine`, `identify`, `require`, `significant`, and mid-sentence `however` all survive untouched here. The first attempt corrupted 14 lines and was reverted rather than patched. The patterns had no leading word boundary, so `it's` matched inside `kit's` and produced "kit is own CSV", `sufficient` matched inside `Insufficient` and produced "Inenough", and `ascertain` matched inside the flag name `--ascertain-sites`. The rewrite added boundaries, plus explicit guards for the three words that appear inside longer words, and was only applied once ten probes covering those exact cases passed. This commit changes no meaning. It is the floor the hand conversion starts from. Co-Authored-By: Claude Opus 5 (1M context) --- crates/navigator-align/src/batch.rs | 4 +- crates/navigator-align/src/index.rs | 2 +- crates/navigator-align/src/map.rs | 4 +- crates/navigator-align/src/map/tests.rs | 2 +- crates/navigator-align/src/output.rs | 2 +- crates/navigator-align/src/pe.rs | 2 +- .../examples/archaic_callable_dump.rs | 2 +- .../examples/archaic_match_probe.rs | 2 +- .../examples/archaic_panel_dump.rs | 2 +- .../examples/reassembly_probe.rs | 2 +- crates/navigator-analysis/src/ancestry.rs | 26 +++--- crates/navigator-analysis/src/archaic.rs | 14 +-- .../navigator-analysis/src/archaic_match.rs | 4 +- .../src/archaic_segments.rs | 10 +- crates/navigator-analysis/src/caller.rs | 6 +- crates/navigator-analysis/src/cancel.rs | 6 +- crates/navigator-analysis/src/contig.rs | 4 +- crates/navigator-analysis/src/coverage.rs | 8 +- crates/navigator-analysis/src/error.rs | 4 +- crates/navigator-analysis/src/gvcf.rs | 4 +- crates/navigator-analysis/src/gzio.rs | 2 +- crates/navigator-analysis/src/haplo.rs | 22 ++--- crates/navigator-analysis/src/ibd_attest.rs | 4 +- crates/navigator-analysis/src/ibd_panel.rs | 8 +- crates/navigator-analysis/src/index.rs | 2 +- crates/navigator-analysis/src/lai.rs | 10 +- crates/navigator-analysis/src/manifest.rs | 2 +- crates/navigator-analysis/src/mask.rs | 2 +- crates/navigator-analysis/src/mastervar.rs | 8 +- .../src/postprocess/cram.rs | 2 +- .../src/postprocess/finalize.rs | 4 +- .../src/postprocess/markdup.rs | 4 +- .../src/postprocess/tests.rs | 4 +- crates/navigator-analysis/src/preflight.rs | 14 +-- crates/navigator-analysis/src/probe.rs | 6 +- crates/navigator-analysis/src/reader.rs | 10 +- crates/navigator-analysis/src/reassembly.rs | 10 +- .../navigator-analysis/src/revert/collate.rs | 2 +- crates/navigator-analysis/src/revert/mod.rs | 2 +- crates/navigator-analysis/src/revert/tests.rs | 4 +- .../src/revert/transform.rs | 4 +- .../navigator-analysis/src/revert/writer.rs | 2 +- crates/navigator-analysis/src/roh.rs | 2 +- crates/navigator-analysis/src/sidecar.rs | 12 +-- crates/navigator-analysis/src/strcaller.rs | 2 +- crates/navigator-analysis/src/strmarker.rs | 4 +- crates/navigator-analysis/src/sv/types.rs | 2 +- crates/navigator-analysis/src/sv/walker.rs | 2 +- crates/navigator-analysis/src/testtype.rs | 8 +- crates/navigator-analysis/src/unified.rs | 8 +- .../navigator-analysis/tests/cancel_real.rs | 2 +- crates/navigator-analysis/tests/genotype.rs | 2 +- .../navigator-app/examples/blocktree_check.rs | 2 +- crates/navigator-app/src/analysis.rs | 28 +++--- crates/navigator-app/src/blocktree.rs | 20 ++-- crates/navigator-app/src/brief.rs | 16 ++-- crates/navigator-app/src/commands.rs | 6 +- crates/navigator-app/src/fastpath.rs | 14 +-- crates/navigator-app/src/ftdna_import.rs | 16 ++-- crates/navigator-app/src/haplogroup.rs | 92 +++++++++---------- crates/navigator-app/src/import_profiles.rs | 8 +- crates/navigator-app/src/import_unified.rs | 44 ++++----- crates/navigator-app/src/lib.rs | 74 +++++++-------- crates/navigator-app/src/llm.rs | 8 +- crates/navigator-app/src/maintenance.rs | 2 +- crates/navigator-app/src/publish.rs | 6 +- crates/navigator-app/src/queries.rs | 12 +-- crates/navigator-app/src/realign.rs | 6 +- crates/navigator-app/src/realign_job.rs | 26 +++--- crates/navigator-app/tests/app.rs | 36 ++++---- .../tests/mastervar_autosomal_real.rs | 2 +- crates/navigator-domain/src/ancestry.rs | 4 +- crates/navigator-domain/src/bisdna.rs | 6 +- crates/navigator-domain/src/brief.rs | 8 +- crates/navigator-domain/src/chipprofile.rs | 8 +- crates/navigator-domain/src/consensus.rs | 6 +- crates/navigator-domain/src/filetype.rs | 8 +- crates/navigator-domain/src/ftdna.rs | 4 +- crates/navigator-domain/src/ftdna_csv.rs | 4 +- crates/navigator-domain/src/i18n.rs | 2 +- crates/navigator-domain/src/identity.rs | 4 +- crates/navigator-domain/src/llm_prompt.rs | 2 +- crates/navigator-domain/src/paths.rs | 2 +- .../navigator-domain/src/results_context.rs | 6 +- crates/navigator-domain/src/roh.rs | 2 +- crates/navigator-domain/src/strchart.rs | 2 +- crates/navigator-domain/src/strprofile.rs | 2 +- crates/navigator-domain/src/testtype.rs | 2 +- crates/navigator-domain/src/variants.rs | 10 +- crates/navigator-domain/src/vendorvcf.rs | 2 +- crates/navigator-domain/src/workspace.rs | 2 +- crates/navigator-domain/src/ysnp_dict.rs | 10 +- .../examples/ascertain_chip.rs | 2 +- .../examples/check_liftover.rs | 2 +- .../examples/filter_sites.rs | 2 +- .../examples/qpadm_selftest.rs | 2 +- crates/navigator-panelbuild/src/archaic.rs | 14 +-- .../navigator-panelbuild/src/archaic_tierb.rs | 2 +- .../navigator-panelbuild/src/genetic_map.rs | 2 +- crates/navigator-panelbuild/src/hap_panel.rs | 2 +- .../navigator-panelbuild/src/lai_validate.rs | 8 +- crates/navigator-panelbuild/src/main.rs | 2 +- crates/navigator-panelbuild/src/pca.rs | 6 +- .../src/validate_ancient.rs | 6 +- crates/navigator-refgenome/src/cache.rs | 2 +- crates/navigator-refgenome/src/gateway.rs | 6 +- crates/navigator-refgenome/src/index.rs | 2 +- crates/navigator-refgenome/src/regions.rs | 2 +- crates/navigator-refgenome/src/registry.rs | 8 +- crates/navigator-refgenome/src/vcf_lift.rs | 10 +- crates/navigator-store/src/ancestry_result.rs | 2 +- .../navigator-store/src/biosample_project.rs | 2 +- crates/navigator-store/src/haplogroup_call.rs | 2 +- crates/navigator-store/src/sync_history.rs | 2 +- crates/navigator-store/src/sync_outbox.rs | 2 +- crates/navigator-store/src/variant_set.rs | 2 +- crates/navigator-store/tests/store.rs | 4 +- crates/navigator-sync/src/device_key.rs | 4 +- crates/navigator-sync/src/lib.rs | 2 +- crates/navigator-sync/src/oauth.rs | 6 +- crates/navigator-sync/src/records.rs | 8 +- crates/navigator-sync/src/secret_store.rs | 6 +- crates/navigator-ui/src/cli.rs | 16 ++-- crates/navigator-ui/src/ui/blocktree.rs | 4 +- crates/navigator-ui/src/ui/central.rs | 14 +-- crates/navigator-ui/src/ui/chrome.rs | 6 +- crates/navigator-ui/src/ui/descent.rs | 12 +-- crates/navigator-ui/src/ui/detail.rs | 26 +++--- crates/navigator-ui/src/ui/events.rs | 14 +-- crates/navigator-ui/src/ui/mod.rs | 24 ++--- crates/navigator-ui/src/ui/modals.rs | 14 +-- crates/navigator-ui/src/ui/simple.rs | 14 +-- crates/navigator-ui/src/ui/sources.rs | 2 +- crates/navigator-ui/src/worker.rs | 34 +++---- 134 files changed, 549 insertions(+), 549 deletions(-) diff --git a/crates/navigator-align/src/batch.rs b/crates/navigator-align/src/batch.rs index 2fdf614a..6e5568f0 100644 --- a/crates/navigator-align/src/batch.rs +++ b/crates/navigator-align/src/batch.rs @@ -132,7 +132,7 @@ impl BatchSize { /// Why [`BatchSize::for_this_machine`] chose what it did, for a log line or a UI tooltip. /// - /// A realignment is a multi-hour job whose memory profile the user cannot see; when it is + /// A realignment is a multi-hour job whose memory profile the user can not see; when it is /// sized automatically, the sizing has to be inspectable rather than a mystery. pub fn explain() -> String { if let Some(bases) = env_override() { @@ -269,7 +269,7 @@ mod tests { /// Detection has to work on whatever machine this runs on — that is the entire point of taking /// the dependency. The assertions are about plausibility rather than a specific number, since - /// the test cannot know the host. + /// the test can not know the host. #[test] fn the_machine_reports_its_own_memory() { let memory = detect_memory().expect("every desktop target sysinfo supports reports memory"); diff --git a/crates/navigator-align/src/index.rs b/crates/navigator-align/src/index.rs index ee48dfd2..76fbae15 100644 --- a/crates/navigator-align/src/index.rs +++ b/crates/navigator-align/src/index.rs @@ -40,7 +40,7 @@ pub type ProgressFn<'a> = &'a mut dyn FnMut(usize, u64); /// Deliberately the same answer `navigator-refgenome::cache::base_dir` gives, reached the same way /// — through `navigator_domain::paths::decodingus_dir`, the one definition of the cache root — so /// `minimap2_index/` lands beside `references/` and `liftover/` rather than in a second location -/// that only this crate knows about. This crate is a leaf and cannot depend on `navigator-refgenome` +/// that only this crate knows about. This crate is a leaf and can not depend on `navigator-refgenome` /// (that would invert the layering), which is why the resolution is repeated rather than imported; /// the shared *definition* is what stops the two drifting. pub fn cache_root() -> PathBuf { diff --git a/crates/navigator-align/src/map.rs b/crates/navigator-align/src/map.rs index bc96fed5..aadf6e04 100644 --- a/crates/navigator-align/src/map.rs +++ b/crates/navigator-align/src/map.rs @@ -29,7 +29,7 @@ //! ## Why not the upstream file-level entry points //! //! `minimap2-pure-rs` ships `map_file_sam_split` and friends, which look like exactly this. They -//! cannot be used: they write to **stdout** (unusable from a desktop app) and they take +//! can not be used: they write to **stdout** (unusable from a desktop app) and they take //! `parts: &[MmIdx]`, holding every part resident — giving up the entire memory bound this design //! exists to buy. What is reused is the per-part record format and the merge; the loop is ours. //! @@ -113,7 +113,7 @@ pub struct MapStats { /// Cancellation, as a callback rather than a shared token type. /// -/// This crate is a leaf — it deliberately does not depend on `navigator-analysis`, so it cannot +/// This crate is a leaf — it deliberately does not depend on `navigator-analysis`, so it can not /// take that crate's `CancelToken` without inverting the layering. A closure lets the caller wire /// whatever cancellation it already has, and costs this crate no dependency. pub type CancelFn<'a> = &'a dyn Fn() -> bool; diff --git a/crates/navigator-align/src/map/tests.rs b/crates/navigator-align/src/map/tests.rs index 3044308c..385a0f18 100644 --- a/crates/navigator-align/src/map/tests.rs +++ b/crates/navigator-align/src/map/tests.rs @@ -590,7 +590,7 @@ fn the_output_format_can_be_read_off_the_path() { assert_eq!(F::from_path(Path::new("x")), F::Bam, "BAM is the default"); } -/// CRAM cannot be written without the reference it is compressed against, and saying so up front +/// CRAM can not be written without the reference it is compressed against, and saying so up front /// beats failing partway through a multi-hour job. #[test] fn cram_without_a_reference_is_refused_before_any_work() { diff --git a/crates/navigator-align/src/output.rs b/crates/navigator-align/src/output.rs index 72e063e2..0d8f18da 100644 --- a/crates/navigator-align/src/output.rs +++ b/crates/navigator-align/src/output.rs @@ -143,7 +143,7 @@ impl AlignmentWriter { /// Parse one SAM line from the mapper and hand it to `edit` before writing. /// - /// `edit` is where paired fields get set. It sees a typed record, so it cannot write a value + /// `edit` is where paired fields get set. It sees a typed record, so it can not write a value /// into the wrong column — which was the entire failure mode this module removes. pub fn write_line_with( &mut self, diff --git a/crates/navigator-align/src/pe.rs b/crates/navigator-align/src/pe.rs index 909e216e..d848d0eb 100644 --- a/crates/navigator-align/src/pe.rs +++ b/crates/navigator-align/src/pe.rs @@ -707,7 +707,7 @@ fn primary(result: &MapResult) -> Option<&AlignReg> { /// Fill in the paired half of a record: flags, `RNEXT`, `PNEXT`, `TLEN`. /// /// The single-end writer produced everything else. This used to patch the formatted SAM text by -/// column position; it now mutates a typed [`RecordBuf`], so a mate position cannot end up in the +/// column position; it now mutates a typed [`RecordBuf`], so a mate position can not end up in the /// template-length field however the formatter's layout changes. /// /// `own` is this record's region (`None` for an unmapped read) and `mate` is the mate's primary. diff --git a/crates/navigator-analysis/examples/archaic_callable_dump.rs b/crates/navigator-analysis/examples/archaic_callable_dump.rs index 7b50bdcb..9f2b6d60 100644 --- a/crates/navigator-analysis/examples/archaic_callable_dump.rs +++ b/crates/navigator-analysis/examples/archaic_callable_dump.rs @@ -1,4 +1,4 @@ -//! Dump the Tier B callability mask as BED, so what the segment caller can and cannot see is +//! Dump the Tier B callability mask as BED, so what the segment caller can and can not see is //! checkable against an external callset rather than assumed. //! //! Windows below `min_frac` of `window_bp` callable are excluded by the caller itself, so the same diff --git a/crates/navigator-analysis/examples/archaic_match_probe.rs b/crates/navigator-analysis/examples/archaic_match_probe.rs index d8552eab..d435d576 100644 --- a/crates/navigator-analysis/examples/archaic_match_probe.rs +++ b/crates/navigator-analysis/examples/archaic_match_probe.rs @@ -83,7 +83,7 @@ fn main() -> Result<(), Box> { GeneticMap::from_bytes(&std::fs::read(&a[3])?).map_err(|e| e.to_string())? }; - // `ARCHAIC_RATIOS=2.0,2.5,3.04` sweeps the emission ratio in one process. It cannot be swept + // `ARCHAIC_RATIOS=2.0,2.5,3.04` sweeps the emission ratio in one process. It can not be swept // post-hoc like the three thresholds — it changes the emissions, so the HMM must be re-decoded — // but the expensive part (reading the reference, walking the diagnostic sites) is per sample, // not per ratio, so doing it here costs one pass instead of one per value. diff --git a/crates/navigator-analysis/examples/archaic_panel_dump.rs b/crates/navigator-analysis/examples/archaic_panel_dump.rs index 52805f8c..de8c9f22 100644 --- a/crates/navigator-analysis/examples/archaic_panel_dump.rs +++ b/crates/navigator-analysis/examples/archaic_panel_dump.rs @@ -3,7 +3,7 @@ //! This is the independent evidence for arbitrating Tier B calls. The segment caller //! ([`navigator_analysis::archaic_match`]) reads only `ArchaicClassify` — a derived base and a //! lineage class per site — and never sees which archaic genome carries what. So the per-genome -//! pattern is information the caller cannot have fitted to, which is what makes it usable as a +//! pattern is information the caller can not have fitted to, which is what makes it usable as a //! referee. //! //! Why a referee is needed: precision has been measured against hmmix's callset, but a call absent diff --git a/crates/navigator-analysis/examples/reassembly_probe.rs b/crates/navigator-analysis/examples/reassembly_probe.rs index 643a3119..10446361 100644 --- a/crates/navigator-analysis/examples/reassembly_probe.rs +++ b/crates/navigator-analysis/examples/reassembly_probe.rs @@ -256,7 +256,7 @@ impl GapParameters for GapParams { } } -/// Semiglobal in the read: free leading/trailing offset so window-edge trimming isn't penalised. +/// Semiglobal in the read: free leading/trailing offset so window-edge trimming is not penalised. struct Semiglobal; impl StartEndGapParameters for Semiglobal { fn free_start_gap_x(&self) -> bool { diff --git a/crates/navigator-analysis/src/ancestry.rs b/crates/navigator-analysis/src/ancestry.rs index 3f3e33c7..d3d3453e 100644 --- a/crates/navigator-analysis/src/ancestry.rs +++ b/crates/navigator-analysis/src/ancestry.rs @@ -293,7 +293,7 @@ impl HaplotypeReference { /// Project a sample's genotypes onto the reference PCA space: centre each site by its panel /// mean and accumulate `centered · loading` into each component. A missing genotype contributes /// 0 (mean-imputed), then the projection is rescaled by `total_sites / sites_used` so a sample -/// with missing genotypes isn't shrunk toward the origin (which would pull it off its true +/// with missing genotypes is not shrunk toward the origin (which would pull it off its true /// cluster). Returns the sample's coordinate in each principal component. pub fn project_pca(genotypes: &[SiteGenotype], pca: &PcaLoadings) -> Vec { let dosage: HashMap<(&str, i64), i32> = genotypes @@ -313,7 +313,7 @@ pub fn project_pca(genotypes: &[SiteGenotype], pca: &PcaLoadings) -> Vec { /// The PCA projection kernel: accumulate `centered · loading` into each component over the sites /// the sample actually has, then un-shrink by `n_sites / used` so a sample with missing genotypes -/// isn't pulled toward the origin (see [`project_pca`]). +/// is not pulled toward the origin (see [`project_pca`]). /// /// `centered` yields `(site index, dosage − site mean)` for each present site; `loading` reads the /// `(site, component)` basis entry. Both are supplied by the caller because the runtime projector @@ -646,7 +646,7 @@ fn haploid_viterbi(sites: &[(i64, Vec, u8)], pi: &[f64], rate: f64, k: usiz /// Paint local ancestry from **phased** genotypes: a haploid ancestry HMM run independently on each /// of the two phased sides, so the two output tracks are internally-consistent parental sides /// (segment `copy` = phased side 0/1, consistent across the whole genome) — the parent-split the -/// unphased [`paint_local_ancestry`] cannot produce. `prior` is the genome-wide composition +/// unphased [`paint_local_ancestry`] can not produce. `prior` is the genome-wide composition /// (anchors the state set); `panel` supplies per-super-pop allele frequencies. pub fn paint_local_ancestry_phased( phased: &crate::phasing::PhasedGenotypes, @@ -731,7 +731,7 @@ impl Default for FineResolveParams { /// **fine** population *within that super-population* from the fine-frequency panel, scoring the /// segment's phased-side alleles by the haploid likelihood under each candidate fine population. /// Sets [`AncestrySegment::fine_population_code`] in place; leaves it `None` when the segment is too -/// short or the best fine call isn't clearly ahead of the runner-up (mirrors the super→fine admixture +/// short or the best fine call is not clearly ahead of the runner-up (mirrors the super→fine admixture /// hierarchy). `fine_panel.populations` are fine-pop codes; each site's `freqs` are per-fine-pop AF. pub fn resolve_fine_populations( segments: &mut [AncestrySegment], @@ -934,9 +934,9 @@ fn collapse_copy( use navigator_domain::seq::complement_base as revcomp_base; /// Alt-allele dosage (0/1/2) for a chip diploid call `(a1,a2)` against a panel site's -/// `ref_allele`/`alt_allele`. When the call's alleles don't both lie in `{ref,alt}`, retry once on +/// `ref_allele`/`alt_allele`. When the call's alleles do not both lie in `{ref,alt}`, retry once on /// the **reverse-complemented** call (the array reported the other strand); `None` if it still -/// doesn't match (no-call / multi-allelic mismatch). The minimal strand-flip logic chip→panel needs. +/// does not match (no-call / multi-allelic mismatch). The minimal strand-flip logic chip→panel needs. pub fn dosage_from_alleles(a1: char, a2: char, ref_allele: char, alt_allele: char) -> Option { let (r, alt) = (ref_allele.to_ascii_uppercase(), alt_allele.to_ascii_uppercase()); let count = |x: char, y: char| -> Option { @@ -1133,10 +1133,10 @@ const ANCIENT_MAX_DISPERSION: f64 = 4.0; /// East Asian, and no term for Sub-Saharan African, so for a person who carries a lot of any of /// those, a three-way decomposition of their *whole genome* is not an approximation — it is a /// category error. A Punjabi fits at Steppe 67% here; their real Steppe ancestry is nearer 20–30%, -/// with the rest Iranian-Neolithic and AASI that this model simply cannot see, so it piles the +/// with the rest Iranian-Neolithic and AASI that this model simply can not see, so it piles the /// unexplained ancestry onto whichever source is least unlike it. /// -/// Dispersion alone cannot catch that (South Asians overlap the European tail), but the *modern* +/// Dispersion alone can not catch that (South Asians overlap the European tail), but the *modern* /// estimate — which is well validated and independent of this panel — separates them cleanly. So /// deep ancestry only runs for samples the modern model already calls predominantly European. const ANCIENT_MIN_WEST_EURASIAN: f64 = 50.0; @@ -1169,7 +1169,7 @@ const QPADM_WEIGHT_TOL: f64 = 0.02; /// ancestry for a WHG/ANF/Steppe decomposition to mean anything, or a fit dispersion above /// [`ANCIENT_MAX_DISPERSION`] (the sample's ancestry lies outside the span of the three sources — a /// Yoruba is not *any* mixture of them). Reporting nothing is the entire point: the EM will always -/// return *some* simplex vector, and presenting that vector for a sample the model cannot express is +/// return *some* simplex vector, and presenting that vector for a sample the model can not express is /// precisely the failure this rebuild exists to prevent. pub fn estimate_ancient_admixture( genotypes: &[SiteGenotype], @@ -2211,7 +2211,7 @@ mod tests { /// Ancestry-HETEROZYGOUS sample: every site het (one copy A, one copy B). Diploid painting must /// put A on one copy and B on the other across the whole chromosome (the case a single-track - /// painter cannot express). + /// painter can not express). #[test] fn painting_diploid_heterozygous_copies_differ() { let n = 60; @@ -2274,7 +2274,7 @@ mod tests { /// Phased painting: two genuine parental sides. Side 0 is ancestry A (alt) on the first half, /// B (ref) on the second; side 1 is the mirror. Each side must paint as a consistent two-segment - /// track (A→B on side 0, B→A on side 1) — the parent-split the unphased painter cannot express. + /// track (A→B on side 0, B→A on side 1) — the parent-split the unphased painter can not express. #[test] fn painting_phased_two_consistent_sides() { use crate::phasing::{PhasedGenotypes, PhasedSite}; @@ -2381,7 +2381,7 @@ mod tests { // // The three-source model is the one that previously shipped fabricated numbers, so these tests // pin the two properties whose absence made that possible: it must recover a mixture it was - // never told, and it must refuse a sample its sources cannot express. + // never told, and it must refuse a sample its sources can not express. /// A deterministic LCG — the simulations below must give the same answer on every run. struct Lcg(u64); @@ -2652,7 +2652,7 @@ mod tests { } /// A symmetric tree `((A,B),(C,D))` has `f4(A,B;C,D) = 0` in expectation (the A–B and C–D drift - /// paths don't overlap), while `f4(A,C;B,D)` sits on the shared internal edge and is non-zero. + /// paths do not overlap), while `f4(A,C;B,D)` sits on the shared internal edge and is non-zero. /// Simulate exactly that and require the jackknife SE to *tell them apart*: the null within a few /// SE of zero, the real edge many SE away. This is the test that the covariance is calibrated — /// the property §5.4 needs and simulation-of-frequencies alone can't fake. diff --git a/crates/navigator-analysis/src/archaic.rs b/crates/navigator-analysis/src/archaic.rs index bb3b1df6..a1d4599d 100644 --- a/crates/navigator-analysis/src/archaic.rs +++ b/crates/navigator-analysis/src/archaic.rs @@ -51,7 +51,7 @@ impl ArchaicCall { /// Which archaic lineage a site's derived allele points to. /// -/// The HMM in Tier B cannot itself separate Neanderthal from Denisovan (they coalesce before either +/// The HMM in Tier B can not itself separate Neanderthal from Denisovan (they coalesce before either /// meets modern humans, design §3); this classification is what lets called segments be labelled /// downstream, so it is stored per site at build time. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -61,7 +61,7 @@ pub enum DiagnosticClass { /// Derived in Denisova, and ≥1 Neanderthal positively **called ancestral**. Denisovan, /// Not attributable to one lineage: derived in both, or the other lineage had no call so its - /// absence cannot be established. + /// absence can not be established. SharedArchaic, } @@ -72,9 +72,9 @@ pub enum DiagnosticClass { /// an earlier version did, and it was the dominant error in Tier B attribution: a site where the /// Neanderthals happened to be masked out and Denisova was called read as Denisovan-*specific*, /// inflating Denisovan-diagnostic sites to 18,551 against 24,077 Neanderthal on chr21+22 and -/// producing ~19 % Denisovan for a European, where design §7 expects approximately zero. +/// producing ~19 % Denisovan for a European, where design §7 expects about zero. /// -/// Sites that cannot be attributed fall to [`DiagnosticClass::SharedArchaic`], which therefore means +/// Sites that can not be attributed fall to [`DiagnosticClass::SharedArchaic`], which therefore means /// "archaic but not attributable" rather than strictly "derived in both". pub fn classify_diagnostic(calls: &[ArchaicCall; 4]) -> DiagnosticClass { let nea_derived = calls @@ -118,7 +118,7 @@ pub struct ArchaicSite { pub reference_allele: char, pub alternate_allele: char, /// The archaic-derived allele — always one of `reference_allele` / `alternate_allele`. Stored as - /// a base, not a ref/alt flag, so a later orientation pass cannot silently invert its meaning. + /// a base, not a ref/alt flag, so a later orientation pass can not silently invert its meaning. pub archaic_derived_allele: char, /// Per-genome state, indexed by [`ARCHAIC_GENOMES`]. pub calls: [ArchaicCall; 4], @@ -1011,7 +1011,7 @@ impl ArchaicOutgroup { /// Asset 3 — genome-wide archaic diagnostic sites, for labelling a called segment Neanderthal vs /// Denisovan (design §5 step 3). /// -/// The HMM itself cannot tell the lineages apart — they coalesce before either meets modern humans +/// The HMM itself can not tell the lineages apart — they coalesce before either meets modern humans /// (§3) — so attribution is a downstream count of derived-allele matches against these sites. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ArchaicClassify { @@ -1104,7 +1104,7 @@ impl ArchaicCallable { /// Callable fraction (0.0–1.0) of the window containing `position`, or 0.0 when the contig or /// window is absent — an unknown region is treated as **not** callable, so the HMM skips it - /// rather than interpreting density it cannot trust. + /// rather than interpreting density it can not trust. pub fn callable_fraction(&self, contig: &str, position: i64) -> f64 { let Some(c) = self.contig(contig) else { return 0.0 }; if position < c.start { diff --git a/crates/navigator-analysis/src/archaic_match.rs b/crates/navigator-analysis/src/archaic_match.rs index 77f9c2f2..955c0c35 100644 --- a/crates/navigator-analysis/src/archaic_match.rs +++ b/crates/navigator-analysis/src/archaic_match.rs @@ -111,7 +111,7 @@ //! An independent arbiter settles this without asking another caller: the Tier A panel records, per //! site, which of the four archaic genomes carries the derived allele, and **this caller never sees //! that** — it reads only a derived base and a lineage class. So per-genome concordance is evidence -//! it cannot have been fitted to. +//! it can not have been fitted to. //! //! Of the sites where a given archaic genome is derived, what fraction does the subject carry //! (best-matching genome): @@ -158,7 +158,7 @@ //! **Still not enough to re-enable.** Beyond the ordering: precision is 34.9 % unfiltered on //! held-out Europeans, the cohort is **chr21+22 only**, and the reference callset is itself weakly //! supported (hmmix's own tracts are enriched just 1.84x for their own archaic SNPs), so agreement -//! with it caps well below 100 % even for a correct caller — F1 alone cannot say when this is done. +//! with it caps well below 100 % even for a correct caller — F1 alone can not say when this is done. use std::collections::BTreeMap; diff --git a/crates/navigator-analysis/src/archaic_segments.rs b/crates/navigator-analysis/src/archaic_segments.rs index c09f933e..6174e5db 100644 --- a/crates/navigator-analysis/src/archaic_segments.rs +++ b/crates/navigator-analysis/src/archaic_segments.rs @@ -5,7 +5,7 @@ //! variants that no African outgroup individual carries. Anything Africans also carry is not //! evidence of introgression, so stripping them is what makes the remaining density informative. //! -//! The HMM **cannot tell Neanderthal from Denisovan** — the two lineages coalesce before either +//! The HMM **can not tell Neanderthal from Denisovan** — the two lineages coalesce before either //! meets modern humans (§3) — so it finds segments and a downstream pass labels them by counting //! derived-allele matches against the archaic genomes (`ArchaicClassify`). //! @@ -27,7 +27,7 @@ pub enum ArchaicSource { Neanderthal, Denisovan, /// Archaic by density, but the diagnostic sites in it do not favour either lineage — the - /// honest label for a segment we cannot attribute, and a substantial share in real data + /// honest label for a segment we can not attribute, and a substantial share in real data /// (Skov 2020 reported ~12 % unknown on Icelanders). Unknown, } @@ -102,7 +102,7 @@ pub struct ArchaicConfig { /// Expected fraction of Neanderthal-diagnostic sites at which a non-archaic-specific genome /// carries the derived allele, and the same for Denisovan-diagnostic sites. /// - /// These base rates are the reason raw match counts cannot attribute a lineage. Measured on the + /// These base rates are the reason raw match counts can not attribute a lineage. Measured on the /// ground-truth European: 4.3 % at Neanderthal-diagnostic sites versus 3.9 % at /// Denisovan-diagnostic ones — a ratio of 1.10, essentially no discrimination. Carrying a /// "Denisovan-diagnostic" allele mostly reflects ordinary shared ancestry, not Denisovan @@ -191,7 +191,7 @@ fn span_cm(gmap: &GeneticMap, chr: &str, start_bp: i64, end_bp: i64) -> f64 { /// Call archaic tracts from a subject's genome-wide diploid calls. /// /// `calls` should be the de-novo diploid variant calls for one alignment (Tier B is gated to -/// WGS/VCF input — a chip cannot supply the density this needs). +/// WGS/VCF input — a chip can not supply the density this needs). pub fn call_archaic_segments( calls: &[SiteGenotype], outgroup: &ArchaicOutgroup, @@ -443,7 +443,7 @@ fn call_contig( if mean_post < cfg.min_posterior { continue; } - // At least half the run's windows must be callable, so a tract cannot be carried by a + // At least half the run's windows must be callable, so a tract can not be carried by a // stretch of uninformative windows riding on the transition prior. let callable_windows = usable[start_w..=end_w].iter().filter(|u| **u).count(); if callable_windows * 2 < end_w - start_w + 1 { diff --git a/crates/navigator-analysis/src/caller.rs b/crates/navigator-analysis/src/caller.rs index 52ced0c4..0fe54290 100644 --- a/crates/navigator-analysis/src/caller.rs +++ b/crates/navigator-analysis/src/caller.rs @@ -477,7 +477,7 @@ fn read_indel_events(record: &RecordBuf, start: i64) -> (Vec<(i64, IndelAllele)> /// guard and vetoes the whole lineage). So indels only ever *confirm* a branch, matching the intent: /// cover the many indel-defined DecodingUs branches when the sample carries them. Requires a /// `reference` (to left-normalize + know deleted bases); returns empty without one, or when the -/// contig isn't in the FASTA. +/// contig is not in the FASTA. pub fn call_indels_at( bam_path: &Path, contig: &str, @@ -855,7 +855,7 @@ pub fn genotype_sites_all_contigs( /// site absent-as-hom-ref in one run is a real vote (resolving "run A het vs run B hom-ref") while a /// genuinely uncovered run abstains. Only **variant** consensus sites (het/hom-alt) are returned; /// hom-ref / no-call consensus is not a variant. Depth/AD are summed and GQ is the max over the -/// supporting alignments; PLs are dropped (the per-run likelihoods don't compose into one PL here). +/// supporting alignments; PLs are dropped (the per-run likelihoods do not compose into one PL here). pub fn reconcile_site_genotypes(per_alignment: &[Vec], min_depth: u32) -> Vec { use std::collections::BTreeMap; struct Acc { @@ -2116,7 +2116,7 @@ mod tests { // An insertion of "A" before anchor 5 (in the A-run) left-aligns to anchor 2. let (a, al) = left_normalize(5, &IndelAllele::Ins(b"A".to_vec()), refc, 1); assert_eq!((a, al), (2, IndelAllele::Ins(b"A".to_vec()))); - // A non-repeat deletion doesn't move: "ACGTC", delete the G (anchor 3). + // A non-repeat deletion does not move: "ACGTC", delete the G (anchor 3). let (a, _) = left_normalize(3, &IndelAllele::Del(1), b"ACGTC", 1); assert_eq!(a, 3); } diff --git a/crates/navigator-analysis/src/cancel.rs b/crates/navigator-analysis/src/cancel.rs index 11b986ce..24d3ca64 100644 --- a/crates/navigator-analysis/src/cancel.rs +++ b/crates/navigator-analysis/src/cancel.rs @@ -2,7 +2,7 @@ //! //! A whole-genome pass takes minutes, and the UI's Cancel button used to do nothing visible for //! all of them: the flag it set lived in `navigator-ui` and was only read *between* pipeline steps, -//! while the step itself ran inside a `spawn_blocking` closure that tokio cannot interrupt. Once a +//! while the step itself ran inside a `spawn_blocking` closure that tokio can not interrupt. Once a //! walk starts, the only thing that can stop it is the walk itself — so the walkers have to ask. //! //! [`CancelToken`] is that question, and the rule for using it is about *where* you ask: often @@ -21,7 +21,7 @@ use std::sync::Arc; use crate::error::AnalysisError; -/// A shared "stop what you're doing" flag, cheap to clone into worker threads. +/// A shared "stop what you are doing" flag, cheap to clone into worker threads. /// /// [`CancelToken::none`] is a token that can never be cancelled. It exists so callers with nothing /// to cancel — tests, CLI one-shots, the non-progress convenience wrappers — pay nothing and read @@ -52,7 +52,7 @@ impl CancelToken { /// Whether cancellation has been requested. /// - /// `Relaxed` is sufficient: this guards no other memory, and the only cost of observing the + /// `Relaxed` is enough: this guards no other memory, and the only cost of observing the /// store one loop iteration late is one more iteration of work. pub fn is_cancelled(&self) -> bool { self.0.as_ref().is_some_and(|flag| flag.load(Ordering::Relaxed)) diff --git a/crates/navigator-analysis/src/contig.rs b/crates/navigator-analysis/src/contig.rs index 1182f123..dcc83ebd 100644 --- a/crates/navigator-analysis/src/contig.rs +++ b/crates/navigator-analysis/src/contig.rs @@ -34,8 +34,8 @@ pub fn is_main_assembly(name: &str) -> bool { } /// **Haploid** contigs: chrY and chrM/MT carry a single allele, so the diploid (het `0/1` + -/// hom-alt `1/1`) model doesn't apply — the haploid caller and Y/mt haplogroup placement own them. -/// (chrX is haploid only in a male; that's left to the sex-aware refinement, not decided here.) +/// hom-alt `1/1`) model does not apply — the haploid caller and Y/mt haplogroup placement own them. +/// (chrX is haploid only in a male; that is left to the sex-aware refinement, not decided here.) pub fn is_haploid(name: &str) -> bool { is_chr_y(name) || is_chr_m(name) } diff --git a/crates/navigator-analysis/src/coverage.rs b/crates/navigator-analysis/src/coverage.rs index 57cd58eb..223d0361 100644 --- a/crates/navigator-analysis/src/coverage.rs +++ b/crates/navigator-analysis/src/coverage.rs @@ -234,7 +234,7 @@ struct CurContig { map_q_total: u64, sum_depth: u128, /// Count of bases dropped because they mapped *before* the finalize frontier — only possible - /// when the input isn't strictly coordinate-sorted (see [`CurContig::add`]). Surfaced as a + /// when the input is not strictly coordinate-sorted (see [`CurContig::add`]). Surfaced as a /// warning in [`CurContig::finish`]; stays 0 for the standard sorted layout. dropped_unsorted: u64, } @@ -312,7 +312,7 @@ impl CurContig { /// Add one covered base at 1-based `pos` to the window. The window only holds positions at or /// after the finalize frontier, so a base *before* it (`pos < emit_cursor`) belongs to an /// already-emitted column and is dropped rather than underflowing `pos - emit_cursor`. This can - /// only happen when the input isn't strictly coordinate-sorted (some vendor CRAMs, e.g. FTDNA + /// only happen when the input is not strictly coordinate-sorted (some vendor CRAMs, e.g. FTDNA /// Big Y): the streaming pileup fundamentally assumes sorted input, so the few out-of-order /// bases are counted as dropped (surfaced in `finish`) instead of crashing the walk. For the /// standard sorted layout `pos >= emit_cursor` always holds and this guard never fires. @@ -490,7 +490,7 @@ impl CoverageState { self.total_tracked } - /// Feed one record. Records that coverage doesn't care about (unmapped/secondary/ + /// Feed one record. Records that coverage does not care about (unmapped/secondary/ /// supplementary/duplicate/qc-fail, or off a tracked contig) are ignored, so the fused /// walker can hand every record here unfiltered. Fires `progress` on contig finalization. pub(crate) fn accept( @@ -682,7 +682,7 @@ fn assemble_coverage_result( let median = median_from_hist(&hist, n); // Exclusion fractions over total observed bases (Picard PCT_EXC_{MAPQ,BASEQ}). `sum_depth` // already counts every observed base (excluded ones included), so it is the denominator. Other - // exclusion reasons (dup/unpaired/overlap/capped) aren't tallied, so these don't sum to a total. + // exclusion reasons (dup/unpaired/overlap/capped) are not tallied, so these do not sum to a total. let (pct_exc_mapq, pct_exc_baseq) = if sum_depth == 0 { (0.0, 0.0) } else { diff --git a/crates/navigator-analysis/src/error.rs b/crates/navigator-analysis/src/error.rs index ef47d261..0dfe748e 100644 --- a/crates/navigator-analysis/src/error.rs +++ b/crates/navigator-analysis/src/error.rs @@ -43,8 +43,8 @@ pub fn panic_text(payload: &(dyn std::any::Any + Send)) -> Option<&str> { /// Run a BAM/CRAM walk, converting a **panic** into a clean [`AnalysisError`] so one undecodable /// file fails gracefully instead of unwinding into a cryptic `JoinError`/aborting a worker. The -/// motivating cases are noodles' `todo!()`/`expect()` on inputs it doesn't handle (an unimplemented -/// CRAM data series, or a decode that needs reference bases it wasn't given): without this, such a +/// motivating cases are noodles' `todo!()`/`expect()` on inputs it does not handle (an unimplemented +/// CRAM data series, or a decode that needs reference bases it was not given): without this, such a /// file panics deep inside the decoder. `what` labels the operation/file for the surfaced message. /// /// The panic's own text is included rather than a guessed explanation — this is a last-resort net, diff --git a/crates/navigator-analysis/src/gvcf.rs b/crates/navigator-analysis/src/gvcf.rs index 5180972c..dd023aa4 100644 --- a/crates/navigator-analysis/src/gvcf.rs +++ b/crates/navigator-analysis/src/gvcf.rs @@ -260,7 +260,7 @@ pub fn read_diploid_calls_from( } let end = info_end(info).unwrap_or(pos); for &t in targets_in_range(sorted, pos, end) { - // Don't overwrite a variant call (variant records are authoritative; in a well-formed + // Do not overwrite a variant call (variant records are authoritative; in a well-formed // gVCF they never overlap a ref block anyway). out.entry((chrom.to_string(), t)).or_insert(GvcfDiploid::HomRef); } @@ -716,7 +716,7 @@ chrM\t100\t.\tC\tT,\t500\t.\tDP=30\tGT:AD:DP:GQ:PL\t1:0,30,0:30:99:510, /// Real-data smoke test: decode the pipeline's actual bgzipped chrY GVCF for HG00096 /// over a dense synthetic target grid across the non-PAR span. Validates real bgzf /// inflation + record parsing at scale (thousands of records). No-ops when the NAS - /// file isn't mounted, so it's safe on any machine. Run with: + /// file is not mounted, so it is safe on any machine. Run with: /// cargo test -p navigator-analysis gvcf -- --ignored --nocapture #[test] #[ignore = "reads a NAS file; run explicitly"] diff --git a/crates/navigator-analysis/src/gzio.rs b/crates/navigator-analysis/src/gzio.rs index 98087306..ac8655b8 100644 --- a/crates/navigator-analysis/src/gzio.rs +++ b/crates/navigator-analysis/src/gzio.rs @@ -25,7 +25,7 @@ enum Compression { /// Open `path` for buffered line reading, transparently decoding gzip/BGZF when the file /// begins with the gzip magic bytes. Plain (uncompressed) text is read directly. /// -/// See [`open_maybe_compressed`] to additionally decode bzip2. +/// See [`open_maybe_compressed`] to also decode bzip2. pub fn open_maybe_gz(path: &Path) -> io::Result> { let mut file = File::open(path)?; match detect_compression(&mut file)? { diff --git a/crates/navigator-analysis/src/haplo.rs b/crates/navigator-analysis/src/haplo.rs index fd4cb28c..84ee1b0e 100644 --- a/crates/navigator-analysis/src/haplo.rs +++ b/crates/navigator-analysis/src/haplo.rs @@ -323,7 +323,7 @@ pub fn normalize_polarity(tree: &mut HaploTree, reference: &HashMap) -> Vec { // |F| — distinct tree sites whose derived allele the sample carries. let mut carried: HashSet = HashSet::new(); @@ -562,7 +562,7 @@ pub fn tree_positions(tree: &HaploTree) -> HashMap { /// Polarity map for the consensus interpreter: **SNP name → (ancestral, derived)** over every /// defining locus in the tree. This is the tree-of-record's per-SNP polarity, applied at read time by /// `navigator_domain::consensus::interpret` so a corrected tree flips states with no re-genotyping. -/// Use for any parsed [`HaploTree`] (mtDNA rCRS, FTDNA) where a JSON polarity map isn't available; +/// Use for any parsed [`HaploTree`] (mtDNA rCRS, FTDNA) where a JSON polarity map is not available; /// for the DecodingUs Y JSON prefer [`decodingus_polarity_map`] (true phylogenetic polarity). Loci /// without a name or derived allele are skipped; a recurrent name keeps its first-seen polarity. pub fn polarity_from_tree(tree: &HaploTree) -> std::collections::BTreeMap { @@ -632,7 +632,7 @@ pub struct SnpEvidence { } /// A child branch below the reported terminal, with the per-SNP evidence that explains why -/// descent did or didn't continue into it. +/// descent did or did not continue into it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BranchEvidence { pub name: String, @@ -695,7 +695,7 @@ pub struct NodeEvidence { /// Group the root→`terminal_id` path into per-node defining-SNP evidence (root→terminal order): /// walk the tree from the terminal up to the root, and for each node attach its loci with the -/// sample's state taken from `state_by_name` (`NoCall` for an equivalent the sample didn't call). +/// sample's state taken from `state_by_name` (`NoCall` for an equivalent the sample did not call). /// Keyed by **SNP name** (build-independent: a name like `M269` is the same across coordinate /// systems), so a cached variant profile placed under any build can colour an FTDNA-tree path. pub fn descent_by_node( @@ -922,7 +922,7 @@ const REDEEM_DERIVED: usize = 4; /// lineage (derived or merely no-call along its length) passes. Used to veto otherwise high-scoring /// tunnel artifacts from the [`score`] ranking. /// -/// A contradicted ancestor only vetoes when it isn't *redeemed* by derived support further down the +/// A contradicted ancestor only vetoes when it is not *redeemed* by derived support further down the /// path: a single stray ancestral at a sparse intermediate node (a Big Y miscall) is overridden /// when ≥[`REDEEM_DERIVED`] derived SNPs below it confirm the branch, while a coincidental tunnel — /// a contradicted branch-point with only a hit or two beneath it — stays vetoed. @@ -1202,7 +1202,7 @@ mod tests { #[test] fn induced_subtree_ignores_terminals_absent_from_the_tree() { let t = parse_ftdna_json(BRANCHY).unwrap(); - // 999 doesn't exist (provider/build skew); the real terminal still resolves. + // 999 does not exist (provider/build skew); the real terminal still resolves. let nodes = induced_subtree(&t, &[4, 999]); let names: Vec<&str> = nodes.iter().map(|n| n.name.as_str()).collect(); assert_eq!(names, vec!["root", "R", "R1", "R1a"]); @@ -1393,7 +1393,7 @@ mod tests { // loci does not drop the node: it stays, named and on the path, with an empty `loci`, and // `descent_by_node` faithfully reports it with no SNPs. A renderer that hides empty blocks // (correctly — the root is genuinely empty) then shows the lineage stopping one branch - // short, while the terminal *name* remains right. Hence `DECODINGUS_NATIVE_BUILD`: parse in + // short, while the terminal *name* remains right. So `DECODINGUS_NATIVE_BUILD`: parse in // hs1 wherever the join is by SNP name. let json = r#"{ "roots": [ @@ -1441,7 +1441,7 @@ mod tests { assert_eq!(terminal.snps[0].state, CallState::Derived); } - /// Mirrors `navigator_app::DECODINGUS_NATIVE_BUILD`, which this crate sits below and so cannot + /// Mirrors `navigator_app::DECODINGUS_NATIVE_BUILD`, which this crate sits below and so can not /// import. The test above is the reason that constant exists. const DECODINGUS_NATIVE_BUILD_FOR_TEST: &str = "hs1"; @@ -1555,7 +1555,7 @@ mod tests { assert!(!locus_carried(&locus, &calls(&[(146, 'T')]))); // Strand-ambiguous SNP (C↔G): the complement of derived G is the ancestral C, so strand - // can't be inferred — keep strict literal matching and don't complement-flip. + // can't be inferred — keep strict literal matching and do not complement-flip. let palindrome = Locus { position: 200, ancestral: "C".into(), @@ -1845,7 +1845,7 @@ mod tests { (700, 'A'), (800, 'A'), ]); - // Deepen enters C from P: it carries 3 derived (≥2) and isn't contradicted (3 anc ≤ 3 der). + // Deepen enters C from P: it carries 3 derived (≥2) and is not contradicted (3 anc ≤ 3 der). // (The "Kulczynski stops at the parent" condition needs a long backbone — validated on // the real WGS229 short-read sample, where the guard stops at R-FGC29067 and deepen // recovers R-FGC29071.) @@ -1867,7 +1867,7 @@ mod tests { (800, 'A'), ]); assert_eq!(deepen_terminal(&t, &lone, id_of(&t, "P")), id_of(&t, "P")); - // 2 derived but 4 ancestral → contradicted (a > d), don't enter even at ≥2 derived. + // 2 derived but 4 ancestral → contradicted (a > d), do not enter even at ≥2 derived. let net_anc = calls(&[ (100, 'G'), (200, 'G'), diff --git a/crates/navigator-analysis/src/ibd_attest.rs b/crates/navigator-analysis/src/ibd_attest.rs index a2c35e64..e404d771 100644 --- a/crates/navigator-analysis/src/ibd_attest.rs +++ b/crates/navigator-analysis/src/ibd_attest.rs @@ -16,7 +16,7 @@ use sha2::{Digest, Sha256}; use crate::ibd::MatchSummary; /// One panel-site dosage on the wire — the minimal input the IBD detector consumes (the heavy -/// [`crate::caller::SiteGenotype`] fields aren't sent). `dosage` is 0/1/2, or -1 for no-call. +/// [`crate::caller::SiteGenotype`] fields are not sent). `dosage` is 0/1/2, or -1 for no-call. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct IbdSite { pub contig: String, @@ -129,7 +129,7 @@ pub enum IbdExchangeMsg { impl IbdExchangeMsg { /// Gzipped JSON bytes for the channel. The dosage payload is large (a panel of sites), and the - /// relay caps an envelope at 1 MiB, so it's compressed (the dosage vector compresses well). + /// relay caps an envelope at 1 MiB, so it is compressed (the dosage vector compresses well). pub fn to_bytes(&self) -> Result, String> { use flate2::{write::GzEncoder, Compression}; use std::io::Write; diff --git a/crates/navigator-analysis/src/ibd_panel.rs b/crates/navigator-analysis/src/ibd_panel.rs index 20ba3c42..523ff90c 100644 --- a/crates/navigator-analysis/src/ibd_panel.rs +++ b/crates/navigator-analysis/src/ibd_panel.rs @@ -1,6 +1,6 @@ //! Multi-build, chip-compatible IBD reference panel (ancestry-ibd-asset-wiring B2/B2c). //! -//! IBD matching needs a neutral, dense SNP set that's also **assayed by consumer arrays** — chip +//! IBD matching needs a neutral, dense SNP set that is also **assayed by consumer arrays** — chip //! kits outnumber WGS by orders of magnitude, so the panel must be where chip and WGS overlap. //! Each site carries its `(contig, pos, REF, ALT)` on **CHM13, GRCh37, and GRCh38** (built once via //! allele-aware GATK liftover, offline), so a chip genotype on *any* build resolves to the canonical @@ -136,13 +136,13 @@ impl IbdPanel { /// Resolve a **whole-genome, variant-only** source (a WGS VCF or CompleteGenomics masterVar) to /// canonical CHM13 dosages over the panel. Unlike a chip — which reports a genotype at every /// array site — such a source lists *only* the non-reference sites, so every panel site the - /// source could have called but didn't is taken as **homozygous reference** (dosage 0). That + /// source could have called but did not is taken as **homozygous reference** (dosage 0). That /// assumption is valid **only** for a source that genotyped the whole genome (absent ⇒ hom-ref, /// not no-call); never pass a targeted panel (Big Y / Sanger) here. /// /// `variant_calls` are the source's variant sites on `build` as `(contig, pos, a1, a2)` /// reference-forward allele pairs. Contigs match `chr`-insensitively (a source's `chr1` lines up - /// with a panel `grch37` locus stored as `1`). A variant whose alleles don't reconcile to the + /// with a panel `grch37` locus stored as `1`). A variant whose alleles do not reconcile to the /// site (multiallelic mismatch) is dropped, not mis-called hom-ref. Palindromic (A/T, C/G) sites /// are skipped — strand-ambiguous across builds, exactly as [`resolve_chip`]. pub fn resolve_whole_genome(&self, build: &str, variant_calls: &[(String, i64, char, char)]) -> Vec { @@ -413,7 +413,7 @@ mod tests { ); // rs1 hom-alt (G/G → dosage 2); rs2 listed but the alleles are internally inconsistent with // the biallelic site: C matches ref directly, A only matches alt(T) under rc — neither a - // pure direct nor a pure rc pair, so it doesn't reconcile → dropped, NOT called hom-ref. + // pure direct nor a pure rc pair, so it does not reconcile → dropped, NOT called hom-ref. let calls = vec![("1".to_string(), 500, 'G', 'G'), ("1".to_string(), 600, 'C', 'A')]; let g = panel.resolve_whole_genome("GRCh37", &calls); let by_pos: std::collections::HashMap = g.iter().map(|s| (s.position, s.dosage)).collect(); diff --git a/crates/navigator-analysis/src/index.rs b/crates/navigator-analysis/src/index.rs index 1ee42f8f..0398855e 100644 --- a/crates/navigator-analysis/src/index.rs +++ b/crates/navigator-analysis/src/index.rs @@ -120,7 +120,7 @@ fn build_bai(path: &Path, dst: &Path, progress: ProgressFn) -> Result<(), Analys .add_record(alignment_context, chunk) .map_err(|e| AnalysisError::io(path, e))?; - // Report on ~32 MB of compressed progress so a multi-GB BAM doesn't flood the channel. + // Report on ~32 MB of compressed progress so a multi-GB BAM does not flood the channel. let done = end_position.compressed(); if done.saturating_sub(last_reported) >= 32_000_000 { last_reported = done; diff --git a/crates/navigator-analysis/src/lai.rs b/crates/navigator-analysis/src/lai.rs index 42a581a0..0bdeedcb 100644 --- a/crates/navigator-analysis/src/lai.rs +++ b/crates/navigator-analysis/src/lai.rs @@ -59,7 +59,7 @@ pub struct CopyingLaiParams { /// populations stop being balanced at all). /// /// Capping is not the only size correction — see [`Self::size_normalize`], which divides by the - /// haplotype count and, on a dense panel, does the work capping cannot. + /// haplotype count and, on a dense panel, does the work capping can not. pub max_ref_haps: usize, /// Runs shorter than this many **centiMorgans** merge into the neighbouring segment. /// @@ -146,7 +146,7 @@ fn kept_super_pops(prior: &[(String, f64)], min_ancestry: f64) -> Option = Vec::with_capacity(n_sites); // GBR, TSI, FIN, YRI diff --git a/crates/navigator-analysis/src/manifest.rs b/crates/navigator-analysis/src/manifest.rs index 734cb195..def9559a 100644 --- a/crates/navigator-analysis/src/manifest.rs +++ b/crates/navigator-analysis/src/manifest.rs @@ -50,7 +50,7 @@ impl AssetManifest { } /// Verify `bytes` for `filename`. `Ok` when the manifest has no entry for the file (advisory — - /// unlisted assets aren't gated) or the digest matches; `Err(expected, got)` on a mismatch. + /// unlisted assets are not gated) or the digest matches; `Err(expected, got)` on a mismatch. pub fn verify(&self, filename: &str, bytes: &[u8]) -> Result<(), (String, String)> { if let Some(e) = self.assets.get(filename) { let got = sha256_hex(bytes); diff --git a/crates/navigator-analysis/src/mask.rs b/crates/navigator-analysis/src/mask.rs index 0686fc85..7a0d69b5 100644 --- a/crates/navigator-analysis/src/mask.rs +++ b/crates/navigator-analysis/src/mask.rs @@ -245,7 +245,7 @@ impl YStructuralRegions { } /// Build from explicit masks (the seam the BED loader + unit tests share). XTR/STR/centromere - /// masks aren't sourced yet — those tiers exist in [`YRegionClass`] for when their data lands. + /// masks are not sourced yet — those tiers exist in [`YRegionClass`] for when their data lands. pub fn from_masks( par: RegionMask, palindrome: RegionMask, diff --git a/crates/navigator-analysis/src/mastervar.rs b/crates/navigator-analysis/src/mastervar.rs index 1f987c0a..ffaccd3a 100644 --- a/crates/navigator-analysis/src/mastervar.rs +++ b/crates/navigator-analysis/src/mastervar.rs @@ -62,7 +62,7 @@ struct Columns { impl Columns { /// Map the masterVar column header (the `>`-prefixed line) to field indices by name. Returns - /// `None` if a required column is missing (so a look-alike table isn't parsed as masterVar). + /// `None` if a required column is missing (so a look-alike table is not parsed as masterVar). fn from_header(line: &str) -> Option { let header = line.strip_prefix('>').unwrap_or(line); let names: Vec<&str> = header.split('\t').map(str::trim).collect(); @@ -170,7 +170,7 @@ enum Hap { } /// Resolve one locus's rows into at most one biallelic SNP [`VariantCall`]. Returns `None` for -/// a locus with no `snp` row (a `ref`/`no-call`/indel span) or one whose SNP alleles aren't a +/// a locus with no `snp` row (a `ref`/`no-call`/indel span) or one whose SNP alleles are not a /// clean single-base substitution. fn locus_call(rows: &[Row]) -> Option { // Site anchor: the first snp row carries the reference base + coordinates. @@ -198,7 +198,7 @@ fn locus_call(rows: &[Row]) -> Option { let hap = |which: Allele| -> Hap { // A compound locus can list several rows for one allele (e.g. a `ref` segment beside the // `snp` segment). Prefer the SNP call for that allele; fall back to ref, else missing — - // so a snp isn't hidden behind a same-allele ref/no-call row and the locus lost. + // so a snp is not hidden behind a same-allele ref/no-call row and the locus lost. let mut result = Hap::Missing; for r in rows.iter().filter(|r| r.allele == which) { match r.var_type { @@ -216,7 +216,7 @@ fn locus_call(rows: &[Row]) -> Option { (Hap::Alt(a), Hap::Alt(_)) => (a.clone(), "1/2"), // tri-allelic het; keep allele 1's alt (Hap::Alt(a), Hap::Ref) | (Hap::Ref, Hap::Alt(a)) => (a.clone(), "0/1"), (Hap::Alt(a), Hap::Missing) | (Hap::Missing, Hap::Alt(a)) => (a.clone(), "1/."), - // No alt on either haplotype — not a variant (shouldn't occur given the snp row above). + // No alt on either haplotype — not a variant (should not occur given the snp row above). _ => return None, }; snp_call(contig, position, reference, &alt, rs_id, Some(genotype.into())) diff --git a/crates/navigator-analysis/src/postprocess/cram.rs b/crates/navigator-analysis/src/postprocess/cram.rs index dbf6e269..54366edc 100644 --- a/crates/navigator-analysis/src/postprocess/cram.rs +++ b/crates/navigator-analysis/src/postprocess/cram.rs @@ -62,7 +62,7 @@ pub struct CramOutput { /// This is not malformed input. SAM permits `SEQ: *` on a secondary alignment, and minimap2 uses /// that permission: only the primary carries the bases, and the secondaries point at other loci /// the same read could have come from. So the reads are not lost by dropping these — the primary -/// holds the sequence — but the records cannot be represented and must not reach the writer. +/// holds the sequence — but the records can not be represented and must not reach the writer. fn is_unencodable(record: &noodles::sam::alignment::RecordBuf) -> bool { !record.cigar().as_ref().is_empty() && record.sequence().as_ref().is_empty() } diff --git a/crates/navigator-analysis/src/postprocess/finalize.rs b/crates/navigator-analysis/src/postprocess/finalize.rs index 42f5691a..4aac16cb 100644 --- a/crates/navigator-analysis/src/postprocess/finalize.rs +++ b/crates/navigator-analysis/src/postprocess/finalize.rs @@ -11,8 +11,8 @@ //! are no bases to difference. [`super::cram`] handles it by dropping those records. //! - **Indexing** then panics on any *multi-reference* slice: `cram::fs::index` decodes records //! with `fasta::Repository::default()` — an empty one, still carrying its `// TODO` upstream — -//! and the reader `expect`s a name that cannot be there. With 25 contigs, every slice that -//! straddles a contig boundary is multi-reference, so a whole-genome CRAM cannot be indexed at +//! and the reader `expect`s a name that can not be there. With 25 contigs, every slice that +//! straddles a contig boundary is multi-reference, so a whole-genome CRAM can not be indexed at //! all. Only the coordinates are actually needed, which is why the decode is a bug rather than //! a requirement. //! diff --git a/crates/navigator-analysis/src/postprocess/markdup.rs b/crates/navigator-analysis/src/postprocess/markdup.rs index 8aa342b3..ca3af515 100644 --- a/crates/navigator-analysis/src/postprocess/markdup.rs +++ b/crates/navigator-analysis/src/postprocess/markdup.rs @@ -161,7 +161,7 @@ struct Signature { mate_reverse: bool, } -/// The signature of an eligible record, or `None` when the record cannot be marked. +/// The signature of an eligible record, or `None` when the record can not be marked. /// /// Unmapped records have no position to compare. Secondary and supplementary records describe an /// alignment of a read whose primary is elsewhere; marking them would double-count a template that @@ -282,7 +282,7 @@ impl SeenSignatures { false } - /// Drop signatures the file has moved past. Anything further back than the window cannot be a + /// Drop signatures the file has moved past. Anything further back than the window can not be a /// duplicate of the current record, because only clipping separates two copies' positions. fn evict(&mut self, position: i64) { while let Some((pos, sig)) = self.order.front().copied() { diff --git a/crates/navigator-analysis/src/postprocess/tests.rs b/crates/navigator-analysis/src/postprocess/tests.rs index 5763dedf..5610d5d1 100644 --- a/crates/navigator-analysis/src/postprocess/tests.rs +++ b/crates/navigator-analysis/src/postprocess/tests.rs @@ -803,7 +803,7 @@ fn cancellation_stops_cram_emission() { } /// A secondary alignment with `SEQ: *` — legal SAM, and what minimap2 emits, since only the -/// primary carries the bases. It cannot be encoded as differences from the reference, so it is +/// primary carries the bases. It can not be encoded as differences from the reference, so it is /// dropped and counted rather than panicking the writer from inside noodles. #[test] fn cram_drops_a_secondary_record_that_carries_no_sequence() { @@ -867,7 +867,7 @@ fn cram_refuses_a_primary_record_that_carries_no_sequence() { } /// Finalising moves the marked BAM into place and indexes it — across *several* contigs, which is -/// the shape that cannot be indexed as CRAM: every slice straddling a contig boundary is +/// the shape that can not be indexed as CRAM: every slice straddling a contig boundary is /// multi-reference, and `cram::fs::index` decodes those against an empty reference repository. #[test] fn finalizing_moves_the_bam_into_place_and_indexes_it() { diff --git a/crates/navigator-analysis/src/preflight.rs b/crates/navigator-analysis/src/preflight.rs index e2f77135..b10ef1ad 100644 --- a/crates/navigator-analysis/src/preflight.rs +++ b/crates/navigator-analysis/src/preflight.rs @@ -74,7 +74,7 @@ pub enum CheckId { } impl CheckId { - /// The human label. Single source of truth, so a check's name and its identity cannot drift. + /// The human label. Single source of truth, so a check's name and its identity can not drift. pub fn label(self) -> &'static str { match self { CheckId::Format => "format", @@ -146,7 +146,7 @@ pub struct Report { } impl Report { - /// Whether any check failed outright (warnings don't count — they have fallbacks). + /// Whether any check failed outright (warnings do not count — they have fallbacks). pub fn failed(&self) -> bool { self.checks.iter().any(|c| c.status == Status::Fail) } @@ -157,7 +157,7 @@ impl Report { self.checks.iter().find(|c| c.status == Status::Fail) } - /// Whether the file cannot be read *at all* — not even by a sequential pass. + /// Whether the file can not be read *at all* — not even by a sequential pass. /// /// This is the question a batch has to answer before deciding to skip a sample. A failure that /// only blocks region queries (a missing or unreadable index) must not skip it: read metrics, @@ -249,7 +249,7 @@ fn probe_file(id: CheckId, path: &Path) -> Check { } Err(e) => { let (status, mut detail, errno) = explain(path, &e); - // A file the OS won't open but that is visible in its own directory listing is being + // A file the OS will not open but that is visible in its own directory listing is being // withheld, not absent — worth saying, because "not found" would send the user looking // for a file that is sitting right there. if e.kind() == std::io::ErrorKind::NotFound && directory_lists(path) { @@ -322,7 +322,7 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { // The index. Its absence is a warning, not a failure: sequential walks (read metrics, coverage, // sex) fall back and succeed, which is exactly why an alignment can look healthy in the UI - // right up until something needs a region query. An index that exists but won't open is a + // right up until something needs a region query. An index that exists but will not open is a // failure, and is the case `has_region_index` silently reports as "no index". let candidates = index_candidates(alignment); let found = candidates.iter().find(|p| directory_lists(p)); @@ -415,7 +415,7 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { v } Err(e) => { - // The whole point of this module: don't repeat the upstream message's mistake of + // The whole point of this module: do not repeat the upstream message's mistake of // blaming the alignment. If we already established there is no index, *that* is the // finding — an `ENOENT` naming the CRAM here means the reader could not autoload a // sibling index, not that the CRAM went missing between two reads of it. @@ -590,7 +590,7 @@ mod tests { assert!(!CheckId::OpenIndexed.blocks_sequential_reads()); assert!(!CheckId::RegionQuery.blocks_sequential_reads()); // A reference problem does not skip the sample on its own — a BAM reads without one, and a - // CRAM that truly cannot use it fails the header read, which does. + // CRAM that truly can not use it fails the header read, which does. assert!(!CheckId::ReferenceFasta.blocks_sequential_reads()); assert!(!CheckId::ReferenceIndex.blocks_sequential_reads()); assert!(CheckId::AlignmentFile.blocks_sequential_reads()); diff --git a/crates/navigator-analysis/src/probe.rs b/crates/navigator-analysis/src/probe.rs index 0fbbfbd2..bd900f37 100644 --- a/crates/navigator-analysis/src/probe.rs +++ b/crates/navigator-analysis/src/probe.rs @@ -124,7 +124,7 @@ fn detect_vendor_hint(header: &sam::Header) -> Option { .map(|(_, canon)| (*canon).to_string()) } -/// Read just the SAM header from a BAM or CRAM (CRAM's header doesn't need the reference). +/// Read just the SAM header from a BAM or CRAM (CRAM's header does not need the reference). fn read_header_only(path: &Path) -> Result { match detect_format(path) { Format::Bam => { @@ -151,7 +151,7 @@ fn s>(v: &T) -> String { fn detect_build(header: &sam::Header) -> Option { // The Y-PAR-masked + rCRS CHM13 analysis set is indistinguishable from plain chm13v2.0 // by `@SQ` (same contig names/lengths), so check the reference filename the aligner - // recorded (`@PG CL` / `@SQ UR`) first. Plain chm13 won't match this signature. + // recorded (`@PG CL` / `@SQ UR`) first. Plain chm13 will not match this signature. if header_mentions_masked_rcrs(header) { return Some("chm13v2.0_maskedY_rCRS".into()); } @@ -355,7 +355,7 @@ mod tests { let by700 = header_from_sam("@HD\tVN:1.4\n@SQ\tSN:chrY\tLN:57227415\n@RG\tID:r\tLB:unknown-library-Big Y-700\tSM:s\n"); assert_eq!(detect_big_y_code(&by700), Some("BIG_Y_700")); - // The library label also marks the vendor (via the "big y" token) when @PG/@CN don't. + // The library label also marks the vendor (via the "big y" token) when @PG/@CN do not. assert_eq!(detect_vendor_hint(&by700).as_deref(), Some("FamilyTreeDNA")); let by500 = diff --git a/crates/navigator-analysis/src/reader.rs b/crates/navigator-analysis/src/reader.rs index 4bc98a2c..e063b8fd 100644 --- a/crates/navigator-analysis/src/reader.rs +++ b/crates/navigator-analysis/src/reader.rs @@ -91,7 +91,7 @@ pub fn build_repository(reference: &Path) -> Result(path: &Path, reference: Option<&'a Path>) -> Result<&'a Path, AnalysisError> { reference.ok_or_else(|| AnalysisError::Message(format!("CRAM {} requires a reference FASTA", path.display()))) } @@ -220,7 +220,7 @@ pub enum IdxReader { /// File offsets of the `.crai` containers that can hold records overlapping `interval` on `ref_id`. /// /// **This is the whole reason CRAM region queries are usable.** A CRAM container is the unit of -/// decode — you cannot decode part of one — so restricting *which containers get decoded* is the +/// decode — you can not decode part of one — so restricting *which containers get decoded* is the /// only place a region query can save work. noodles' own `Query` (and, before this, our `for_each`) /// selected containers by reference sequence alone and then discarded non-overlapping records /// *after* decoding them, which made every query cost a whole chromosome no matter how small the @@ -228,7 +228,7 @@ pub enum IdxReader { /// same query on a BAM. chr21 of a 30x WGS holds 1,140 containers and a point query needs exactly /// one of them. /// -/// A container whose `alignment_start` is absent is **kept**: that is a container this index cannot +/// A container whose `alignment_start` is absent is **kept**: that is a container this index can not /// place, and dropping it would silently lose records. Skipping is only ever done on positive /// evidence that the container lies outside the interval. fn cram_container_offsets(index: &cram::crai::Index, ref_id: usize, interval: Interval) -> Vec { @@ -571,7 +571,7 @@ pub fn has_crai_index(path: &Path) -> bool { } /// Whether the file has a coordinate index supporting **per-contig region queries** — a BAM `.bai` -/// or a CRAM `.crai`. The prerequisite for the parallel per-contig walker (CRAM additionally can't +/// or a CRAM `.crai`. The prerequisite for the parallel per-contig walker (CRAM also can't /// region-query the unmapped tail; callers handle that separately). pub fn has_region_index(path: &Path) -> bool { has_bai_index(path) || has_crai_index(path) @@ -696,7 +696,7 @@ mod tests { fn container_offsets_select_only_overlapping_containers() { let p = |n: usize| Position::new(n).unwrap(); // ref 0 containers spanning [1000,1099], [2000,2099], [3000,3099]; one on ref 1; and one - // the index cannot place. + // the index can not place. let idx: cram::crai::Index = vec![ cram::crai::Record::new(Some(0), Some(p(1000)), 100, 10, 0, 0), cram::crai::Record::new(Some(0), Some(p(2000)), 100, 20, 0, 0), diff --git a/crates/navigator-analysis/src/reassembly.rs b/crates/navigator-analysis/src/reassembly.rs index 525b559a..033cce9b 100644 --- a/crates/navigator-analysis/src/reassembly.rs +++ b/crates/navigator-analysis/src/reassembly.rs @@ -43,7 +43,7 @@ pub struct ReassemblyParams { pub min_alt_fragments: u32, /// v2: assemble the alternate haplotype from the alt-supporting reads (majority consensus over /// the reference frame — [`assemble_alt_haplotype`]) so linked variants the true reads carry - /// don't penalise them against reference. **Default off**: it helps the synthetic linked-variant + /// do not penalise them against reference. **Default off**: it helps the synthetic linked-variant /// case but on real WGS229 it perturbs marginal ~50/50 sites (regressed `chrY:4284195`), and /// there is no real linked-variant truth site yet to validate the benefit. The mechanism is /// unit-tested and opt-in (this flag / `NAVIGATOR_REASSEMBLY_ASSEMBLE=1`) pending that validation; @@ -168,7 +168,7 @@ fn genotype_candidate( let kept = dedup_spanning_fragments(reads, ci, params); // Stage C — alternate haplotype. v2: POA-assemble the alt-supporting reads so linked variants - // they carry don't penalise them against reference; fall back to reference-plus-one-substitution + // they carry do not penalise them against reference; fall back to reference-plus-one-substitution // when assembly is degenerate. v1 behaviour is the fallback, so simple sites are unchanged. let mut single_snv = ref_window.to_vec(); if off < single_snv.len() { @@ -298,7 +298,7 @@ fn assemble_alt_haplotype( hap[pos] = BASES[bi]; } } - // The candidate substitution is why we're here — force it (its column may be exactly 50/50). + // The candidate substitution is why we are here — force it (its column may be exactly 50/50). if site_off < hap.len() { hap[site_off] = alt_base; } @@ -329,7 +329,7 @@ fn argmax4(counts: &[u32; 4]) -> (usize, u32) { } /// Add `seq`'s bases to the per-reference-position `counts`/`cover` tallies by semiglobally aligning -/// it to `ref_window` (only aligned match/mismatch columns contribute; insertions/deletions don't). +/// it to `ref_window` (only aligned match/mismatch columns contribute; insertions/deletions do not). fn project_read_onto_ref(seq: &[u8], ref_window: &[u8], counts: &mut [[u32; 4]], cover: &mut [u32]) { let score = |a: u8, b: u8| if a == b { 1i32 } else { -4i32 }; let mut aligner = PwAligner::new(-5, -1, score); @@ -416,7 +416,7 @@ impl GapParameters for GapParams { } } -/// Semiglobal in the read: free leading/trailing offset so window-edge trimming isn't penalised. +/// Semiglobal in the read: free leading/trailing offset so window-edge trimming is not penalised. struct Semiglobal; impl StartEndGapParameters for Semiglobal { fn free_start_gap_x(&self) -> bool { diff --git a/crates/navigator-analysis/src/revert/collate.rs b/crates/navigator-analysis/src/revert/collate.rs index 38640bee..e66ee9ce 100644 --- a/crates/navigator-analysis/src/revert/collate.rs +++ b/crates/navigator-analysis/src/revert/collate.rs @@ -26,7 +26,7 @@ use super::transform::{Mate, RevertedRead}; use crate::error::AnalysisError; /// Buffer size for run spill/read-back. Large enough that the merge's per-run reads stay -/// sequential, small enough that the runs a WGS produces don't add up to real memory when the merge +/// sequential, small enough that the runs a WGS produces do not add up to real memory when the merge /// holds all of them open at once. const RUN_IO_BUFFER: usize = 256 * 1024; diff --git a/crates/navigator-analysis/src/revert/mod.rs b/crates/navigator-analysis/src/revert/mod.rs index 607be822..05112308 100644 --- a/crates/navigator-analysis/src/revert/mod.rs +++ b/crates/navigator-analysis/src/revert/mod.rs @@ -58,7 +58,7 @@ const CANCEL_CHECK_INTERVAL: u64 = 4096; /// What to do with a **primary** record whose CIGAR contains a hard clip. /// -/// Hard clipping means the aligner discarded sequence from the record, so the read cannot be fully +/// Hard clipping means the aligner discarded sequence from the record, so the read can not be fully /// recovered. Mainstream aligners hard-clip only supplementary records (which we drop anyway), but /// some pipelines emit hard-clipped primaries, and emitting those as if whole would silently feed /// a truncated read to the mapper. diff --git a/crates/navigator-analysis/src/revert/tests.rs b/crates/navigator-analysis/src/revert/tests.rs index 21dceb26..042a2a87 100644 --- a/crates/navigator-analysis/src/revert/tests.rs +++ b/crates/navigator-analysis/src/revert/tests.rs @@ -82,7 +82,7 @@ fn a_reverse_strand_read_is_restored_to_sequencer_orientation() { } /// A forward-strand read must be passed through untouched — the mirror of the test above, so a -/// bug that reverse-complements unconditionally cannot pass both. +/// bug that reverse-complements unconditionally can not pass both. #[test] fn a_forward_strand_read_is_left_alone() { let dir = scratch("forward"); @@ -262,7 +262,7 @@ fn a_read_whose_mate_was_dropped_becomes_a_singleton() { assert_eq!(read_lines(&out.singletons)[0], "@b"); } -/// Flags that claim "paired" but not which end cannot be placed in a synchronized file. +/// Flags that claim "paired" but not which end can not be placed in a synchronized file. #[test] fn a_paired_record_with_contradictory_segment_flags_is_a_singleton() { let dir = scratch("contradictory"); diff --git a/crates/navigator-analysis/src/revert/transform.rs b/crates/navigator-analysis/src/revert/transform.rs index 27dfb274..7323ff00 100644 --- a/crates/navigator-analysis/src/revert/transform.rs +++ b/crates/navigator-analysis/src/revert/transform.rs @@ -23,7 +23,7 @@ const SYNTHETIC_PHRED: u8 = 40; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Mate { /// Not part of a pair (`0x1` clear), or paired but with neither/both segment bits set — a - /// record whose flags contradict themselves cannot be placed in an R1/R2 file. + /// record whose flags contradict themselves can not be placed in an R1/R2 file. Unpaired, /// First segment (`0x40`). One, @@ -186,7 +186,7 @@ fn mate_of(flags: noodles::sam::alignment::record::Flags) -> Mate { match (flags.is_first_segment(), flags.is_last_segment()) { (true, false) => Mate::One, (false, true) => Mate::Two, - // Both or neither: the record claims to be paired but won't say which end. It cannot go in + // Both or neither: the record claims to be paired but will not say which end. It can not go in // a synchronized R1/R2 file, so it becomes a singleton rather than corrupting the pairing. _ => Mate::Unpaired, } diff --git a/crates/navigator-analysis/src/revert/writer.rs b/crates/navigator-analysis/src/revert/writer.rs index 182b328b..a561bf37 100644 --- a/crates/navigator-analysis/src/revert/writer.rs +++ b/crates/navigator-analysis/src/revert/writer.rs @@ -2,7 +2,7 @@ //! //! The invariant the mapper depends on: `_1.fastq` and `_2.fastq` must stay in lockstep, record //! for record. A template only reaches those files if it has exactly one R1 and exactly one R2; -//! anything else — an unpaired library, a mate lost to hard clipping, flags that don't say which +//! anything else — an unpaired library, a mate lost to hard clipping, flags that do not say which //! end a read is — goes to singletons. Silently writing an unmatched read into `_1` would shift //! every later pair by one and mis-pair the entire rest of the file, so the check is per-template //! rather than a trailing reconciliation. diff --git a/crates/navigator-analysis/src/roh.rs b/crates/navigator-analysis/src/roh.rs index c8e36b2f..a7e8c907 100644 --- a/crates/navigator-analysis/src/roh.rs +++ b/crates/navigator-analysis/src/roh.rs @@ -12,7 +12,7 @@ //! the density class array-based ROH tools (PLINK, BCFtools/RoH, detectRUNS) assume. Segment cM //! lengths and the F_ROH denominator come from the same [`GeneticMap`] the IBD path already loads. //! -//! **What's deliberately simplified in the spike** (see the module tests + the follow-up notes): +//! **What is deliberately simplified in the spike** (see the module tests + the follow-up notes): //! - The Normal-state heterozygosity expectation is a single `baseline_het` knob. A production //! version should derive it per-site from panel allele frequencies (2·f·(1−f)), which //! `AncestryPanel`/`IbdPanel` already carry, so the emission is properly frequency-aware. diff --git a/crates/navigator-analysis/src/sidecar.rs b/crates/navigator-analysis/src/sidecar.rs index 2937e76b..2045a1af 100644 --- a/crates/navigator-analysis/src/sidecar.rs +++ b/crates/navigator-analysis/src/sidecar.rs @@ -10,7 +10,7 @@ //! - `coverage.txt` (`samtools coverage`) + `callable.summary.txt` (GATK `CallableLoci`) → //! a **lite** [`CoverageResult`]: genome-wide mean depth (length-weighted) + per-contig //! stats + callable-base counts. The depth histogram and `pct_Nx` / median need the -//! per-base walk, so they're left zeroed and the result is flagged `partial` by the caller. +//! per-base walk, so they are left zeroed and the result is flagged `partial` by the caller. //! //! Unknown numeric fields are `0.0` (not `NaN`) because the cache round-trips through //! `serde_json`, which encodes `NaN` as `null` and then fails to read it back. @@ -53,7 +53,7 @@ pub fn parse_sex(text: &str) -> SexInferenceResult { /// Parse `samtools stats` output into a fully-populated [`ReadMetrics`]. `SN` lines give the /// scalar counts; `RL`/`IS` lines give the read-length / insert-size histograms (and thus -/// their median/std/min/max). `mean_mapping_quality` isn't emitted by samtools stats → 0.0. +/// their median/std/min/max). `mean_mapping_quality` is not emitted by samtools stats → 0.0. pub fn parse_samtools_stats(text: &str) -> ReadMetrics { let mut sn: BTreeMap<&str, f64> = BTreeMap::new(); let mut rl: BTreeMap = BTreeMap::new(); @@ -125,7 +125,7 @@ pub fn parse_samtools_stats(text: &str) -> ReadMetrics { min_insert_size: is_min, max_insert_size: is_max, insert_size_histogram: is, - // samtools stats doesn't classify orientation; Illumina paired-end is FR. + // samtools stats does not classify orientation; Illumina paired-end is FR. pair_orientation: PairOrientation::Fr, // Picard-style chimera rate: read pairs mapping to different chromosomes / total pairs. pct_chimeras: pct(pairs_diff_chrom, total_reads / 2), @@ -364,7 +364,7 @@ pub fn parse_flagstat(text: &str) -> ReadMetrics { /// Parse a Picard metrics table: skip to the header line beginning with `header_key`, then read the /// tab-separated data rows until a blank line (Picard appends a histogram section after a blank). -/// Returns `(headers, rows)`. `None` if the header isn't found. +/// Returns `(headers, rows)`. `None` if the header is not found. fn parse_picard_rows(text: &str, header_key: &str) -> Option<(Vec, Vec>)> { let mut lines = text.lines(); let header = lines.by_ref().find(|l| l.trim_start().starts_with(header_key))?; @@ -421,7 +421,7 @@ pub fn parse_wgs_metrics(text: &str) -> Option { /// Parse Picard `CollectAlignmentSummaryMetrics` → a [`ReadMetrics`] (the `PAIR` summary row, /// else `UNPAIRED`, else the first). Counts + alignment percentages + mean read length + chimera -/// rate; read-length / insert-size histograms aren't in this metrics class, so they stay 0. Picard +/// rate; read-length / insert-size histograms are not in this metrics class, so they stay 0. Picard /// `PCT_*` are 0–1 fractions → scaled to the `ReadMetrics` 0–100 convention. `None` if no table. pub fn parse_alignment_summary(text: &str) -> Option { let (keys, rows) = parse_picard_rows(text, "CATEGORY")?; @@ -623,7 +623,7 @@ chrY\t1\t500\t20\t400\t80.0\t10.0\t29.0\t40.0 } /// Real-data smoke test: parse HG00096's actual pipeline sidecars off the NAS. No-ops - /// when the share isn't mounted. Run: `cargo test -p navigator-analysis sidecar -- --ignored --nocapture`. + /// when the share is not mounted. Run: `cargo test -p navigator-analysis sidecar -- --ignored --nocapture`. #[test] #[ignore = "reads NAS files; run explicitly"] fn real_sidecars_parse() { diff --git a/crates/navigator-analysis/src/strcaller.rs b/crates/navigator-analysis/src/strcaller.rs index 255e4ce1..237f1013 100644 --- a/crates/navigator-analysis/src/strcaller.rs +++ b/crates/navigator-analysis/src/strcaller.rs @@ -134,7 +134,7 @@ fn call_diploid(observed: &[i32], p: &StrCallerParams) -> Option<(i32, i32)> { best.map(|(g, _)| g) } -/// The repeat copies observed in one enclosing read at `locus`, or `None` if the read isn't a clean +/// The repeat copies observed in one enclosing read at `locus`, or `None` if the read is not a clean /// enclosing read (not anchored `flank` bp of indel-free reference on both sides). Reads the length /// off the CIGAR: `tract_bp + insertions − deletions` within the tract, ÷ period. fn observed_copies(ops: &[(Kind, usize)], aln_start: i64, locus: &StrLocus, flank: i64) -> Option { diff --git a/crates/navigator-analysis/src/strmarker.rs b/crates/navigator-analysis/src/strmarker.rs index 8ae39413..b54a75c0 100644 --- a/crates/navigator-analysis/src/strmarker.rs +++ b/crates/navigator-analysis/src/strmarker.rs @@ -3,7 +3,7 @@ //! The HipSTR reference already names the loci (DYS393, DYS19/DYS394, …; see [`crate::strref`]), so //! the caller emits DYS names directly. What differs is the **counting convention**: FTDNA reports a //! per-marker value that is the caller's repeat count plus a fixed offset (0 for most, ±1–3 for some), -//! and a set of markers whose HipSTR tract doesn't correspond 1:1 to the FTDNA marker (large tract +//! and a set of markers whose HipSTR tract does not correspond 1:1 to the FTDNA marker (large tract //! mismatches, plus multi-copy/nested markers like DYS385/DYS464/DYS389II) which can't be mapped by a //! single offset. //! @@ -123,7 +123,7 @@ static OFFSETS: &[(&str, i32)] = &[ /// Markers whose HipSTR tract can't be mapped to the FTDNA value by a single offset: large tract /// mismatches and multi-copy/nested markers (DYS385/DYS464 split sub-loci, DYS389II nesting). Reported -/// as `Excluded` — the enclosing-read caller doesn't yield a vendor-comparable value here (yet). +/// as `Excluded` — the enclosing-read caller does not yield a vendor-comparable value here (yet). static EXCLUDE: &[&str] = &[ // Multi-copy / nested (split sub-loci, never a single vendor-comparable value). "DYS385", diff --git a/crates/navigator-analysis/src/sv/types.rs b/crates/navigator-analysis/src/sv/types.rs index 0b12c472..bbcf1398 100644 --- a/crates/navigator-analysis/src/sv/types.rs +++ b/crates/navigator-analysis/src/sv/types.rs @@ -75,7 +75,7 @@ pub struct SvCallerConfig { pub min_total_support: u32, pub min_quality: f64, /// Ceiling on retained discordant pairs, and separately on retained split reads, for one walk. - /// A safety valve, not a filter: the point is that a pathological library cannot take the whole + /// A safety valve, not a filter: the point is that a pathological library can not take the whole /// process — and in a batch, the other 147 samples — down with an OOM. See the default. pub max_evidence_records: u64, } diff --git a/crates/navigator-analysis/src/sv/walker.rs b/crates/navigator-analysis/src/sv/walker.rs index 7a27b73f..9426cc4f 100644 --- a/crates/navigator-analysis/src/sv/walker.rs +++ b/crates/navigator-analysis/src/sv/walker.rs @@ -2,7 +2,7 @@ //! collecting per-bin read depth (CNV), discordant read pairs (BreakDancer-style), and //! split reads from the SA tag (Pindel-style). //! -//! Two walks share one per-record body ([`EvidenceSink::accept_read`]), so they cannot drift: +//! Two walks share one per-record body ([`EvidenceSink::accept_read`]), so they can not drift: //! [`collect_evidence_parallel`] fans one region query per contig across a decode-safe rayon pool, //! and [`collect_evidence`] makes a single sequential pass for files with no coordinate index. //! Prefer the parallel entry point — it falls back to the sequential one on its own. diff --git a/crates/navigator-analysis/src/testtype.rs b/crates/navigator-analysis/src/testtype.rs index e78af856..4b4213cc 100644 --- a/crates/navigator-analysis/src/testtype.rs +++ b/crates/navigator-analysis/src/testtype.rs @@ -100,14 +100,14 @@ pub fn coverage_profile_from_bai(bam_path: &Path, mean_read_length: Option) /// Map a free-text vendor hint to a specific targeted-Y test code (else the honest generic). fn targeted_y_for_vendor(vendor_hint: Option<&str>) -> &'static str { match vendor_hint.map(|v| v.to_lowercase()) { - // FTDNA only sells Big Y, but the *generation* (500 vs 700) isn't in the vendor token — + // FTDNA only sells Big Y, but the *generation* (500 vs 700) is not in the vendor token — // it comes from the `@RG LB` label ([`crate::probe`], passed as `big_y_label`) or, on older // headers that omit it, from the callable-chrY footprint resolved after analysis. Stay // generic here so neither generation is guessed from the vendor name alone. Some(v) if v.contains("ftdna") || v.contains("familytreedna") => "TARGETED_Y", Some(v) if v.contains("full genomes") || v.contains("fullgenomes") => "Y_ELITE", Some(v) if v.contains("yseq") => "Y_PRIME", - // An unknown vendor isn't mislabeled to a specific product. + // An unknown vendor is not mislabeled to a specific product. _ => "TARGETED_Y", } } @@ -139,7 +139,7 @@ pub fn infer_test_type( mean_read_length: Option, big_y_label: Option<&str>, ) -> Option { - // An explicit FTDNA Big Y generation from the header (`@RG LB`) is authoritative — it's FTDNA's + // An explicit FTDNA Big Y generation from the header (`@RG LB`) is authoritative — it is FTDNA's // own product label, so it overrides the coverage-shape guess entirely. if let Some(code) = big_y_label { return Some(code.to_string()); @@ -197,7 +197,7 @@ mod tests { fn targeted_y_maps_vendor_or_generic() { // Y-only reference (no autosomes) — clean targeted-Y. let p = prof(0.0, 35.0, 0.0, false); - // FTDNA without a generation label stays generic — 500 vs 700 isn't in the vendor token + // FTDNA without a generation label stays generic — 500 vs 700 is not in the vendor token // (the header `@RG LB` or the callable-chrY footprint decides it). assert_eq!( infer_test_type(Some(&p), Some("ILLUMINA"), Some("FamilyTreeDNA"), None, None).as_deref(), diff --git a/crates/navigator-analysis/src/unified.rs b/crates/navigator-analysis/src/unified.rs index 8f9d9c39..ee46149b 100644 --- a/crates/navigator-analysis/src/unified.rs +++ b/crates/navigator-analysis/src/unified.rs @@ -75,7 +75,7 @@ struct ContigSink<'a> { impl RecordSink for ContigSink<'_> { fn accept(&mut self, record: &impl AlnRead) { - // Only this contig's own records (drop a multi-reference slice's foreign records — they're + // Only this contig's own records (drop a multi-reference slice's foreign records — they are // processed by their own contig's query). Keeps every per-record tally counted exactly once. if record.reference_sequence_id() != Some(self.ref_id) { return; @@ -123,7 +123,7 @@ pub const UNIFIED_VERSION: &str = "unified-1"; /// The three quality-metric results collected in one pass. Sex is `None` when inference /// can't be computed for the input (no autosomes/chrX, or no autosomal reads — e.g. a /// targeted panel or chrY-only test); coverage + read-metrics are unaffected, mirroring the -/// pipeline where sex is an independent step whose failure doesn't kill the others. +/// pipeline where sex is an independent step whose failure does not kill the others. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct UnifiedMetricsResult { pub coverage: CoverageResult, @@ -266,7 +266,7 @@ struct ContigPartial { /// Like [`collect_unified_metrics_parallel`], reporting `progress(megabases_done, megabases_total)` /// — base-pair position walked across all contigs, so the bar advances continuously rather than /// stepping once per finished contig (the big autosomes run first and finish in a late burst, -/// freezing a contig-count bar at 0 for ~half the run). The callback is `Fn + Sync` because it's +/// freezing a contig-count bar at 0 for ~half the run). The callback is `Fn + Sync` because it is /// invoked concurrently from worker threads. pub fn collect_unified_metrics_parallel_with_progress( bam_path: &Path, @@ -415,7 +415,7 @@ pub fn collect_unified_metrics_parallel_with_progress( // noodles' CRAM decoder can recurse deeply enough to blow rayon's default 2 MiB worker stack // (the main thread's larger stack handles the same file in the sequential walker). CRAM 3.1 // files (new range/arithmetic + fqzcomp + name-tokenizer codecs) recurse deeper still. Give the - // workers a generous decode-safe stack so the per-contig CRAM decode doesn't overflow — an + // workers a generous decode-safe stack so the per-contig CRAM decode does not overflow — an // overflow aborts the whole process, so this must not be marginal. let pool = reader::decode_pool(n_threads)?; diff --git a/crates/navigator-analysis/tests/cancel_real.rs b/crates/navigator-analysis/tests/cancel_real.rs index a8bb1952..008ace75 100644 --- a/crates/navigator-analysis/tests/cancel_real.rs +++ b/crates/navigator-analysis/tests/cancel_real.rs @@ -16,7 +16,7 @@ fn env_path(key: &str) -> Option { /// A whole-genome walk over a real WGS file takes minutes. Cancel it a second in and assert it /// returns in well under that — the entire point of threading the token into the walkers, and the -/// thing that a unit test on the token alone cannot demonstrate. +/// thing that a unit test on the token alone can not demonstrate. #[test] #[ignore] fn cancelling_a_whole_genome_walk_returns_promptly() { diff --git a/crates/navigator-analysis/tests/genotype.rs b/crates/navigator-analysis/tests/genotype.rs index a3dc26d9..3359cf7b 100644 --- a/crates/navigator-analysis/tests/genotype.rs +++ b/crates/navigator-analysis/tests/genotype.rs @@ -203,7 +203,7 @@ fn call_indels_at_without_reference_is_empty() { fn denovo_diploid_calls_a_multiallelic_snv() { // snv_multi.bam (chr1): 10 reads carry G at pos 2, 10 carry T (ref C) → compound het 1/2. let dir = fixtures(); - // A private chr1 reference dir (distinct from chr1_reference()'s, so parallel tests don't race). + // A private chr1 reference dir (distinct from chr1_reference()'s, so parallel tests do not race). let refdir = std::env::temp_dir().join(format!("dun-snvmulti-ref-{}", std::process::id())); std::fs::create_dir_all(&refdir).unwrap(); let reference = refdir.join("chr1.fa"); diff --git a/crates/navigator-app/examples/blocktree_check.rs b/crates/navigator-app/examples/blocktree_check.rs index d438e9e3..5a53b503 100644 --- a/crates/navigator-app/examples/blocktree_check.rs +++ b/crates/navigator-app/examples/blocktree_check.rs @@ -72,7 +72,7 @@ async fn main() -> Result<(), Box> { } // Split the unplaced: "no placement at all" is expected (STR-only kits), but "has a terminal - // this tree doesn't carry" is provider/build skew worth naming. + // this tree does not carry" is provider/build skew worth naming. let (skew, unplaced_none): (Vec<_>, Vec<_>) = tree.unplaced.iter().partition(|u| u.terminal.is_some()); println!( "unplaced: {} with no Y placement · {} with a terminal absent from this tree", diff --git a/crates/navigator-app/src/analysis.rs b/crates/navigator-app/src/analysis.rs index f775195d..8c2c0691 100644 --- a/crates/navigator-app/src/analysis.rs +++ b/crates/navigator-app/src/analysis.rs @@ -53,7 +53,7 @@ impl App { ) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let bam = Self::alignment_file(&aln)?; - // The reference isn't asked for at import — resolve the alignment's build via the gateway + // The reference is not asked for at import — resolve the alignment's build via the gateway // (cached, else download) when no FASTA was stored. let reference = match aln.reference_path { Some(p) => PathBuf::from(p), @@ -153,7 +153,7 @@ impl App { Ok(result) } - /// Write the inferred sex back to the biosample when the user didn't provide one, so it + /// Write the inferred sex back to the biosample when the user did not provide one, so it /// shows in the subjects table + header instead of "Unknown". No-op for Unknown sex or /// when the biosample already carries a sex. pub(crate) async fn write_back_inferred_sex( @@ -286,7 +286,7 @@ impl App { match tokio::task::spawn_blocking(move || copy_with_index(&remote_owned, &local2, remote_len)).await { // Registering can still fail if the copy was removed in the gap (a concurrent holder // finishing and dropping to zero). Returning an `owned` handle to a missing path would - // fail the walk with a confusing ENOENT and then "clean up" a file that isn't there. + // fail the walk with a confusing ENOENT and then "clean up" a file that is not there. Ok(Ok(())) if LocalAlignment::retain(&local, remote_len) => LocalAlignment::owned(local), Ok(Ok(())) => { eprintln!( @@ -320,7 +320,7 @@ impl App { /// Like [`run_unified_metrics`], reporting `progress(contigs_done, contigs_total)` as the /// (slow) whole-genome coverage portion finalizes each contig. Uses the per-contig parallel /// walker (falling back to a sequential pass for CRAM / unindexed BAM); the callback is - /// `Fn + Sync` because it's invoked concurrently from the fan-out's worker threads. + /// `Fn + Sync` because it is invoked concurrently from the fan-out's worker threads. pub async fn run_unified_metrics_with_progress( &self, alignment_id: i64, @@ -377,7 +377,7 @@ impl App { .await?; self.write_back_read_stats(alignment_id, &result.read_metrics).await?; // Sex: a Y-targeted test (Big Y, Y Elite, …) sequences the donor's Y chromosome — he is male - // by definition. The chrX/autosome ratio the inference needs isn't present in a chrY-scoped + // by definition. The chrX/autosome ratio the inference needs is not present in a chrY-scoped // walk, and is unreliable even whole-genome (a Big Y's off-target chrX ≈ autosome ≈ 0.4× // reads as *female*). So force Male for a Y-targeted test, overriding the inference + any // prior auto-assignment; WGS / mt-targeted keep the walk's result. @@ -510,7 +510,7 @@ impl App { /// Genotype short tandem repeats on `contig` from the alignment, via the enclosing-read caller /// over the HipSTR reference tracts (haploid for chrY/chrM, diploid elsewhere). Persisted as a - /// `str:{contig}` artifact (so it's cached + source-invalidated like other analyses). Errors if + /// `str:{contig}` artifact (so it is cached + source-invalidated like other analyses). Errors if /// no STR reference is configured for the alignment's build (the tracts are build-specific — /// CHM13/GRCh37 need their own reference or liftover, not yet wired). pub async fn run_str_calls( @@ -531,7 +531,7 @@ impl App { )) })?; // Resolve the reference for decode (see alignment_reference_for_decode): required for a CRAM, - // None for a BAM. STR region-genotyping reads the alignment; it doesn't consult reference bases. + // None for a BAM. STR region-genotyping reads the alignment; it does not consult reference bases. let (bam, reference) = self.alignment_reference_for_decode(alignment_id).await?; // chrY / chrM are haploid (one allele); autosomes + chrX (in a female) are diploid. We // genotype chrY/chrM haploid and everything else diploid — sex-aware chrX is a refinement. @@ -751,7 +751,7 @@ impl App { /// A **whole-genome** diploid VCF: de-novo SNV + indel calls over the diploid primary /// chromosomes (1–22, X) of the alignment, per-contig cached. chrY and chrM are **excluded** — - /// they're haploid, so the diploid (het 0/1) model is wrong for them; their variants come from + /// they are haploid, so the diploid (het 0/1) model is wrong for them; their variants come from /// the haploid caller and the Y/mt haplogroup + mtDNA-mutation features. Heavy (a real WGS /// calling pass); the caller runs it off the UI thread (the export path). pub async fn diploid_vcf_genome(&self, alignment_id: i64, cancel: CancelToken) -> Result { @@ -912,7 +912,7 @@ impl App { /// without the reference, so resolve it (stored path, else from the build via the gateway, /// cache-first); a BAM decodes without one, so return the stored path as-is (usually `None`) and /// never force a reference download. Use this for record/pileup reads and SNP-site genotyping - /// that don't consult reference bases; use [`alignment_bam_reference`](Self::alignment_bam_reference) + /// that do not consult reference bases; use [`alignment_bam_reference`](Self::alignment_bam_reference) /// for calling paths (de-novo SNV/indel) that need the reference even on a BAM. pub(crate) async fn alignment_reference_for_decode( &self, @@ -941,7 +941,7 @@ impl App { /// The distinct reference builds across a subject's alignments — the builds whose FASTA an /// analysis of this subject may need. Used to pre-resolve references (with a progress bar) after - /// import and before a subject-level analysis, so on-demand downloads aren't silent. + /// import and before a subject-level analysis, so on-demand downloads are not silent. pub async fn reference_builds_for_subject(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alns = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut builds: Vec = alns.into_iter().map(|a| a.reference_build).collect(); @@ -1076,7 +1076,7 @@ fn partial_path(local: &Path) -> PathBuf { /// last (via a temp + rename), so a present `local` always implies its index is present too — the /// cache check in [`App::localize`] can't see a half-copied pair. /// -/// `expect_len` is the remote's size; when known, a copy that doesn't match it is rejected rather +/// `expect_len` is the remote's size; when known, a copy that does not match it is rejected rather /// than published. A short copy is otherwise indistinguishable from corrupt data: it surfaces as a /// decode error ("unexpected end of file", a bad container checksum) tens of gigabytes into a walk, /// naming the cache path, and the copy is deleted on drop before anyone can look at it. @@ -1131,7 +1131,7 @@ fn copy_with_index(remote: &Path, local: &Path, expect_len: Option) -> std: /// One step of a full analysis of a single alignment, in the order [`App::plan_full_analysis`] /// returns them. The variants carry whatever the step needs, so a caller's dispatch is total and it -/// cannot silently run a step the plan excluded. +/// can not silently run a step the plan excluded. #[derive(Debug, Clone, PartialEq)] pub enum AnalysisStep { /// Coverage + callable, read-level QC, and sex inference in one pass over the alignment. @@ -1330,7 +1330,7 @@ impl LocalAlignment { /// /// A file with no registry entry is a **leftover from an earlier process** — `Drop` never ran, /// so the run was killed — and is validated against `expect_len` (the remote's size) before - /// being trusted, then discarded if it doesn't match. Adopting a leftover on its existence + /// being trusted, then discarded if it does not match. Adopting a leftover on its existence /// alone is how a truncated copy gets read as though it were the alignment: the failure then /// appears as a decode error deep into a walk, pointing at a cache path whose file is deleted /// moments later. A wrong-sized copy is worth exactly one re-copy to be rid of. @@ -1559,7 +1559,7 @@ mod local_alignment_tests { #[test] fn the_original_is_never_removed() { - // A path we didn't copy (local disk, or a failed copy falling back to the remote) must be + // A path we did not copy (local disk, or a failed copy falling back to the remote) must be // left alone — deleting the user's own alignment would be catastrophic. let d = scratch("borrowed"); let original = d.join("c.cram"); diff --git a/crates/navigator-app/src/blocktree.rs b/crates/navigator-app/src/blocktree.rs index c0f0c7d3..2de5d04a 100644 --- a/crates/navigator-app/src/blocktree.rs +++ b/crates/navigator-app/src/blocktree.rs @@ -11,7 +11,7 @@ //! reconciliation the subjects table and project report use. Nothing here can move a subject. //! - **A member that can't be placed is reported, not dropped** ([`UnplacedMember`]). On a multi-lab //! cohort provider/build skew is expected; silently omitting those members would make the tree -//! look like it accounts for the whole project when it doesn't. +//! look like it accounts for the whole project when it does not. //! //! Design: `documents/design/project-block-tree.md`. @@ -156,7 +156,7 @@ impl App { evidence: Vec::new(), }) .collect(); - // Stable leaf order, so the layout doesn't reshuffle between opens. + // Stable leaf order, so the layout does not reshuffle between opens. for b in &mut blocks { b.members.sort_by(|x, y| (&x.name, x.guid.0).cmp(&(&y.name, y.guid.0))); } @@ -185,7 +185,7 @@ impl App { /// A cohort spans builds, so there is no per-subject answer as there is in `descent_report`. /// Picking one is safe because node names and topology are build-independent — only the loci /// *positions* are, and the aggregate carries the key so the view can say which it means. Ties - /// break on the key name, so the choice doesn't depend on map iteration order. + /// break on the key name, so the choice does not depend on map iteration order. async fn project_build_key(&self, members: &[Biosample]) -> &'static str { let guids: Vec = members.iter().map(|b| b.guid).collect(); let Ok(alns) = alignment::list_for_biosamples(self.store.pool(), &guids).await else { @@ -278,7 +278,7 @@ fn drop_clustered(positions: &BTreeSet) -> BTreeSet { /// positions were carried by *all* 111 donors with private-Y — those are reference-vs-population /// differences, real but not private, and the bundled cohort-shared blocklist (derived from a /// 3,352-sample CHM13 cohort that predates this collection) does not list them. Deriving the -/// exclusion from the cohort in hand catches what a bundled list cannot anticipate. +/// exclusion from the cohort in hand catches what a bundled list can not anticipate. const COHORT_SHARED_FRACTION: f64 = 0.25; /// Donors required before the frequency rule engages at all. @@ -365,7 +365,7 @@ fn clustered_candidate_positions(blocks: &[Block], private: &HashMap) -> BTreeSet { let mut blocks_per_position: HashMap> = HashMap::new(); @@ -403,7 +403,7 @@ fn recurrent_positions(blocks: &[Block], private: &HashMap = candidate_positions(&spread).into_iter().collect(); assert_eq!(kept, vec![1_000_000], "only the pair within 100 bp is dropped"); @@ -1026,7 +1026,7 @@ mod tests { #[test] fn a_position_defining_branches_under_two_parents_is_rejected() { // 11311865 was shared by two members under one block *and* two under another. A variant that - // arose twice cannot mark a new branch, and the laminar check can't see it — it reasons + // arose twice can not mark a new branch, and the laminar check can't see it — it reasons // inside a single block. let mut left = block(1, "R-A", 0, &["a", "b"]); left.subtree_members = 2; diff --git a/crates/navigator-app/src/brief.rs b/crates/navigator-app/src/brief.rs index 820aa43d..55936821 100644 --- a/crates/navigator-app/src/brief.rs +++ b/crates/navigator-app/src/brief.rs @@ -58,7 +58,7 @@ const HAPLO_ENRICH_TTL_DAYS: u64 = 30; /// Live haplogroup content fetched from the AppView, cached per (dna-type, name). `found = false` is /// a negative-cache marker (the endpoint answered but had nothing) so a definitively-absent -/// haplogroup isn't re-requested every rebuild. +/// haplogroup is not re-requested every rebuild. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] struct HaploEnrichment { found: bool, @@ -175,7 +175,7 @@ impl App { _ => None, }; - // Runs-of-homozygosity (relatedness / endogamy). Read-only: only surfaced when it's already + // Runs-of-homozygosity (relatedness / endogamy). Read-only: only surfaced when it is already // been computed and cached (the brief must stay cheap — ROH computation is on-demand). let roh = self.cached_roh(biosample_guid).await?.map(|r| { brief::roh_brief( @@ -248,8 +248,8 @@ impl App { /// nothing else — ancestry, IBD and the autosomes already handle GRCh37/38 and give the same /// answer either way. With no Y placed there is no payoff, so no offer. /// - **Reads to re-map.** A chip or VCF-only subject has no alignment; an alignment row without - /// a file cannot be read. - /// - **No CHM13 alignment already.** The offer claims part of their paternal line cannot + /// a file can not be read. + /// - **No CHM13 alignment already.** The offer claims part of their paternal line can not /// currently be read; for someone who already has data on the complete assembly, by any route, /// that claim is simply false — even if some older file of theirs has never been realigned. /// - **Reads the job would actually act on** — not already on CHM13, not itself a realignment, @@ -370,8 +370,8 @@ impl App { /// Best-effort live enrichment for one haplogroup: cache-first (30-day TTL), else a short-timeout /// `GET {appview}/api/v1/haplogroup/{name}`. A definitive answer (200 / 404) is cached — including - /// "not found" — so it isn't re-requested each rebuild; a transient network error is *not* cached, - /// so enrichment self-heals once connectivity returns. Returns content only when there's something + /// "not found" — so it is not re-requested each rebuild; a transient network error is *not* cached, + /// so enrichment self-heals once connectivity returns. Returns content only when there is something /// worth folding in (an age or narrative). async fn enrich_haplogroup(&self, name: &str, dna_type: DnaType) -> Option { if name.trim().is_empty() { @@ -404,7 +404,7 @@ impl App { } // The endpoint answered but had nothing (404 etc.) → cache a negative result. Ok(_) => HaploEnrichment::default(), - // Network/timeout error → don't cache (retry next time). + // Network/timeout error → do not cache (retry next time). Err(_) => return None, }; @@ -625,7 +625,7 @@ fn build_test( } } -/// Plain-language test description when the pack doesn't cover the code, derived from what the test +/// Plain-language test description when the pack does not cover the code, derived from what the test /// targets. fn fallback_test_text(lang: Lang, target: TargetType) -> (String, Option) { let (what, limits) = match target { diff --git a/crates/navigator-app/src/commands.rs b/crates/navigator-app/src/commands.rs index b557c0af..e84e14a6 100644 --- a/crates/navigator-app/src/commands.rs +++ b/crates/navigator-app/src/commands.rs @@ -159,7 +159,7 @@ impl App { /// A Y-targeted test (Big Y, Targeted Y, a Y-SNP pack, …) or any Y-STR profile is definitive /// evidence of a male subject. Set the biosample's sex to "Male" when such data is present and - /// it isn't already recorded as male. Best-effort and idempotent — safe to call after any run + /// it is not already recorded as male. Best-effort and idempotent — safe to call after any run /// or STR-profile import (it re-derives the verdict from the stored data each time). pub(crate) async fn assign_male_for_y_evidence(&self, guid: SampleGuid) -> Result<(), AppError> { use navigator_domain::testtype::{by_code, TargetType}; @@ -308,7 +308,7 @@ impl App { /// Merge `secondary` sequence run into `primary` (both must belong to `biosample_guid`): /// reparent the secondary run's alignments onto the primary, then delete the now-empty secondary - /// (its analysis artifacts travel with the alignments — they're alignment-keyed). Destructive + + /// (its analysis artifacts travel with the alignments — they are alignment-keyed). Destructive + /// irreversible. Returns the number of alignments moved. pub async fn merge_sequence_runs( &self, @@ -516,7 +516,7 @@ impl App { /// Persist a marker that a Navigator walk failed for this alignment (e.g. an undecodable / /// corrupt CRAM). Stored as the `error`/`"1"` artifact so the project report can surface a /// "Failed" cell instead of a silent blank; cleared by [`clear_analysis_error`] on the next - /// successful walk. Best-effort — a failure to record the marker is swallowed (it's diagnostic). + /// successful walk. Best-effort — a failure to record the marker is swallowed (it is diagnostic). pub async fn record_analysis_error(&self, alignment_id: i64, step: &str, message: &str) { let mut message = message.to_string(); message.truncate(500); // keep the payload small; the head carries the cause diff --git a/crates/navigator-app/src/fastpath.rs b/crates/navigator-app/src/fastpath.rs index 9bcd6109..ce97f3e6 100644 --- a/crates/navigator-app/src/fastpath.rs +++ b/crates/navigator-app/src/fastpath.rs @@ -262,7 +262,7 @@ impl App { /// Assign an mtDNA haplogroup from a precomputed chrM GVCF — no CRAM walk. Places against /// the FTDNA mt tree; on CHM13 the tree's rCRS positions are lifted onto `chrM` (the cheap - /// self-generated rCRS↔chrM map), on GRCh38 they're read directly. Recorded under the CRAM + /// self-generated rCRS↔chrM map), on GRCh38 they are read directly. Recorded under the CRAM /// path's mt source key (`aln:{id}:mt`) with a `gv:`-prefixed fingerprint. pub async fn assign_mt_from_gvcf(&self, alignment_id: i64, gvcf: &Path) -> Result { let tree_json = self.fetch_ftdna_mt_tree().await?; @@ -319,7 +319,7 @@ impl App { // placement made from a GVCF against the tree of the day could never be re-derived, and the // resulting `haplogroup_call` row outlived every tree it was placed against. See // `App::replace_against_current_tree`, which replays this. Best-effort — a workspace that - // cannot record the paths should still get the ingest. + // can not record the paths should still get the ingest. let _ = self .save_analysis_with_provenance( alignment_id, @@ -418,7 +418,7 @@ impl App { } else { return Ok(false); }; - // Don't downgrade a full deep walk on reimport — keep it if it's already equal-or-fuller. + // Do not downgrade a full deep walk on reimport — keep it if it is already equal-or-fuller. let wrote = self .save_analysis_no_downgrade( alignment_id, @@ -515,7 +515,7 @@ impl App { .map_err(Into::into) } - /// The **private bucket**: de-novo SNP calls on chrY that the Y placement doesn't + /// The **private bucket**: de-novo SNP calls on chrY that the Y placement does not /// explain (not on the assigned backbone), classified as off-path-known (a finer/ /// sibling FTDNA branch) or novel (a new-branch candidate). With `callable_bed` (e.g. /// the Poznik/1KG `b38_sites.bed`), calls outside reliable regions are dropped. @@ -892,7 +892,7 @@ const VCF_PRIVATE_MIN_GQ: u32 = 20; /// Derived-allele fraction a call must reach to count as **deterministic** on a haploid chromosome. /// chrY carries one copy, so a genuine call is essentially all-alt; a middling fraction is an -/// ambiguous locus, and an ambiguous call cannot support a private-variant claim. +/// ambiguous locus, and an ambiguous call can not support a private-variant claim. const VCF_PRIVATE_MIN_AF: f64 = 0.95; /// Depth ceiling, as a multiple of the donor's own typical depth at good calls. @@ -948,7 +948,7 @@ impl App { // Without per-call evidence every quality gate below is a no-op, and the result is a list of // whatever the vendor's caller emitted — on a real set that is 400-550 "novel" calls against - // ~70 for the same donor's evidence-bearing set. A call we cannot judge is the most + // ~70 for the same donor's evidence-bearing set. A call we can not judge is the most // non-deterministic kind there is, so refuse rather than publish a number that looks like a // finding. Re-importing the source populates `CallEvidence` (migration 0042). if !set.has_evidence() { @@ -968,7 +968,7 @@ impl App { .collect(); // `pv2`: the chrY structural masks are now lifted to the set's own build, so a `pv1` // bucket was classified with **no** structural mask on anything but CHM13 and its counts - // are inflated. The version is the invalidation — `--force` cannot reach this cache. + // are inflated. The version is the invalidation — `--force` can not reach this cache. format!("pv2:{}", crate::haplogroup::genotype_cache_key("chrY", None, &targets)) }; if let Ok(Some(json)) = variant_set_private_y::get(self.store.pool(), set.id, &cache_key).await { diff --git a/crates/navigator-app/src/ftdna_import.rs b/crates/navigator-app/src/ftdna_import.rs index a50e3226..df39997c 100644 --- a/crates/navigator-app/src/ftdna_import.rs +++ b/crates/navigator-app/src/ftdna_import.rs @@ -142,7 +142,7 @@ pub enum FtdnaResolution { Merge(SampleGuid), /// Treat as a new Subject. New, - /// Don't import this kit at all. + /// Do not import this kit at all. Skip, } @@ -335,7 +335,7 @@ impl App { continue; } } - // Skip samples whose name isn't a recognizable catalog alias unless `--all`. + // Skip samples whose name is not a recognizable catalog alias unless `--all`. if !all && navigator_domain::identity::catalog_ids_from_provenance(&b.donor_identifier, None).is_empty() { continue; } @@ -357,7 +357,7 @@ impl App { out.resolved += 1; let fetched_acc = sample.accession.as_deref().map(str::trim).filter(|a| !a.is_empty()); // One pass: the catalog *name* id (from the donor id) + the authoritative INSDC *accession* - // (from the API, when it's a real one) — the union of both sources via the shared helper. + // (from the API, when it is a real one) — the union of both sources via the shared helper. let ids = navigator_domain::identity::catalog_ids_from_provenance(&b.donor_identifier, fetched_acc); if ids.is_empty() { continue; @@ -406,7 +406,7 @@ impl App { /// Re-publish a subject's biosample anchor after its identifier set changed, so the AppView's /// mirror (which full-replaces `external_ids`) honors the add/remove. Deterministic rkey → the /// re-publish overwrites in place. **Only for a subject already federated** and while signed in — - /// signed out, or a never-published subject, is a no-op (we don't newly federate a donor just + /// signed out, or a never-published subject, is a no-op (we do not newly federate a donor just /// because a local id was attached). async fn republish_biosample_ids(&self, guid: SampleGuid) -> Result<(), AppError> { let Some(did) = self.current_account() else { @@ -530,7 +530,7 @@ impl App { label: display_label(&kit, &input), kit_number: kit, y_terminal, - // Orphan only when a roster was provided but this kit isn't in it. + // Orphan only when a roster was provided but this kit is not in it. in_roster: !roster_provided || roster.contains(&input.kit_number), ystr_count: input.ystr_markers.len(), kind, @@ -648,7 +648,7 @@ impl App { ) .await?; - // MDKA from paternal (Y) + maternal (Mt) ancestry, when there's anything worth storing. + // MDKA from paternal (Y) + maternal (Mt) ancestry, when there is anything worth storing. let mut wrote = 0; if let Some(m) = input.paternal.as_ref().and_then(|a| mdka_from(a, Lineage::Y)) { mdka::upsert(pool, guid, &m, now).await?; @@ -883,7 +883,7 @@ struct ExistingSubject { guid: SampleGuid, donor_identifier: String, /// Terminal SNP of the subject's computed Y consensus (may be an ISOGG long-form label that - /// doesn't reduce to an SNP — then Y-STR is the reliable signal). + /// does not reduce to an SNP — then Y-STR is the reliable signal). y_terminal: Option, /// The subject's merged Y-STR markers (across all imported profiles), for genetic-distance match. ystr: Vec, @@ -919,7 +919,7 @@ fn display_label(kit: &str, input: &FtdnaSubjectInput) -> String { } } -/// Drop FTDNA redaction/placeholder names so they don't pollute identifiers or matching. +/// Drop FTDNA redaction/placeholder names so they do not pollute identifiers or matching. fn clean_name(name: Option<&str>) -> Option { let n = name?.trim(); if n.is_empty() || n.eq_ignore_ascii_case("REDACTED") { diff --git a/crates/navigator-app/src/haplogroup.rs b/crates/navigator-app/src/haplogroup.rs index 036285b9..1d67354b 100644 --- a/crates/navigator-app/src/haplogroup.rs +++ b/crates/navigator-app/src/haplogroup.rs @@ -67,7 +67,7 @@ fn anchor_side_to_parent(phased: &navigator_analysis::phasing::PhasedGenotypes, } /// `(this-side, other-side)` labels from a parent's recorded sex — once one parent is anchored the -/// other side is definitionally the other parent. `None` if the sex isn't a clear male/female. +/// other side is definitionally the other parent. `None` if the sex is not a clear male/female. fn parent_labels_for_sex(sex: Option<&str>) -> Option<(&'static str, &'static str)> { match sex.map(|s| s.trim().to_ascii_lowercase()).as_deref() { Some("female") | Some("f") => Some(("Mother", "Father")), @@ -96,7 +96,7 @@ fn build_side_labels(phased: bool, anchor: Option, parent: Option<&(Option>> = std::sync::OnceLock::new(); fn tree_memo() -> &'static std::sync::Mutex> { @@ -130,7 +130,7 @@ pub(crate) fn genotype_cache_key(contig: &str, source_build: Option<&str>, targe } // `g3`: chrY native genotyping now also resolves indel loci (additive derived sentinels), so the // cached result differs from the SNP-only `g1` payload — bump on any genotyping-logic change so a - // stale payload isn't reused (the site-set hash alone doesn't capture logic changes). + // stale payload is not reused (the site-set hash alone does not capture logic changes). format!("g3:{contig}:{}:{h:016x}", sorted.len()) } @@ -149,7 +149,7 @@ impl App { // ---- result exports (gap §6) ------------------------------------------- /// Format a cached result as a shareable file body (TSV / HTML / BED). The UI writes the - /// returned string to the user-chosen path. Errors when the source result hasn't been computed + /// returned string to the user-chosen path. Errors when the source result has not been computed /// yet (`NotFound`). [`ExportRequest::CallableBed`] re-walks the BAM (no cached intervals). pub async fn export_content(&self, req: &ExportRequest) -> Result { match req { @@ -500,14 +500,14 @@ impl App { let has_external = calls.iter().any(|(p, _)| *p == CallProvenance::External); // Per-run label reconciliation supplies the lineage / compatibility / divergence warnings — // honoring provenance: when the user prefers the external caller and one placed this subject, - // it wins the vote (a damaged ancient-DNA CRAM walk cannot out-score it). + // it wins the vote (a damaged ancient-DNA CRAM walk can not out-score it). let mut consensus = reconciliation::reconcile_with_provenance(&calls, prefer_external); // …the genome-level PLACED call (consensus_profile.consensus_label, from build_{y,mt}_profile) // is normally authoritative. Phase 2 makes that placement GVCF-sourced on preferred-external // subjects (place_{y,mt}_consensus → consensus_base_calls, no CRAM walk), so a freshly built // label already agrees with the external call. We still skip it here so a *stale* label left - // by a pre-Phase-2 (CRAM-pooled) build cannot resurface before the profile is rebuilt — the + // by a pre-Phase-2 (CRAM-pooled) build can not resurface before the profile is rebuilt — the // external reconcile is the safe authority for these subjects. // // …*unless* the reconcile has no branch name to offer. A call whose stored haplogroup is a @@ -696,7 +696,7 @@ impl App { return Ok(None); }; let value: serde_json::Value = serde_json::from_str(&row.payload)?; - // New payloads carry `schema_version`; legacy baked profiles don't. + // New payloads carry `schema_version`; legacy baked profiles do not. if value.get("schema_version").is_some() { Ok(Some(serde_json::from_value(value)?)) } else { @@ -821,11 +821,11 @@ impl App { /// per-SNP calls — every alignment's haplogroup placement, the combined chip/BISDNA placement, /// and the private-Y bucket — into one concordance view (confirmed / novel / conflict / /// single-source per SNP, with per-source provenance + per-observation quality weighting). - /// Expensive (re-genotypes each alignment), so it's an explicit action; the result is persisted + /// Expensive (re-genotypes each alignment), so it is an explicit action; the result is persisted /// so [`cached_y_profile`](Self::cached_y_profile) reloads it instantly. Sources without Y data /// are skipped. pub async fn build_y_profile(&self, biosample_guid: SampleGuid) -> Result { - // Females have no Y chromosome — don't build or persist a Y variant profile for them. + // Females have no Y chromosome — do not build or persist a Y variant profile for them. if !self.subject_has_y_dna(biosample_guid).await? { return Ok(YProfile { variants: Vec::new(), @@ -995,7 +995,7 @@ impl App { /// sequence's placement, and the combined chip mtDNA placement) into one concordance view, /// keyed by phylotree **mutation name** (rCRS-coordinate, build-independent). Persisted with /// `dna_type='Mt'` so [`cached_mt_profile`](Self::cached_mt_profile) reloads it instantly. - /// Expensive (re-places each alignment's chrM), so it's an explicit action; mt-less sources skip. + /// Expensive (re-places each alignment's chrM), so it is an explicit action; mt-less sources skip. pub async fn build_mt_profile(&self, biosample_guid: SampleGuid) -> Result { // One mt tree in rCRS coordinates (DecodingUs remapped from hs1, FTDNA fallback), shared by // the per-source placements below and the pooled terminal — so the variants and the terminal @@ -1069,7 +1069,7 @@ impl App { /// then place that pooled set on one canonical tree **once** via [`assemble_assignment`]. This /// replaces voting among the per-run terminal *labels*: a sparse run no longer drags the call /// shallow, and a branch confirmed by any source informs the placement. `Ok(None)` when the - /// subject has no Y-bearing source. Re-genotypes each source (like [`build_y_profile`]), so it's + /// subject has no Y-bearing source. Re-genotypes each source (like [`build_y_profile`]), so it is /// only run as part of that explicit action. pub async fn place_y_consensus(&self, biosample_guid: SampleGuid) -> Result, AppError> { // Females have no Y chromosome — no genome consensus to place. @@ -1114,7 +1114,7 @@ impl App { sources.push((SourceType::WgsShortRead, calls)); } } - // Dense GRCh38 vendor Y-NGS VCFs pool alongside the WGS; non-GRCh38 sets wouldn't match. + // Dense GRCh38 vendor Y-NGS VCFs pool alongside the WGS; non-GRCh38 sets would not match. let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; for set in &vsets { if set.source_type == SourceType::Chip || !is_grch38_build(&set.reference_build) { @@ -1303,7 +1303,7 @@ impl App { let json = self.fetch_decodingus_y_tree().await?; let tree = haplo::parse_decodingus_json(&json, bk).map_err(AppError::Import)?; // Native-build genotyping (no liftover) — the same walk place_y_consensus uses; the cache hit - // means `base_calls` returns the identical winning bases we're auditing here. + // means `base_calls` returns the identical winning bases we are auditing here. let calls = self.base_calls(alignment_id, "chrY", &tree, None).await?; let assignment = assemble_assignment(&tree, &calls); if assignment.lineage.is_empty() { @@ -1623,7 +1623,7 @@ impl App { /// Rank every other workspace subject against `query_guid` by Y relatedness (gap §2) — shared /// derived/novel SNPs, divergence haplogroup, Y-STR genetic distance, and rough SNP/STR TMRCA. /// One-vs-all over the workspace (or one project when `project_id` is set); local-only. Consumes - /// **cached** profiles so it's cheap over hundreds of subjects (no re-genotyping). `Ok(vec![])` + /// **cached** profiles so it is cheap over hundreds of subjects (no re-genotyping). `Ok(vec![])` /// when the query subject has no matchable Y data. pub async fn y_matches(&self, query_guid: SampleGuid, project_id: Option) -> Result, AppError> { // The tree only supplies the divergence haplogroup; shared-SNP and STR matching work without @@ -1788,7 +1788,7 @@ impl App { /// persisted** variant profile — no re-genotyping. Reads the cached profile for its terminal + /// per-SNP states (keyed by build-independent SNP name), then walks the FTDNA tree from the /// terminal to the root, attaching each node's defining SNPs with the sample's call (`NoCall` for - /// an untested equivalent). `Ok(None)` when the profile isn't built yet or has no terminal — the + /// an untested equivalent). `Ok(None)` when the profile is not built yet or has no terminal — the /// UI then offers to build it (one expensive, persisted step that also powers the variant tabs). pub async fn descent_report( &self, @@ -1965,7 +1965,7 @@ impl App { .into_iter() .map(|r| { let pos = r.snp.position; - // Either allele being multi-base (or empty) means this isn't a clean SNV. + // Either allele being multi-base (or empty) means this is not a clean SNV. let is_indel = r.snp.derived.chars().count() != 1 || r.snp.ancestral.chars().count() != 1; let (source, dp, ad, gq) = match evidence.get(&pos).copied() { Some(e) if e.refblock => ("gvcf_refblock", None, None, e.gq), @@ -2046,10 +2046,10 @@ impl App { /// adapter over the generic [`navigator_domain::consensus`] engine. Genotypes every WGS alignment /// and imported chip over the canonical CHM13 **IBD panel** ([`ibd_panel_dosages`](Self::ibd_panel_dosages)) /// and reconciles the per-site dosages into a voted genotype (confirmed where sources agree, - /// conflict where they don't), keyed by rsID. Persisted with `dna_type='Auto'`. Requires the IBD - /// panel asset (built with `panelbuild ibd-panel`); errors if it's missing. + /// conflict where they do not), keyed by rsID. Persisted with `dna_type='Auto'`. Requires the IBD + /// panel asset (built with `panelbuild ibd-panel`); errors if it is missing. pub async fn build_autosomal_profile(&self, biosample_guid: SampleGuid) -> Result { - // Full build: genotype any alignment whose panel dosages aren't cached yet. + // Full build: genotype any alignment whose panel dosages are not cached yet. self.build_autosomal_profile_inner(biosample_guid, false).await } @@ -2068,7 +2068,7 @@ impl App { .await .map(Some) .or_else(|e| match e { - // "no source" isn't an error for a refresh — the subject just has nothing cached yet. + // "no source" is not an error for a refresh — the subject just has nothing cached yet. AppError::Import(_) => Ok(None), other => Err(other), }) @@ -2160,7 +2160,7 @@ impl App { // One source per WGS alignment (panel-genotyped, cached per alignment). The IBD panel carries // every build's coordinates, so `ibd_panel_dosages` genotypes a CHM13 alignment at its native // loci and a GRCh37/GRCh38 alignment at that build's loci, re-keying the result to canonical - // CHM13. A build the panel doesn't cover yields no genotypes and is skipped downstream. + // CHM13. A build the panel does not cover yields no genotypes and is skipped downstream. let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; for a in &alignments { // Progressive refresh reduces over cached dosages only — an uncached alignment is skipped @@ -2240,7 +2240,7 @@ impl App { return Err(e); // e.g. the IBD panel asset isn't built yet } if cached_alignments_only { - // A progressive refresh with nothing available yet: don't persist an empty consensus + // A progressive refresh with nothing available yet: do not persist an empty consensus // (refresh_autosomal_consensus maps this to Ok(None)). return Err(AppError::Import("no cached autosomal sources yet".into())); } @@ -2488,7 +2488,7 @@ impl App { } /// The **mtDNA placement tree in rCRS coordinates**, with the provider tag. Honors the - /// configured Y-tree provider (the Preferences toggle / `NAVIGATOR_Y_TREE_PROVIDER`): when it's + /// configured Y-tree provider (the Preferences toggle / `NAVIGATOR_Y_TREE_PROVIDER`): when it is /// set to FTDNA, use the FTDNA mt tree (already rCRS) directly. Otherwise prefer the DecodingUs /// mt tree remapped from its native `hs1` (CHM13 `chrM`) positions onto rCRS — so it drops /// straight into the existing rCRS mt pipeline (FASTA/chip sources and the `chrM` genotyper all @@ -2506,13 +2506,13 @@ impl App { } /// The DecodingUs mt tree parsed and remapped from `hs1` (CHM13 `chrM`) coordinates onto rCRS. - /// `None` (→ FTDNA fallback) when the tree can't be fetched, or the CHM13 reference isn't cached + /// `None` (→ FTDNA fallback) when the tree can't be fetched, or the CHM13 reference is not cached /// to build the `hs1`↔rCRS map. Best-effort so an offline / reference-less workspace still works. async fn decodingus_mt_tree_rcrs(&self) -> Option { let json = self.fetch_decodingus_mt_tree().await.ok()?; let mut tree = navigator_analysis::haplo::parse_decodingus_json(&json, "hs1").ok()?; let hs1_to_rcrs = self.hs1_to_rcrs_mt_map().await?; - // Remap each defining locus from hs1 (CHM13 chrM) to rCRS; drop any that don't map (indel + // Remap each defining locus from hs1 (CHM13 chrM) to rCRS; drop any that do not map (indel // regions near the rotation wrap). An emptied node still exists in the topology. for node in tree.nodes.values_mut() { node.loci.retain_mut(|l| match hs1_to_rcrs.get(&l.position) { @@ -2528,7 +2528,7 @@ impl App { /// The `hs1` (CHM13 `chrM`, 1-based) → rCRS (1-based) position map, memoized for the process. /// Built by aligning the bundled rCRS to the cached CHM13 reference's `chrM` (rotation-aware). - /// `None` when the CHM13 reference isn't cached (never forces a multi-GB download for this). + /// `None` when the CHM13 reference is not cached (never forces a multi-GB download for this). async fn hs1_to_rcrs_mt_map(&self) -> Option> { static MAP: std::sync::OnceLock>> = std::sync::OnceLock::new(); if let Some(m) = MAP.get() { @@ -2590,7 +2590,7 @@ impl App { /// footprint: a Big Y-500 covers ≤ ~10 Mb of callable chrY, and only the newer Big Y-700 /// consistently exceeds it. Only acts on an FTDNA `TARGETED_Y` run — a header `@RG LB` label /// already pins the generation at import (those are `BIG_Y_500`/`BIG_Y_700`, never `TARGETED_Y`, - /// so they're never second-guessed here), and a non-FTDNA targeted-Y stays generic. Idempotent. + /// so they are never second-guessed here), and a non-FTDNA targeted-Y stays generic. Idempotent. pub(crate) async fn refine_big_y_generation(&self, run: &SequenceRun, callable_chr_y: u64) -> Option<&'static str> { const BIG_Y_500_MAX_CALLABLE: u64 = 10_000_000; if run.test_type != "TARGETED_Y" { @@ -2661,7 +2661,7 @@ impl App { }, }; // A run we now know is FTDNA but typed as the generic TARGETED_Y is a Big Y — pick - // its generation (500/700) from cached coverage when it's already been analyzed, so + // its generation (500/700) from cached coverage when it is already been analyzed, so // pre-existing runs get corrected on startup without a re-analysis. let _ = facility; // (resolved above; the refine reads facility off the run record) if run.test_type == "TARGETED_Y" { @@ -2678,7 +2678,7 @@ impl App { } /// Cached coverage for a run, via its first alignment that has one (Big Y runs have a single - /// alignment). `None` when the run hasn't been analyzed yet. + /// alignment). `None` when the run has not been analyzed yet. async fn cached_coverage_for_run(&self, run_id: i64) -> Result, AppError> { for aln in alignment::list_for_run(self.store.pool(), run_id).await? { if let Some(cov) = self.cached_coverage(aln.id).await? { @@ -2922,7 +2922,7 @@ impl App { /// cached autosomal [`DiploidProfile`] (reconciled 0/1/2 dosages over the probe panel, pooled /// across all WGS + chip sources), bridges it to genotypes, and runs the same estimators as the /// per-alignment path used to. Persisted under the consensus pseudo-source - /// ([`CONSENSUS_SOURCE_ID`]). Errors if the autosomal consensus hasn't been built yet. + /// ([`CONSENSUS_SOURCE_ID`]). Errors if the autosomal consensus has not been built yet. pub async fn estimate_ancestry_from_consensus( &self, biosample_guid: SampleGuid, @@ -3099,7 +3099,7 @@ impl App { let bytes = read_verified_asset(build, &path)?.ok_or_else(|| AppError::AncestryPanelMissing(path.clone()))?; let panel = AncestryPanel::from_bytes(&bytes)?; // The super-pop panel too: deep ancestry is scoped by the modern estimate, so each view has - // to be scored by both models or the diagnostic wouldn't be reproducing the shipped policy. + // to be scored by both models or the diagnostic would not be reproducing the shipped policy. let super_path = ancestry_panel_path(build); let super_bytes = read_verified_asset(build, &super_path)? .ok_or_else(|| AppError::AncestryPanelMissing(super_path.clone()))?; @@ -3295,7 +3295,7 @@ impl App { /// projected coordinate (`AncestryResult::pca_coordinates`) is always computed against the CHM13 /// PCA asset (the canonical consensus frame — see [`estimate_ancestry_from_consensus`]), so the /// backdrop centroids must come from that same asset regardless of which source is selected. - /// Returns an empty vec when the asset isn't installed (the caller shows "reference not built"). + /// Returns an empty vec when the asset is not installed (the caller shows "reference not built"). pub async fn ancestry_pca_reference(&self) -> Result, AppError> { let build = ReferenceBuild::Chm13v2; let Ok(bytes) = std::fs::read(ancestry_pca_path(build)) else { @@ -3447,7 +3447,7 @@ impl App { phased, }; - // Cache keyed to the consensus signature so it's reused until the consensus is rebuilt. + // Cache keyed to the consensus signature so it is reused until the consensus is rebuilt. sig_cache::PAINTING .upsert( self.store.pool(), @@ -3477,7 +3477,7 @@ impl App { child_genotypes: &[SiteGenotype], ) -> Option<(Vec, Option, String)> { // Auto-anchoring is for small family/trio projects; larger sets are skipped (a research - // corpus shouldn't trigger a many-way scan, and parent detection there isn't meaningful). + // corpus should not trigger a many-way scan, and parent detection there is not meaningful). const MAX_CANDIDATES: usize = 32; // Opposite-homozygous fraction below which a pair is treated as parent-child (Mendel forbids // opposite homozygotes for a true parent-child pair; the slack absorbs genotyping error). @@ -3583,7 +3583,7 @@ impl App { }) .await?; - // Cache keyed to the consensus signature so it's reused until the consensus is rebuilt. + // Cache keyed to the consensus signature so it is reused until the consensus is rebuilt. sig_cache::ROH .upsert( self.store.pool(), @@ -4211,7 +4211,7 @@ impl App { /// `include_unknown` adds the subjects whose calls predate the fingerprint field, which is a /// provenance backfill rather than a response to a tree change — see the store function. /// - /// A tree that cannot be fetched yields no subjects for that DNA type rather than an error: the + /// A tree that can not be fetched yields no subjects for that DNA type rather than an error: the /// point of the sweep is to act on a *known* new tree, and "the network is down" is not one. pub async fn subjects_placed_against_another_tree( &self, @@ -4250,7 +4250,7 @@ impl App { /// A consensus is *derived* from the per-source calls and persisted separately, with no tree /// stamp of its own. So it can rot while every call beneath it is current: `GMWOF5428705` holds /// a call placed against today's tree (`E-C116698`) under a consensus of `E-FT400514:n0` last - /// reconciled four weeks earlier. A sweep keyed on call fingerprints alone cannot see that. + /// reconciled four weeks earlier. A sweep keyed on call fingerprints alone can not see that. /// /// Testing the label against the tree's node names catches it directly and needs no schema /// change: a label absent from the tree is stale by definition, whatever the cause — and it is @@ -4314,7 +4314,7 @@ impl App { pub async fn assign_y_haplogroup(&self, alignment_id: i64) -> Result { let bio = self.biosample_of_alignment(alignment_id).await.ok(); - // Females have no Y chromosome — don't genotype chrY or record a Y call for them. + // Females have no Y chromosome — do not genotype chrY or record a Y call for them. if let Some(guid) = bio { if !self.subject_has_y_dna(guid).await? { return Ok(HaploAssignment { @@ -4432,7 +4432,7 @@ impl App { aln.reference_build )) })?; - // Before the tree fetch, not after: a gone alignment file cannot be genotyped against any + // Before the tree fetch, not after: a gone alignment file can not be genotyped against any // tree, so downloading one first is pure waste — and its io error surfacing from *below* the // fetch is what let `y_assignment_full` mistake it for the tree being unavailable. Self::alignment_file(&aln)?; @@ -4450,7 +4450,7 @@ impl App { /// DecodingUs tree at the native build (FTDNA fallback only on GRCh38, where positions /// match), and the chip-robust terminal selection ([`assemble_assignment_robust`]). The /// call is recorded as a reconciliation source. Only derived (positive) calls drive the - /// Kulczynski ranking, so the stored positives-only variant set is sufficient. + /// Kulczynski ranking, so the stored positives-only variant set is enough. pub async fn assign_y_bisdna( &self, biosample_guid: SampleGuid, @@ -4567,7 +4567,7 @@ impl App { /// Tree-position genotypes for a variant set — the VCF counterpart of [`Self::base_calls`]. /// /// [`Self::vset_chr_y_calls`] can only report the stored rows, which are the donor's *derived* - /// calls: the workspace never recorded where he is confidently ancestral, so placement cannot + /// calls: the workspace never recorded where he is confidently ancestral, so placement can not /// separate "ancestral" from "not covered" and every backbone node scores as no-call. Re-reading /// the source VCF at the tree's positions recovers the hom-ref rows it already contains, which is /// what the CRAM path gets for free by genotyping every target. @@ -4668,7 +4668,7 @@ impl App { } /// Fetch + parse the Y haplotree for a chip placement on `build`. DecodingUs is native multi-build - /// (no liftover); the FTDNA tree is GRCh38-only, so it's a fallback only when the calls are GRCh38. + /// (no liftover); the FTDNA tree is GRCh38-only, so it is a fallback only when the calls are GRCh38. /// Shared by the combined [`assign_y_bisdna`](Self::assign_y_bisdna) placement and the per-panel /// Y-profile sources, so the tree is fetched once. pub(crate) async fn chip_y_tree(&self, build: &str) -> Result { @@ -4767,7 +4767,7 @@ impl App { /// Full Y-haplogroup placement **report** for an alignment (gap §8): the ranked candidate /// haplogroups (with score / matched-vs-expected) + the defining-SNP evidence along the reported /// lineage (each SNP's derived / ancestral / no-call state). A fresh placement against the - /// configured provider tree — heavier than the cached terminal label, so it's button-driven. + /// configured provider tree — heavier than the cached terminal label, so it is button-driven. pub async fn y_haplogroup_report( &self, alignment_id: i64, @@ -4877,7 +4877,7 @@ impl App { /// alignment's external sidecar GVCF (no CRAM decode) when the "prefer external caller" policy is /// on and the GVCF is present; otherwise the cached CRAM walk ([`base_calls`]). A drop-in for the /// per-alignment genotype in `place_{y,mt}_consensus`, so a preferred-external (e.g. ancient-DNA) - /// subject's damaged CRAM is not re-walked and cannot dilute the pooled placement (Phase 2 of + /// subject's damaged CRAM is not re-walked and can not dilute the pooled placement (Phase 2 of /// `documents/design/external-caller-precedence.md` §4.5). `tree_source_build` matches what `base_calls` /// receives — `None` for a native-build tree (DecodingUs Y, rCRS mt), the tree's build for a lift. pub(crate) async fn consensus_base_calls( @@ -4962,7 +4962,7 @@ impl App { // Indel loci (multi-base ancestral/derived) on the tree — genotyped separately on the native // chrY path (VCF left-anchored; needs the reference to normalize + know deleted bases). Their // resolved sentinel overlays the (meaningless) base call at the anchor. Liftover of indel - // coordinates isn't handled, so only the native path (no lift) contributes them. + // coordinates is not handled, so only the native path (no lift) contributes them. let indel_targets: Vec<(i64, String, String)> = if contig.eq_ignore_ascii_case("chrY") { tree.nodes .values() @@ -5303,7 +5303,7 @@ mod lifted_targets_tests { } /// A real build that matches the alignment's build needs no lift — still `Ok(None)`, not an - /// error. Pins that the new guard didn't narrow the chrY path. + /// error. Pins that the new guard did not narrow the chrY path. #[tokio::test] async fn a_matching_reference_build_still_means_no_lift() { let app = App::new(Store::open_in_memory().await.unwrap()); diff --git a/crates/navigator-app/src/import_profiles.rs b/crates/navigator-app/src/import_profiles.rs index 1331e0ae..50683edc 100644 --- a/crates/navigator-app/src/import_profiles.rs +++ b/crates/navigator-app/src/import_profiles.rs @@ -221,7 +221,7 @@ impl App { /// Import an FTDNA Big Y CSV variant report (Named or Private Variants) — the data a project /// admin gets when their access tier exposes the browser CSVs but not the BAM/CRAM/VCF. The - /// rows are GRCh38 chrY derived-allele calls, so they're stored as a `TargetedNgs` variant set + /// rows are GRCh38 chrY derived-allele calls, so they are stored as a `TargetedNgs` variant set /// on GRCh38 (FTDNA's native Y-tree build) and placed via the vendor path on import — the Named /// report lands a Y haplogroup directly (positions match the tree, no liftover). Private /// Variants are stored too (novel loci, off-tree) for the record. @@ -310,7 +310,7 @@ impl App { } /// Ensure a Y-SNP dictionary is present, downloading the full catalog (`dictionary.tsv`, - /// ~208 MB) from the asset release on first use — it's too big and too volatile (~weekly YBrowse + /// ~208 MB) from the asset release on first use — it is too big and too volatile (~weekly YBrowse /// refresh) to bundle in the installer. No-op when a dictionary (the chromo2 panel or the full /// catalog) is already installed, or the user pointed `NAVIGATOR_YSNP_DIR` at one. The download /// is verified against a small published manifest (`ysnp_manifest.json`, the ancestry @@ -516,7 +516,7 @@ impl App { // Pull the haploid Y/MT genotype rows and store them as Chip-source variant calls so the // haplogroup placement (and later re-placement) has them without re-reading the file. The - // observed allele goes in both `reference` and `alternate` (we don't know the ancestral); + // observed allele goes in both `reference` and `alternate` (we do not know the ancestral); // the placement reads `alternate`. let haplo = chipprofile::haplo_calls(&text); if !haplo.is_empty() { @@ -556,7 +556,7 @@ impl App { eprintln!("chip Y placement deferred ({e})"); } } - // AncestryDNA's stray MT rows aren't a usable mtDNA panel — only place mtDNA when the + // AncestryDNA's stray MT rows are not a usable mtDNA panel — only place mtDNA when the // array carries a real MT marker set (23andMe has thousands; the threshold filters noise). const MIN_MT_CALLS: usize = 20; if mt_count >= MIN_MT_CALLS { diff --git a/crates/navigator-app/src/import_unified.rs b/crates/navigator-app/src/import_unified.rs index 783f53bf..ab1e13a3 100644 --- a/crates/navigator-app/src/import_unified.rs +++ b/crates/navigator-app/src/import_unified.rs @@ -37,8 +37,8 @@ impl App { /// Auto-import an alignment file by probing its header: create the sequencing run (test type, /// platform, instrument) and the alignment (reference build + aligner) with no questions - /// asked. The reference FASTA is **not** required — it's resolved from the build on demand; - /// if already cached it's stored so every analysis step has it immediately. + /// asked. The reference FASTA is **not** required — it is resolved from the build on demand; + /// if already cached it is stored so every analysis step has it immediately. async fn import_alignment_file( &self, biosample_guid: SampleGuid, @@ -257,7 +257,7 @@ impl App { /// Batch [`add_data`]: expand any directories among `paths` into their recognized data files, /// then auto-detect + import each into the subject, collecting a [`BatchImportSummary`]. A - /// failed/unrecognized file is recorded (not propagated) so one bad file doesn't abort the + /// failed/unrecognized file is recorded (not propagated) so one bad file does not abort the /// batch. `progress(done, total)` ticks per file. The unified multi-file / folder importer /// behind the GUI's Add Data button + drag-and-drop. (Distinct from [`import_project_dir`], /// which builds a *new* multi-subject project from a NAS layout; this adds to *this* subject.) @@ -269,7 +269,7 @@ impl App { ) -> Result { let mut files = Vec::new(); for p in &paths { - // Guard against a single picked folder that's really a *parent* of several per-sample + // Guard against a single picked folder that is really a *parent* of several per-sample // folders (e.g. an FTDNA download root): recursing it would silently merge sibling // samples into this one subject. Refuse with guidance rather than import the wrong data. if p.is_dir() { @@ -404,7 +404,7 @@ impl App { // Import bundled variant files ONLY when there is no haplogroup GVCF. When a GVCF is present // the fast path below is the authoritative Y/mt source, so a called `chrY.vcf.gz` sitting // beside it (the GATK repo layout ships both) is redundant — importing it would fire a second - // Y placement and, because variant-set import isn't content-idempotent, would duplicate the + // Y placement and, because variant-set import is not content-idempotent, would duplicate the // set on a resumable re-run. Non-GVCF tiers (e.g. the b38 aengine `variants.vcf.gz`) still // import here: there the VCF *is* the Y source. GVCFs themselves are `.g.vcf.gz`, which `scan` // also lists as variant files — the guard keeps them out of this loop too. @@ -475,7 +475,7 @@ impl App { /// plus its Biosample → SequenceRun → Alignment rows. The reference is resolved per /// alignment: pass `Some(fasta)` to use a specific FASTA (validated with its `.fai`) for /// every alignment, or `None` to let the gateway resolve each file's inferred build from - /// the cache. If a needed build isn't cached, returns [`AppError::ReferenceNeeded`] + /// the cache. If a needed build is not cached, returns [`AppError::ReferenceNeeded`] /// **before any DB writes** so the UI can prompt + download, then retry. Idempotent: an /// existing project (by name), biosample (by donor id), or alignment (by path) is reused. /// Coverage is NOT computed here — run it per alignment or via the project report. @@ -493,7 +493,7 @@ impl App { /// Re-run the sidecar fast path for every alignment of a subject whose source directory still /// carries the pipeline GVCFs — restoring external (GATK4) Y/mt calls that an older build's /// internal walk had overwritten before provenance existed. Cheap: reads the small GVCFs, never - /// the CRAM. The external calls land on their own `:ext` keys (they cannot clobber, and with the + /// the CRAM. The external calls land on their own `:ext` keys (they can not clobber, and with the /// "prefer external caller" policy they win the consensus). Returns `(y_placed, mt_placed)`. /// This is the operational fix for a workspace imported before external-caller precedence. pub async fn reingest_external_for_biosample( @@ -553,9 +553,9 @@ impl App { let scan_dir = dir.to_path_buf(); let discovered = tokio::task::spawn_blocking(move || navigator_analysis::scan::scan(&scan_dir)).await??; - // Detect each alignment's reference build from its **header** (only the header, so it's + // Detect each alignment's reference build from its **header** (only the header, so it is // cheap and needs no reference FASTA). The filename is an unreliable signal — most NAS - // project layouts don't put the build in the name — so probe first, fall back to the + // project layouts do not put the build in the name — so probe first, fall back to the // filename, and record how each build was decided for the import diagnostics. let all_paths: Vec = discovered .samples @@ -575,7 +575,7 @@ impl App { // Resolve each *distinct* detected build to a reference path. A build the gateway can't // canonicalize falls back to the CHM13v2.0 default rather than aborting the whole batch; - // a known build that isn't cached is surfaced as a recoverable download need. `effective_of` + // a known build that is not cached is surfaced as a recoverable download need. `effective_of` // maps a detected build to the one actually stored on the alignment (after any fallback). let explicit = reference.as_ref().map(|p| p.to_string_lossy().into_owned()); let mut resolved: HashMap = HashMap::new(); // effective build -> FASTA path @@ -792,7 +792,7 @@ impl App { variant_caller: None, bam_path: Some(path_str), reference_path, - // Batch import: hash lazily on first analysis (don't stall a bulk NAS import + // Batch import: hash lazily on first analysis (do not stall a bulk NAS import // hashing every multi-GB file up front). content_sha256: None, // An imported alignment is an original; see above. @@ -862,7 +862,7 @@ impl App { } /// Re-hash a cached reference against its integrity sidecar (gap §7) — detects on-disk - /// corruption of the cached `.fa`. Runs on a blocking thread (re-reads the whole FASTA), so it's + /// corruption of the cached `.fa`. Runs on a blocking thread (re-reads the whole FASTA), so it is /// an explicit, user-triggered check (Settings), not the hot path. pub async fn verify_reference(&self, build: &str) -> Result { let gw = self.gateway.clone(); @@ -921,14 +921,14 @@ impl App { /// See [`asset_action`] for the present/stale/absent decision this drives. /// /// Ensure a prebuilt ancestry/IBD asset at `path` is present **and current**, downloading it — - /// and the asset manifest it's verified against — from the published GitHub release. End users + /// and the asset manifest it is verified against — from the published GitHub release. End users /// get the panels this way instead of running the offline `panelbuild` tool. /// /// Three cases, all manifest-driven: /// /// * **Absent** → download it, provided the manifest lists it (an unpublished optional asset /// simply stays absent and its feature degrades). - /// * **Manifest doesn't list it** → re-fetch the manifest once, then re-check. The cached + /// * **Manifest does not list it** → re-fetch the manifest once, then re-check. The cached /// manifest is otherwise never refreshed, so an install that predates an asset's publication /// would never learn the asset exists — which is exactly what happened to `ancestry_haps`. /// * **Present but the wrong size** → the published asset was revised; replace it. Without this @@ -951,9 +951,9 @@ impl App { let manifest_name = format!("ancestry_manifest_{}.json", build.as_str()); let manifest_path = default.with_file_name(&manifest_name); - // (1) The manifest: fetch when absent, and re-fetch when it doesn't list this asset (a + // (1) The manifest: fetch when absent, and re-fetch when it does not list this asset (a // manifest cached before the asset was published). Keep the old copy in memory so a - // failed refresh doesn't cost us the integrity data we already had. + // failed refresh does not cost us the integrity data we already had. let listed = |m: &Option| { m.as_ref().is_some_and(|m| m.assets.contains_key(&name)) }; @@ -984,7 +984,7 @@ impl App { return Ok(()); }; - // (2) What to do with what's on disk. Content is verified at read time by + // (2) What to do with what is on disk. Content is verified at read time by // `read_verified_asset`; hashing every asset here would cost seconds per paint. let on_disk = std::fs::metadata(&default).ok().map(|m| m.len()); let action = asset_action(Some(&entry), on_disk); @@ -1066,7 +1066,7 @@ impl App { /// Resolve an imported chip's genotypes to canonical CHM13 **IBD-panel** dosages — the chip→IBD /// path (no alignment, no runtime liftover: the multi-build panel pre-computes coordinates). The /// output [`SiteGenotype`]s are over the same CHM13 sites a WGS caller would hit, so a chip and a - /// WGS sample compare uniformly. Errors if the IBD panel asset isn't built yet. + /// WGS sample compare uniformly. Errors if the IBD panel asset is not built yet. pub async fn chip_ibd_dosages(&self, chip_profile_id: i64) -> Result, AppError> { let chip = chip_profile::get(self.store.pool(), chip_profile_id) .await? @@ -1305,7 +1305,7 @@ impl App { } // Resolve the reference for decode (see alignment_reference_for_decode): required for // a CRAM, None for a BAM. Panel genotyping tallies SNP sites (ref/alt come from the - // panel), so a BAM consults no reference bases — don't force a download for it. + // panel), so a BAM consults no reference bases — do not force a download for it. let build = self.alignment_or_err(id).await?.reference_build; let (bam, reference) = self.alignment_reference_for_decode(id).await?; let panel = self.load_ibd_panel().await?; @@ -1384,7 +1384,7 @@ impl App { .await??; panel.resolve_alignment(&build, &raw) } else { - // A build the panel doesn't carry — nothing to genotype (degrade gracefully rather + // A build the panel does not carry — nothing to genotype (degrade gracefully rather // than probe the wrong loci). Vec::new() }; @@ -1396,7 +1396,7 @@ impl App { } /// Cached IBD-panel dosages for an alignment, **without genotyping** — `Ok(None)` when they - /// haven't been computed yet (so callers can reduce over what's available progressively rather + /// have not been computed yet (so callers can reduce over what is available progressively rather /// than triggering a whole-genome decode). [`Self::ibd_panel_dosages`] is the compute-and-cache /// path; this is the read-only companion used by the progressive-consensus refresh. pub async fn cached_alignment_panel_dosages( @@ -1442,7 +1442,7 @@ impl App { } /// What [`App::ensure_ancestry_asset`] must do for one asset, from the manifest entry (`None` when -/// the manifest doesn't list it) and the on-disk size (`None` when the file is absent). +/// the manifest does not list it) and the on-disk size (`None` when the file is absent). /// /// Size, not hash: a published asset is revised by rebuilding it, which changes its length, and the /// authoritative content check already happens at read time. Hashing a 133 MB panel on every call diff --git a/crates/navigator-app/src/lib.rs b/crates/navigator-app/src/lib.rs index 9cbb657e..57ae4dec 100644 --- a/crates/navigator-app/src/lib.rs +++ b/crates/navigator-app/src/lib.rs @@ -67,7 +67,7 @@ pub struct HaploAssignment { pub ranked: Vec, pub branches: Vec, /// Per-SNP evidence along the placed lineage (root→terminal): every defining mutation the - /// sample carries (or doesn't), Derived/Ancestral/NoCall. This is the set the multi-source + /// sample carries (or does not), Derived/Ancestral/NoCall. This is the set the multi-source /// variant/mutation **profile** reconciles — distinct from `branches`, which is the *untaken* /// child branches (explaining why descent stopped, hence largely ancestral/no-call). pub lineage: Vec, @@ -92,7 +92,7 @@ pub struct DescentReport { /// to each other. Built by [`App::project_block_tree`]; see /// `documents/design/project-block-tree.md`. /// -/// This view **reads** placements and never re-places, so it cannot introduce a placement error. +/// This view **reads** placements and never re-places, so it can not introduce a placement error. #[derive(Debug, Clone)] pub struct ProjectBlockTree { pub dna: DnaType, @@ -114,7 +114,7 @@ pub struct ProjectBlockTree { /// phylogenetic conflict in the cohort, which is worth knowing about. pub candidate_conflicts: usize, /// Positions rejected as **recurrent** — each would have defined a candidate branch under more - /// than one parent block, so it arose more than once and cannot mark a new branch. Counted + /// than one parent block, so it arose more than once and can not mark a new branch. Counted /// rather than hidden: a high number says the cohort's private calls carry systematic noise. pub candidate_recurrent: usize, } @@ -143,7 +143,7 @@ pub struct Block { /// True when this is a **candidate branch** — not a node in the published tree, but a grouping /// inferred from private (unnamed) variants that two or more members share. `node_id` is /// synthetic and negative for these; `name` is empty, because the label is the view's to - /// localize. This is the thing a published tree cannot tell you and we can: a branch that is + /// localize. This is the thing a published tree can not tell you and we can: a branch that is /// real in the data but has not been named yet. pub candidate: bool, /// For a candidate branch: every carrier's evidence at each shared position, so it can be @@ -280,7 +280,7 @@ pub enum PrivateClass { Novel, } -/// A derived variant the sample carries that the haplogroup placement doesn't explain. +/// A derived variant the sample carries that the haplogroup placement does not explain. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct PrivateVariant { pub position: i64, @@ -511,7 +511,7 @@ pub struct IbdComparison { pub segments: Vec, /// Sites called in **both** samples — the effective comparison size. Sparse overlap (a /// chip↔chip pair, or chip↔WGS limited to the chip's sites) weakens short-segment calls, so - /// it's surfaced rather than hidden. + /// it is surfaced rather than hidden. pub overlapping_sites: usize, } @@ -556,7 +556,7 @@ pub struct AssetStatus { /// /// `target_sample_guid` is the AppView's handle for **our own** sample the candidate was ranked /// against. We already own it, so it discloses nothing — but a self-publishing client has no other -/// way to learn its server-side sample handle, and [`App::ibd_attest`] cannot report a completed +/// way to learn its server-side sample handle, and [`App::ibd_attest`] can not report a completed /// comparison without it. `None` when talking to an AppView that predates that field. #[derive(Debug, Clone, PartialEq)] pub struct IbdSuggestion { @@ -764,7 +764,7 @@ pub struct EstablishedSession { } /// Jetstream-ingest retry budget for a freshly-published device key: a 403 right after -/// publishing means the AppView hasn't ingested our `deviceKey` record yet. Exponential +/// publishing means the AppView has not ingested our `deviceKey` record yet. Exponential /// backoff 1+2+4+8 s ≈ 15 s total before giving up. const DEVICE_KEY_INGEST_RETRIES: u32 = 4; @@ -922,7 +922,7 @@ pub mod sync_reconcile; pub use settings::AppSettings; pub use update::UpdateInfo; -/// Artifact kind for de-novo calls, keyed per contig so different contigs don't +/// Artifact kind for de-novo calls, keyed per contig so different contigs do not /// overwrite each other in the cache. fn denovo_kind(contig: &str) -> String { format!("denovo_snps:{contig}") @@ -976,7 +976,7 @@ fn tree_cache_is_fresh(path: &Path) -> bool { /// /// The Kulczynski `score` ranks the candidates by proportional similarity (and supplies the /// alternatives list), but the *reported terminal* is chosen in two steps: (1) the best-ranked -/// candidate the path-supported parsimony guard admits — i.e. whose lineage doesn't tunnel +/// candidate the path-supported parsimony guard admits — i.e. whose lineage does not tunnel /// through a branch the sample contradicts (the distal-Y paralog artifact); then (2) /// [`haplo::deepen_terminal`] descends further into any child the sample clearly entered, /// correcting under-calls at **unsplit tree nodes** (a half-ancestral SNP block scores below @@ -1063,7 +1063,7 @@ fn assemble_assignment_robust( let mut ranked = haplo::score(tree, calls); if let Some(top_id) = ranked.first().map(|r| r.id) { let terminal_id = haplo::deepen_terminal(tree, calls, top_id); - // Parsimony back-off: don't report a deeper terminal than the evidence supports. Trim any + // Parsimony back-off: do not report a deeper terminal than the evidence supports. Trim any // net-contradicted tail of the lineage (sparse-panel / damaged-aDNA over-deepening) while // a lone contradiction outweighed by deeper derived support still reaches the deep terminal. let chosen_id = support_backoff_terminal(tree, calls, terminal_id); @@ -1086,7 +1086,7 @@ fn assemble_assignment_robust( } } -/// The root→`target` path of node ids (inclusive), or empty if `target` isn't reachable. +/// The root→`target` path of node ids (inclusive), or empty if `target` is not reachable. fn lineage_ids(tree: &navigator_analysis::haplo::HaploTree, target: i64) -> Vec { fn dfs(tree: &navigator_analysis::haplo::HaploTree, id: i64, target: i64, acc: &mut Vec) -> bool { let Some(node) = tree.nodes.get(&id) else { return false }; @@ -1113,7 +1113,7 @@ fn lineage_ids(tree: &navigator_analysis::haplo::HaploTree, target: i64) -> Vec< Vec::new() } -/// Root→`name` lineage of haplogroup names from the tree (empty if the name isn't found). Used to +/// Root→`name` lineage of haplogroup names from the tree (empty if the name is not found). Used to /// derive a placed terminal's lineage path for cross-subject divergence/LCA without re-genotyping. fn lineage_names(tree: &navigator_analysis::haplo::HaploTree, name: &str) -> Vec { let Some(id) = tree.nodes.values().find(|n| n.name == name).map(|n| n.id) else { @@ -1178,8 +1178,8 @@ fn support_backoff_terminal( /// ancestral/derived convention. For each call at a tree position: keep the observed base if it /// already equals the ancestral or derived allele; else substitute its complement when *that* /// matches; else keep it (a genuine no-match the scorer will count against the branch). Positions -/// absent from the tree pass through unchanged (they don't affect scoring). This is a no-op for -/// dictionary-reconciled BISDNA calls (their base is always the derived allele), so it's safe to +/// absent from the tree pass through unchanged (they do not affect scoring). This is a no-op for +/// dictionary-reconciled BISDNA calls (their base is always the derived allele), so it is safe to /// apply on the shared chip-placement path. fn strand_reconcile_to_tree( tree: &navigator_analysis::haplo::HaploTree, @@ -1442,10 +1442,10 @@ pub struct SeedSummary { pub skipped: usize, } -/// Copy every regular file in `src_dir` into `dest_dir` that isn't already present there. Never +/// Copy every regular file in `src_dir` into `dest_dir` that is not already present there. Never /// overwrites an existing file — a CDN-refreshed asset must win over the bundled one. Creates /// `dest_dir`. A missing/unreadable `src_dir` is a no-op (returns the empty summary). Pure over the -/// two directories (no globals) so it's unit-testable. +/// two directories (no globals) so it is unit-testable. pub fn seed_assets_from(src_dir: &Path, dest_dir: &Path) -> std::io::Result { let mut summary = SeedSummary::default(); let Ok(entries) = std::fs::read_dir(src_dir) else { @@ -1801,7 +1801,7 @@ impl ExportRequest { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct StrConcordanceRow { pub marker: String, - /// Called FTDNA-convention value, or `None` if the marker wasn't called from sequence. + /// Called FTDNA-convention value, or `None` if the marker was not called from sequence. pub called: Option, /// Calibration status: `Reliable` | `ConventionOffset` | `Excluded` | `Uncalibrated` | `NotCalled`. pub status: String, @@ -1939,7 +1939,7 @@ fn collect_data_files(path: &Path, out: &mut Vec, depth: usize) { /// "this folder holds several samples" signal. A single sample's folder fans data into at most one /// subdirectory (e.g. FTDNA `//.bam` plus a top-level results CSV → just the kit /// dir); a *parent* of many per-sample folders spreads it across several. Files sitting directly in -/// `root` aren't counted (they belong to the picked folder itself). +/// `root` are not counted (they belong to the picked folder itself). fn contributing_subdirs(root: &std::path::Path, files: &[PathBuf]) -> std::collections::BTreeSet { use std::path::Component; let mut set = std::collections::BTreeSet::new(); @@ -1965,7 +1965,7 @@ pub struct DrainOutcome { pub published: Vec<(String, String)>, /// Rows that hit a non-transient error and were marked FAILED. pub failed: usize, - /// Whether a transient failure rescheduled a row (i.e. we're likely offline). + /// Whether a transient failure rescheduled a row (i.e. we are likely offline). pub retry_scheduled: usize, /// Rows still awaiting a successful push after this pass. pub pending: i64, @@ -2206,7 +2206,7 @@ fn peek_vcf_header(path: &Path) -> (String, Vec) { /// homozygous-reference (`GT 0/0`) — e.g. `chrY 2781955 C T … 0/0`, where the sample is C, not T. /// Taking `ALT[0]` blindly (as a sites-only VCF parser does) records that T as a derived call, and /// a Big Y export carries thousands of such reference sites → the placement deepens into branches -/// the sample doesn't actually carry. So when a genotyped sample column is present we read its `GT` +/// the sample does not actually carry. So when a genotyped sample column is present we read its `GT` /// and keep a single-base ALT only when the genotype selects it (the first non-zero allele, /// multi-allelic-aware); `0/0` and `./.` rows are dropped. A VCF with no FORMAT/sample column /// (a sites-only list) keeps its old meaning: every listed ALT is one of the subject's variants. @@ -2264,7 +2264,7 @@ fn parse_vcf_subject_snps(path: &Path) -> Result, App }; // Evidence the source supplies. Every field stays `None` when absent — a missing DP means - // "the vendor didn't say", and recording it as 0 would make a good call look unsupported. + // "the vendor did not say", and recording it as 0 would make a good call look unsupported. let ad: Option> = sample_field("AD").map(|v| v.split(',').map(|x| x.parse().unwrap_or(0)).collect()); let evidence = variants::CallEvidence { qual: f.get(5).and_then(|q| q.parse::().ok()), @@ -2422,7 +2422,7 @@ use navigator_domain::seq::complement_base; /// (`chrM`) is rCRS and stays a direct query (no chain), so it returns `None`. /// Whether a stored reference-build string denotes GRCh38 (the FTDNA Y tree's native coordinate /// space). `None` → assumed GRCh38 (the vendor-Y-VCF import default). Used by the FTDNA-provider Y -/// consensus to admit only GRCh38 vendor sets (others wouldn't match the GRCh38 tree positions). +/// consensus to admit only GRCh38 vendor sets (others would not match the GRCh38 tree positions). fn is_grch38_build(build: &Option) -> bool { match build { None => true, @@ -2646,7 +2646,7 @@ const OAUTH_SCOPE: &str = "atproto transition:generic"; /// Resolve Navigator's OAuth client config (pure). `DECODINGUS_OAUTH_CLIENT_ID` overrides the /// hosted default: the literal `loopback` selects the atproto dev loopback client (for logging in -/// against a local / test PDS that hasn't registered the production document); any other non-blank +/// against a local / test PDS that has not registered the production document); any other non-blank /// value is treated as a hosted client-metadata URL. fn resolve_oauth_config(env_client_id: Option) -> OAuthConfig { match env_client_id.map(|s| s.trim().to_string()).filter(|s| !s.is_empty()) { @@ -3021,7 +3021,7 @@ fn decodingus_build_key(reference_build: &str) -> Option<&'static str> { } /// Whether an alignment's reference build matches a GVCF name's build token (e.g. `chm13`), -/// compared on the canonical build so `chm13`/`chm13v2`/`hs1` all agree. A token that doesn't +/// compared on the canonical build so `chm13`/`chm13v2`/`hs1` all agree. A token that does not /// resolve to a known build is treated as a non-match (fall back to the first alignment). fn build_hint_matches(reference_build: &str, hint: &str) -> bool { match (canonical_build(reference_build), canonical_build(hint)) { @@ -3115,7 +3115,7 @@ fn archaic_panel_cache_kind() -> String { } /// Count of sites called (dosage within ploidy) in **both** samples — the effective IBD comparison -/// size, surfaced so a sparse chip↔chip / chip↔WGS overlap isn't mistaken for a confident result. +/// size, surfaced so a sparse chip↔chip / chip↔WGS overlap is not mistaken for a confident result. fn overlapping_called_sites(a: &[SiteGenotype], b: &[SiteGenotype]) -> usize { let called = |g: &SiteGenotype| (0..=g.ploidy as i32).contains(&g.dosage); let set: std::collections::HashSet<(&str, i64)> = a @@ -3386,7 +3386,7 @@ pub struct ProjectSampleReport { /// `false` when full (or no coverage yet). pub coverage_partial: bool, /// The last analysis attempt on the primary alignment failed (e.g. a corrupt/undecodable CRAM), - /// carrying the failure message. `Some` distinguishes a genuinely-failed sample from one that's + /// carrying the failure message. `Some` distinguishes a genuinely-failed sample from one that is /// merely un-analyzed, so the report shows "Failed" rather than a silent blank. pub decode_error: Option, } @@ -3478,7 +3478,7 @@ pub struct StrChartRow { /// Member's reached STR panel/tier (member rows only). pub test: String, /// Per-marker cells, aligned to [`ProjectStrChart::markers`]; empty for non-member rows that - /// don't fill every column. + /// do not fill every column. pub cells: Vec, } @@ -3493,7 +3493,7 @@ pub struct ProjectStrChart { pub group_count: usize, } -/// A reference build an import needs but doesn't have cached — surfaced so the UI can +/// A reference build an import needs but does not have cached — surfaced so the UI can /// prompt and download it before retrying. #[derive(Debug, Clone)] pub struct BuildNeed { @@ -3511,7 +3511,7 @@ pub struct AnalyzeSummary { pub y_done: usize, pub sex_done: usize, pub metrics_done: usize, - /// Per-sample failures (best-effort: one sample's error doesn't abort the rest). + /// Per-sample failures (best-effort: one sample's error does not abort the rest). pub errors: Vec, } @@ -3530,7 +3530,7 @@ pub struct SampleAnalyzeOutcome { } /// Outcome of a BISDNA chromo2 Y-SNP import: the variant set created plus a per-category -/// tally so the UI/CLI can surface coverage and any names the dictionary couldn't place. +/// tally so the UI/CLI can surface coverage and any names the dictionary could not place. #[derive(Debug, Clone)] pub struct BisdnaImportSummary { pub variant_set: VariantSet, @@ -3546,7 +3546,7 @@ pub struct BisdnaImportSummary { pub no_call: usize, /// Back-mutated markers — flagged and excluded from placement. pub back_mutated: usize, - /// Markers whose name was absent from the dictionary on this build (cannot be placed). + /// Markers whose name was absent from the dictionary on this build (can not be placed). pub unresolved: usize, /// A sample of unresolved names for diagnostics (capped). pub unresolved_names: Vec, @@ -3555,7 +3555,7 @@ pub struct BisdnaImportSummary { pub strand_mismatches: usize, } -/// Outcome of a batch project-directory import (idempotent — counts only what's new). +/// Outcome of a batch project-directory import (idempotent — counts only what is new). #[derive(Debug, Clone)] pub struct ProjectImportSummary { pub project: Project, @@ -3744,7 +3744,7 @@ mod placement_tests { assert_eq!(dirs.len(), 2); assert!(dirs.contains("42048") && dirs.contains("166433")); - // Files sitting directly in the picked folder don't count as a subdir. + // Files sitting directly in the picked folder do not count as a subdir. let flat = [PathBuf::from("/data/FTDNA/a.csv"), PathBuf::from("/data/FTDNA/b.bam")]; assert!(contributing_subdirs(root, &flat).is_empty()); } @@ -4115,7 +4115,7 @@ mod publish_tests { let value = app.sequence_run_record("did:plc:test", &reloaded).await.unwrap(); assert_eq!(value.get("instrumentId").and_then(|v| v.as_str()), Some("A00182")); // The known sequencing lab is published so the AppView can display it (its instrument→lab - // map doesn't cover every serial, e.g. PacBio). + // map does not cover every serial, e.g. PacBio). assert_eq!( value.get("sequencingFacility").and_then(|v| v.as_str()), Some("Dante Labs") @@ -4229,7 +4229,7 @@ mod ymatch_tests { seed_str(&app, q.guid, &[("DYS393", "13")]).await; // A subject with no STR / Y data at all. let _empty = app.add_biosample(None, "Empty", None, None).await.unwrap(); - // A subject whose markers don't overlap the query's. + // A subject whose markers do not overlap the query's. let other = app.add_biosample(None, "Other", None, None).await.unwrap(); seed_str(&app, other.guid, &[("DYS999", "10")]).await; @@ -4323,7 +4323,7 @@ mod ibd_attest_tests { ); } - /// Two peers computing the same summary produce the same agreement hash; different summaries don't. + /// Two peers computing the same summary produce the same agreement hash; different summaries do not. #[test] fn summary_hash_drives_agreement() { use navigator_analysis::ibd_attest::summary_hash; diff --git a/crates/navigator-app/src/llm.rs b/crates/navigator-app/src/llm.rs index 08c70e0c..c950e10f 100644 --- a/crates/navigator-app/src/llm.rs +++ b/crates/navigator-app/src/llm.rs @@ -294,7 +294,7 @@ impl App { /// prose to `on_chunk`. Grounded in only that signal's curated section (see /// [`navigator_domain::results_context::signal_section`]); cached and health-guarded like brief /// narration. `Err` when the assistant is off / unreachable / the subject has nothing for that - /// signal — the UI then just doesn't show an explanation. + /// signal — the UI then just does not show an explanation. pub async fn narrate_signal_streaming( &self, guid: SampleGuid, @@ -400,7 +400,7 @@ impl App { if !cfg.enabled { return Err(AppError::Llm("The AI assistant is turned off.".into())); } - // Incoming scope guard: don't even ask the model a medical question. + // Incoming scope guard: do not even ask the model a medical question. if llm_prompt::mentions_health(&question) { return Ok(llm_prompt::health_deflection().to_string()); } @@ -473,7 +473,7 @@ impl App { }; // Y-STR panels — name + marker count only (never the raw values: token cost, no answerable - // gain, and they're lineage patterns not facts to recite). + // gain, and they are lineage patterns not facts to recite). let ystr: Vec = self .list_str_profiles(guid) .await @@ -586,7 +586,7 @@ impl App { max_tokens: cfg.max_tokens, stream: true, // Grounded "explain my results" never needs chain-of-thought — disable it at the server - // so Gemma 4 et al. don't waste tokens/latency on a reasoning channel we'd discard. + // so Gemma 4 et al. do not waste tokens/latency on a reasoning channel we'd discard. chat_template_kwargs: Some(serde_json::json!({ "enable_thinking": false })), }; let url = format!("{}/chat/completions", cfg.base_url.trim_end_matches('/')); diff --git a/crates/navigator-app/src/maintenance.rs b/crates/navigator-app/src/maintenance.rs index 2939e9b4..3789c4fb 100644 --- a/crates/navigator-app/src/maintenance.rs +++ b/crates/navigator-app/src/maintenance.rs @@ -49,7 +49,7 @@ pub struct ChoreSurvey { /// Items it would consider — `due` of `total` is what makes "0 due" readable as "nothing to /// do" rather than "nothing found". pub total: usize, - /// Why the chore cannot run at all (not signed in, no tree). `Some` disables it. + /// Why the chore can not run at all (not signed in, no tree). `Some` disables it. pub blocked: Option, } diff --git a/crates/navigator-app/src/publish.rs b/crates/navigator-app/src/publish.rs index 4c3a713f..d26a7d4c 100644 --- a/crates/navigator-app/src/publish.rs +++ b/crates/navigator-app/src/publish.rs @@ -168,7 +168,7 @@ impl App { Utc::now().to_rfc3339(), ) // Publish the known lab so the AppView can display it (and learn the instrument→lab map — - // many serials, e.g. PacBio, aren't in its dataset). See [`SequenceRun::sequencing_facility`]. + // many serials, e.g. PacBio, are not in its dataset). See [`SequenceRun::sequencing_facility`]. .with_facility(run.sequencing_facility.clone()) // Exact sequenced yield + read chemistry back the standardized DTC test label the AppView // renders/groups by (`du_domain::testprofile`). Both `Option`al — older records omit them. @@ -207,7 +207,7 @@ impl App { // reference-build mismatch — e.g. a GRCh38 alignment, whose chrY reference is far noisier // and whose shared-lineage variants the hs1-native tree can't fully resolve), the whole // set is suspect. Publish nothing rather than flood curators with candidates from a sample - // we've already flagged; the variants still show in the in-app DISPLAY under the banner. + // we have already flagged; the variants still show in the in-app DISPLAY under the banner. if let Some(warn) = bucket.qc_banner() { eprintln!("private-variants publish skipped for alignment {alignment_id}: {warn}"); Vec::new() @@ -428,7 +428,7 @@ impl App { /// Fold a [`CoverageResult`]'s two per-contig views (samtools-style stats + /// callable-state counts) into the shared lexicon's `contigs[]`, paired by contig /// name — the same join `export::coverage_tsv` uses. Contigs present in the stats -/// but missing callable counts (shouldn't happen) fall back to zeros. +/// but missing callable counts (should not happen) fall back to zeros. fn contig_metrics(cov: &CoverageResult) -> Vec { cov.contig_coverage_stats .iter() diff --git a/crates/navigator-app/src/queries.rs b/crates/navigator-app/src/queries.rs index 99337b54..5c53c3c7 100644 --- a/crates/navigator-app/src/queries.rs +++ b/crates/navigator-app/src/queries.rs @@ -5,7 +5,7 @@ use super::*; /// Every analysis artifact of a set of alignments, pre-loaded and indexed for the report builders. /// /// Reading one cached result through [`App::load_analysis`] costs two queries — the artifact, then -/// the `alignment` row it needs in order to stat the BAM for staleness — plus that stat. A project +/// the `alignment` row it needs to stat the BAM for staleness — plus that stat. A project /// report reads five kinds per alignment for every member, so the per-cell form meant thousands of /// round-trips to open one tab. This loads them all in a single `IN` query and stats each BAM once. /// @@ -135,7 +135,7 @@ impl App { /// Batch-populate the read-profile fields backing the standardized test label /// ([`du_domain::testprofile`]) on runs imported before those fields existed — for the CLI - /// `backfill-profiles` command. Idempotent; only fills what's missing. + /// `backfill-profiles` command. Idempotent; only fills what is missing. /// /// - **`total_bases`** — recovered for free from a cached `read_metrics` artifact on any of the /// run's alignments (`Σ read_length_histogram`), no file walk. @@ -319,7 +319,7 @@ impl App { /// A specific persisted consensus ancestry estimate (keyed on the consensus pseudo-source + /// `method`) — e.g. `"FINE_ADMIXTURE"` (detailed modern populations) or `"PCA_PROJECTION_GMM"` - /// (ancient components). Filtered per-subject (alignment_id 0 isn't biosample-unique on its own). + /// (ancient components). Filtered per-subject (alignment_id 0 is not biosample-unique on its own). pub async fn consensus_ancestry( &self, biosample_guid: SampleGuid, @@ -574,7 +574,7 @@ impl App { median_insert_size: metrics.as_ref().map(|m| m.median_insert_size), sv_count, coverage_partial, - // Surface a persisted failure (corrupt/undecodable file) only when there's no + // Surface a persisted failure (corrupt/undecodable file) only when there is no // coverage to show — a successful re-walk clears the marker anyway. Read without a // freshness check, as `analysis_error` does: the marker stands until a success clears it. decode_error: match (coverage.is_none(), primary_alignment_id) { @@ -918,7 +918,7 @@ impl App { // Coverage + read-metrics + sex in ONE pass (the unified walker) instead of three separate // reads of the BAM/CRAM — a 3x I/O cut per subject, which dominates the batch on a slow / - // network volume (the single-subject Full Analysis already does this; the batch path didn't). + // network volume (the single-subject Full Analysis already does this; the batch path did not). // Walk only when something's missing: a full, correctly-scoped coverage (a stale whole-genome // result for a targeted-Y test is recomputed) plus cached read-metrics and sex = all done. let coverage_full = matches!( @@ -1096,7 +1096,7 @@ fn read_type_from_mean_len(mean: f64) -> Option<&'static str> { /// alignment (see [`App::default_alignment_for_subject`]). Higher is broader: a whole-genome test /// carries paternal, maternal *and* autosomal ancestry; an autosomal/chip test carries the ancestry /// composition the brief leads with; a Y/mt/X test is a single-lineage close-up. `None` is an -/// unrecognized test type — ranked above targeted (it may be a broad test whose label we didn't +/// unrecognized test type — ranked above targeted (it may be a broad test whose label we did not /// recognize) but below anything we know is genome-wide. fn test_breadth_rank(target: Option) -> u8 { use navigator_domain::testtype::TargetType::*; diff --git a/crates/navigator-app/src/realign.rs b/crates/navigator-app/src/realign.rs index 6c45c9ff..358c95da 100644 --- a/crates/navigator-app/src/realign.rs +++ b/crates/navigator-app/src/realign.rs @@ -14,7 +14,7 @@ //! //! ## The reference is part of the file //! -//! A CRAM cannot be read without the reference it was compressed against, so `reference_path` is +//! A CRAM can not be read without the reference it was compressed against, so `reference_path` is //! recorded on the row rather than resolved by convention later. An alignment whose reference has //! moved is unreadable, and the row is the only place that knows which one it was. @@ -43,7 +43,7 @@ impl App { /// required, because a CRAM without its reference is unreadable rather than merely awkward. /// /// `backend` and `preset` are taken separately rather than as a ready-made derivation string - /// so the recorded format stays authoritative here; a caller cannot invent its own spelling + /// so the recorded format stays authoritative here; a caller can not invent its own spelling /// that later queries then fail to recognise. pub async fn register_realigned_alignment( &self, @@ -212,7 +212,7 @@ pub fn is_target_build(build: &str) -> bool { /// the input is. /// /// Shared by the project-wide count and the Simple-mode single-subject offer, so that the two -/// cannot drift apart. If they disagreed, a user would be told a batch covers a sample it then +/// can not drift apart. If they disagreed, a user would be told a batch covers a sample it then /// silently skips — or be offered four hours of work that the job itself would refuse. pub(crate) fn realignable_for_subject(alignments: &[Alignment], target_build: &str) -> Vec { alignments diff --git a/crates/navigator-app/src/realign_job.rs b/crates/navigator-app/src/realign_job.rs index 649ca7aa..6834bb4e 100644 --- a/crates/navigator-app/src/realign_job.rs +++ b/crates/navigator-app/src/realign_job.rs @@ -100,7 +100,7 @@ pub struct RealignProgress { /// /// The counts are optional because a resumed job did not necessarily run the stage that produces /// them. A run that picks up from a previous attempt's sorted BAM never reverted anything, so it -/// cannot report how many unmapped reads that revert saw; [`ScratchState`] carries the figure +/// can not report how many unmapped reads that revert saw; [`ScratchState`] carries the figure /// across when the earlier attempt recorded it, and `None` says plainly that nobody measured it /// rather than reporting a zero that reads like a finding. #[derive(Debug, Clone)] @@ -128,7 +128,7 @@ pub struct RealignParams { /// Pick up from a previous attempt's intermediates instead of starting over. /// /// Off by default, because reusing files a *different* job left behind would be a correctness - /// bug, and the scratch path alone cannot prove they came from this source and this target. + /// bug, and the scratch path alone can not prove they came from this source and this target. /// The caller opting in is what supplies that knowledge. See [`Resumed`]. pub resume: bool, } @@ -161,7 +161,7 @@ impl Resumed { } } -/// Counts a resumed job cannot re-derive, left beside the intermediates they describe. +/// Counts a resumed job can not re-derive, left beside the intermediates they describe. /// /// Each stage's contribution to [`RealignOutcome`] is measured while that stage runs and is gone /// once it has. A resumed job skips stages by design, so the numbers are written down as they are @@ -175,7 +175,7 @@ struct ScratchState { duplicates_marked: Option, /// The furthest stage that *returned successfully*, as opposed to merely leaving a file behind. /// - /// Written after the stage returns, never before, so it says something the file itself cannot: + /// Written after the stage returns, never before, so it says something the file itself can not: /// see [`discard_partial`] for why a finished-looking BAM is not proof on its own. A scratch /// directory predating this field has `None`, and then the marker is all there is to go on. completed_through: Option, @@ -200,7 +200,7 @@ impl ScratchState { /// Remove a stage's output when the stage did not finish. /// -/// The BGZF end-of-file marker cannot carry the whole weight of "this file is complete", and +/// The BGZF end-of-file marker can not carry the whole weight of "this file is complete", and /// finding that out the hard way is what this function exists to prevent. noodles' multithreaded /// writer finishes its stream from `Drop`, so a stage that *unwinds* — cancelled, failed, panicked /// — leaves a partial file wearing a finished file's marker. Measured: a merge cancelled at 13.2 GB @@ -232,7 +232,7 @@ async fn stage(output: &Path, work: impl std::future::Future Resumed { let by_marker = resumable_by_marker(scratch); @@ -257,7 +257,7 @@ fn resumable_by_marker(scratch: &Path) -> Resumed { /// Clear a stage's working directory before that stage runs. /// -/// A resumed job re-runs the first stage it cannot skip, and that stage's leftovers from the +/// A resumed job re-runs the first stage it can not skip, and that stage's leftovers from the /// killed attempt are pure cost: the sort ignores run files it did not write itself, so stale ones /// are not a correctness problem, but at WGS scale they are tens of GB held against a disk that /// the same job is about to need. Best-effort — a directory that will not clear is not a reason to @@ -646,7 +646,7 @@ fn resume_preflight(scratch: &Path, mapped: &Path, sorted: &Path, marked: &Path) plan_for(scratch, largest.saturating_mul(3), "resume the realignment") } -/// Measure the disk, refuse a job that cannot finish on it, and describe what was decided. +/// Measure the disk, refuse a job that can not finish on it, and describe what was decided. /// /// The two preflights differ only in how they size `needed`; everything after that — probing free /// space, the refusal, the wording, the plan — was written out twice and had to be kept in step by @@ -696,7 +696,7 @@ fn has_room(needed: u64, free: u64) -> bool { free == 0 || free >= needed } -/// Free bytes on the filesystem holding `path`, or 0 when it cannot be determined. +/// Free bytes on the filesystem holding `path`, or 0 when it can not be determined. /// /// Zero means "unknown", and preflight treats it as "do not block" — refusing a job because the /// free-space call failed would be worse than letting it run and fail honestly on a real write. @@ -856,11 +856,11 @@ mod tests { } } - /// Preflight must refuse a job that cannot finish, and say how much is needed rather than + /// Preflight must refuse a job that can not finish, and say how much is needed rather than /// leaving the user to guess. /// /// Runs wherever [`fs_free_space`] has a real implementation, which is now both desktop - /// families. Platforms without one report "unknown" and cannot refuse anything — pinned + /// families. Platforms without one report "unknown" and can not refuse anything — pinned /// separately by `preflight_cannot_refuse_where_free_space_is_unknown`. #[cfg(any(unix, windows))] #[test] @@ -926,7 +926,7 @@ mod tests { /// What the platforms without a free-space probe actually do, stated as a test rather than left /// as the absence of one. /// - /// `preflight` cannot refuse a job it has no measurement for, and refusing on an unknown would + /// `preflight` can not refuse a job it has no measurement for, and refusing on an unknown would /// block every realignment on that platform. So a job that would obviously not fit is allowed /// to start and fail honestly on a real write. Windows used to be in this bucket; it now has /// `GetDiskFreeSpaceExW` and is tested by the two above. @@ -1017,7 +1017,7 @@ mod resume_tests { ); } - /// The record cannot promise more than the files deliver either — a scratch directory whose + /// The record can not promise more than the files deliver either — a scratch directory whose /// `marked.bam` was removed must not be resumed from just because a stale record mentions it. #[test] fn the_record_cannot_outrun_the_files() { diff --git a/crates/navigator-app/tests/app.rs b/crates/navigator-app/tests/app.rs index ecf5be19..6372f640 100644 --- a/crates/navigator-app/tests/app.rs +++ b/crates/navigator-app/tests/app.rs @@ -26,7 +26,7 @@ fn tests_never_touch_the_os_keychain() { /// Serializes tests that mutate the process-global `NAVIGATOR_TREE_DIR`: one test's `remove_var` /// would otherwise yank the seeded tree dir out from under another running concurrently. Held for -/// the whole test body; ignores poisoning so a panicking test doesn't wedge the rest. +/// the whole test body; ignores poisoning so a panicking test does not wedge the rest. static TREE_DIR_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Reuse the analysis crate's committed fixtures (workspace-relative). @@ -35,7 +35,7 @@ fn fixtures() -> PathBuf { } /// Serializes the `NAVIGATOR_REFGENOME_DIR` env write (read once in `App::new`) so -/// parallel tests pointing the gateway cache at different temp dirs don't race. +/// parallel tests pointing the gateway cache at different temp dirs do not race. static REF_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// An `App` whose reference-gateway cache is `cache`. The store is opened first (async), then @@ -185,7 +185,7 @@ async fn import_mtdna_fasta_derives_variants() { app.import_mtdna_from_fasta(subject.guid, &path).await.unwrap(); // The import derived + persisted an rCRS-relative variant set (haplogroup placement needs the - // network, so it's best-effort and not asserted here). + // network, so it is best-effort and not asserted here). let sets = app.list_variant_sets(subject.guid).await.unwrap(); let mt = sets .iter() @@ -620,7 +620,7 @@ async fn validate_gfx_decodingus_y() { eprintln!("set GFX_CHM13_BAM + GFX_CHM13_REF (+ DECODINGUS_APPVIEW_URL) to run this"); return; }; - // Force the DecodingUs provider (it's the default, but be explicit); host from env or :9000. + // Force the DecodingUs provider (it is the default, but be explicit); host from env or :9000. std::env::set_var("NAVIGATOR_Y_TREE_PROVIDER", "decodingus"); let app = app().await; @@ -1068,7 +1068,7 @@ async fn add_data_detects_and_routes() { assert_eq!(app.list_chip_profiles(subject.guid).await.unwrap().len(), 1); // A BAM/CRAM auto-imports: it creates a sequencing run + alignment (header probed - // best-effort; here the bytes aren't a real BAM so detection falls back to defaults). + // best-effort; here the bytes are not a real BAM so detection falls back to defaults). let bam = dir.join(format!("data-{}.bam", subject.guid.0)); std::fs::write(&bam, b"\x1f\x8b").unwrap(); assert_eq!(app.add_data(subject.guid, &bam).await.unwrap(), DetectedData::Alignment); @@ -1077,9 +1077,9 @@ async fn add_data_detects_and_routes() { let alns = app.list_alignments(runs[0].id).await.unwrap(); assert_eq!(alns.len(), 1); // The content hash is deferred (not computed at import) so a multi-GB alignment imports - // instantly; it's filled in lazily on the first analysis that needs it. + // instantly; it is filled in lazily on the first analysis that needs it. assert_eq!(alns[0].content_sha256, None, "content hash is deferred at import"); - // Idempotent: re-adding the same path doesn't duplicate the run/alignment. + // Idempotent: re-adding the same path does not duplicate the run/alignment. assert_eq!(app.add_data(subject.guid, &bam).await.unwrap(), DetectedData::Alignment); assert_eq!(app.list_sequence_runs(subject.guid).await.unwrap().len(), 1); @@ -1266,7 +1266,7 @@ async fn run_coverage_persists_and_reads_back_from_cache() { assert_eq!(result.callable_bases, 10); // now cached for this version (integer fields exact; floats survive round-trip to - // ~1 ULP, so compare those approximately rather than with fragile float ==) + // ~1 ULP, so compare those about rather than with fragile float ==) let cached = app.cached_coverage(aln).await.unwrap().unwrap(); assert_eq!(cached.genome_territory, result.genome_territory); assert_eq!(cached.callable_bases, result.callable_bases); @@ -1566,7 +1566,7 @@ async fn reimport_under_different_project_name_reuses_subject() { ); assert_eq!(s2.samples_created, 0, "the subject is reused, not duplicated"); - // Exactly one subject in the workspace, and it's a roster member of BOTH projects. + // Exactly one subject in the workspace, and it is a roster member of BOTH projects. assert_eq!(app.list_all_biosamples().await.unwrap().len(), 1); assert_eq!(app.list_biosamples(s1.project.id).await.unwrap().len(), 1); assert_eq!(app.list_biosamples(s2.project.id).await.unwrap().len(), 1); @@ -2107,7 +2107,7 @@ async fn sex_and_read_metrics_persist_and_reload() { } /// Live: GFX0457637 carries a Y haplogroup (R-FGC29071), so sex inference should call Male. -/// Uses the BAI fast-path, so it's quick. Requires GFX_CHM13_BAM. +/// Uses the BAI fast-path, so it is quick. Requires GFX_CHM13_BAM. #[tokio::test] #[ignore = "requires GFX_CHM13_BAM"] async fn gfx_sex_is_male() { @@ -2425,7 +2425,7 @@ async fn ftdna_matches_existing_subject_by_ystr_distance() { .await .unwrap(); - // Plan with roster + Y-STR. B5163's SNP terminal won't match the ISOGG label, but the Y-STR + // Plan with roster + Y-STR. B5163's SNP terminal will not match the ISOGG label, but the Y-STR // genetic distance (GD 0) must surface KANE-0001 as a candidate. let plan = app .plan_ftdna_import( @@ -2457,7 +2457,7 @@ async fn ftdna_matches_existing_subject_by_ystr_distance() { // Commit the merge into the (new) project. KANE-0001 has NO home project (`project_id` is NULL) — // the merge adds an M:N membership row only. The project report must still surface it (regression - // for "matched samples don't appear in the Project report" — it reads membership ∪ home column). + // for "matched samples do not appear in the Project report" — it reads membership ∪ home column). let mut res = std::collections::BTreeMap::new(); res.insert("B5163".to_string(), navigator_app::FtdnaResolution::Merge(kane.guid)); let summary = app.commit_ftdna_import(&plan, &res).await.unwrap(); @@ -2481,7 +2481,7 @@ async fn ftdna_matches_existing_subject_by_ystr_distance() { } /// Deleting a sequencing run purges the haplogroup calls + consensus placement derived from its -/// alignments, so a wrong haplogroup doesn't linger after the run is removed. +/// alignments, so a wrong haplogroup does not linger after the run is removed. #[tokio::test] async fn deleting_run_purges_derived_haplogroup_and_consensus() { use navigator_app::DnaType; @@ -2627,7 +2627,7 @@ async fn branch_report_genotypes_the_mt_subtree_end_to_end() { let _ = std::fs::remove_dir_all(&trees); } -/// `pick_mt_alignment` skips a Y-only run so the mtDNA report isn't genotyped against an +/// `pick_mt_alignment` skips a Y-only run so the mtDNA report is not genotyped against an /// alignment that carries no chrM reads, while `pick_y_debug_alignment` still targets it. #[tokio::test] async fn mt_alignment_pick_skips_a_y_only_run() { @@ -2639,7 +2639,7 @@ async fn mt_alignment_pick_skips_a_y_only_run() { .await .unwrap(); - // A Big-Y (Y-only) run, recorded first so it's a candidate for both pickers. + // A Big-Y (Y-only) run, recorded first so it is a candidate for both pickers. let y_run = app .record_sequence_run(NewSequenceRun::new(b.guid, "ILLUMINA", "BIG_Y_700")) .await @@ -2763,7 +2763,7 @@ async fn add_sample_dir_falls_back_to_per_file_for_a_loose_bundle() { #[tokio::test] async fn add_sample_dir_skips_called_vcf_when_gvcf_present() { // GATK repo layout: a bare `chrY.g.vcf.gz` (the Y source → fast path) sits beside the called - // `chrY.vcf.gz`. With the GVCF present the called VCF must NOT be imported — it's redundant and + // `chrY.vcf.gz`. With the GVCF present the called VCF must NOT be imported — it is redundant and // (variant-set import not being content-idempotent) would duplicate on a resumable re-run. The // GVCF is a stub here, so the placement itself is a best-effort no-op; this pins the routing. let app = app().await; @@ -2940,7 +2940,7 @@ async fn maintenance_survey_reports_every_chore() { let app = app().await; let survey = app.maintenance_survey().await.expect("survey"); - // Every chore is present, in a fixed order, so the panel cannot silently lose one. + // Every chore is present, in a fixed order, so the panel can not silently lose one. let chores: Vec<_> = survey.iter().map(|s| s.chore).collect(); assert_eq!(chores, navigator_app::Chore::ALL.to_vec()); @@ -2948,7 +2948,7 @@ async fn maintenance_survey_reports_every_chore() { assert!(s.due <= s.total || s.total == 0, "{:?}: due exceeds total", s.chore); } - // Nothing to publish, and no account to publish with — the chore reports *why* it cannot run + // Nothing to publish, and no account to publish with — the chore reports *why* it can not run // rather than offering a button that would fail. let publish = survey .iter() diff --git a/crates/navigator-app/tests/mastervar_autosomal_real.rs b/crates/navigator-app/tests/mastervar_autosomal_real.rs index 5578d10f..8a63cec0 100644 --- a/crates/navigator-app/tests/mastervar_autosomal_real.rs +++ b/crates/navigator-app/tests/mastervar_autosomal_real.rs @@ -83,7 +83,7 @@ async fn mastervar_feeds_autosomal_and_ancestry() { } } } - // Don't fail the whole check if an ancestry asset is absent — the autosomal consensus (the + // Do not fail the whole check if an ancestry asset is absent — the autosomal consensus (the // thing this PR wires up) already proved the masterVar feeds the pipeline. Err(e) => println!("[{:>7.1?}] ancestry skipped: {e}", t.elapsed()), } diff --git a/crates/navigator-domain/src/ancestry.rs b/crates/navigator-domain/src/ancestry.rs index 5407304f..eae9760c 100644 --- a/crates/navigator-domain/src/ancestry.rs +++ b/crates/navigator-domain/src/ancestry.rs @@ -5,7 +5,7 @@ //! Phase 1 works at **super-population** granularity (AFR/AMR/EAS/EUR/SAS), the resolution //! the 1000G-on-CHM13 INFO allele counts give us directly. The fine-grained 26/33-population //! catalog (and PCA coordinates) is deferred to phase 2 — the `pca_coordinates` field is -//! already carried so the result shape doesn't change when PCA lands. +//! already carried so the result shape does not change when PCA lands. use serde::{Deserialize, Serialize}; @@ -26,7 +26,7 @@ pub fn super_populations() -> Vec { ("EAS", "East Asian", "#00CC00"), ("EUR", "European", "#0066CC"), ("SAS", "South Asian", "#9900CC"), - // Added with the SGDP diversity panel (continents 1000G doesn't cover). + // Added with the SGDP diversity panel (continents 1000G does not cover). ("MEA", "Middle Eastern", "#996633"), ("CAS", "Central Asian & Siberian", "#66CCCC"), ("OCE", "Oceanian", "#009999"), diff --git a/crates/navigator-domain/src/bisdna.rs b/crates/navigator-domain/src/bisdna.rs index 959250f6..dcfa010c 100644 --- a/crates/navigator-domain/src/bisdna.rs +++ b/crates/navigator-domain/src/bisdna.rs @@ -145,7 +145,7 @@ pub struct ResolveOutcome { pub no_call: usize, /// Back-mutated markers — flagged, excluded from placement. pub back_mutated: usize, - /// Positive markers whose name the dictionary couldn't place on this build. + /// Positive markers whose name the dictionary could not place on this build. pub unresolved: usize, /// A capped sample of unresolved names (for diagnostics). pub unresolved_names: Vec, @@ -336,7 +336,7 @@ S163\ths1\tchrY\t15000000\t+\tA\tC let calls = parse(SAMPLE).unwrap(); // Apt-, CTS10003-, CTS10149+, CTS12633+, CTS3281 no_call, S163 (positive) let out = resolve_calls(&calls, &dict(), "hs1", 10); - // Apt + CTS10003 are negative → counted, not emitted; CTS10003 also isn't in the dict. + // Apt + CTS10003 are negative → counted, not emitted; CTS10003 also is not in the dict. assert_eq!(out.ancestral, 2); assert_eq!(out.no_call, 1); // CTS3281 // Three positives (CTS10149, CTS12633, S163) are all in the dict → three calls. @@ -351,7 +351,7 @@ S163\ths1\tchrY\t15000000\t+\tA\tC #[test] fn unresolved_positive_is_tallied_not_emitted() { - // A positive whose name isn't in the dictionary. + // A positive whose name is not in the dictionary. let f = "SNPID\tgenotype\tresult\nUNKNOWNSNP\tGG\tpositive\nCTS10149\tGG\tpositive\n"; let out = resolve_calls(&parse(f).unwrap(), &dict(), "hs1", 10); assert_eq!(out.calls.len(), 1); diff --git a/crates/navigator-domain/src/brief.rs b/crates/navigator-domain/src/brief.rs index fb780760..3ce6d1f4 100644 --- a/crates/navigator-domain/src/brief.rs +++ b/crates/navigator-domain/src/brief.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; // Reference pack (narrative content) // --------------------------------------------------------------------------------------------- -/// One haplogroup's narrative content: when it formed, where it's associated with, and a short +/// One haplogroup's narrative content: when it formed, where it is associated with, and a short /// curated story. Every field is optional so a sparse pack still contributes what it has. #[derive(Debug, Clone, Default, PartialEq, Deserialize, Serialize)] pub struct HaploEntry { @@ -286,7 +286,7 @@ pub struct SubjectBrief { #[serde(default)] pub archaic: Option, pub test: TestBrief, - /// True when the subject has a sequencing alignment that hasn't been analyzed yet (data present, + /// True when the subject has a sequencing alignment that has not been analyzed yet (data present, /// no coverage computed) — the signal for the Simple-mode one-click "Analyze" prompt. False for /// an already-analyzed subject or one with no alignment (chip/VCF-only, nothing to analyze). #[serde(default)] @@ -327,7 +327,7 @@ fn group_thousands(n: i64) -> String { } } -/// Round an age to a friendly magnitude so a precise estimate doesn't read as false precision +/// Round an age to a friendly magnitude so a precise estimate does not read as false precision /// (4,237 → "about 4,200"; 63,500 → "about 64,000"; 850 → "about 850"). fn round_age(ybp: i32) -> i64 { let y = ybp.max(0) as i64; @@ -383,7 +383,7 @@ pub fn confidence_phrase(lang: Lang, confidence: f64, run_count: usize, conflict /// Sequencing-depth quality, gated by what the test targets. Returns the phrase and an ok flag /// (drives a ✓/⚠ chip). A targeted test (Y/mt) is judged on its own target depth, which is much -/// higher than a WGS average, so the WGS thresholds don't apply. +/// higher than a WGS average, so the WGS thresholds do not apply. pub fn quality_phrase(lang: Lang, mean_coverage: f64, target: TargetType) -> (String, bool) { let (label_key, ok) = match target { // Whole-genome / autosomal / exome: judged on genome-wide average depth. diff --git a/crates/navigator-domain/src/chipprofile.rs b/crates/navigator-domain/src/chipprofile.rs index 2a5827ed..7dee323c 100644 --- a/crates/navigator-domain/src/chipprofile.rs +++ b/crates/navigator-domain/src/chipprofile.rs @@ -1,6 +1,6 @@ //! Genotyping-array (chip) profiles — the QC summary of a vendor raw-data export //! (23andMe, AncestryDNA, MyHeritage, …), a pragmatic port of the Scala `ChipProfile`. -//! We don't keep every genotype (a chip is ~600–700k markers); we keep the call/no-call/ +//! We do not keep every genotype (a chip is ~600–700k markers); we keep the call/no-call/ //! het summary and per-region counts that drive quality and downstream eligibility. //! [`summarize`] is a pure pass over the file text (no IO) that also guesses the vendor. @@ -223,8 +223,8 @@ pub struct ChipHaploCall { pub base: char, } -/// The single haploid base of a genotype token, or `None` if it's a no-call, an indel -/// (`I`/`D`), or heterozygous (two different bases — on a true haploid Y/MT that's +/// The single haploid base of a genotype token, or `None` if it is a no-call, an indel +/// (`I`/`D`), or heterozygous (two different bases — on a true haploid Y/MT that is /// contamination, so we drop it rather than guess). fn haploid_base(genotype: &str) -> Option { let mut bases = genotype @@ -337,7 +337,7 @@ pub fn autosomal_calls(text: &str) -> Vec { } /// The reference build a vendor export is reported on. Consumer arrays (23andMe v4/v5, -/// AncestryDNA v1/v2) are GRCh37, so that's the default; a header naming build 38 / GRCh38 / +/// AncestryDNA v1/v2) are GRCh37, so that is the default; a header naming build 38 / GRCh38 / /// hg38 overrides it. Scans only the comment header. pub fn detect_build(text: &str) -> String { for raw in text.lines() { diff --git a/crates/navigator-domain/src/consensus.rs b/crates/navigator-domain/src/consensus.rs index d34f6d42..3e7d324c 100644 --- a/crates/navigator-domain/src/consensus.rs +++ b/crates/navigator-domain/src/consensus.rs @@ -204,7 +204,7 @@ pub struct ObservedProfile { } /// One source's call at a variant, fed into [`reconcile`]. Quality fields refine the concordance -/// weight (see [`obs_weight`]); sources that don't carry them (chip, tree placement) leave them +/// weight (see [`obs_weight`]); sources that do not carry them (chip, tree placement) leave them /// `None` / `1.0` and fall back to the plain source-type weight. #[derive(Debug, Clone, PartialEq)] pub struct ConsensusObs { @@ -672,7 +672,7 @@ pub fn summarize(variants: &[ConsensusVariant]) -> ConsensusSummary { ConsensusStatus::Novel => s.novel += 1, ConsensusStatus::Conflict => s.conflict += 1, ConsensusStatus::SingleSource => s.single_source += 1, - // Pending / NoCoverage aren't headline counts; they fold into `total` only. + // Pending / NoCoverage are not headline counts; they fold into `total` only. ConsensusStatus::Pending | ConsensusStatus::NoCoverage => {} } } @@ -850,7 +850,7 @@ pub fn reconcile_diploid(sources: &[(String, SourceType, Vec)]) -> V out } -/// Per-status counts + overall confidence over a reconciled diploid variant list. `Novel` doesn't +/// Per-status counts + overall confidence over a reconciled diploid variant list. `Novel` does not /// apply to autosomal sites, so the confidence is `(confirmed − 0.5·conflict) / total`. pub fn summarize_diploid(variants: &[DiploidVariant]) -> ConsensusSummary { let mut s = ConsensusSummary { diff --git a/crates/navigator-domain/src/filetype.rs b/crates/navigator-domain/src/filetype.rs index f947f377..0ba09fbb 100644 --- a/crates/navigator-domain/src/filetype.rs +++ b/crates/navigator-domain/src/filetype.rs @@ -157,7 +157,7 @@ pub fn detect(file_name: &str, head: &str) -> DetectedData { /// chrY/chrM product never does. /// /// Returns `false` when there is **no contig evidence at all** (an empty or unreadable head). Absence -/// of evidence is not evidence of absence: a `.g.vcf` we couldn't read should keep the claim its +/// of evidence is not evidence of absence: a `.g.vcf` we could not read should keep the claim its /// extension makes rather than be demoted on a guess. /// /// This is the guard that keeps a haploid-lineage call set off the autosomal panel pipeline. @@ -256,7 +256,7 @@ fn count_token(haystack: &str, prefix: &str, min_digits: usize, max_digits: usiz /// Recognize a named Y-SNP panel (BISDNA chromo2): either the exact /// `SNPIDgenotyperesult` header, or — lacking it — several tab rows whose third /// column is a positive/negative/no_call/back-mutated verdict. Tolerant of the multi-line -/// prose preamble BISDNA prepends (those lines aren't tab-delimited and never match). +/// prose preamble BISDNA prepends (those lines are not tab-delimited and never match). fn looks_like_ysnp_panel(lines: &[&str]) -> bool { let is_verdict = |s: &str| { let v = s.trim().trim_matches(|c| c == '"').to_ascii_lowercase(); @@ -497,7 +497,7 @@ chr1\t246193\trs3094315\tG\tA\t225\t.\tDP=29\tGT:DP\t0/0:29 #[test] fn an_unreadable_head_keeps_the_gvcf_extensions_claim() { - // No contig evidence is not evidence of no autosomes — don't demote on a guess. + // No contig evidence is not evidence of no autosomes — do not demote on a guess. assert_eq!(detect("sample.g.vcf.gz", ""), DetectedData::GvcfCallSet); } @@ -527,7 +527,7 @@ chr1\t246193\trs3094315\tG\tA\t225\t.\tDP=29\tGT:DP\t0/0:29 #TYPE\tVAR-ANNOTATION\n\ >locus\tploidy\tallele\tchromosome\tbegin\tend\tvarType\treference\talleleSeq\tvarScoreVAF\tvarScoreEAF\tvarQuality\thapLink\txRef\n\ 1\t2\tall\tchr1\t0\t10000\tno-ref\t=\t?\t\t\t\t\t\n"; - // Both the raw name and a `.tsv.bz2` (extension isn't consulted for this format) detect. + // Both the raw name and a `.tsv.bz2` (extension is not consulted for this format) detect. assert_eq!( detect("var-GS00253-DNA_A01_200_37-ASM.tsv", head), DetectedData::CompleteGenomicsVar diff --git a/crates/navigator-domain/src/ftdna.rs b/crates/navigator-domain/src/ftdna.rs index 44bd3250..18735cab 100644 --- a/crates/navigator-domain/src/ftdna.rs +++ b/crates/navigator-domain/src/ftdna.rs @@ -54,7 +54,7 @@ pub enum FtdnaFileKind { YdnaOverview, } -/// Classify an FTDNA export from its header row. `None` if it doesn't look like one of ours. +/// Classify an FTDNA export from its header row. `None` if it does not look like one of ours. /// Disambiguators: the marker block (`DYS…`) is unique to the Y-STR overview; the roster has the /// `Publicly Share DNA Results` consent column; the ancestry files use `Sub Group` (with a space) /// and a `Paternal`/`Maternal Ancestor Name`. @@ -292,7 +292,7 @@ fn parse_ancestor_name(raw: &str) -> (Option, Option, Option) } /// Byte offset of a `b.`/`d.` date marker, requiring a word boundary before it (so the `b` in -/// "Abbett" doesn't match). Returns the offset of the marker letter. +/// "Abbett" does not match). Returns the offset of the marker letter. fn find_marker(lower: &str, marker: &str) -> Option { let bytes = lower.as_bytes(); let mut from = 0; diff --git a/crates/navigator-domain/src/ftdna_csv.rs b/crates/navigator-domain/src/ftdna_csv.rs index 6c6d0981..ac2f8e6d 100644 --- a/crates/navigator-domain/src/ftdna_csv.rs +++ b/crates/navigator-domain/src/ftdna_csv.rs @@ -43,7 +43,7 @@ fn cells(line: &str) -> Vec { .collect() } -/// Recognize the report flavor from a header row's columns, or `None` if it isn't an FTDNA Big Y +/// Recognize the report flavor from a header row's columns, or `None` if it is not an FTDNA Big Y /// Named/Private Variants header. Case-insensitive; quotes already stripped by [`cells`]. pub fn report_of_header(cols: &[String]) -> Option { let norm: Vec = cols.iter().map(|c| c.to_ascii_lowercase()).collect(); @@ -67,7 +67,7 @@ pub fn looks_like_ftdna_variant_csv(text: &str) -> bool { } /// Parse an FTDNA Big Y Named/Private Variants CSV into chrY derived-allele SNP calls, returning -/// the report flavor alongside. Errors if the header isn't a recognized FTDNA report or no SNP +/// the report flavor alongside. Errors if the header is not a recognized FTDNA report or no SNP /// rows parse. pub fn parse(text: &str) -> Result<(FtdnaReport, Vec), String> { let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty()); diff --git a/crates/navigator-domain/src/i18n.rs b/crates/navigator-domain/src/i18n.rs index 12a36e05..052ffb9f 100644 --- a/crates/navigator-domain/src/i18n.rs +++ b/crates/navigator-domain/src/i18n.rs @@ -148,7 +148,7 @@ mod tests { fn translates_and_falls_back() { assert_eq!(tr(Lang::En, "nav.subjects"), "Subjects"); assert_eq!(tr(Lang::Es, "nav.subjects"), "Sujetos"); - // Missing in Es → English fallback (assuming this key isn't translated). + // Missing in Es → English fallback (assuming this key is not translated). assert_eq!(tr(Lang::Es, "status.label"), tr(Lang::Es, "status.label")); // Unknown key → the key itself. assert_eq!(tr(Lang::En, "totally.unknown.key"), "totally.unknown.key"); diff --git a/crates/navigator-domain/src/identity.rs b/crates/navigator-domain/src/identity.rs index 5603f5c9..005362eb 100644 --- a/crates/navigator-domain/src/identity.rs +++ b/crates/navigator-domain/src/identity.rs @@ -118,7 +118,7 @@ fn is_hgdp_name(s: &str) -> bool { matches!(rest, Some(r) if !r.is_empty() && r.bytes().all(|b| b.is_ascii_digit())) } -/// The INSDC **sample**-accession namespace for `acc`, if it's a real one (not a friendly name): +/// The INSDC **sample**-accession namespace for `acc`, if it is a real one (not a friendly name): /// `SAM*` → BIOSAMPLE, `ERS…` → ENA, `SRS…` → SRA. `None` for anything else (a plain friendly name). /// Used both by [`catalog_ids_from_provenance`] and by the API-driven accession backfill. pub fn insdc_sample_namespace(acc: &str) -> Option<&'static str> { @@ -138,7 +138,7 @@ pub fn insdc_sample_namespace(acc: &str) -> Option<&'static str> { } } -/// FTDNA-reported member labels only (the batch-file metadata we don't otherwise model). Computed +/// FTDNA-reported member labels only (the batch-file metadata we do not otherwise model). Computed /// haplogroups stay in the haplogroup-call store — different provenance (design §4.2). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FtdnaMember { diff --git a/crates/navigator-domain/src/llm_prompt.rs b/crates/navigator-domain/src/llm_prompt.rs index fa82aa27..b3045ce4 100644 --- a/crates/navigator-domain/src/llm_prompt.rs +++ b/crates/navigator-domain/src/llm_prompt.rs @@ -10,7 +10,7 @@ use crate::brief::{LineageBrief, SubjectBrief}; /// guardrails have a single reviewed source. It carries the facts-only / no-new-claims / no-health / /// preserve-uncertainty rules — but **no output-format** rules (those differ: narration writes a /// story, Q&A answers a question). The explicit "no medical disclaimers" clause matters: a model -/// that volunteers a "this isn't medical advice" hedge trips the post-generation [`mentions_health`] +/// that volunteers a "this is not medical advice" hedge trips the post-generation [`mentions_health`] /// guard and gets its otherwise-fine answer replaced by the deflection. fn grounding_rules() -> String { "Stay grounded in the facts given in the user message. You may interpret, connect, and add \ diff --git a/crates/navigator-domain/src/paths.rs b/crates/navigator-domain/src/paths.rs index e3c3b0f5..8468cf2d 100644 --- a/crates/navigator-domain/src/paths.rs +++ b/crates/navigator-domain/src/paths.rs @@ -14,7 +14,7 @@ use std::ffi::OsString; use std::path::PathBuf; -/// The current user's home directory, or `None` if the platform's variables don't say. +/// The current user's home directory, or `None` if the platform's variables do not say. /// /// Unix: `$HOME`. Windows: `%USERPROFILE%`, else `%HOMEDRIVE%%HOMEPATH%`. pub fn home_dir() -> Option { diff --git a/crates/navigator-domain/src/results_context.rs b/crates/navigator-domain/src/results_context.rs index a2e1fc77..67c72a50 100644 --- a/crates/navigator-domain/src/results_context.rs +++ b/crates/navigator-domain/src/results_context.rs @@ -90,7 +90,7 @@ pub struct IbdFact { /// The brief plus curated summaries of the other signals — the grounding context for the M4 chat. /// Absent signals are `None` / empty and are simply omitted from the fact sheet (so the model can't -/// restate what isn't there), exactly like the brief's own optional sections. +/// restate what is not there), exactly like the brief's own optional sections. #[derive(Debug, Clone, PartialEq)] pub struct ResultsContext { pub brief: SubjectBrief, @@ -300,7 +300,7 @@ pub fn results_fact_sheet(ctx: &ResultsContext) -> String { mt_section(&ctx.mt_mutations), ibd_section(&ctx.ibd), // ROH already reaches the sheet via narrate_fact_sheet (it lives on the brief); the archaic - // block does not, so add it explicitly or the chat cannot answer about it. + // block does not, so add it explicitly or the chat can not answer about it. archaic_section(&ctx.brief), ] .into_iter() @@ -452,7 +452,7 @@ mod tests { assert!(section.contains("longest 7 Mb")); assert!(!mentions_health(§ion), "ROH must not read as a health result"); - // Absent when ROH hasn't been computed. + // Absent when ROH has not been computed. ctx.brief.roh = None; assert!(signal_section(&ctx, SignalKind::Roh).is_none()); } diff --git a/crates/navigator-domain/src/roh.rs b/crates/navigator-domain/src/roh.rs index 612af5b7..f172f633 100644 --- a/crates/navigator-domain/src/roh.rs +++ b/crates/navigator-domain/src/roh.rs @@ -1,6 +1,6 @@ //! Runs-of-homozygosity domain types. //! -//! The pattern read is a *classification*, not a rendering: it's computed once by +//! The pattern read is a *classification*, not a rendering: it is computed once by //! `navigator_analysis::roh` (which re-exports this enum) and then consumed by both the Advanced ROH //! chart and the Simple-mode brief. It lives here, below the analysis engine, so the brief builder //! in [`crate::brief`] can switch on the canonical verdict instead of re-deriving its own. diff --git a/crates/navigator-domain/src/strchart.rs b/crates/navigator-domain/src/strchart.rs index 0593b052..61a81000 100644 --- a/crates/navigator-domain/src/strchart.rs +++ b/crates/navigator-domain/src/strchart.rs @@ -84,7 +84,7 @@ pub enum Deviation { Below, /// Strictly above the modal value (more repeats). Above, - /// Differs from the mode but isn't strictly orderable (multi-copy with mixed direction). + /// Differs from the mode but is not strictly orderable (multi-copy with mixed direction). Differs, } diff --git a/crates/navigator-domain/src/strprofile.rs b/crates/navigator-domain/src/strprofile.rs index 7ad29adb..1ad4afa4 100644 --- a/crates/navigator-domain/src/strprofile.rs +++ b/crates/navigator-domain/src/strprofile.rs @@ -126,7 +126,7 @@ fn clean_cell(s: &str) -> &str { s.trim().trim_matches('"').trim() } -/// A value is "missing" when it's blank or a placeholder dash. +/// A value is "missing" when it is blank or a placeholder dash. fn is_blank_value(v: &str) -> bool { v.is_empty() || v == "-" } diff --git a/crates/navigator-domain/src/testtype.rs b/crates/navigator-domain/src/testtype.rs index 5f7d5e93..c9a43a39 100644 --- a/crates/navigator-domain/src/testtype.rs +++ b/crates/navigator-domain/src/testtype.rs @@ -176,7 +176,7 @@ pub fn by_code(code: &str) -> Option<&'static TestType> { CATALOG.iter().find(|t| t.code == code) } -/// Classify a stored `test_type` into its [`TargetType`] — tolerant of values that aren't a +/// Classify a stored `test_type` into its [`TargetType`] — tolerant of values that are not a /// canonical [`by_code`] code. A bulk import or a `--test-type` override may store a human label /// like `"Big Y"` rather than `BIG_Y_500`/`BIG_Y_700`; without recognizing it the targeted-Y /// scoping is lost and coverage walks the whole genome (slow on a targeted multi-reference CRAM). diff --git a/crates/navigator-domain/src/variants.rs b/crates/navigator-domain/src/variants.rs index 5ec3fbfd..d254a782 100644 --- a/crates/navigator-domain/src/variants.rs +++ b/crates/navigator-domain/src/variants.rs @@ -17,7 +17,7 @@ pub const CALL_SCHEMA_BASIC: i64 = 1; pub const CALL_SCHEMA_EVIDENCE: i64 = 2; /// Per-call evidence carried over from the source VCF. Every field is optional — a sites-only VCF -/// has no FORMAT column, and vendors vary in what they emit — so absence means "the source didn't +/// has no FORMAT column, and vendors vary in what they emit — so absence means "the source did not /// say", never "zero". #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct CallEvidence { @@ -155,7 +155,7 @@ pub struct VariantSet { pub calls: Vec, /// Which call schema this set was stored under — [`CALL_SCHEMA_BASIC`] or /// [`CALL_SCHEMA_EVIDENCE`]. Derived from what was captured, not from the importer version, so - /// it never promises evidence the source didn't supply. Check it before applying a quality gate: + /// it never promises evidence the source did not supply. Check it before applying a quality gate: /// a `BASIC` set can't satisfy one, and treating its absent DP/GQ as zero would silently reject /// every call. pub call_schema: i64, @@ -212,7 +212,7 @@ pub fn snp_call( /// [`snp_call`] carrying the source's [`CallEvidence`]. Separate rather than a seventh parameter on /// `snp_call` because most call sites (CSV tables, chip exports, hand entry) have no evidence to -/// give and shouldn't have to say so. +/// give and should not have to say so. pub fn snp_call_with_evidence( contig: &str, position: i64, @@ -256,7 +256,7 @@ impl Layout { } } - /// Map columns by a recognized header row, or `None` if the row isn't a header. + /// Map columns by a recognized header row, or `None` if the row is not a header. fn from_header(cols: &[&str]) -> Option { let find = |names: &[&str]| { cols.iter().position(|c| { @@ -298,7 +298,7 @@ pub fn parse_csv(text: &str) -> Result, String> { let first_cols: Vec<&str> = first.split(sep).map(str::trim).collect(); let layout = Layout::from_header(&first_cols); let mut calls = Vec::new(); - // If the first row wasn't a header, it's data — parse it positionally too. + // If the first row was not a header, it is data — parse it positionally too. let header_layout = match layout { Some(l) => l, None => { diff --git a/crates/navigator-domain/src/vendorvcf.rs b/crates/navigator-domain/src/vendorvcf.rs index d7c351b9..2bf15869 100644 --- a/crates/navigator-domain/src/vendorvcf.rs +++ b/crates/navigator-domain/src/vendorvcf.rs @@ -60,7 +60,7 @@ pub fn classify(meta: &str, contigs: &[String], filename: &str, readme: Option<& let mt_only = only(is_mt_contig); let y_only = only(is_y_contig); - // FTDNA: the Arpeggi caller (`aengine`) or an explicit "big y" mention. mtFull if it's chrM-only. + // FTDNA: the Arpeggi caller (`aengine`) or an explicit "big y" mention. mtFull if it is chrM-only. if hay.contains("aengine") || hay.contains("big y") || hay.contains("bigy") || hay.contains("mtfull") { return if mt_only { VendorVcf::FtdnaMtFull diff --git a/crates/navigator-domain/src/workspace.rs b/crates/navigator-domain/src/workspace.rs index 9d831b69..0caa1bb4 100644 --- a/crates/navigator-domain/src/workspace.rs +++ b/crates/navigator-domain/src/workspace.rs @@ -130,7 +130,7 @@ impl SequenceRun { } /// The standardized, vendor-neutral test label (`WGS150 45Gbases`, `HiFi 90Gbases`, `BigY-700`), - /// or `None` when this isn't a yield/product test we standardize (chips, panels) — the caller + /// or `None` when this is not a yield/product test we standardize (chips, panels) — the caller /// falls back to the raw `test_type`. See [`du_domain::testprofile`]. pub fn standardized_label(&self) -> Option { du_domain::testprofile::standardized_label(&du_domain::testprofile::RunProfile { diff --git a/crates/navigator-domain/src/ysnp_dict.rs b/crates/navigator-domain/src/ysnp_dict.rs index 155a98fb..58ce4044 100644 --- a/crates/navigator-domain/src/ysnp_dict.rs +++ b/crates/navigator-domain/src/ysnp_dict.rs @@ -2,7 +2,7 @@ //! its missing coordinates. A SNP name like `CTS10003` resolves to a position plus its //! ancestral/derived alleles — **per reference build**, so the codebase stays build-agnostic: //! `coordinates` is keyed by build label (`"GRCh38"`, `"GRCh37"`, `"hs1"`, …), exactly the -//! convention the DecodingUs Y-tree uses. The importer is handed the build it's placing +//! convention the DecodingUs Y-tree uses. The importer is handed the build it is placing //! against and reads that coordinate; nothing here is CHM13-specific. //! //! The bulk data is a generated asset (built from YBrowse + liftover by @@ -17,7 +17,7 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; /// One SNP's locus on a specific reference build. Alleles are on that build's + strand, so a -/// strand-flipping liftover stores its own (complemented) alleles — they're per-coordinate, +/// strand-flipping liftover stores its own (complemented) alleles — they are per-coordinate, /// not per-SNP. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Coord { @@ -151,12 +151,12 @@ impl YsnpDictionary { /// Candidate dictionary filenames in `load` preference order: the full ~200 MB / ~2M-name /// catalog first, then the small per-chip panel only as a fallback. The chromo2 chip panel is a /// stale ~14k-name subset that would shadow current names present in the full catalog, so the - /// catalog wins whenever it's installed (it's the one downloaded on first use). + /// catalog wins whenever it is installed (it is the one downloaded on first use). pub const ASSET_FILENAMES: &'static [&'static str] = &["dictionary.tsv", "chromo2-panel.tsv"]; /// Read the asset from `dir`: the first of [`Self::ASSET_FILENAMES`] that exists, plus an /// optional sibling `aliases.tsv`. Prefers the full catalog for the widest, current name - /// coverage; the chromo2 panel is only used when the catalog isn't present. + /// coverage; the chromo2 panel is only used when the catalog is not present. pub fn load(dir: &Path) -> Result { let dict_path = Self::ASSET_FILENAMES .iter() @@ -259,7 +259,7 @@ M269\tCTS10003 #[test] fn load_prefers_full_dictionary_over_chromo2_panel() { // Both present → the full `dictionary.tsv` wins; the stale ~14k-name chromo2 chip panel must - // not shadow current names in the full catalog. With only the panel, it's the fallback. + // not shadow current names in the full catalog. With only the panel, it is the fallback. let dir = std::env::temp_dir().join(format!("dun-ysnp-pref-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); diff --git a/crates/navigator-panelbuild/examples/ascertain_chip.rs b/crates/navigator-panelbuild/examples/ascertain_chip.rs index 6816720e..770f88a0 100644 --- a/crates/navigator-panelbuild/examples/ascertain_chip.rs +++ b/crates/navigator-panelbuild/examples/ascertain_chip.rs @@ -1,5 +1,5 @@ //! Throwaway (Option A′ concept test): emit a copy of the ancient AF panel restricted to -//! consumer-array-ascertained sites. Reads the rsIDs assayed by one or more consumer chip files, +//! consumer-array-found out sites. Reads the rsIDs assayed by one or more consumer chip files, //! maps them to CHM13 (contig,pos) via the IBD panel (which carries rsid + CHM13 locus), and keeps //! only ancient-panel sites at those positions. Usage: //! ascertain_chip [chip2.txt ...] diff --git a/crates/navigator-panelbuild/examples/check_liftover.rs b/crates/navigator-panelbuild/examples/check_liftover.rs index 49ce6e3c..adc38871 100644 --- a/crates/navigator-panelbuild/examples/check_liftover.rs +++ b/crates/navigator-panelbuild/examples/check_liftover.rs @@ -32,7 +32,7 @@ fn main() -> anyhow::Result<()> { let mut chm = open(&chm13_fa)?; let mut g38 = open(&grch38_fa)?; - // Try the stored contig name, then chr/bare variants, so a naming mismatch doesn't masquerade + // Try the stored contig name, then chr/bare variants, so a naming mismatch does not masquerade // as a coordinate error. let g38_base = |g38: &mut _, l: &navigator_analysis::ibd_panel::Locus| -> Option { let bare = navigator_analysis::contig::bare(&l.contig).to_string(); diff --git a/crates/navigator-panelbuild/examples/filter_sites.rs b/crates/navigator-panelbuild/examples/filter_sites.rs index cce985d6..aecf28df 100644 --- a/crates/navigator-panelbuild/examples/filter_sites.rs +++ b/crates/navigator-panelbuild/examples/filter_sites.rs @@ -1,7 +1,7 @@ //! Apply an ascertainment floor to an already-built ancient AF panel: keep only sites whose CHM13 //! (contig,pos) appears in a `contigpos` set. Equivalent to rebuilding the panel with //! `ancient-panel --ascertain-sites` on the same inputs (the per-site frequencies are unchanged; -//! only the site set is restricted), for when the source matrices aren't at hand but the full panel +//! only the site set is restricted), for when the source matrices are not at hand but the full panel //! is. Usage: filter_sites use navigator_analysis::ancestry::AncestryPanel; use std::collections::HashSet; diff --git a/crates/navigator-panelbuild/examples/qpadm_selftest.rs b/crates/navigator-panelbuild/examples/qpadm_selftest.rs index 472bf2a0..f314afa7 100644 --- a/crates/navigator-panelbuild/examples/qpadm_selftest.rs +++ b/crates/navigator-panelbuild/examples/qpadm_selftest.rs @@ -4,7 +4,7 @@ //! //! For each site, target frequency = Σ wᵢ·sourceᵢ(site) with a known w, drawn as a diploid genome. //! qpadm_fit must return ~w, feasible, not rejected. If it can't recover a self-consistent mixture, -//! the outgroups don't resolve WHG/ANF/Steppe (a construction problem, not a target problem). +//! the outgroups do not resolve WHG/ANF/Steppe (a construction problem, not a target problem). //! qpadm_selftest use navigator_analysis::ancestry::{qpadm_fit, AncestryPanel, F4_BLOCK_BP}; use navigator_analysis::caller::SiteGenotype; diff --git a/crates/navigator-panelbuild/src/archaic.rs b/crates/navigator-panelbuild/src/archaic.rs index 380c1f4e..eb5c4378 100644 --- a/crates/navigator-panelbuild/src/archaic.rs +++ b/crates/navigator-panelbuild/src/archaic.rs @@ -90,10 +90,10 @@ fn split_list(s: &str) -> Vec { /// /// Returns `None` when the ancestral base is unusable — either low-confidence/absent (the EPO /// sequence lower-cases low-confidence calls and uses `.`/`-`/`N` for gaps) or matching neither -/// allele, which means the site cannot be polarized and must be dropped rather than guessed. +/// allele, which means the site can not be polarized and must be dropped rather than guessed. /// /// Only **upper-case** ancestral bases are accepted: in the EPO alignment lower case marks a -/// low-confidence call, and polarity is the one thing this panel cannot afford to get wrong — an +/// low-confidence call, and polarity is the one thing this panel can not afford to get wrong — an /// inverted site turns an archaic-derived allele into its opposite. fn derived_allele(ancestral: char, reference_allele: char, alternate_allele: char) -> Option { if !matches!(ancestral, 'A' | 'C' | 'G' | 'T') { @@ -254,7 +254,7 @@ pub fn build_archaic_candidates(args: ArchaicCandidatesArgs) -> Result<()> { continue; } // The site's allele pair comes from the genomes that actually carry a variant; a - // reference-confident record states only the REF base and cannot define the pair. At least + // reference-confident record states only the REF base and can not define the pair. At least // one genome must vary, otherwise the site is invariant across all four and carries no // information regardless of polarity. let Some((reference_allele, alternate_allele)) = present.iter().find_map(|(r, a, _)| a.map(|alt| (*r, alt))) @@ -502,7 +502,7 @@ fn load_outgroup_af(path: &Path) -> Result { /// The outgroup table and the candidate need not label ref/alt the same way, so the frequency is /// re-expressed against the *derived base* rather than against whichever allele happened to be /// called ALT. Returns `None` when the candidate's derived base is absent from the outgroup's -/// allele pair, which means the two sources disagree about the site and it cannot be filtered. +/// allele pair, which means the two sources disagree about the site and it can not be filtered. fn derived_freq(derived: char, og_ref: char, og_alt: char, af_alt: f32) -> Option { let d = derived.to_ascii_uppercase(); if d == og_alt.to_ascii_uppercase() { @@ -618,7 +618,7 @@ pub fn build_archaic_panel(args: ArchaicPanelArgs) -> Result<()> { continue; } - // Strand-ambiguous sites cannot be reconciled against a chip's unknown strand. + // Strand-ambiguous sites can not be reconciled against a chip's unknown strand. if is_palindromic(cand.reference_allele, cand.alternate_allele) { palindromic += 1; continue; @@ -659,7 +659,7 @@ pub fn build_archaic_panel(args: ArchaicPanelArgs) -> Result<()> { }), grch38: hg38.get(&idx).cloned(), // Unchanged by the swap: the derived allele is stored as a base precisely so orientation - // cannot invert its meaning. + // can not invert its meaning. archaic_derived_allele: cand.derived, calls: cand.calls, diagnostic_class: classify_diagnostic(&cand.calls), @@ -738,7 +738,7 @@ mod tests { // Ancestral == REF → ALT is derived, and vice versa. assert_eq!(derived_allele('A', 'A', 'G'), Some('G')); assert_eq!(derived_allele('G', 'A', 'G'), Some('A')); - // Matching neither allele cannot be polarized. + // Matching neither allele can not be polarized. assert_eq!(derived_allele('C', 'A', 'G'), None); // Gaps / unknowns. assert_eq!(derived_allele('N', 'A', 'G'), None); diff --git a/crates/navigator-panelbuild/src/archaic_tierb.rs b/crates/navigator-panelbuild/src/archaic_tierb.rs index b725262a..063bdbf6 100644 --- a/crates/navigator-panelbuild/src/archaic_tierb.rs +++ b/crates/navigator-panelbuild/src/archaic_tierb.rs @@ -112,7 +112,7 @@ pub struct ArchaicClassifyArgs { pub fn build_archaic_classify(args: ArchaicClassifyArgs) -> Result<()> { // The classification track is the genome-wide superset of the marker panel: every polarized // candidate with an archaic hom-derived call, BEFORE the frequency filters that select markers. - // Segment attribution wants maximum diagnostic density, not the ascertained marker subset. + // Segment attribution wants maximum diagnostic density, not the found out marker subset. let want: Option> = args .contigs .as_ref() diff --git a/crates/navigator-panelbuild/src/genetic_map.rs b/crates/navigator-panelbuild/src/genetic_map.rs index ec0251e3..299f85ae 100644 --- a/crates/navigator-panelbuild/src/genetic_map.rs +++ b/crates/navigator-panelbuild/src/genetic_map.rs @@ -3,7 +3,7 @@ //! IBD segment lengths in cM (and thus the relationship bands) are only as good as the map, so this //! replaces the app's flat 1 cM/Mb stand-in with a real sex-averaged map (deCODE 2019 / HapMap II), //! **already lifted to CHM13**. The lift is coordinate-only (no alleles), so a stage-2 CrossMap BED -//! lift of the map's positions is sufficient — this step just parses the lifted text and serializes +//! lift of the map's positions is enough — this step just parses the lifted text and serializes //! it to the bincode [`navigator_analysis::ibd::GeneticMap`] the app loads. //! //! Input is whitespace/tab-delimited with columns `chromosome position(bp) … cumulative_cM` diff --git a/crates/navigator-panelbuild/src/hap_panel.rs b/crates/navigator-panelbuild/src/hap_panel.rs index bbd1e244..a069aedd 100644 --- a/crates/navigator-panelbuild/src/hap_panel.rs +++ b/crates/navigator-panelbuild/src/hap_panel.rs @@ -256,7 +256,7 @@ mod tests { assert_eq!(parse_gt_phased("1|0"), (1, 0, false)); assert_eq!(parse_gt_phased("1|1"), (1, 1, false)); assert_eq!(parse_gt_phased("0|0"), (0, 0, false)); - // Unphased separator still parses (defensive), phase just isn't meaningful. + // Unphased separator still parses (defensive), phase just is not meaningful. assert_eq!(parse_gt_phased("0/1"), (0, 1, false)); // Missing → ref with the missing flag set. assert_eq!(parse_gt_phased(".|1"), (0, 1, true)); diff --git a/crates/navigator-panelbuild/src/lai_validate.rs b/crates/navigator-panelbuild/src/lai_validate.rs index dcd00421..3b9013c2 100644 --- a/crates/navigator-panelbuild/src/lai_validate.rs +++ b/crates/navigator-panelbuild/src/lai_validate.rs @@ -2,11 +2,11 @@ //! [`navigator_analysis::lai::paint_copying_lai`]. //! //! The painter's calibration knobs ([`CopyingLaiParams`]) were tuned by *looking* at one kit's -//! painted chromosomes, which cannot distinguish "the smear is gone" from "the smear moved". This +//! painted chromosomes, which can not distinguish "the smear is gone" from "the smear moved". This //! runs the shipping painter against ground truth we control and prints the numbers: //! //! 1. **Leave-one-out gate** — take a real reference individual (both of its haplotypes), remove -//! them from the reference so it cannot copy itself, paint, and score every site against the +//! them from the reference so it can not copy itself, paint, and score every site against the //! individual's known population. A NW-European reference individual painted as Finnish is the //! exact defect the recent recalibration commits were chasing, and here it is a number. //! 2. **Simulated-admixture gate** — splice held-out donor haplotypes from two (or more) @@ -677,7 +677,7 @@ fn build_cases( ); continue; } - // Evenly spaced picks so replicates aren't all neighbours in the panel's sample order. + // Evenly spaced picks so replicates are not all neighbours in the panel's sample order. let step = (pool.len() / args.replicates.max(1)).max(1); for r in 0..args.replicates.min(pool.len()) { let ind = pool[(r * step) % pool.len()]; @@ -1078,7 +1078,7 @@ fn contig_count(reference: &HaplotypeReference, sel: &[usize]) -> usize { } /// Base-pair weight of each selected site: half the distance to each neighbour within its contig, -/// so a sparse region doesn't count the same as a dense one and the reported composition matches +/// so a sparse region does not count the same as a dense one and the reported composition matches /// the painted (bp-proportioned) chromosomes. fn site_weights(reference: &HaplotypeReference, sel: &[usize]) -> Vec { let pos = |i: usize| reference.sites[sel[i]].position as f64; diff --git a/crates/navigator-panelbuild/src/main.rs b/crates/navigator-panelbuild/src/main.rs index f67727bb..3851618c 100644 --- a/crates/navigator-panelbuild/src/main.rs +++ b/crates/navigator-panelbuild/src/main.rs @@ -283,7 +283,7 @@ fn scan_file( Ok((seen, kept)) } -/// Parse a VCF data line into a [`Candidate`], or `None` if it's not a biallelic SNP with full +/// Parse a VCF data line into a [`Candidate`], or `None` if it is not a biallelic SNP with full /// per-population allele-count data. fn parse_record(line: &str) -> Option { let mut f = line.split('\t'); diff --git a/crates/navigator-panelbuild/src/pca.rs b/crates/navigator-panelbuild/src/pca.rs index 731a9785..9413093d 100644 --- a/crates/navigator-panelbuild/src/pca.rs +++ b/crates/navigator-panelbuild/src/pca.rs @@ -62,7 +62,7 @@ pub struct AncientPanelArgs { #[arg(long)] samples: String, /// `samplepopulation` for every sample across the matrices (the pipeline's pop map; - /// samples whose population isn't in `--components` are ignored). + /// samples whose population is not in `--components` are ignored). #[arg(long)] pops: PathBuf, /// The deep source (**left**) populations, comma-separated and **in panel-axis order** @@ -95,7 +95,7 @@ pub struct AncientPanelArgs { /// **Ascertainment floor (Option A′).** Restrict the panel to the CHM13 `contigpos` sites in /// this file — a consumer-array manifest. Allele-frequency admixture is only valid when the /// sample and the reference share ascertainment; the AADR/1240k universe includes capture sites - /// consumer chips don't assay, and on those the deep estimate is unstable (a WGS sample reads + /// consumer chips do not assay, and on those the deep estimate is unstable (a WGS sample reads /// ~90% Steppe where its own chip reads ~58%). Intersecting with the sites arrays actually assay /// makes the estimate agree across data sources. See `documents/design/ancient-ancestry-rebuild.md` §4. /// Optional: omit to build the full (unascertained) panel. @@ -595,7 +595,7 @@ pub fn build_ancient_panel(args: AncientPanelArgs) -> Result<()> { .collect(); let pop_of = load_fine_map(&args.pops)?; - // No global call-rate filter: the AADR matrix is mostly individuals we don't reference, so a + // No global call-rate filter: the AADR matrix is mostly individuals we do not reference, so a // matrix-wide call rate says nothing about the sources. The per-component floor below is the // filter that matters. let (samples, metas, rows) = load_combined(&split_paths(&args.matrix), &split_paths(&args.samples), 0.0)?; diff --git a/crates/navigator-panelbuild/src/validate_ancient.rs b/crates/navigator-panelbuild/src/validate_ancient.rs index 41180a11..28a2b9b7 100644 --- a/crates/navigator-panelbuild/src/validate_ancient.rs +++ b/crates/navigator-panelbuild/src/validate_ancient.rs @@ -16,7 +16,7 @@ //! 2. **Density.** `--downsample` re-runs on a random half of the sites; the answer must barely //! move, or the fit is ill-conditioned. //! 3. **Fit residual.** The dispersion is reported for every population, so the separation between -//! "fits" and "doesn't fit" is visible rather than asserted. +//! "fits" and "does not fit" is visible rather than asserted. use std::collections::HashMap; use std::path::PathBuf; @@ -38,7 +38,7 @@ pub struct ValidateAncientArgs { reference: PathBuf, /// The modern super-population panel (`ancestry_panel_.bin`). Deep ancestry is scoped by /// the modern estimate, so the validator must score both models or it would be validating a - /// policy the app doesn't actually run. + /// policy the app does not actually run. #[arg(long)] panel: PathBuf, /// Reference populations to simulate, comma-separated. @@ -46,7 +46,7 @@ pub struct ValidateAncientArgs { /// **Use 1000G populations only.** The `freq_global` asset's SGDP-derived columns (Sardinian, /// Basque, French, Orcadian, Han, …) record `0.0` where a population had no called sample — /// indistinguishable from a true zero — so 60%+ of their sites are fake zeros and an individual - /// simulated from them is not that population, it's noise. The 1000G columns are called + /// simulated from them is not that population, it is noise. The 1000G columns are called /// essentially everywhere and are the only trustworthy simulation source in that asset. /// (This is the same no-data-as-zero defect the ancient panel exists to avoid; see /// `pca::build_ancient_panel`.) diff --git a/crates/navigator-refgenome/src/cache.rs b/crates/navigator-refgenome/src/cache.rs index 0c3e235a..cedbe55f 100644 --- a/crates/navigator-refgenome/src/cache.rs +++ b/crates/navigator-refgenome/src/cache.rs @@ -115,7 +115,7 @@ pub fn regions_path(base: &Path, build: Build) -> PathBuf { base.join("regions").join(format!("{}.json", build.as_str())) } -/// Age of a cached file in days (for TTL checks); `None` if it doesn't exist or its mtime is +/// Age of a cached file in days (for TTL checks); `None` if it does not exist or its mtime is /// unreadable / in the future. pub fn age_days(path: &Path) -> Option { let modified = std::fs::metadata(path).ok()?.modified().ok()?; diff --git a/crates/navigator-refgenome/src/gateway.rs b/crates/navigator-refgenome/src/gateway.rs index 85b2772b..f485603b 100644 --- a/crates/navigator-refgenome/src/gateway.rs +++ b/crates/navigator-refgenome/src/gateway.rs @@ -199,7 +199,7 @@ impl ReferenceGateway { // The cache stores chains as plain text (`load_liftover` reads them with `read_to_string`). // Every chain flows through the same path: if the downloaded artifact is gzipped (UCSC // serves `.over.chain.gz`; the curated bucket serves plain `.chain`), decompress it in place - // — detected by the gzip magic bytes, so the source URL/extension doesn't matter. + // — detected by the gzip magic bytes, so the source URL/extension does not matter. maybe_gunzip_in_place(&path)?; write_sidecar(&path, &sha); Ok(path) @@ -349,7 +349,7 @@ impl ReferenceGateway { } /// Parse the cached chain for a build pair into a `du-bio` `Liftover` (call - /// [`resolve_chain`](Self::resolve_chain) first to ensure it's present). + /// [`resolve_chain`](Self::resolve_chain) first to ensure it is present). pub fn load_liftover(&self, from_name: &str, to_name: &str) -> Result { let (from, to) = self.chain_builds(from_name, to_name)?; let path = cache::chain_path(&self.base, from, to); @@ -582,7 +582,7 @@ impl ReferenceGateway { /// Re-hash a cached reference and compare to its integrity sidecar (TOFU, written at download /// time). Detects on-disk corruption of the cached `.fa`. Re-reads the whole FASTA, so call it - /// from a blocking context (it's an explicit, user-triggered check, not the hot path). A + /// from a blocking context (it is an explicit, user-triggered check, not the hot path). A /// user-pinned local FASTA has no sidecar → [`VerifyOutcome::NoSidecar`]. pub fn verify_reference(&self, build_name: &str) -> Result { let fa = match self.reference_status(build_name) { diff --git a/crates/navigator-refgenome/src/index.rs b/crates/navigator-refgenome/src/index.rs index 5eec4209..4c74fe14 100644 --- a/crates/navigator-refgenome/src/index.rs +++ b/crates/navigator-refgenome/src/index.rs @@ -30,7 +30,7 @@ fn is_gzip(path: &Path) -> Result { Ok(n == 2 && magic == [0x1f, 0x8b]) } -/// Read until the buffer is full or EOF; returns bytes read (a short final read isn't EOF). +/// Read until the buffer is full or EOF; returns bytes read (a short final read is not EOF). fn read_up_to(r: &mut impl Read, buf: &mut [u8]) -> io::Result { let mut filled = 0; while filled < buf.len() { diff --git a/crates/navigator-refgenome/src/regions.rs b/crates/navigator-refgenome/src/regions.rs index a46cd446..bfb780b0 100644 --- a/crates/navigator-refgenome/src/regions.rs +++ b/crates/navigator-refgenome/src/regions.rs @@ -143,7 +143,7 @@ impl GenomeRegions { regions } - /// Overlay the chrY pseudoautosomal regions for the build (PAR isn't in cytoBand). Best-known + /// Overlay the chrY pseudoautosomal regions for the build (PAR is not in cytoBand). Best-known /// constants for the builds we resolve; other builds get none. fn overlay_par(&mut self, build: &str) { let par = crate::registry::canonical_build(build) diff --git a/crates/navigator-refgenome/src/registry.rs b/crates/navigator-refgenome/src/registry.rs index 5ee5e8c3..be5c7cb8 100644 --- a/crates/navigator-refgenome/src/registry.rs +++ b/crates/navigator-refgenome/src/registry.rs @@ -109,7 +109,7 @@ pub fn canonical_build(name: &str) -> Option { /// Where a reference FASTA is fetched from, with a rough size for the download prompt and an /// optional pinned SHA-256 of the downloaded artifact (publisher's hash, when known) used to -/// verify the download before it's accepted. `None` = no authoritative hash to pin against yet. +/// verify the download before it is accepted. `None` = no authoritative hash to pin against yet. #[derive(Debug, Clone)] pub struct ReferenceSource { pub build: Build, @@ -186,7 +186,7 @@ pub struct BuildOverride { #[serde(skip_serializing_if = "Option::is_none", default)] pub url: Option, /// Pin an authoritative SHA-256 (lowercase hex) of the downloaded artifact; the download is - /// rejected if it doesn't match. Lets a user supply a publisher checksum we don't ship. + /// rejected if it does not match. Lets a user supply a publisher checksum we do not ship. #[serde(skip_serializing_if = "Option::is_none", default)] pub sha256: Option, /// Whether a missing reference may be auto-downloaded for this build (default `true`). @@ -209,7 +209,7 @@ impl UserConfig { /// Load the config if present; a missing or unreadable file yields the empty default (overrides /// are advisory, never fatal — a novice with no config just gets the self-managed auto-download). /// - /// A file that **exists but doesn't parse** also falls back to defaults, but is **warned about**: + /// A file that **exists but does not parse** also falls back to defaults, but is **warned about**: /// silently dropping it is how a power user's `local_path` override vanishes and the app surprises /// them with a full reference download (issue #26 — the config had been corrupted by a racing /// non-atomic write; see [`crate::cache::atomic_write`]). Say so instead of reverting in silence. @@ -240,7 +240,7 @@ impl UserConfig { /// (temp + rename, see [`crate::cache::atomic_write`]) — this file is rewritten from spawned worker /// tasks that can race, and a plain non-atomic write corrupts it into head-of-new + tail-of-old /// garbage. Callers should still avoid concurrent read-modify-write (prefer one bulk save) so an - /// update isn't lost; atomicity only guarantees the file is never *torn*. + /// update is not lost; atomicity only guarantees the file is never *torn*. pub fn save(&self, path: &Path) -> std::io::Result<()> { let json = serde_json::to_string_pretty(self).map_err(std::io::Error::other)?; crate::cache::atomic_write(path, json.as_bytes()) diff --git a/crates/navigator-refgenome/src/vcf_lift.rs b/crates/navigator-refgenome/src/vcf_lift.rs index c424c40a..597e325a 100644 --- a/crates/navigator-refgenome/src/vcf_lift.rs +++ b/crates/navigator-refgenome/src/vcf_lift.rs @@ -4,7 +4,7 @@ //! POS (via the UCSC chain), and on a reverse-strand (inverted) lift the REF/ALT alleles are //! reverse-complemented. REF/ALT-swap recovery reads the target reference base and, when the lifted //! REF no longer matches it, swaps REF↔ALT (flipping a biallelic single-sample GT). Records whose -//! position doesn't map, whose multi-base REF straddles a chain break, or that can't be safely +//! position does not map, whose multi-base REF straddles a chain break, or that can't be safely //! recovered are dropped and tallied. Output is coordinate-sorted (a lift can reorder/invert). //! //! Reuses the `du_bio` chain primitives and mirrors the drop-with-stats shape of @@ -36,9 +36,9 @@ pub struct VcfLiftStats { pub unmapped: usize, /// A multi-base REF whose endpoints lifted to different target contigs (straddled a break). pub split: usize, - /// The lifted REF matched neither the target base nor any ALT (couldn't recover). + /// The lifted REF matched neither the target base nor any ALT (could not recover). pub ref_mismatch: usize, - /// A REF/ALT swap was needed but couldn't be applied safely (multiallelic or multi-sample). + /// A REF/ALT swap was needed but could not be applied safely (multiallelic or multi-sample). pub swap_ambiguous: usize, /// Dropped in the target PAR (only when `filter_par`). pub par: usize, @@ -301,7 +301,7 @@ fn lift_record( if let Some(tbase) = ref_base_at(fasta_reader, &target_contig, q_pos) { let tb = (tbase as char).to_string(); if !new_ref.eq_ignore_ascii_case(&tb) { - // REF doesn't match the target base — try to recover by swapping with a matching ALT. + // REF does not match the target base — try to recover by swapping with a matching ALT. if let Some(idx) = new_alts.iter().position(|a| a.eq_ignore_ascii_case(&tb)) { if new_alts.len() != 1 { stats.swap_ambiguous += 1; // multiallelic swap — ambiguous to relabel @@ -344,7 +344,7 @@ fn lift_record( } /// Flip the allele indices of a biallelic single-sample genotype (0↔1) in the first sample column, -/// after a REF/ALT swap. No-op when there's no FORMAT/sample (sites-only VCF). +/// after a REF/ALT swap. No-op when there is no FORMAT/sample (sites-only VCF). fn flip_biallelic_gt(f: &mut [String]) { if f.len() < 10 { return; // no FORMAT + sample columns diff --git a/crates/navigator-store/src/ancestry_result.rs b/crates/navigator-store/src/ancestry_result.rs index 0d90bfd9..65c11039 100644 --- a/crates/navigator-store/src/ancestry_result.rs +++ b/crates/navigator-store/src/ancestry_result.rs @@ -124,7 +124,7 @@ pub async fn get_for_alignment_method( } /// Delete every ancestry estimate recorded for `alignment_id` (all methods). Used when an -/// alignment is deleted, so its per-alignment ancestry doesn't outlive it. +/// alignment is deleted, so its per-alignment ancestry does not outlive it. pub async fn delete_for_alignment(pool: &SqlitePool, alignment_id: i64) -> Result<(), StoreError> { sqlx::query("DELETE FROM ancestry_result WHERE alignment_id = ?") .bind(alignment_id) diff --git a/crates/navigator-store/src/biosample_project.rs b/crates/navigator-store/src/biosample_project.rs index 6fdb3522..6549c8db 100644 --- a/crates/navigator-store/src/biosample_project.rs +++ b/crates/navigator-store/src/biosample_project.rs @@ -141,7 +141,7 @@ mod tests { add(pool, s, p1, Some("subgroupX"), t).await.unwrap(); add(pool, s, p2, None, t).await.unwrap(); - // Re-add the same pair updates role, doesn't duplicate. + // Re-add the same pair updates role, does not duplicate. add(pool, s, p1, Some("subgroupY"), t).await.unwrap(); let projects = list_projects_for(pool, s).await.unwrap(); diff --git a/crates/navigator-store/src/haplogroup_call.rs b/crates/navigator-store/src/haplogroup_call.rs index 10e80b17..aed84c93 100644 --- a/crates/navigator-store/src/haplogroup_call.rs +++ b/crates/navigator-store/src/haplogroup_call.rs @@ -104,7 +104,7 @@ pub async fn stored_fingerprint( /// is the first 16 hex of the current tree's SHA-256. /// /// `include_unknown` decides what to do with a call carrying **no tree tag** — placed before the -/// fingerprint existed, so which tree it used cannot be established. These are a different job from +/// fingerprint existed, so which tree it used can not be established. These are a different job from /// a tree change, and much more expensive: in this workspace 3,780 Y calls are *provably* on a /// superseded tree while 15,648 have no fingerprint at all, and most of the latter own a BAM that a /// re-placement would re-walk. Default them out so the routine "a new tree landed" sweep stays diff --git a/crates/navigator-store/src/sync_history.rs b/crates/navigator-store/src/sync_history.rs index a29587c2..5027a329 100644 --- a/crates/navigator-store/src/sync_history.rs +++ b/crates/navigator-store/src/sync_history.rs @@ -1,6 +1,6 @@ //! Append-only audit trail of completed PDS-publish attempts (sync durability, gap §5). One row is //! written per terminal outcome — a successful push (with the resulting `at://` URI + CID) or a -//! non-transient failure. Transient retries don't write history (they stay in [`crate::sync_outbox`]). +//! non-transient failure. Transient retries do not write history (they stay in [`crate::sync_outbox`]). use sqlx::SqlitePool; diff --git a/crates/navigator-store/src/sync_outbox.rs b/crates/navigator-store/src/sync_outbox.rs index b349c280..7bce7533 100644 --- a/crates/navigator-store/src/sync_outbox.rs +++ b/crates/navigator-store/src/sync_outbox.rs @@ -1,6 +1,6 @@ //! Persistent PDS-publish outbox (sync durability, gap §5). A publish enqueues a fully-built //! record; a background drain pushes it with exponential backoff. A transient/offline failure -//! reschedules the row (so it isn't lost); a non-transient failure marks it `FAILED`; a success +//! reschedules the row (so it is not lost); a non-transient failure marks it `FAILED`; a success //! removes it (its outcome is logged in [`crate::sync_history`]). use sqlx::SqlitePool; diff --git a/crates/navigator-store/src/variant_set.rs b/crates/navigator-store/src/variant_set.rs index a26b5195..f4bfa76b 100644 --- a/crates/navigator-store/src/variant_set.rs +++ b/crates/navigator-store/src/variant_set.rs @@ -123,7 +123,7 @@ pub async fn create(pool: &SqlitePool, new: &NewVariantSet) -> Result Result, StoreError> { let Some(r) = sqlx::query_as::<_, SetRow>( "SELECT id, biosample_guid, source_label, source_type, reference_build, call_schema, source_path FROM variant_set WHERE id = ?", diff --git a/crates/navigator-store/tests/store.rs b/crates/navigator-store/tests/store.rs index e70c584c..6af15d32 100644 --- a/crates/navigator-store/tests/store.rs +++ b/crates/navigator-store/tests/store.rs @@ -456,7 +456,7 @@ async fn delete_cascades_run_to_alignments_and_artifacts() { #[tokio::test] async fn set_sequence_run_reparents_an_alignment() { // The merge primitive: an alignment's owning run can be changed (then the empty run deleted), - // and its artifacts travel with it (they're alignment-keyed). + // and its artifacts travel with it (they are alignment-keyed). let s = store().await; let b = sample(None); biosample::create(s.pool(), &b).await.unwrap(); @@ -809,7 +809,7 @@ async fn variant_call_evidence_round_trips_and_tags_the_schema() { source_label: "big-y".into(), source_type: SourceType::TargetedNgs, reference_build: Some("GRCh38".into()), - // One call carries evidence, one doesn't — a real VCF mixes both. + // One call carries evidence, one does not — a real VCF mixes both. calls: vec![call(100, evidence.clone()), call(200, CallEvidence::default())], source_path: Some("/tmp/big-y.vcf.gz".into()), }, diff --git a/crates/navigator-sync/src/device_key.rs b/crates/navigator-sync/src/device_key.rs index f063b85e..6ec1df52 100644 --- a/crates/navigator-sync/src/device_key.rs +++ b/crates/navigator-sync/src/device_key.rs @@ -1,6 +1,6 @@ //! Per-device Ed25519 signing key for authenticated Edge↔AppView calls. //! -//! Navigator cannot sign as its `did:plc` account (the PDS custodies that signing key), so +//! Navigator can not sign as its `did:plc` account (the PDS custodies that signing key), so //! each installation generates its own Ed25519 *device key*, persists the 32-byte seed in //! the OS keychain beside the OAuth session (keyed by account DID), and publishes the //! public half once as a `com.decodingus.atmosphere.deviceKey` record in the user's PDS @@ -217,7 +217,7 @@ mod tests { ); } - /// Keys are namespaced by account, so two identities on one install don't collide. + /// Keys are namespaced by account, so two identities on one install do not collide. #[test] fn device_keys_are_per_account() { let svc = "device-key-per-account-test"; diff --git a/crates/navigator-sync/src/lib.rs b/crates/navigator-sync/src/lib.rs index 0e212ebd..393db6d3 100644 --- a/crates/navigator-sync/src/lib.rs +++ b/crates/navigator-sync/src/lib.rs @@ -33,7 +33,7 @@ pub use sync::{AsyncSync, RetryPolicy}; pub use tokens::{Session, TokenStore}; // Federated atproto wire records — the single source of truth lives in the shared -// `du-domain::fed` module so the AppView's Jetstream consumer cannot drift from us. +// `du-domain::fed` module so the AppView's Jetstream consumer can not drift from us. // (Its `RecordMeta` is intentionally not re-exported to avoid colliding with the // reconciliation record's `RecordMeta`; `::new` builds it internally.) pub use du_domain::fed::{ diff --git a/crates/navigator-sync/src/oauth.rs b/crates/navigator-sync/src/oauth.rs index c30d0698..a140637b 100644 --- a/crates/navigator-sync/src/oauth.rs +++ b/crates/navigator-sync/src/oauth.rs @@ -224,7 +224,7 @@ async fn post_with_dpop( } /// The server's error body (usually `{"error":"invalid_grant",...}`), trimmed to keep the message -/// readable — it's the difference between "your session expired, re-sign-in" and a real fault. +/// readable — it is the difference between "your session expired, re-sign-in" and a real fault. fn truncate_body(body: &str) -> String { let b = body.trim(); if b.len() > 200 { @@ -436,7 +436,7 @@ mod tests { /// Live discovery + public-client PAR (DPoP + use_dpop_nonce) against a local atproto /// PDS container (see documents/atmosphere/13-Local-PDS-Testing.md). Exercises this /// crate's `post_with_dpop` (canonical htu vs transport URL) and loopback client_id. - /// The full browser→token loop needs HTTPS at the canonical host, so it's a manual smoke. + /// The full browser→token loop needs HTTPS at the canonical host, so it is a manual smoke. #[tokio::test] #[ignore = "requires PDS_TEST_URL (local atproto PDS container)"] async fn discovery_and_par_against_live_pds() { @@ -449,7 +449,7 @@ mod tests { let pds = pds.trim_end_matches('/').to_string(); let http = reqwest::Client::new(); - // Fetch metadata from the reachable base (not the https issuer a container won't terminate). + // Fetch metadata from the reachable base (not the https issuer a container will not terminate). let meta: AuthServerMetadata = http .get(format!("{pds}/.well-known/oauth-authorization-server")) .send() diff --git a/crates/navigator-sync/src/records.rs b/crates/navigator-sync/src/records.rs index ae5d1e54..5dda3212 100644 --- a/crates/navigator-sync/src/records.rs +++ b/crates/navigator-sync/src/records.rs @@ -385,7 +385,7 @@ pub struct OriginExternalId { /// context and may be published; a given name is not, and the living tester never is. The fields /// here are the complete list of what may leave the workspace — there is deliberately no /// `ancestorName`, no `notes`, and no donor identifier. [`Self::build`] is the only constructor, -/// so the gates cannot be bypassed by assembling one field-by-field. +/// so the gates can not be bypassed by assembling one field-by-field. /// /// **No floats:** DAG-CBOR has none, so the coordinate is a pair of strings (see the module /// header). The AppView parses numbers or numeric strings either way. @@ -430,7 +430,7 @@ pub const ANCESTOR_BIRTH_YEAR_MAX: i32 = 1900; const ANCESTOR_BIRTH_YEAR_MIN: i32 = 1000; /// Coarsen a coordinate to ~1 km before it leaves the workspace. A rooftop coordinate plus a -/// surname narrows to one family; a county-scale view cannot use the precision anyway. +/// surname narrows to one family; a county-scale view can not use the precision anyway. fn coarsen(v: f64) -> String { format!("{:.2}", (v * 100.0).round() / 100.0) } @@ -554,8 +554,8 @@ mod ancestral_origin_tests { assert_eq!(r.death_year, None, "an undated ancestor has no dates at all"); } - /// A rooftop coordinate plus a surname narrows to one family; a county-scale view cannot use - /// the precision anyway. Floats also cannot cross DAG-CBOR, hence the strings. + /// A rooftop coordinate plus a surname narrows to one family; a county-scale view can not use + /// the precision anyway. Floats also can not cross DAG-CBOR, hence the strings. #[test] fn coordinates_are_coarsened_and_sent_as_strings() { let r = build(Some("Thomas Kane"), Some(1830)).expect("publishable"); diff --git a/crates/navigator-sync/src/secret_store.rs b/crates/navigator-sync/src/secret_store.rs index 8425259e..e2ed470b 100644 --- a/crates/navigator-sync/src/secret_store.rs +++ b/crates/navigator-sync/src/secret_store.rs @@ -7,14 +7,14 @@ //! through [`get`], [`set`], and [`delete`] here. //! //! **The backend is in-memory unless a process explicitly opts in.** A test binary, a CI runner, a -//! doctest, or an `examples/` probe therefore *cannot* reach the login keychain no matter what it -//! constructs or in what order — the capability simply isn't switched on. The production binary +//! doctest, or an `examples/` probe therefore *can not* reach the login keychain no matter what it +//! constructs or in what order — the capability simply is not switched on. The production binary //! turns it on once, at the top of `main`, via [`use_os_keychain`]. //! //! This is the safe direction for the default to fail. Under the old opt-*out* scheme a test had to //! remember to call an escape hatch, and forgetting meant silently reading the user's real //! credentials under the production service name (and, on macOS, an interactive unlock prompt that -//! hangs CI). Under this scheme forgetting means a session doesn't persist across restarts — loud, +//! hangs CI). Under this scheme forgetting means a session does not persist across restarts — loud, //! local to the one binary that owns `main`, and impossible to miss on first launch. use std::collections::HashMap; diff --git a/crates/navigator-ui/src/cli.rs b/crates/navigator-ui/src/cli.rs index dff40ae1..2e7a2cbe 100644 --- a/crates/navigator-ui/src/cli.rs +++ b/crates/navigator-ui/src/cli.rs @@ -109,7 +109,7 @@ pub enum Command { /// node's descendant subtree (observed base + derived/ancestral status + evidence). For /// spot-checking placement and exchanging observations. Table by default; `--tsv` / `--json`. BranchReport(BranchReportArgs), - /// Diagnostic: explain why an alignment cannot be read. Probes the BAM/CRAM, its coordinate + /// Diagnostic: explain why an alignment can not be read. Probes the BAM/CRAM, its coordinate /// index, the reference FASTA and that FASTA's `.fai` **separately**, so a failure names the /// file actually at fault instead of whichever path the failing call happened to be handed — /// and reports the raw errno, which on macOS is the only thing distinguishing a privacy (TCC) @@ -428,7 +428,7 @@ pub struct BackfillArgs { #[derive(Args)] pub struct AccessionArgs { - /// Actually attach the accessions and correct the local `sample_accession`. Without this it's a + /// Actually attach the accessions and correct the local `sample_accession`. Without this it is a /// dry run (queries the API read-only, writes nothing). #[arg(long)] apply: bool, @@ -479,7 +479,7 @@ pub struct LoginArgs { #[derive(Args)] pub struct PruneArgs { /// Actually delete the orphans. Without this flag the command is a dry run (lists what it would - /// remove and touches nothing) — a PDS delete is irreversible, so it's opt-in. + /// remove and touches nothing) — a PDS delete is irreversible, so it is opt-in. #[arg(long)] apply: bool, /// Emit the outcome as JSON. @@ -802,7 +802,7 @@ async fn rebuild_signatures(args: RebuildArgs) -> i32 { } } // Re-place the per-alignment calls *and* rebuild the signatures built from them — the same - // `replace_against_current_tree` the GUI chore runs, so the two surfaces cannot drift. + // `replace_against_current_tree` the GUI chore runs, so the two surfaces can not drift. // Rebuilding only the profiles (what this did) left every `haplogroup_call` row on the tree // it was placed against, which is both the "sources diverge" conflicts on the Y card and the // reason `--stale-tree` re-selected the same subjects forever: it selects *by* those call @@ -914,7 +914,7 @@ async fn compare_callers(args: ShowArgs) -> i32 { let ext = c.external.as_deref().unwrap_or("(none)"); let nav = c.navigator.as_deref().unwrap_or("(none)"); // Only a real disagreement (both present, different) is flagged — a missing side - // is just "the other caller didn't produce a call here". + // is just "the other caller did not produce a call here". let differ = c.external.is_some() && c.navigator.is_some() && !c.agree(); if differ { diverged += 1; @@ -942,7 +942,7 @@ async fn analyze(args: AnalyzeArgs) -> i32 { let id = args.alignment; // The step list comes from `App::plan_full_analysis` — the same one the GUI's Full Analysis - // uses, so the two cannot drift again. In particular this is what stops a `navigator analyze` + // uses, so the two can not drift again. In particular this is what stops a `navigator analyze` // from re-genotyping Y over a trusted external call the user asked to prefer. let mut steps = match app.plan_full_analysis(id, args.ancestry, args.sv, None).await { Ok(s) => s, @@ -966,7 +966,7 @@ async fn analyze(args: AnalyzeArgs) -> i32 { let total = steps.len(); let t = Instant::now(); // Each arm renders its own one-line summary; `Err` is reported and the run continues, since - // a failed step (e.g. SV below the depth threshold) doesn't invalidate the rest. + // a failed step (e.g. SV below the depth threshold) does not invalidate the rest. let outcome: Result = match &step { AnalysisStep::QualityMetrics => match app.run_unified_metrics(id).await { Ok(r) => { @@ -2032,7 +2032,7 @@ pub struct DoctorArgs { /// gate in a script and not just by eye. /// /// `--file` deliberately skips opening the workspace: the file being undiagnosable is often *why* -/// the user cannot import it, so requiring a workspace record first would make the diagnostic +/// the user can not import it, so requiring a workspace record first would make the diagnostic /// unavailable in the case it exists for. async fn doctor(args: DoctorArgs) -> i32 { let diagnosis = if let Some(file) = args.file { diff --git a/crates/navigator-ui/src/ui/blocktree.rs b/crates/navigator-ui/src/ui/blocktree.rs index 22278b21..1c1596bc 100644 --- a/crates/navigator-ui/src/ui/blocktree.rs +++ b/crates/navigator-ui/src/ui/blocktree.rs @@ -160,7 +160,7 @@ pub(crate) struct Layout { /// The ticks are computed rather than spaced evenly, because evenly spaced would be wrong: each /// block spends one row on its name, so a fixed pixels-per-SNP scale drifts by a row per generation. /// Walking the lineage and placing each graduation inside the block that contains it keeps the axis -/// honest — the ticks come out *nearly* regular, and where they don't, the irregularity is real. +/// honest — the ticks come out *nearly* regular, and where they do not, the irregularity is real. fn ruler_ticks(blocks: &[Block], placed: &[Placed], row_h: f32, pad: f32) -> Vec { // Cumulative mutations to the bottom of each block, so "deepest" means most mutations, not most // generations — a long slow branch outranks several short ones. @@ -220,7 +220,7 @@ fn lines_for(b: &Block) -> usize { /// /// The test is the *fold*, not whether men sit on it. A collapsed run is by construction more than /// one branch, so its height is a sum across the tree above the cohort rather than one branch's -/// elapsed time — the one place where height-as-time doesn't hold. A root that was never collapsed +/// elapsed time — the one place where height-as-time does not hold. A root that was never collapsed /// is a single genuine branch and stays in the canvas at full height like any other. /// /// Men parked on the backbone (shallow kits, typically) keep their roster: the breadcrumb selects diff --git a/crates/navigator-ui/src/ui/central.rs b/crates/navigator-ui/src/ui/central.rs index e7723e66..e18ee810 100644 --- a/crates/navigator-ui/src/ui/central.rs +++ b/crates/navigator-ui/src/ui/central.rs @@ -90,7 +90,7 @@ impl NavigatorApp { }); ui.separator(); } - // Members vs Report on tabs — both can run to thousands of rows, so they don't stack. + // Members vs Report on tabs — both can run to thousands of rows, so they do not stack. ui.add_space(4.0); self.project_tab = self.sub_bar(ui, self.project_tab, &ProjectTab::ALL); match self.project_tab { @@ -194,7 +194,7 @@ impl NavigatorApp { }); } - /// When the selected subject has imported data that hasn't been analyzed yet, show a prominent + /// When the selected subject has imported data that has not been analyzed yet, show a prominent /// call-to-action to run the analysis pipeline — the brief stays empty until it runs. Shown only /// in the `Pending` state (has alignments, coverage not yet computed); hidden once analysis /// completes (→ `Complete`) or when the subject has nothing to analyze (no status row). @@ -421,7 +421,7 @@ impl NavigatorApp { asset_status_line(ui, &self.asset_status); self.donor_ancestry_summary(ui); // Publish the subject's consensus ancestry breakdown (one record per method) - // — available once it's been estimated. + // — available once it is been estimated. if self.donor_ancestry.is_some() { self.publish_row(ui, "Publish ancestry to PDS", Command::PublishAncestry { biosample_guid: guid }); } @@ -534,7 +534,7 @@ impl NavigatorApp { draw_roh(ui, result, regions); } }); - // Per-tab AI explanation of the ROH result (M5) — only once it's been computed. + // Per-tab AI explanation of the ROH result (M5) — only once it is been computed. if self.roh.is_some() { ui.add_space(8.0); self.ai_explain(ui, guid, SignalKind::Roh); @@ -601,7 +601,7 @@ impl NavigatorApp { } _ => { // Sparse input (a chip covers a few % of the panel, biased to its - // common tail) cannot be ranked against the WGS-scored cohort. + // common tail) can not be ranked against the WGS-scored cohort. ui.label( egui::RichText::new(self.tr("archaic.noPercentile")) .weak() @@ -617,7 +617,7 @@ impl NavigatorApp { ); } }); - // Per-tab AI explanation of the archaic result — only once it's been computed. + // Per-tab AI explanation of the archaic result — only once it is been computed. if self.archaic.is_some() { ui.add_space(8.0); self.ai_explain(ui, guid, SignalKind::Archaic); @@ -675,7 +675,7 @@ impl NavigatorApp { // footnote: this figure is measured against four sequenced archaic // genomes that represent some ancestries better than others, so // comparing it between people of different ancestry is the one use it - // cannot support. Stated where the number is read, or it will not be. + // can not support. Stated where the number is read, or it will not be. ui.label( egui::RichText::new(self.tr("archaicSegments.withinPopulation")) .color(egui::Color32::from_rgb(230, 180, 90)), diff --git a/crates/navigator-ui/src/ui/chrome.rs b/crates/navigator-ui/src/ui/chrome.rs index 0b3dd5e6..3a8a4a56 100644 --- a/crates/navigator-ui/src/ui/chrome.rs +++ b/crates/navigator-ui/src/ui/chrome.rs @@ -50,7 +50,7 @@ impl NavigatorApp { /// The cancel control shown beside a running analysis: it disables itself once clicked. /// - /// The disable matters — cancellation is cooperative and doesn't take effect until the walk + /// The disable matters — cancellation is cooperative and does not take effect until the walk /// reaches its next check, so a live button invited repeat clicks and made a working cancel /// look ignored. Every place that can start a walk needs the same behaviour, so it lives here /// rather than being re-derived per tab. @@ -440,7 +440,7 @@ impl NavigatorApp { self.start_ftdna_import(paths); } } - // Panels (import sites VCF) moved to Settings (⚙) → they're a workspace-wide asset, not a + // Panels (import sites VCF) moved to Settings (⚙) → they are a workspace-wide asset, not a // per-project action. } @@ -500,7 +500,7 @@ impl NavigatorApp { ui.heading(self.tr("nav.myDna")); ui.separator(); - // Add New (kept above the scroll body so it's always reachable). + // Add New (kept above the scroll body so it is always reachable). if ui .add( egui::Button::new(egui::RichText::new(self.tr("subjects.addNew")).color(egui::Color32::WHITE)) diff --git a/crates/navigator-ui/src/ui/descent.rs b/crates/navigator-ui/src/ui/descent.rs index 766b8aec..9582dd6f 100644 --- a/crates/navigator-ui/src/ui/descent.rs +++ b/crates/navigator-ui/src/ui/descent.rs @@ -15,8 +15,8 @@ const NOCALL: egui::Color32 = egui::Color32::from_rgb(110, 110, 110); // no conf impl NavigatorApp { /// Render the descent report for `dna`, loading it lazily off the worker thread on first view and /// caching the result. `compact` = the Simple-view path chain; otherwise the full Advanced report. - /// Additive and self-contained: shows a spinner while loading and a plain note when there's no - /// placement, so it's safe to drop into any tab. + /// Additive and self-contained: shows a spinner while loading and a plain note when there is no + /// placement, so it is safe to drop into any tab. pub(crate) fn descent_card(&mut self, ui: &mut egui::Ui, guid: SampleGuid, dna: DnaType, compact: bool) { self.ensure_descent(guid, dna); let entry = self @@ -53,7 +53,7 @@ impl NavigatorApp { } } } - // Loaded but empty → the variant profile isn't built yet (or has no placement). Offer the + // Loaded but empty → the variant profile is not built yet (or has no placement). Offer the // one-time build, which persists and then feeds this report instantly. Some(false) => self.descent_build_prompt(ui, guid, dna), None => { @@ -65,8 +65,8 @@ impl NavigatorApp { } } - /// Shown when there's no cached report: a one-time "Build" affordance that runs (and persists) - /// the variant profile this report is drawn from, or a plain note if it's built but unplaced. + /// Shown when there is no cached report: a one-time "Build" affordance that runs (and persists) + /// the variant profile this report is drawn from, or a plain note if it is built but unplaced. fn descent_build_prompt(&mut self, ui: &mut egui::Ui, guid: SampleGuid, dna: DnaType) { let (built, loading) = match dna { DnaType::Y => (self.y_profile.is_some(), self.y_profile_loading), @@ -129,7 +129,7 @@ impl NavigatorApp { } } - /// Fire a `LoadDescentReport` command if this (subject, DNA) report isn't already loaded or in + /// Fire a `LoadDescentReport` command if this (subject, DNA) report is not already loaded or in /// flight. Idempotent — safe to call every frame. pub(crate) fn ensure_descent(&mut self, guid: SampleGuid, dna: DnaType) { let loaded = self.descent_reports.iter().any(|(g, d, _)| *g == guid && *d == dna); diff --git a/crates/navigator-ui/src/ui/detail.rs b/crates/navigator-ui/src/ui/detail.rs index 072aeca5..2b451fff 100644 --- a/crates/navigator-ui/src/ui/detail.rs +++ b/crates/navigator-ui/src/ui/detail.rs @@ -420,7 +420,7 @@ impl NavigatorApp { .filter(|(a, _)| *a == key) .map(|(_, r)| r.as_slice()) .unwrap_or(&[]); - // Don't render a degenerate one-point plot: without the reference cloud the scatter + // Do not render a degenerate one-point plot: without the reference cloud the scatter // auto-zooms onto the donor alone and is meaningless. Surface the missing asset instead. if reference.is_empty() { ui.label(egui::RichText::new(self.tr("pca.referenceMissing")).weak()); @@ -640,7 +640,7 @@ impl NavigatorApp { /// A per-tab "Explain this" affordance (M5): a small button that asks the local model to explain /// just one signal (`kind`) for the selected subject in plain language, plus the streamed/finalized /// explanation rendered below it. Additive — the structured facts in the tab always remain, and - /// it's a no-op when the AI assistant is off. Only one explanation runs at a time (one worker). + /// it is a no-op when the AI assistant is off. Only one explanation runs at a time (one worker). pub(crate) fn ai_explain(&mut self, ui: &mut egui::Ui, guid: SampleGuid, kind: SignalKind) { if !self.ai_enabled { return; @@ -700,7 +700,7 @@ impl NavigatorApp { } /// "Ask about your results" chat (M2): a subject-scoped Q&A grounded in the brief. Sign-in is not - /// required (it's local), but the AI assistant must be enabled. Answers are AI-generated from the + /// required (it is local), but the AI assistant must be enabled. Answers are AI-generated from the /// results, so a persistent banner says to verify against the data. pub(crate) fn simple_chat_section(&mut self, ui: &mut egui::Ui, guid: SampleGuid) { if !self.ai_enabled { @@ -883,7 +883,7 @@ impl NavigatorApp { }); ui.add_space(10.0); // The subject anchor: publish the anonymized biosample summary + its sequence runs to the - // signed-in PDS. Every derived record (coverage / ancestry) links back to this, so it's the + // signed-in PDS. Every derived record (coverage / ancestry) links back to this, so it is the // one to publish first. card(ui, "Publish to PDS", |ui| { ui.label( @@ -909,7 +909,7 @@ impl NavigatorApp { /// the Add affordances are reachable). Edits are collected in locals and dispatched after the /// render closure (which borrows `self.tr` immutably). fn genealogy_card(&mut self, ui: &mut egui::Ui, guid: SampleGuid) { - // Clone the loaded bundle for this subject so the closure doesn't hold a borrow of `self`. + // Clone the loaded bundle for this subject so the closure does not hold a borrow of `self`. let Some(data) = self .genealogy .as_ref() @@ -1093,7 +1093,7 @@ impl NavigatorApp { ui.add_space(10.0); card(ui, self.tr("card.yHaplogroup"), |ui| self.y_haplogroup_section(ui, id)); // Hide the mtDNA sections when the selected alignment's coverage shows no chrM reads (a - // targeted-Y test without mitochondrial reads). Shown when coverage hasn't been run yet. + // targeted-Y test without mitochondrial reads). Shown when coverage has not been run yet. let no_mtdna = self.coverage.as_ref().is_some_and(|c| { !c.contig_coverage_stats .iter() @@ -1195,7 +1195,7 @@ impl NavigatorApp { ui.label(format!("{}>{}", v.reference, v.alternate)); ui.label(v.depth.to_string()); match &v.class { - // A "novel" call that lands on a catalogued Y-SNP: surface that name (it's not + // A "novel" call that lands on a catalogued Y-SNP: surface that name (it is not // on the placed lineage, but it is a known site, not a brand-new variant). PrivateClass::Novel => match names.get(&v.position) { Some(name) => ui @@ -1672,7 +1672,7 @@ impl NavigatorApp { // `all_alignments` — the whole workspace — and label the result "in this project": a // project whose alignments were every one already on the target build was still offered 35 // of them. The rule also lives in exactly one place now (`realignable_in_project`), so the - // number shown and the batch the button starts cannot disagree. + // number shown and the batch the button starts can not disagree. if self.project_realignable_asked != Some(project_id) { self.project_realignable_asked = Some(project_id); self.project_realignable = None; @@ -1863,7 +1863,7 @@ impl NavigatorApp { row.col(|ui| { ui.label(r.alignment_count.to_string()); }); - // Mean coverage, with a "lite" badge when it's a partial sidecar estimate that a + // Mean coverage, with a "lite" badge when it is a partial sidecar estimate that a // deep walk (the per-row coverage button) would upgrade. row.col(|ui| { if let Some(err) = &r.decode_error { @@ -1956,7 +1956,7 @@ impl NavigatorApp { use navigator_app::{StrChartCell, StrRowKind}; use navigator_domain::strchart::Deviation; - // Header labels (pulled before borrowing the chart, so closures don't re-borrow `self.tr`). + // Header labels (pulled before borrowing the chart, so closures do not re-borrow `self.tr`). let (h_name, h_kit, h_hap, h_test) = ( self.tr("ystr.col.name").to_string(), self.tr("ystr.col.kit").to_string(), @@ -2208,7 +2208,7 @@ impl NavigatorApp { /// Y-STR clusters: members grouped by branch, with confirmed placements and STR-only branch /// suggestions (the "autocluster + propagate" view). fn samples_clustered(&mut self, ui: &mut egui::Ui) { - // Clone the lightweight view data so the render closures don't hold a borrow of `self` while + // Clone the lightweight view data so the render closures do not hold a borrow of `self` while // we also call `self.tr` / dispatch a selection. let Some((_, clustering)) = self.project_clustering.clone() else { return; @@ -2226,7 +2226,7 @@ impl NavigatorApp { }; let mut pick = None; - // The parent (samples_section) owns the scroll area — no nested one here (that's the widget-ID + // The parent (samples_section) owns the scroll area — no nested one here (that is the widget-ID // clash and double-scrollbar). Render clusters directly. for cluster in &clustering.clusters { // A cluster shows if its branch matches the filter (→ all members) or any member matches. @@ -2528,7 +2528,7 @@ impl NavigatorApp { ui.label(egui::RichText::new(&a.reference_build).color(ACCENT).strong()); // A realigned alignment sits next to the one it came from, on the // same run, often with the same aligner — so without a mark the two - // rows are indistinguishable and the user cannot tell which is the + // rows are indistinguishable and the user can not tell which is the // vendor's file. if let Some(source) = a.derived_from_alignment_id { chip(ui, "realigned", ui.visuals().selection.bg_fill, egui::Color32::WHITE) diff --git a/crates/navigator-ui/src/ui/events.rs b/crates/navigator-ui/src/ui/events.rs index dcdb9694..31ce5110 100644 --- a/crates/navigator-ui/src/ui/events.rs +++ b/crates/navigator-ui/src/ui/events.rs @@ -263,7 +263,7 @@ impl NavigatorApp { Err(msg) => format!("{} {msg}", self.tr("brief.aiUnavailable")), }; // Set the authoritative answer on the pending assistant turn (pre-pushed on - // send); fall back to appending one if it's missing. + // send); fall back to appending one if it is missing. match self.chat_history.last_mut().filter(|t| !t.from_user) { Some(turn) => turn.text = text, None => self.chat_history.push(ChatTurn { from_user: false, text }), @@ -769,7 +769,7 @@ impl NavigatorApp { self.y_matches = Some((biosample_guid, matches)); } Event::DefaultAlignment { run_id, alignment_id } => { - // Only auto-select if the user hasn't already chosen an alignment. + // Only auto-select if the user has not already chosen an alignment. if self.selected_alignment.is_none() { self.pending_alignment = Some(alignment_id); self.select_run(run_id); // loads the run's alignments → applied below @@ -826,7 +826,7 @@ impl NavigatorApp { self.y_snp_names_requested = false; // re-resolve names incl. the new positions // A rebuild re-places the genome consensus (consensus_label); refresh the - // Overview's cached Y/mt consensus so it doesn't lag until the next reload. + // Overview's cached Y/mt consensus so it does not lag until the next reload. let _ = self.tx.send(Command::LoadConsensus(biosample_guid)); // The descent report is drawn from this profile — drop its cache so it rebuilds. self.descent_reports @@ -1204,7 +1204,7 @@ impl NavigatorApp { self.pca_reference = Some((alignment_id, points)); } Event::SourceFilesVerified { missing } => { - // Don't clobber a live import's progress status with this workspace-wide sweep + // Do not clobber a live import's progress status with this workspace-wide sweep // (the sweep and the import are unrelated; overwriting made imports look stalled). if !self.importing { self.status = if missing == 0 { @@ -1315,7 +1315,7 @@ impl NavigatorApp { self.status = format!("Error: {message}"); self.diagnosis = Some(report); // Open it unprompted: the whole point is that the one-line message is the part - // that isn't actionable, so making the user go find the detail would reproduce + // that is not actionable, so making the user go find the detail would reproduce // the original problem. self.show_diagnosis = true; self.clear_in_flight(); @@ -1374,7 +1374,7 @@ impl NavigatorApp { pub(crate) fn select_sample(&mut self, guid: SampleGuid) { self.selected_sample = Some(guid); - // A plain selection isn't "from a project" — the project opener re-sets this after. + // A plain selection is not "from a project" — the project opener re-sets this after. self.return_to_project = None; self.y_sub = YSub::default(); self.y_snp_sub = YSnpSub::default(); @@ -1530,7 +1530,7 @@ impl NavigatorApp { let _ = self.tx.send(Command::LoadReadMetrics(id)); let _ = self.tx.send(Command::LoadSv(id)); // Load cached chrM de-novo (mtDNA tab). chrY variant discovery is the masked private-Y - // pass, not a raw whole-chrY de-novo, so it isn't loaded here. + // pass, not a raw whole-chrY de-novo, so it is not loaded here. let _ = self.tx.send(Command::LoadDenovo { alignment_id: id, contig: "chrM".into(), diff --git a/crates/navigator-ui/src/ui/mod.rs b/crates/navigator-ui/src/ui/mod.rs index 474af610..17f88953 100644 --- a/crates/navigator-ui/src/ui/mod.rs +++ b/crates/navigator-ui/src/ui/mod.rs @@ -434,8 +434,8 @@ fn resolved_ui_scale() -> f32 { /// One-shot auto UI-scale probe (the "behave like a native app" default). On the first frame the /// monitor size is known, derive a zoom when the OS reports a ~1.0 scale factor on a clearly -/// high-resolution panel (e.g. native-4K, where macOS itself doesn't up-scale). A Retina / scaled -/// display (native ppp > 1) is already handled by egui's native scaling, so it's left at 1.0. Skipped +/// high-resolution panel (e.g. native-4K, where macOS itself does not up-scale). A Retina / scaled +/// display (native ppp > 1) is already handled by egui's native scaling, so it is left at 1.0. Skipped /// entirely when a manual scale is persisted (`probed` starts `true`). The result fills the Settings /// slider but is not persisted until the user saves — re-probed each launch otherwise. fn run_auto_scale(probed: &mut bool, form: &mut SettingsForm, ctx: &egui::Context) { @@ -515,7 +515,7 @@ struct EditProject { } /// Editable copy of a sequence run, driving the run Edit modal (Some ⇒ the dialog is shown). -/// Read-metric columns are not editable here, so they're not carried. +/// Read-metric columns are not editable here, so they are not carried. #[derive(Clone)] struct EditRun { id: i64, @@ -862,7 +862,7 @@ pub struct NavigatorApp { consensus_y: Option, consensus_mt: Option, /// YFull-style descent reports for the selected subject, loaded lazily per `DnaType` and cached - /// as `Some(report)` / `None` (placed-but-empty), so a built-once result isn't re-fetched; plus + /// as `Some(report)` / `None` (placed-but-empty), so a built-once result is not re-fetched; plus /// the (guid, dna) pairs currently loading. All cleared on subject switch. descent_reports: Vec<(SampleGuid, DnaType, Option)>, descent_loading: Vec<(SampleGuid, DnaType)>, @@ -916,7 +916,7 @@ pub struct NavigatorApp { ancient_ancestry: Option, /// Reference PC1/PC2 centroids for the PCA scatter, keyed by alignment_id (lazy-loaded). pca_reference: Option<(i64, PcaCentroids)>, - /// Which PCA-reference key we've already dispatched a load for (avoids re-sending every frame). + /// Which PCA-reference key we have already dispatched a load for (avoids re-sending every frame). pca_reference_attempted: Option, /// Donor-level private-Y union across the subject's sources. donor_private_y: Option, @@ -933,7 +933,7 @@ pub struct NavigatorApp { /// Catalogued Y-SNP names at variant positions (`position → name`), used to annotate the two /// Y-SNP tables' position-only / novel calls. Resolved once per subject from the Y-SNP dictionary. y_snp_names: std::collections::HashMap, - /// True once we've dispatched the Y-SNP-name resolution for the current subject (avoids re-sending). + /// True once we have dispatched the Y-SNP-name resolution for the current subject (avoids re-sending). y_snp_names_requested: bool, /// True while the (expensive) Y-variant profile is being built. y_profile_loading: bool, @@ -987,7 +987,7 @@ pub struct NavigatorApp { genome_regions: Option<(i64, std::sync::Arc)>, /// True while the cytoBand fetch is in flight. loading_regions: bool, - /// The alignment we've already kicked off (or completed) a region load for — avoids re-firing + /// The alignment we have already kicked off (or completed) a region load for — avoids re-firing /// the fetch every frame, including after a failure. regions_attempted: Option, /// Which contig's depth histogram the coverage view charts: `None` = whole-genome histogram, @@ -1092,9 +1092,9 @@ pub struct NavigatorApp { /// The admin's per-kit resolutions for the fuzzy rows in [`Self::ftdna_plan`]. ftdna_resolutions: std::collections::BTreeMap, /// The selected subject's imported genealogy (vendor ids + FTDNA member + MDKA), for the - /// Overview card. `(guid, data)` so a stale bundle from a prior subject isn't shown. + /// Overview card. `(guid, data)` so a stale bundle from a prior subject is not shown. genealogy: Option<(SampleGuid, FtdnaGenealogy)>, - /// The current project's Y-STR clustering, keyed by project id (so a stale one isn't shown). + /// The current project's Y-STR clustering, keyed by project id (so a stale one is not shown). project_clustering: Option<(i64, YstrClustering)>, /// True while the project Y-STR clustering is computing. clustering_running: bool, @@ -1282,12 +1282,12 @@ impl NavigatorApp { let dark = !matches!(settings.theme.as_deref(), Some("light")); apply_theme(&cc.egui_ctx, dark); // Persisted UI scale (egui zoom) — fixes tiny text on a native-4K display the OS reports at - // scale factor 1.0. egui's keyboard zoom (Cmd +/-/0) also works but isn't persisted. + // scale factor 1.0. egui's keyboard zoom (Cmd +/-/0) also works but is not persisted. cc.egui_ctx.set_zoom_factor(resolved_ui_scale()); // Restore the last navigation position (view / focused subject / detail tab). The subject is // applied once the list loads (see the `AllBiosamples` handler); nav/tab apply immediately // (nav is then reconciled to the interface mode by `normalize_for_mode`). Seed `saved_ui_sig` - // with the restored intent so a matching restore doesn't trigger a redundant re-save. + // with the restored intent so a matching restore does not trigger a redundant re-save. let restore = &settings; let restored_nav = restore .last_nav @@ -2475,7 +2475,7 @@ mod icon_glyph_tests { /// Every string literal this crate draws must be drawable. /// /// `every_translated_string_is_renderable` covers the catalogs, which is where user-facing copy - /// belongs — but icons don't live there. A button label like `ui.small_button("✎")` is a bare + /// belongs — but icons do not live there. A button label like `ui.small_button("✎")` is a bare /// literal in the source, invisible to the catalog scan, and that is where the second round of /// tofu boxes was found: the MDKA edit/remove buttons and the kit-remove button on the /// Genealogy card, every clear-filter `✕`, the Y-STR agreement `✓`/`✗`, the sortable-table diff --git a/crates/navigator-ui/src/ui/modals.rs b/crates/navigator-ui/src/ui/modals.rs index 25825403..55ecc8db 100644 --- a/crates/navigator-ui/src/ui/modals.rs +++ b/crates/navigator-ui/src/ui/modals.rs @@ -399,7 +399,7 @@ impl NavigatorApp { /// The diagnosis modal: why the last alignment command actually failed, file by file. /// /// Shown when a command fails *and* the preflight found a concrete cause, because the one-line - /// status-bar message is exactly the part that isn't actionable — the reader helpers report + /// status-bar message is exactly the part that is not actionable — the reader helpers report /// whichever path the failing call was handed, which is routinely not the file at fault. The /// report is selectable and copyable so it can go straight into a bug report; that is the /// primary job of this modal, not a convenience. @@ -482,7 +482,7 @@ impl NavigatorApp { let mut lift_request = false; let mut test_llm: Option = None; let mut refresh_trees = false; - // While the scale slider is being dragged, DON'T live-apply the zoom (see the live-apply + // While the scale slider is being dragged, Do not live-apply the zoom (see the live-apply // block below): changing the zoom factor rescales the slider's own rail mid-drag, so the // cursor maps to a runaway value that collapses to a bound. Apply only once the drag ends. let mut scale_dragging = false; @@ -875,7 +875,7 @@ impl NavigatorApp { self.dark_mode = theme_dark; apply_theme(ctx, self.dark_mode); } - // Apply the zoom only when the slider isn't mid-drag (typed/committed/button changes still + // Apply the zoom only when the slider is not mid-drag (typed/committed/button changes still // apply immediately). Applying during a drag would rescale the rail and make the value run // away to a bound — the reported "only 0.8 or 2.5" symptom. if !scale_dragging && (ctx.zoom_factor() - form.ui_scale).abs() > f32::EPSILON { @@ -891,7 +891,7 @@ impl NavigatorApp { if save { let appview = form.appview_url.trim().to_string(); - // The fields this dialog doesn't own are carried over from disk; read once rather than + // The fields this dialog does not own are carried over from disk; read once rather than // re-reading and re-parsing settings.json for each of them. let kept = AppSettings::load(); let settings = AppSettings { @@ -1154,7 +1154,7 @@ impl NavigatorApp { }); }); if skip { - // Persist the skip so this exact version doesn't notify again (a newer one still will). + // Persist the skip so this exact version does not notify again (a newer one still will). let mut settings = AppSettings::load(); settings.skip_update_version = Some(info.latest_version.clone()); match settings.save() { @@ -1895,7 +1895,7 @@ impl NavigatorApp { /// The consent decision for an inbound matching request. /// /// A modal rather than an Accept button in a table row, because consenting does two things the - /// row cannot say: it reveals our DID to the counterpart, and it puts our IBD-panel dosages on + /// row can not say: it reveals our DID to the counterpart, and it puts our IBD-panel dosages on /// the encrypted channel. Neither is undoable. The three headings below are the whole point of /// the dialog — what we send, what they learn, and what never leaves the device. pub(crate) fn consent_modal(&mut self, ctx: &egui::Context) { @@ -1967,7 +1967,7 @@ impl NavigatorApp { /// Review a candidate branch: the shared position(s) and every carrier's read evidence. /// - /// A candidate is inferred, not published, and "1 SNP shared by three men" cannot be judged from + /// A candidate is inferred, not published, and "1 SNP shared by three men" can not be judged from /// the canvas. What decides it is the evidence behind each call — depth, and how cleanly the /// derived allele dominates on a chromosome carrying one copy. A middling fraction or a thin /// depth is the signature of the mapping artefacts this view is most at risk of presenting as diff --git a/crates/navigator-ui/src/ui/simple.rs b/crates/navigator-ui/src/ui/simple.rs index 69005eda..0f11632a 100644 --- a/crates/navigator-ui/src/ui/simple.rs +++ b/crates/navigator-ui/src/ui/simple.rs @@ -28,7 +28,7 @@ impl NavigatorApp { /// /// The reference-download and "not analyzed yet" prompts render *above* the split rather than in /// a panel — both block every panel equally, so burying either one behind a rail click would let - /// a user wander an empty view without being told why it's empty. + /// a user wander an empty view without being told why it is empty. pub(crate) fn simple_subject_view(&mut self, ui: &mut egui::Ui, guid: SampleGuid) { ui.separator(); self.reference_prompt(ui); @@ -119,7 +119,7 @@ impl NavigatorApp { self.simple_panel = panel; } - // The bridge to the full power-user view lives at the foot of the rail, where it's reachable + // The bridge to the full power-user view lives at the foot of the rail, where it is reachable // from every panel rather than only from the bottom of one long scroll. ui.add_space(12.0); ui.separator(); @@ -294,7 +294,7 @@ impl NavigatorApp { /// The at-a-glance grid: one tile per remaining panel, each showing its headline value and /// opening that panel when clicked. This is the landing screen's index — it is why the story - /// panel doesn't need to restate the paternal line, the ancestry donut, and the match list. + /// panel does not need to restate the paternal line, the ancestry donut, and the match list. fn simple_glance_grid(&mut self, ui: &mut egui::Ui, guid: SampleGuid) { let tiles: Vec<(SimplePanel, &'static str, &'static str, Option)> = SimplePanel::ALL .iter() @@ -352,7 +352,7 @@ impl NavigatorApp { /// One lineage panel. Renders the brief's lineage card (haplogroup, age, origin, story, /// confidence, descent trail) or, when this subject has no such line, an explanation of *why* - /// there isn't one — "no data" with no reason is the complaint this redesign exists to fix. + /// there is not one — "no data" with no reason is the complaint this redesign exists to fix. fn simple_lineage_panel(&mut self, ui: &mut egui::Ui, guid: SampleGuid, kind: LineageKind) { let Some(brief) = self.subject_brief.as_ref().filter(|(g, _)| *g == guid).map(|(_, b)| b) else { self.simple_brief_placeholder(ui); @@ -491,7 +491,7 @@ impl NavigatorApp { ui.add_space(10.0); // The fine breakdown gets its own card rather than a collapsed row inside the one above: - // the panel has the vertical room the old single scroll didn't, and it is the section most + // the panel has the vertical room the old single scroll did not, and it is the section most // readers came for. if !a.fine_pops.is_empty() { card(ui, detail_title, |ui| { @@ -533,7 +533,7 @@ impl NavigatorApp { /// a **confirmed** relative, where an encrypted segment exchange has actually run and produced a /// shared-cM total, and a **candidate**, where the AppView has only scored the signals we both /// published. They are grouped by the same three bands, but the band is chosen from measured cM - /// where we have it and from the suggestion's tier where we don't — and the row says which, + /// where we have it and from the suggestion's tier where we do not — and the row says which, /// because "close family" inferred from a score is a much weaker claim than the same words /// backed by 1,800 shared centimorgans. fn simple_relatives_panel(&mut self, ui: &mut egui::Ui, guid: SampleGuid) { @@ -649,7 +649,7 @@ impl NavigatorApp { } }); }); - // The evidence line. Measured sharing beats a score, so it's what's shown when we + // The evidence line. Measured sharing beats a score, so it is what is shown when we // have it; the tier note above already said which kind of row this is. match &row.shared { Some(s) => { diff --git a/crates/navigator-ui/src/ui/sources.rs b/crates/navigator-ui/src/ui/sources.rs index cd2b9943..72024244 100644 --- a/crates/navigator-ui/src/ui/sources.rs +++ b/crates/navigator-ui/src/ui/sources.rs @@ -280,7 +280,7 @@ impl NavigatorApp { ui.label(format!("Already on {target} — there is nothing to realign.")); return; } - // A row with no file cannot be re-mapped; the job fails at `MissingPaths`. The app's + // A row with no file can not be re-mapped; the job fails at `MissingPaths`. The app's // `realignable_for_subject` has always excluded these — this card did not. if alignment.bam_path.is_none() { ui.label("This alignment has no file to re-map."); diff --git a/crates/navigator-ui/src/worker.rs b/crates/navigator-ui/src/worker.rs index 31f08ab1..0e12acab 100644 --- a/crates/navigator-ui/src/worker.rs +++ b/crates/navigator-ui/src/worker.rs @@ -836,23 +836,23 @@ pub enum Event { dna: DnaType, result: Result, String>, }, - /// A streamed slice of narration text as it's generated (live preview; the final BriefNarration + /// A streamed slice of narration text as it is generated (live preview; the final BriefNarration /// is authoritative). BriefNarrationChunk { guid: SampleGuid, text: String, }, - /// AI-assisted narration of a subject's brief (or a plain-language reason it's unavailable). + /// AI-assisted narration of a subject's brief (or a plain-language reason it is unavailable). BriefNarration { guid: SampleGuid, result: Result, }, - /// A streamed slice of a chat answer as it's generated (live preview). + /// A streamed slice of a chat answer as it is generated (live preview). ChatAnswerChunk { guid: SampleGuid, text: String, }, - /// Answer to an "ask my results" question (or a plain-language reason it's unavailable). + /// Answer to an "ask my results" question (or a plain-language reason it is unavailable). ChatAnswer { guid: SampleGuid, result: Result, @@ -863,7 +863,7 @@ pub enum Event { kind: SignalKind, text: String, }, - /// AI-assisted explanation of one result signal (or a plain-language reason it's unavailable). + /// AI-assisted explanation of one result signal (or a plain-language reason it is unavailable). SignalNarration { guid: SampleGuid, kind: SignalKind, @@ -1847,7 +1847,7 @@ pub async fn handle(app: &App, cmd: Command, cancel: &CancelToken) -> Event { } Command::EstimateDeepAncestry { biosample_guid } => { // Heavy: genotypes the best CHM13 alignment at ~1.15M sites, then fits qpAdm f4. Persists - // the ANCIENT_ADMIXTURE result (or nothing, when the model doesn't apply). + // the ANCIENT_ADMIXTURE result (or nothing, when the model does not apply). ev(app.estimate_deep_ancestry(biosample_guid).await, |result| { Event::DeepAncestryEstimated { biosample_guid, @@ -2164,7 +2164,7 @@ pub async fn handle(app: &App, cmd: Command, cancel: &CancelToken) -> Event { Command::RunIbdExchange { info, biosample_guid } => { let cfg = IbdDetectorConfig::default(); // A failure is recorded on the conversation rather than only surfaced as a transient - // toast — otherwise the request sits at READY and the user cannot tell it was tried. + // toast — otherwise the request sits at READY and the user can not tell it was tried. let outcome = match app.open_exchange_session(&info).await { Ok(session) => { app.exchange_ibd_for_subject(&session, biosample_guid, &info.request_uri, None, cfg) @@ -2364,7 +2364,7 @@ async fn resolve_reference_streaming( wake: &(dyn Fn() + Send + Sync), ) { // The progress closure must be Send (it runs in a task) — capture an owned Sender clone - // and a label, not borrows. Throttle to ~every 25 MB so a multi-GB pull doesn't flood. + // and a label, not borrows. Throttle to ~every 25 MB so a multi-GB pull does not flood. let tx = evt_tx.clone(); let label = build.clone(); let mut last_sent = 0u64; @@ -2390,7 +2390,7 @@ async fn resolve_reference_streaming( /// [`resolve_reference_streaming`]). Cached builds are skipped silently. The reference FASTA is a /// required artifact for any BAM/CRAM analysis and is fetched on demand (cache-first, else a /// multi-GB download) — without this the download runs deep inside a pure `App` method with a no-op -/// callback, so the UI shows nothing and a first import looks like it "didn't register". Call this +/// callback, so the UI shows nothing and a first import looks like it "did not register". Call this /// from the worker after an import and before a reference-needing analysis so the pull is visible. async fn ensure_references_streaming( app: &App, @@ -2558,7 +2558,7 @@ async fn run_realign_streaming( /// Run the full per-alignment analysis pipeline, emitting `AnalysisProgress` before each step /// and forwarding each step's own result event (so the detail tabs fill in live). `cancel` is -/// checked between steps. Per-step errors are forwarded but don't abort the pipeline (best-effort). +/// checked between steps. Per-step errors are forwarded but do not abort the pipeline (best-effort). async fn run_full_analysis_streaming( app: &App, alignment_id: i64, @@ -2593,9 +2593,9 @@ async fn run_full_analysis_streaming( }); wake(); // Reuse cached sub-results instead of re-scanning the whole genome (minutes) — only when - // all three are present, since they're persisted together by the unified walker. The + // all three are present, since they are persisted together by the unified walker. The // coverage must also be at the right scope (a stale whole-genome result for a targeted-Y - // test reads as a miss) so it's recomputed restricted to the target contigs. + // test reads as a miss) so it is recomputed restricted to the target contigs. let cached = match ( app.cached_coverage_for_analysis(alignment_id).await, app.cached_read_metrics(alignment_id).await, @@ -3169,7 +3169,7 @@ pub fn spawn(db_path: PathBuf, wake: impl Fn() + Send + Sync + 'static) -> (Unbo } // Import, then eagerly resolve the imported alignments' reference(s) with a // visible progress bar — a first CRAM/BAM that needs a multi-GB reference - // download otherwise looks like it didn't register (§ ensure_references_streaming). + // download otherwise looks like it did not register (§ ensure_references_streaming). Command::AddDataBatch { biosample_guid, paths } => { let event = handle( &app, @@ -3216,7 +3216,7 @@ pub fn spawn(db_path: PathBuf, wake: impl Fn() + Send + Sync + 'static) -> (Unbo } } // Pre-resolve the subject's / alignment's reference and coordinate index (with a - // progress bar) so a query-driven analysis doesn't trigger a silent download or + // progress bar) so a query-driven analysis does not trigger a silent download or // index build partway through. Command::BuildAutosomalProfile { biosample_guid } => { if let Ok(builds) = app.reference_builds_for_subject(biosample_guid).await { @@ -3567,11 +3567,11 @@ pub fn spawn(db_path: PathBuf, wake: impl Fn() + Send + Sync + 'static) -> (Unbo Command::DrainOutbox => { emit_drain(&app, &evt_tx, &*wake).await; } - // Streams narration text as it's generated, then a final BriefNarration. + // Streams narration text as it is generated, then a final BriefNarration. Command::NarrateBrief(guid) => { narrate_brief_streaming(&app, guid, &evt_tx, &*wake).await; } - // Streams a chat answer as it's generated, then a final ChatAnswer. + // Streams a chat answer as it is generated, then a final ChatAnswer. Command::AskQuestion { guid, history, @@ -3579,7 +3579,7 @@ pub fn spawn(db_path: PathBuf, wake: impl Fn() + Send + Sync + 'static) -> (Unbo } => { ask_question_streaming(&app, guid, history, question, &evt_tx, &*wake).await; } - // Streams a per-signal explanation as it's generated, then a final SignalNarration. + // Streams a per-signal explanation as it is generated, then a final SignalNarration. Command::NarrateSignal { guid, kind } => { narrate_signal_streaming(&app, guid, kind, &evt_tx, &*wake).await; } From 95db9eb98b293de10447f35bbe25430d5203290e Mon Sep 17 00:00:00 2001 From: James Kane Date: Tue, 18 Aug 2026 10:08:40 -0500 Subject: [PATCH 03/33] docs(ste): navigator-resource and sig_cache, converted in full MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first two files taken to zero violations, and the proof that the rule set survives contact with this codebase's harder prose. `navigator-resource/src/lib.rs` was the real test: its module header is a post-mortem of the 2026-08-13 WGS sort that a WindowServer watchdog killed. Narrative, em-dashes throughout, and the whole argument for the crate carried in two long paragraphs. Under STE it becomes a dated sequence of short factual statements. Every fact survives — the 549 GB of dirty file-backed memory, the 1.4x write-back limit, the 40-second watchdog, the login session going down — and so does every reason. What goes is the compression and the voice. That is the trade this standard makes on purpose. 134 violations in that file, four passes, roughly 34 targeted rewrites. `sig_cache.rs` took two. Both crates still build and `cargo fmt --all --check` is clean. Co-Authored-By: Claude Opus 5 (1M context) --- crates/navigator-resource/src/lib.rs | 284 +++++++++++++----------- crates/navigator-store/src/sig_cache.rs | 86 +++---- 2 files changed, 198 insertions(+), 172 deletions(-) diff --git a/crates/navigator-resource/src/lib.rs b/crates/navigator-resource/src/lib.rs index 055b1c7e..0f5b00eb 100644 --- a/crates/navigator-resource/src/lib.rs +++ b/crates/navigator-resource/src/lib.rs @@ -1,42 +1,53 @@ -//! Watching what a long stage is doing to the machine. +//! A record of the load that a long stage puts on the machine. //! -//! A realignment runs for hours on a machine its user is still trying to use, and the failure that -//! motivated this module was not the one anybody was watching for. On 2026-08-13 a WGS-scale sort -//! was killed six hours in, and the obvious suspect was memory. It was not: the compressor was -//! idle, no jetsam report was filed, and anonymous memory sat at 32 GB of 128. What the operating -//! system *did* complain about was writes — it filed a resource notice against the process for -//! dirtying 549 GB of file-backed memory at nearly 1.4x the system's sustained write-back limit. -//! WindowServer's main thread then missed a 40-second watchdog check-in, the watchdog killed it, -//! and every process in the login session went down with it, this job included. +//! A realignment runs for hours. The user continues to use the machine during this time. The fault +//! that made this module necessary was not the fault that we monitored. //! -//! So the thing worth watching is not one number. Memory is cheap to sample and would have -//! exonerated itself immediately; the write rate is the number that was actually extreme, and -//! nothing was recording it. This samples both, on a cadence, and hands the caller a -//! [`ResourceSample`] to log. +//! On 2026-08-13, a WGS-scale sort stopped after six hours. We first thought that memory was the +//! cause. It was not. The compressor was idle. The system wrote no jetsam report. Anonymous memory +//! stayed at 32 GB of 128 GB. //! -//! ## It reports; it does not intervene +//! The operating system reported a different problem. The problem was the writes. The system filed +//! a resource notice against the process, because the process made 549 GB of file-backed memory +//! dirty. That rate is almost 1.4 times the sustained write-back limit of the system. //! -//! Nothing here aborts a job. A stage that is writing hard is doing its job — the sort *is* a -//! hundreds-of-GB write — and a watchdog that killed a six-hour run for going fast would be worse -//! than the problem. Bounding the damage belongs where the writes happen, on a byte cadence -//! ([`PacedFile`]); this exists so that the next time something goes wrong there is a record of -//! what the machine looked like, instead of an inference from a crash report. +//! The main thread of WindowServer then missed a 40-second watchdog check. The watchdog stopped +//! WindowServer. All processes in the login session stopped, and this job was one of them. //! -//! ## Why a crate of its own +//! So one number is not enough. Memory is easy to sample, and a memory sample shows immediately +//! that memory is not the cause. The write rate was the extreme value, but no code recorded it. +//! This module samples both values at an interval. It gives a [`ResourceSample`] to the caller, and +//! the caller writes the sample to the log. //! -//! Because a byte counter only means anything if there is exactly one of it, and the pipeline's -//! writers are split across crates that do not depend on each other: `navigator-align` maps, -//! `navigator-analysis` reverts, sorts, marks, and compresses. This first shipped as a module in -//! the latter, which meant the mapping stage — the single longest in the job, and the one that -//! writes the ~60 GB `mapped.bam` — was neither paced nor counted. The run log read `0 MB/s` -//! straight through it, which is not a quiet stage but an unmeasured one. +//! ## This module reports. It does not intervene. +//! +//! No code here stops a job. A stage that writes at a high rate does the correct work. The sort +//! must write hundreds of GB. A watchdog that stopped a six-hour run for high speed is worse than +//! the initial problem. +//! +//! To limit the damage, control the writes at the point where the code makes them. [`PacedFile`] +//! does this at a byte interval. This module gives you a record of the state of the machine. Before +//! this module, you could only make a deduction from a crash report. +//! +//! ## Why this is a crate +//! +//! A byte counter is correct only if there is one counter. The writers of the pipeline are in +//! crates that do not depend on each other. `navigator-align` maps the reads. `navigator-analysis` +//! reverts, sorts, marks, and compresses them. +//! +//! At first, this code was a module in `navigator-analysis`. So the mapping stage had no pace +//! control and no counter. That stage is the longest stage in the job, and it writes the 60 GB +//! `mapped.bam` file. The run log showed `0 MB/s` for the full stage. The stage was not quiet. No +//! code measured it. //! //! ## Portability //! -//! Every probe here is `sysinfo`, which binds the platform APIs through pure-Rust crates on all -//! three desktop targets — the same reason `navigator-align` picked it for RAM detection. There is -//! deliberately no `fcntl`/`ioctl`/`/proc` in this crate: the guard has to hold on Windows, and a -//! guard that only arms itself on macOS would have been no guard at all for most users. +//! Each probe uses `sysinfo`. That crate binds the platform APIs with pure-Rust code on all three +//! desktop targets. `navigator-align` uses `sysinfo` to find the quantity of RAM for the same +//! reason. +//! +//! This crate has no `fcntl` code, no `ioctl` code, and no `/proc` code. The guard must work on +//! Windows. A guard that starts only on macOS is not a guard for most users. use std::fs::File; use std::io::{self, Write}; @@ -45,23 +56,25 @@ use std::sync::Arc; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; -/// Bytes handed to disk by the pipeline's paced writers since the process started. +/// The quantity of bytes that the paced writers of the pipeline sent to disk. The count starts +/// when the process starts. /// -/// Self-accounted rather than read back from the OS: every platform exposes per-process I/O -/// counters differently (and Windows' are not in `sysinfo`'s default surface), whereas the writers -/// already know exactly how much they wrote. It costs one relaxed add per buffer. +/// This crate counts the bytes. It does not read a count from the operating system. Each platform +/// gives the I/O counters for a process in a different form, and `sysinfo` does not show the +/// Windows counters by default. The writers already know the exact quantity. The count costs one +/// relaxed add for each buffer. /// -/// One counter for the whole process, which is the reason this lives in a crate of its own: a -/// realignment's writes come from two crates that cannot see each other, and two counters would -/// have been two half-answers. +/// There is one counter for the full process. This is the reason for a separate crate. The writes +/// of a realignment come from two crates that can not see each other. Two counters would give two +/// incomplete answers. static BYTES_WRITTEN: AtomicU64 = AtomicU64::new(0); -/// Account `n` bytes written. Called from the write path; must stay this cheap. +/// Add `n` bytes to the count. The write path calls this function, so it must stay small. pub fn record_bytes_written(n: u64) { BYTES_WRITTEN.fetch_add(n, Ordering::Relaxed); } -/// Total bytes the post-processing writers have written this process. +/// The total quantity of bytes that the post-process writers wrote in this process. pub fn bytes_written() -> u64 { BYTES_WRITTEN.load(Ordering::Relaxed) } @@ -69,9 +82,9 @@ pub fn bytes_written() -> u64 { /// Bytes written between forced flushes to disk. See [`PacedFile`]. const DEFAULT_SYNC_MB: u64 = 256; -/// How much may be left dirty before a paced stream pushes it to disk. +/// The quantity of data that can stay dirty before a paced stream sends it to disk. /// -/// `NAVIGATOR_IO_SYNC_MB=0` turns the pacing off and restores the unbounded behaviour. +/// `NAVIGATOR_IO_SYNC_MB=0` stops the pace control. The stream then has no limit. fn sync_interval() -> u64 { std::env::var("NAVIGATOR_IO_SYNC_MB") .ok() @@ -81,30 +94,33 @@ fn sync_interval() -> u64 { * 1024 } -/// A file that will not let an unbounded amount of its output sit dirty in the page cache. +/// A file that limits the quantity of its output that stays dirty in the page cache. /// -/// Without this, a stage writes as fast as it can into the page cache and leaves write-back to the -/// operating system, which sounds like the right division of labour and is, until the volume gets -/// far enough out of scale. One WGS realignment dirtied 549 GB of file-backed memory — enough that -/// macOS filed a disk-writes resource notice against the process for exceeding its sustained -/// write-back limit by 1.4x, and enough that WindowServer's main thread missed a 40-second watchdog -/// check-in and was killed, taking the login session and a six-hour job with it. +/// Without this control, a stage writes to the page cache at its maximum speed. The operating +/// system then does the write-back. This division of work is correct until the volume becomes very +/// large. /// -/// Flushing on a byte cadence caps how much can be outstanding at once. The write path pays for its -/// own I/O as it goes instead of handing the machine a debt to settle later, which is also why this -/// is not obviously a slowdown: the same bytes reach the same disk, in steadier instalments rather -/// than in storms. +/// One WGS realignment made 549 GB of file-backed memory dirty. The process went past its +/// sustained write-back limit by 1.4 times, so macOS filed a disk-writes resource notice. The main +/// thread of WindowServer then missed a 40-second watchdog check and stopped. The login session +/// and a six-hour job stopped with it. /// -/// It lives here rather than beside any one stage because every stage that writes tens of GB wants -/// it: the mapper's output and its 8.93 GB minimizer index, the revert's spill runs and FASTQ -/// (where the scratch peak actually is), the sort's runs and merged output, and the final CRAM. The -/// byte accounting that [`ResourceWatch`] reports comes from the same place, so a stream that is -/// paced is also a stream that is counted — and a writer nobody wrapped is a writer that shows up -/// in neither. +/// A flush at a byte interval limits the quantity of data that waits in the cache. The write path +/// does its own I/O during the run. It does not give the machine a debt to pay later. For this +/// reason the control is not a large delay: the same bytes go to the same disk at a more constant +/// rate. /// -/// `sync_data` rather than `sync_all` — the contents must be durable, the metadata need not be, and -/// on a stream this size that is many thousands of inode updates. It is std's portable spelling: -/// `fdatasync` where there is one, `FlushFileBuffers` on Windows. +/// This type is in this crate, not beside one stage, because each stage that writes tens of GB +/// needs it. These stages are the mapper with its 8.93 GB minimizer index, and the revert stage +/// with its spill runs and FASTQ. The sort with its runs and merged output and the last CRAM also +/// need it. The byte +/// count that [`ResourceWatch`] reports comes from the same place. So a paced stream is also a +/// counted stream, and a writer without this type is in neither total. +/// +/// This type calls `sync_data`, not `sync_all`. The contents must be durable, but the metadata does +/// not need to be durable. On a stream of this size, the metadata is many thousands of inode +/// updates. `sync_data` is the portable name in std. It calls `fdatasync` where that call exists, +/// and `FlushFileBuffers` on Windows. pub struct PacedFile { file: File, since_sync: u64, @@ -120,10 +136,10 @@ impl PacedFile { } } - /// Push everything still outstanding to disk. + /// Send all data that waits in the cache to disk. /// - /// Called at the end of a stream whose end-of-file marker a later run may trust: a marker - /// sitting in the page cache is a promise the disk has not made. + /// Call this at the end of a stream when a later run can trust the end-of-file marker of that + /// stream. A marker in the page cache is a promise that the disk did not make. pub fn sync(&self) -> io::Result<()> { self.file.sync_data() } @@ -150,32 +166,34 @@ impl Write for PacedFile { } } -/// The budget an external-sort stage holds in memory before spilling a run to disk. +/// The quantity of memory that an external-sort stage holds before it spills a run to disk. +/// +/// The pipeline has two spill-to-disk stages: the collator of the revert stage and the coordinate +/// sort. A constant gave the size of each one, 256 MB and 512 MB. Nobody had run a WGS sample +/// through the two stages when they chose these values. /// -/// Both of the pipeline's spill-to-disk stages — the revert's collator and the coordinate sort — -/// were sized by a constant, 256 MB and 512 MB, chosen when nobody had run a WGS through them. -/// The measured cost of that: a 30x WGS coordinate sort spilled **688 runs**, all of which the -/// merge then opens at once. It is bounded memory by design and it does work, but on a machine with -/// 128 GB of RAM it is a lot of fan-in bought for no reason, and the constant that produced it was -/// the same on a laptop that genuinely needed it. +/// The measured result was bad. A 30x WGS coordinate sort spilled **688 runs**, and the merge +/// opens all of the runs at the same time. The memory has a limit by design, and the code works. +/// But on a machine with 128 GB of RAM, this fan-in is very large and gives no advantage. A laptop +/// that needs a small value uses the same constant. /// -/// So the number comes from the machine. `var` still wins when it is set — an explicit MB count is -/// the escape hatch for a run that has to be reproduced or squeezed — and the sizing is otherwise: +/// So the machine gives the number. An explicit MB count in `var` has priority, because a run that +/// you must repeat or limit needs a fixed value. If `var` is empty, the rules are: /// -/// - **A quarter of installed RAM.** Total rather than free, because free fluctuates with whatever -/// the user happens to have open, and a stage whose run count depends on the browser is a stage -/// whose behaviour cannot be reproduced from a bug report. -/// - **Never below 512 MB.** That is the sort's existing default, so no machine sorts with less -/// than it does today. -/// - **Never above 8 GB.** Past that the returns are gone — 88 runs against 44 is nothing next to -/// 688 against 88 — while the costs are not: the stable sort allocates half the buffer again as -/// scratch, and growing the record vector doubles its allocation while still holding the old one. -/// - **Never more than half of what is free right now.** The stable part of the rule assumes a -/// machine that is otherwise idle. When it is not, spilling an extra run is cheap and swapping -/// the buffer is not. +/// - **One quarter of the installed RAM.** Use the total RAM, not the free RAM. The free value +/// changes with the other applications of the user. The run count of a stage must not depend on +/// the browser. You can not repeat such behaviour from a bug report. +/// - **512 MB minimum.** This value is the current default of the sort. So no machine sorts with a +/// smaller value than it uses today. +/// - **8 GB maximum.** A larger value gives almost no advantage. The step from 688 runs to 88 runs +/// is large, but the step from 88 runs to 44 runs is small. The costs stay. The stable sort takes +/// half of the buffer again for scratch space. A larger record vector needs a second allocation. +/// The first allocation stays in memory at the same time. +/// - **Half of the free memory, maximum.** The rules above are for an idle machine. On a busy +/// machine, one more spilled run costs little, and a swap of the buffer costs much. /// -/// Memory the platform will not report reads as zero, and zero means *unknown*, which must not be -/// read as "no memory" — the same rule [`classify`] follows. An unknown machine gets the floor. +/// A platform that does not report the memory returns zero. Zero means *unknown*. Do not read zero +/// as "no memory". [`classify`] uses the same rule. An unknown machine gets the minimum value. pub fn spill_budget(var: &str) -> u64 { if let Some(mb) = std::env::var(var).ok().and_then(|s| s.parse::().ok()) { return mb.max(1) * 1024 * 1024; @@ -185,26 +203,26 @@ pub fn spill_budget(var: &str) -> u64 { budget(system.total_memory(), system.available_memory()) } -/// What one heap allocation costs beyond the bytes asked for: the allocator's own bookkeeping plus -/// rounding up to a size class. +/// The cost of one heap allocation above the quantity of bytes that the caller asked for. The cost +/// is the internal data of the allocator, and the increase to the next size class. /// -/// It lives beside [`spill_budget`] because the two are one contract. A budget is only as honest as -/// the tally that fills it, and a stage that counts only payload bytes will hold well over its -/// budget in real memory — which is fine against a hand-picked 512 MB constant chosen with a margin -/// nobody wrote down, and not fine against a fraction of the machine. +/// This constant is beside [`spill_budget`] because the two are one contract. A budget is correct +/// only if the total that fills it is correct. A stage that counts only the payload bytes holds +/// much more than its budget in real memory. This error was acceptable against a fixed 512 MB +/// constant with an unwritten margin. It is not acceptable against a fraction of the machine. /// -/// Sixteen bytes is the conventional figure for the allocators on the three desktop targets. This -/// is a budget estimate rather than an audit; the point is that a record with four small vectors -/// costs meaningfully more than the sum of their lengths. +/// Sixteen bytes is the usual value for the allocators on the three desktop targets. This value is +/// an estimate, not an exact audit. It shows that a record with four small vectors costs much more +/// than the sum of the lengths of those vectors. pub const ALLOCATION_OVERHEAD: usize = 16; /// The floor, and the answer for a machine that will not say how much memory it has. const MIN_SPILL_BUDGET: u64 = 512 << 20; -/// The ceiling. See [`spill_budget`] for why bigger stops paying. +/// The maximum value. [`spill_budget`] gives the reason why a larger value has no advantage. const MAX_SPILL_BUDGET: u64 = 8 << 30; -/// The sizing decision, split from the probe so it is testable on any machine — the same split -/// [`classify`] makes, and for the same reason. +/// The size decision. It is separate from the probe, so a test can call it on any machine. +/// [`classify`] has the same separation for the same reason. fn budget(total_memory: u64, available_memory: u64) -> u64 { if total_memory == 0 { return MIN_SPILL_BUDGET; @@ -213,16 +231,16 @@ fn budget(total_memory: u64, available_memory: u64) -> u64 { if available_memory == 0 { return budget; } - // The floor holds even here: a machine this short of memory would have taken 512 MB under the - // old constant anyway, so honouring the busy-machine guard past that point would be a - // regression dressed as caution. + // The minimum value applies here also. With the old constant, a machine with this little + // memory took 512 MB. To apply the busy-machine guard below that value gives a worse result, + // and only looks careful. budget.min((available_memory / 2).max(MIN_SPILL_BUDGET)) } -/// How hard the machine is being leaned on. +/// The level of load on the machine. /// -/// Bands, not a single threshold, because the interesting reading is the trend: a stage that -/// spends an hour at [`Pressure::Elevated`] is a different story from one that touches it once. +/// There are bands, not one threshold, because the trend is the important measurement. A stage that +/// stays at [`Pressure::Elevated`] for one hour is not the same as a stage that reaches it once. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Pressure { Normal, @@ -246,8 +264,9 @@ const ELEVATED_FREE_FRACTION: f64 = 0.15; const CRITICAL_FREE_FRACTION: f64 = 0.07; /// Swap growth (bytes, since the watch started) that counts as [`Pressure::Elevated`]. /// -/// Growth, not absolute use: a desktop that has been up for a week has swap in use for reasons -/// that have nothing to do with this job, and blaming the job for it would cry wolf on every run. +/// This value is the growth, not the absolute use. A desktop that ran for a week has swap in use +/// for other reasons. If the code reports that swap as a fault of this job, every run gives a false +/// alarm. const ELEVATED_SWAP_GROWTH: u64 = 2 << 30; /// Swap growth that counts as [`Pressure::Critical`]. const CRITICAL_SWAP_GROWTH: u64 = 8 << 30; @@ -271,10 +290,10 @@ pub struct ResourceSample { } impl ResourceSample { - /// A one-line rendering for a log. + /// One line of text for a log. /// - /// Deliberately compact: at a 30-second cadence a six-hour stage produces 720 of these, and - /// they have to stay skimmable next to the stage timings they sit between. + /// The line is short by design. At an interval of 30 seconds, a six-hour stage makes 720 of + /// these lines. A user must be able to read them quickly between the stage times. pub fn summary(&self) -> String { format!( "[{:>7.0}s] mem {:.1}/{:.1} GB free, swap +{:.1} GB, wrote {:.1} GB @ {:.0} MB/s — {}", @@ -296,8 +315,8 @@ fn gib(bytes: u64) -> f64 { /// Classify a reading. Split from the probe so the decision is testable without a machine in a /// particular state. fn classify(total_memory: u64, available_memory: u64, swap_growth: u64) -> Pressure { - // A platform that will not report its memory reports zero; that is "unknown", and unknown must - // not read as "critical" — see `navigator_app::realign_job::has_room` for the same rule. + // A platform that does not report its memory returns zero. Zero means "unknown". Do not read + // "unknown" as "critical". `navigator_app::realign_job::has_room` uses the same rule. let free_fraction = if total_memory == 0 { 1.0 } else { @@ -315,25 +334,24 @@ fn classify(total_memory: u64, available_memory: u64, swap_growth: u64) -> Press /// How often to sample, when the caller does not say. /// -/// Thirty seconds is chosen against the thing being watched: the WindowServer watchdog fires at -/// 40 seconds, so a cadence slower than that could step over the entire window in which the -/// machine was in trouble. +/// The value of 30 seconds comes from the event that we must see. The WindowServer watchdog starts +/// at 40 seconds. A longer interval can miss the full period in which the machine has a problem. pub const DEFAULT_INTERVAL: Duration = Duration::from_secs(30); -/// A background sampler, running for as long as it is held. +/// A background sampler. It operates while the caller holds it. /// -/// Dropping it stops the thread and joins it, so a stage cannot outlive its own watch or leave a -/// thread behind on the way out of an error. +/// When the caller drops it, the thread stops and joins. So a stage can not continue after its own +/// watch stops. An error can not leave a thread behind. pub struct ResourceWatch { stop: Arc, handle: Option>, } impl ResourceWatch { - /// Start sampling every `interval`, passing each reading to `report`. + /// Take a sample at each `interval` and give each sample to `report`. /// - /// `report` runs on the sampler thread and should do no real work — writing a line is the - /// intended use. + /// `report` runs on the sampler thread, so it must do very little work. The intended use is to + /// write one line. pub fn start(interval: Duration, mut report: impl FnMut(ResourceSample) + Send + 'static) -> Self { let stop = Arc::new(AtomicBool::new(false)); let flag = Arc::clone(&stop); @@ -347,8 +365,8 @@ impl ResourceWatch { let mut last_bytes = bytes_written(); let mut last_at = started; - // Poll the stop flag far more often than the sample cadence, so dropping the watch - // returns promptly instead of blocking a job's teardown for up to `interval`. + // Read the stop flag much more often than the sample interval. A drop of the watch + // then returns quickly. If not, the exit of a job waits for up to `interval`. const TICK: Duration = Duration::from_millis(250); while !flag.load(Ordering::Relaxed) { @@ -416,8 +434,8 @@ mod tests { assert_eq!(classify(128 << 30, 12 << 30, 0), Pressure::Elevated); } - /// Swap that appeared *during* the job is the signal; swap that was already there is not, and - /// the sample carries growth rather than absolute use for exactly that reason. + /// New swap during the job is the signal. Swap that was present before the job is not a signal. + /// For this reason the sample carries the growth, not the absolute use. #[test] fn swap_growth_raises_pressure_on_an_otherwise_idle_machine() { assert_eq!(classify(128 << 30, 100 << 30, 3 << 30), Pressure::Elevated); @@ -430,7 +448,7 @@ mod tests { assert_eq!(classify(0, 0, 0), Pressure::Normal); } - /// The case the autosizing exists for: a big machine should stop spilling hundreds of runs. + /// The reason for the automatic size: a large machine must not spill hundreds of runs. #[test] fn a_large_machine_gets_the_ceiling() { assert_eq!(budget(128 << 30, 100 << 30), MAX_SPILL_BUDGET); @@ -441,8 +459,8 @@ mod tests { assert_eq!(budget(16 << 30, 12 << 30), 4 << 30); } - /// A machine whose memory is already spoken for gets a smaller buffer, because an extra spilled - /// run costs a file and swapping the buffer costs the run. + /// A machine with little free memory gets a smaller buffer. One more spilled run costs one + /// file. A swap of the buffer costs the full run. #[test] fn a_busy_machine_is_held_to_half_of_what_is_free() { assert_eq!(budget(64 << 30, 6 << 30), 3 << 30); @@ -455,15 +473,15 @@ mod tests { assert_eq!(budget(64 << 30, 100 << 20), MIN_SPILL_BUDGET); } - /// Unknown is not zero. A platform that will not report memory must not be sized as if it had - /// none — and must not be sized as if it had plenty either. + /// Unknown is not zero. For a platform that does not report the memory, do not calculate a + /// size for a machine with no memory. Do not calculate a size for a large machine either. #[test] fn unknown_memory_gets_the_floor() { assert_eq!(budget(0, 0), MIN_SPILL_BUDGET); assert_eq!(budget(64 << 30, 0), MAX_SPILL_BUDGET.min(16 << 30)); } - /// The escape hatch has to win, or a run cannot be reproduced on a different machine. + /// The manual value must have priority. If not, you can not repeat a run on another machine. #[test] fn an_explicit_override_beats_the_machine() { let var = "NAVIGATOR_TEST_SPILL_MB_OVERRIDE"; @@ -472,8 +490,8 @@ mod tests { std::env::remove_var(var); } - /// A nonsense override must not produce a zero-byte budget, which would spill one run per - /// record. + /// An invalid manual value must not give a budget of zero bytes. Such a budget spills one run + /// for each record. #[test] fn a_zero_override_is_floored_at_one_megabyte() { let var = "NAVIGATOR_TEST_SPILL_MB_ZERO"; @@ -489,7 +507,7 @@ mod tests { assert_eq!(bytes_written(), before + 1024); } - /// The watch must stop when dropped, rather than leaving a sampler thread behind. + /// The watch must stop at the drop. It must not leave a sampler thread in the process. #[test] fn dropping_the_watch_stops_it() { let watch = ResourceWatch::start(Duration::from_millis(50), |_| {}); diff --git a/crates/navigator-store/src/sig_cache.rs b/crates/navigator-store/src/sig_cache.rs index 53a87793..423468ee 100644 --- a/crates/navigator-store/src/sig_cache.rs +++ b/crates/navigator-store/src/sig_cache.rs @@ -1,24 +1,29 @@ -//! Signature-keyed result caches — one row per biosample, holding an opaque JSON result plus the -//! signature of the input it was computed from. +//! Result caches with a signature key. Each row holds one result for one biosample. The result is +//! JSON text. The row also holds the signature of the input data that made the result. //! -//! Four tables share this exact shape, and the app uses them the same way every time: read the row, -//! compare its signature against what the current inputs hash to, and recompute on a mismatch. They -//! were four hand-copied modules until this one replaced them; the copies had already drifted (the -//! two purge paths in the app each forgot a different table), which is the argument for having one. +//! Four tables have this shape. The app uses each of the four tables in the same sequence: //! -//! The columns are *named* differently per table for historical reasons — `consensus_sig` vs -//! `source_sig`, `roh` vs `archaic` vs `segments`, `computed_at` vs `painted_at` — so each cache -//! carries its column names and `get` aliases them back to the common [`Cached`] shape. The schema -//! is untouched; only the Rust side is unified. +//! 1. Read the row for the biosample. +//! 2. Compare the signature in the row with the signature of the current input data. +//! 3. If the two signatures are different, calculate the result again. +//! +//! Before this module, there were four modules with the same code. The four copies became +//! different. Each of the two purge paths in the app forgot a different table. One module prevents +//! this fault. +//! +//! Each table gives different names to its columns. One table has `consensus_sig` and another table +//! has `source_sig`. So each cache keeps its own column names, and `get` changes these names to the +//! names in the [`Cached`] structure. This module does not change the database schema. It changes +//! only the Rust code. use du_domain::ids::SampleGuid; use sqlx::SqlitePool; use crate::StoreError; -/// A cached result: the signature of the inputs it came from, the result itself as opaque JSON, -/// and when it was computed. The caller compares [`Cached::sig`] against the current inputs to -/// decide whether the payload is still good. +/// One cached result. `sig` is the signature of the input data. `payload` is the result as JSON +/// text. `computed_at` is the time of the calculation. The caller compares [`Cached::sig`] with the +/// signature of the current input data. If the two are different, the payload is out of date. #[derive(Debug, Clone, PartialEq, sqlx::FromRow)] pub struct Cached { pub biosample_guid: String, @@ -27,38 +32,39 @@ pub struct Cached { pub computed_at: String, } -/// One signature-keyed cache table, identified by its name and its three non-key columns. +/// One cache table. The table name and the three columns that are not the key define it. #[derive(Debug, Clone, Copy)] pub struct SigCache { - /// The table name. A compile-time constant in every case — never caller input — so - /// interpolating it into SQL is safe. + /// The table name. This value is always a constant in the code. The caller never supplies it. + /// For this reason, it is safe to put the value into the SQL text. table: &'static str, sig_col: &'static str, payload_col: &'static str, at_col: &'static str, } -/// Cached chromosome painting (local-ancestry segments), keyed to the autosomal consensus's -/// `last_reconciled_at`. +/// The cached chromosome painting. The painting holds the local ancestry segments. The key is the +/// `last_reconciled_at` value of the autosomal consensus. pub const PAINTING: SigCache = SigCache::new("consensus_painting", "consensus_sig", "segments", "painted_at"); -/// Cached runs-of-homozygosity result (segments + summary), keyed to the autosomal consensus. +/// The cached runs-of-homozygosity result. The result holds the segments and a summary. The key is +/// the autosomal consensus. pub const ROH: SigCache = SigCache::new("consensus_roh", "consensus_sig", "roh", "computed_at"); -/// Cached archaic (Neanderthal / Denisovan) **Tier A** marker count, keyed to the autosomal -/// consensus. +/// The cached archaic **Tier A** marker count for Neanderthal and Denisovan. The key is the +/// autosomal consensus. pub const ARCHAIC: SigCache = SigCache::new("consensus_archaic", "consensus_sig", "archaic", "computed_at"); -/// Cached archaic **Tier B** segment calls. Keyed to the *alignment* they were called from rather -/// than to the consensus: segments come from genome-wide de-novo diploid calls on one alignment, -/// whereas the consensus only carries the 1240k panel loci. The signature is the alignment id plus -/// the caller's genotype version, so re-calling with a newer caller invalidates the cache. +/// The cached archaic **Tier B** segment calls. The key is the alignment, not the consensus. The +/// caller finds these segments from de-novo diploid calls on one alignment across the genome. The +/// consensus holds only the 1240k panel loci. The signature is the alignment id and the genotype +/// version of the caller. So a newer caller makes the cache out of date. pub const ARCHAIC_SEGMENTS: SigCache = SigCache::new("consensus_archaic_segments", "source_sig", "segments", "computed_at"); -/// Every signature-keyed cache, in one list — so a purge that means "drop this subject's derived -/// results" drops *all* of them. Both purge paths used to enumerate tables by hand and both had -/// fallen behind the set. +/// All of the caches, in one list. A purge that must remove the derived results of a subject +/// removes all of them. Before this list, the two purge paths named the tables one by one. Each +/// path did not name all of the tables. pub const ALL: [SigCache; 4] = [PAINTING, ROH, ARCHAIC, ARCHAIC_SEGMENTS]; impl SigCache { @@ -71,13 +77,14 @@ impl SigCache { } } - /// The table this cache lives in — for callers that must fold it into a wider delete inside - /// their own transaction, where [`SigCache::delete`]'s pool-level call would not enlist. + /// The table of this cache. A caller needs the name when it deletes many tables in its own + /// transaction. The [`SigCache::delete`] function uses the pool, so that function can not join + /// such a transaction. pub const fn table(&self) -> &'static str { self.table } - /// Insert or replace this biosample's cached result. + /// Insert or replace the cached result for this biosample. pub async fn upsert( &self, pool: &SqlitePool, @@ -101,8 +108,8 @@ impl SigCache { Ok(()) } - /// This biosample's cached result, if one exists. The caller checks [`Cached::sig`] for - /// staleness — a row here is not by itself a usable result. + /// The cached result for this biosample, if a result exists. The caller must check + /// [`Cached::sig`]. A row is not a usable result until that check passes. pub async fn get(&self, pool: &SqlitePool, guid: SampleGuid) -> Result, StoreError> { let (table, s, p, a) = (self.table, self.sig_col, self.payload_col, self.at_col); let row: Option = sqlx::query_as(&format!( @@ -115,7 +122,7 @@ impl SigCache { Ok(row) } - /// Remove this biosample's cached result. `false` means there was nothing to remove. + /// Remove the cached result for this biosample. `false` shows that there was no result. pub async fn delete(&self, pool: &SqlitePool, guid: SampleGuid) -> Result { let affected = sqlx::query(&format!("DELETE FROM {} WHERE biosample_guid = ?", self.table)) .bind(guid.0.to_string()) @@ -131,9 +138,9 @@ mod tests { use super::*; use uuid::Uuid; - /// Every cache round-trips, and upsert replaces rather than duplicating (a recompute after the - /// inputs changed). Running the same body over [`ALL`] is what keeps a newly added table from - /// silently going untested. + /// Each cache keeps and returns the same data. `upsert` replaces a row and does not add a + /// second row. This occurs when the input data changes and the app calculates the result again. + /// The test uses [`ALL`], so a new table always gets a test. #[tokio::test] async fn every_cache_round_trips_and_upsert_replaces() { let pool = crate::Store::open_in_memory().await.unwrap(); @@ -166,8 +173,9 @@ mod tests { } } - /// The caches are independent: writing one must not disturb another that happens to share a - /// column name (`segments` is `consensus_painting`'s *and* `consensus_archaic_segments`'s). + /// The caches are independent. A write to one cache must not change a different cache. Two of + /// the tables use the column name `segments`: `consensus_painting` and + /// `consensus_archaic_segments`. #[tokio::test] async fn caches_do_not_alias_each_other() { let pool = crate::Store::open_in_memory().await.unwrap(); From ab3447668f66dc05b8488d38208622fe1ea60b18 Mon Sep 17 00:00:00 2001 From: James Kane Date: Tue, 18 Aug 2026 10:08:41 -0500 Subject: [PATCH 04/33] docs(ste): navigator-app, the first thirteen files appview, auth, dm, error, export, ibd_exchange, matching, recruitment, settings, social, sync, sync_reconcile and update, all at zero violations. The crate goes 3,893 to 3,521; twenty files remain, of which haplogroup.rs (788) and lib.rs (685) are two fifths. Nothing here is a summary. Where a comment explained why the code is shaped as it is, the explanation is still present and usually longer: the reason a device key signs each call rather than the HTTP request, why a federated post is deliberately absent from `PUBLISHED_COLLECTIONS`, why `AlignmentFileMissing` earns a variant of its own, why the update check compared versions wrongly for sixteen alphas. Short sentences, active voice, one idea each. Worth recording for whoever picks this up: a first pass reliably leaves violations behind. sync.rs went 39 to 19, ibd_exchange 51 to 29, update 54 to 17, each needing a second and sometimes a third pass. Splitting one long sentence into three explanatory ones almost always produces a new twenty-six-word sentence. It converges at roughly two and a half passes per file, not one. `cargo check -p navigator-app --all-targets` passes. Co-Authored-By: Claude Opus 5 (1M context) --- crates/navigator-app/src/appview.rs | 59 +++++--- crates/navigator-app/src/auth.rs | 38 +++-- crates/navigator-app/src/dm.rs | 56 ++++--- crates/navigator-app/src/error.rs | 73 +++++---- crates/navigator-app/src/export.rs | 100 +++++++----- crates/navigator-app/src/ibd_exchange.rs | 168 +++++++++++++-------- crates/navigator-app/src/matching.rs | 123 ++++++++------- crates/navigator-app/src/recruitment.rs | 18 ++- crates/navigator-app/src/settings.rs | 106 ++++++++----- crates/navigator-app/src/social.rs | 72 +++++---- crates/navigator-app/src/sync.rs | 166 ++++++++++++-------- crates/navigator-app/src/sync_reconcile.rs | 42 ++++-- crates/navigator-app/src/update.rs | 138 ++++++++++------- 13 files changed, 703 insertions(+), 456 deletions(-) diff --git a/crates/navigator-app/src/appview.rs b/crates/navigator-app/src/appview.rs index 926fce29..cd36c7ca 100644 --- a/crates/navigator-app/src/appview.rs +++ b/crates/navigator-app/src/appview.rs @@ -1,29 +1,35 @@ -//! The one way Navigator talks to the AppView's `/api/v1/*` Edge API. +//! The one way that Navigator speaks to the `/api/v1/*` Edge API of the AppView. //! -//! Three clients grew here independently — IBD exchange, social, recruitment — and each arrived at -//! the same two shapes: an unauthenticated-looking POST whose body carries the device-key -//! signature, and a replay-guarded signed GET whose `did`/`ts`/`sig` ride on the query string. The -//! IBD and social versions were byte-for-byte identical, and the remaining one-off calls in -//! `sync.rs` / `matching.rs` open-coded the same thing a fourth and fifth time. They are all this -//! module now, so the error mapping, the signing-query layout, and the non-2xx classification are -//! decided once. +//! Three clients grew here separately: IBD exchange, social, and recruitment. Each client made the +//! same two request shapes. //! -//! What travels: a DID, a timestamp, a signature, and whatever the caller chose to send. Never -//! genotypes, never coordinates. +//! The first shape is a POST. The body of the POST holds the device-key signature, so the request +//! looks unauthenticated. The second shape is a signed GET with a replay guard. Its `did`, `ts`, +//! and `sig` values go on the query string. +//! +//! The IBD version and the social version were the same code. The single calls in `sync.rs` and +//! `matching.rs` wrote the same request a fourth time and a fifth time. All of this code is now in +//! this module. So the error map, the layout of the signature query, and the class of a non-2xx +//! response have one definition. +//! +//! These values cross the network: a DID, a timestamp, a signature, and the content that the caller +//! chose to send. A genotype never crosses. A coordinate never crosses. use super::*; -/// A transport failure (connection refused, timeout, TLS) on an AppView call. +/// A transport failure on a call to the AppView. Examples are a refused connection, a timeout, and +/// a TLS fault. /// -/// The AppView is reached with a bare `reqwest` client rather than through the sync engine, but a -/// network failure means the same thing either way, so it lands in the same error variant the PDS -/// paths use and the offline indicator already understands. +/// A plain `reqwest` client makes these calls. The calls do not go through the sync engine. But a +/// network failure has the same result on both paths. So this function returns the error variant +/// that the PDS paths use, and the offline indicator already knows that variant. pub(crate) fn transport(e: reqwest::Error) -> AppError { AppError::Sync(navigator_sync::SyncError::from(e)) } -/// Classify a non-2xx AppView response into a user-facing [`AppError::AppView`]. Consumes `resp` to -/// read the body (so capture the status first at the call site if it is also needed). +/// Change a non-2xx response from the AppView into an [`AppError::AppView`] for the user. The +/// function consumes `resp` to read the body. So the caller must keep the status first, if the +/// caller also needs it. pub(crate) async fn status_error(api: &str, resp: reqwest::Response) -> AppError { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); @@ -44,12 +50,13 @@ impl App { format!("{}/api/v1/{path}", decodingus_appview_url()) } - /// POST a JSON body to an `/api/v1/` endpoint and return the decoded response. + /// Send a JSON body to an `/api/v1/` endpoint with POST, and return the decoded + /// response. /// - /// The signature (and the DID it is over) belongs in `body` — these endpoints authenticate the - /// device key per call, not the HTTP request — so this deliberately takes an already-signed - /// body rather than signing on the caller's behalf: the canonical string differs per endpoint - /// and only the caller knows it. + /// The signature and the DID for that signature belong in `body`. These endpoints authenticate + /// the device key for each call. They do not authenticate the HTTP request. For this reason the + /// function takes a body that the caller signed. It does not sign the body, because the + /// canonical string is different at each endpoint and only the caller knows it. pub(crate) async fn appview_post( &self, path: &str, @@ -69,11 +76,13 @@ impl App { resp.json().await.map_err(transport) } - /// Device-key-signed GET to an `/api/v1/` endpoint, decoded into `T`. + /// Send a GET to an `/api/v1/` endpoint with a device-key signature, and decode the + /// response into `T`. /// - /// `build_msg(did, ts)` produces the canonical string to sign — the one thing that varies - /// between a poll, a thread read, and an exchange pull. `did`/`ts`/`sig` plus `extra` go on the - /// query; the timestamp is what makes the signature replay-guarded. + /// `build_msg(did, ts)` makes the canonical string for the signature. That string is the only + /// difference between a poll, a thread read, and an exchange pull. The `did`, `ts`, and `sig` + /// values go on the query, together with `extra`. The timestamp gives the signature its replay + /// guard. pub(crate) async fn appview_get_signed( &self, path: &str, diff --git a/crates/navigator-app/src/auth.rs b/crates/navigator-app/src/auth.rs index 3ed35f40..9a78c79a 100644 --- a/crates/navigator-app/src/auth.rs +++ b/crates/navigator-app/src/auth.rs @@ -5,9 +5,11 @@ use super::*; impl App { // ---- authentication ---------------------------------------------------- - /// Run the public-client OAuth login for `handle` (handle or DID): browser authorize → - /// loopback callback → token exchange. On success the DPoP-bound session is persisted - /// to the OS keychain and becomes the active account. Returns the authenticated DID. + /// Do the public-client OAuth login for `handle`. The value can be a handle or a DID. + /// + /// The sequence is: the browser authorizes, the loopback receives the callback, and the client + /// exchanges the token. After a good login, the code writes the DPoP-bound session to the OS + /// keychain, and that session becomes the active account. The function returns the DID. pub async fn login(&self, handle: &str) -> Result { let session = login_default(&self.auth.http, &self.auth.config, handle).await?; let did = session.did.clone(); @@ -22,17 +24,20 @@ impl App { self.auth.active.lock().unwrap().clone() } - /// The signed-in account's DID, or [`AppError::NotAuthenticated`] — the cheap auth guard publish - /// methods run before building a record / touching the DB. + /// The DID of the active account, or [`AppError::NotAuthenticated`]. This is the small guard + /// that the publish methods call first, before they make a record or read the database. pub(crate) fn require_account(&self) -> Result { self.current_account().ok_or(AppError::NotAuthenticated) } - /// Adopt a **local `did:key` identity** as the active account: the device key *is* the identity, - /// so AppView calls self-certify (`verify_signed` accepts `did:key` directly — no PDS record). - /// This is the desktop bootstrap for the federated edge: device-key-signed calls (IBD suggestions, - /// the encrypted exchange) work with no OAuth/PDS. Reuses an existing local identity if one is - /// active; otherwise generates + persists a fresh device key. Returns the `did:key`. + /// Use a **local `did:key` identity** as the active account. The device key is the identity. + /// So a call to the AppView certifies itself, because `verify_signed` accepts a `did:key` + /// directly and needs no PDS record. + /// + /// This function is the desktop start point for the federated edge. Calls that the device key + /// signs, such as IBD suggestions and the encrypted exchange, then work with no OAuth and no + /// PDS. If a local identity is already active, this function uses it. If not, it makes a new + /// device key and writes it to the keychain. It returns the `did:key`. pub fn use_local_identity(&self) -> Result { if let Some(did) = self.current_account() { if did.starts_with("did:key:") && DeviceKey::load(KEYCHAIN_SERVICE, &did)?.is_some() { @@ -47,8 +52,9 @@ impl App { Ok(did) } - /// Switch the active account to an already-known DID (in-memory; the keychain marker too). For - /// multi-identity flows — e.g. driving both sides of an exchange from one process. + /// Change the active account to a known DID. The change applies to memory and to the keychain + /// marker. Use this for a flow with more than one identity. One example is a test that operates + /// both sides of an exchange in one process. pub fn set_active_account(&self, did: &str) { let _ = self.auth.tokens.set_active(did); *self.auth.active.lock().unwrap() = Some(did.to_string()); @@ -64,9 +70,11 @@ impl App { Ok(()) } - /// Build the resilient sync engine for the active account, loading its session from the - /// keychain. Errors with [`AppError::NotAuthenticated`] when no one is signed in. The - /// engine auto-refreshes on 401 and retries transient failures with backoff. + /// Make the sync engine for the active account and read its session from the keychain. + /// + /// The function returns [`AppError::NotAuthenticated`] if no account is active. On a 401 + /// response, the engine refreshes the token. It also tries again after a temporary failure. + /// The delay becomes longer after each try. pub(crate) fn sync_engine(&self) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let session = self.auth.tokens.load(&did)?.ok_or(AppError::NotAuthenticated)?; diff --git a/crates/navigator-app/src/dm.rs b/crates/navigator-app/src/dm.rs index 14126625..d8ec4091 100644 --- a/crates/navigator-app/src/dm.rs +++ b/crates/navigator-app/src/dm.rs @@ -1,15 +1,21 @@ -//! `impl App` methods for **peer direct messages** (social roadmap 3a) — a thin DM layer over the -//! generic D1 encrypted exchange (`ibd_exchange.rs` + `navigator_sync::exchange`). The crypto, -//! discovery, consent, and relay are reused unchanged; this adds the DM `purpose`, persistence of the -//! established session key (so a conversation is async + survives restart), and the per-message -//! store. The AppView only ever relays ciphertext — message plaintext never leaves the device. +//! `impl App` methods for **peer direct messages** (social roadmap 3a). +//! +//! This module is a thin DM layer on the generic D1 encrypted exchange, which is `ibd_exchange.rs` +//! and `navigator_sync::exchange`. It uses the cryptography, the discovery, the consent, and the +//! relay of that exchange without a change. +//! +//! This module adds three parts: the DM `purpose`, storage for the session key, and a store for +//! each message. Storage of the session key makes a conversation asynchronous, and the conversation +//! continues after a restart. The AppView relays only ciphertext. The plaintext of a message never +//! leaves the device. use super::*; use base64::engine::general_purpose::STANDARD; use base64::Engine; -/// Exchange `purpose` tag for a peer DM (the AppView titles its consent notification from this; IBD -/// requests use `IBD_*`, so filtering on it keeps the DM inbox separate from the IBD tab). +/// The exchange `purpose` tag for a peer DM. The AppView makes the title of its consent +/// notification from this tag. An IBD request uses a tag that starts with `IBD_`. So a filter on +/// the tag keeps the DM inbox separate from the IBD tab. pub const DM_PURPOSE: &str = "GENEALOGY_PII"; impl App { @@ -19,7 +25,8 @@ impl App { self.exchange_request(partner_did, DM_PURPOSE, None).await } - /// Inbound DM requests awaiting our consent (symmetric-blind; DM-purpose only). + /// The DM requests that arrived and that need our consent. The view is symmetric-blind, and it + /// holds only requests with the DM purpose. pub async fn dm_incoming(&self) -> Result, AppError> { Ok(self .exchange_incoming() @@ -29,8 +36,9 @@ impl App { .collect()) } - /// Consent-ready DM sessions (both parties consented) that we have not yet connected — i.e. no - /// persisted conversation/key yet. DM-purpose only. + /// The DM sessions that both parties agreed to, but that we did not connect. These sessions + /// have no conversation and no key in the store. The list holds only sessions with the DM + /// purpose. pub async fn dm_ready(&self) -> Result, AppError> { let mut out = Vec::new(); for info in self.exchange_pending().await? { @@ -52,10 +60,14 @@ impl App { self.exchange_consent(request_uri, given).await } - /// Connect a consent-ready DM session: run the X3DH-lite handshake (both peers must be online for - /// this one step) and persist the derived session key + partner, so all later send/receive is - /// async and restart-safe. Idempotent — re-connecting refreshes the key without resetting the - /// conversation's seq counters. + /// Connect a DM session that both parties agreed to. + /// + /// The function does the X3DH-lite handshake. Both peers must be online for this one step. It + /// then writes the derived session key and the partner to the store. So each later send and + /// receive is asynchronous, and it continues after a restart. + /// + /// A second call is safe. It makes a new key, and the seq counters of the conversation keep + /// their values. pub async fn dm_connect(&self, info: &ExchangeSessionInfo) -> Result<(), AppError> { let did = self.require_account()?; let session = self.open_exchange_session(info).await?; @@ -93,8 +105,8 @@ impl App { Ok(navigator_store::dm::messages(self.store.pool(), session_id).await?) } - /// Encrypt + relay a message on an established conversation, persisting it locally. Returns the - /// seq it was sent under. + /// Encrypt a message, relay it on an open conversation, and write it to the local store. The + /// function returns the seq of the message. pub async fn dm_send(&self, session_id: &str, text: &str) -> Result { let convo = self.dm_conversation_or_err(session_id).await?; let session = self.rebuild_session(&convo)?; @@ -107,8 +119,8 @@ impl App { Ok(seq) } - /// Pull, decrypt, persist, and ack any messages waiting on a conversation. Returns the count of - /// newly-stored (non-duplicate) messages. + /// Read, decrypt, store, and acknowledge each message that a conversation holds. The function + /// returns the count of new messages. It does not count a duplicate message. pub async fn dm_sync(&self, session_id: &str) -> Result { let did = self.require_account()?; let convo = self.dm_conversation_or_err(session_id).await?; @@ -118,12 +130,13 @@ impl App { let Ok(parsed) = exchange::Envelope::from_blob(&env.blob) else { continue; }; - // A leftover handshake (seq 0) can't decrypt as data — ack and drop it. + // An old handshake has seq 0, and the code can not decrypt it as data. Acknowledge + // the handshake and remove it. if matches!(parsed, exchange::Envelope::Handshake { .. }) { let _ = self.exchange_relay_ack(env.id).await; continue; } - // AAD binds the sender's routing: from = the partner (sender), to = us. + // The AAD binds the route of the sender. `from` is the partner, and `to` is us. let aad = exchange::relay_aad(session_id, &env.from_did, &did, env.seq); let Ok(pt) = exchange::open(&session.key, &aad, &parsed) else { continue; // not for this session key / tampered — leave un-acked @@ -182,7 +195,8 @@ mod tests { let back: [u8; 32] = STANDARD.decode(&stored).unwrap().try_into().unwrap(); assert_eq!(key, back); - // And it actually works as an AES session key: seal here, open with the rebuilt key. + // The value also works as an AES session key. Seal the data here, then open the data with + // the key that the code makes again. let aad = exchange::relay_aad("sess", "did:plc:a", "did:plc:b", 1); let blob = exchange::seal(&key, &aad, b"hello").and_then(|e| e.to_blob()).unwrap(); let parsed = exchange::Envelope::from_blob(&blob).unwrap(); diff --git a/crates/navigator-app/src/error.rs b/crates/navigator-app/src/error.rs index 794d934c..1f49e980 100644 --- a/crates/navigator-app/src/error.rs +++ b/crates/navigator-app/src/error.rs @@ -8,36 +8,45 @@ pub enum AppError { #[error(transparent)] Analysis(#[from] navigator_analysis::AnalysisError), - /// Read mapping (realignment stage B). Its own variant rather than folded into `Analysis` - /// because `navigator-align` is a separate crate with its own error type, and a mapping - /// failure points somewhere different from an analysis one. + /// A fault in read mapping, which is realignment stage B. + /// + /// This is a separate variant, not part of `Analysis`. `navigator-align` is a separate crate + /// with its own error type. Also, a mapping fault has a different cause from an analysis + /// fault. #[error("{0}")] Align(#[from] navigator_align::AlignError), #[error("serialization error: {0}")] Serde(#[from] serde_json::Error), - /// A blocking analysis task failed to join (panicked or was cancelled). + /// An analysis task on a blocked thread did not join. The task had a panic, or the user + /// stopped it. #[error("analysis task failed: {0}")] Join(String), #[error("alignment {0} has no BAM/reference path recorded")] MissingPaths(i64), - /// The alignment's recorded file is no longer on disk — a superseded vendor download, a - /// deleted import, an unmounted volume. Distinct from [`AppError::MissingPaths`], which means - /// no path was ever recorded: here there is one, it just no longer resolves. + /// The file of the alignment is no longer on disk. The cause can be a newer vendor download, a + /// deleted import, or a volume that the user removed. + /// + /// This variant is different from [`AppError::MissingPaths`]. That variant means that the + /// alignment never had a path. Here the alignment has a path, but the path no longer points to + /// a file. /// - /// Worth its own variant because it is the one read failure that is *expected* in a long-lived - /// workspace and is nobody's fault. Sweeps over many alignments skip on it - /// ([`AppError::is_missing_alignment_file`]) rather than counting a failure, and it is raised - /// before the expensive setup a walk implies rather than surfacing as an opaque io error from - /// deep inside the reader — where it had been misread as the *haplotree* being unavailable. + /// This fault has its own variant for two reasons. First, it is the one read fault that a long + /// life workspace can expect, and no user made a mistake. So a sweep across many alignments + /// skips the alignment with [`AppError::is_missing_alignment_file`] and does not count a + /// failure. + /// + /// Second, the code raises this error before the large setup that a walk needs. Before this + /// variant, the reader gave an unclear io error from deep in its code, and a user read that + /// error as an absent haplotree. #[error("alignment {id} file is no longer at {path}")] AlignmentFileMissing { id: i64, path: String }, - /// The ancestry reference panel file is missing — build it with `navigator-panelbuild` - /// and install it (or set `$NAVIGATOR_ANCESTRY_PANEL`). + /// The ancestry reference panel file is absent. Make the file with `navigator-panelbuild` and + /// install it. As an alternative, set `$NAVIGATOR_ANCESTRY_PANEL`. #[error("ancestry panel not found at {0} — build it with navigator-panelbuild")] AncestryPanelMissing(std::path::PathBuf), @@ -52,7 +61,7 @@ pub enum AppError { #[error("not signed in — log in to a PDS account first")] NotAuthenticated, - /// An AppView API call failed (e.g. federated IBD). 403 → the device key isn't + /// An AppView API call failed (e.g. federated IBD). 403 → the device key is not /// registered/verified yet; 422 → clock skew; otherwise the server's reason. #[error("appview error: {0}")] AppView(String), @@ -69,13 +78,14 @@ pub enum AppError { #[error(transparent)] Refgenome(#[from] navigator_refgenome::RefgenomeError), - /// Import needs reference build(s) that aren't cached — the UI prompts, downloads via - /// the gateway, then retries. No DB writes happened. + /// The import needs one or more reference builds that the cache does not hold. The UI asks the + /// user, downloads the builds through the gateway, and then tries again. The code wrote nothing + /// to the database. #[error("reference download required: {0:?}")] ReferenceNeeded(Vec), - /// A requested mutation is refused because of current state (e.g. deleting a subject that - /// still has sequencing data or profiles). + /// The app refuses a change because of the current state. One example is a request to delete a + /// subject that still has sequence data or a profile. #[error("{0}")] Conflict(String), @@ -91,18 +101,21 @@ pub enum AppError { } impl AppError { - /// Whether this is a user-requested cancellation rather than a genuine failure. + /// Shows that the user stopped the job. It does not show a fault. + /// + /// A stop moves through the code as an error, because an error unwinds the walk from any point. + /// But the caller must know the difference between the two. A run that the user stopped holds a + /// partial result, and the app must not write that result to the store. The UI must also show + /// the word "cancelled", not an error. /// - /// Cancellation travels as an error so it unwinds the walk from wherever it was, but callers - /// must be able to tell the two apart: a cancelled run holds a partial result that must not be - /// persisted, and the UI has to say "cancelled" instead of showing an error. Lives here rather - /// than in the UI so the layers above never have to reach past `navigator-app` for it. + /// This method is in this crate, not in the UI. So a layer above `navigator-app` never needs to + /// look inside this crate. pub fn is_cancellation(&self) -> bool { matches!(self, AppError::Analysis(navigator_analysis::AnalysisError::Cancelled)) } - /// Whether this failure is only "the alignment's file is gone", which a sweep should skip past - /// rather than record as a failure. See [`AppError::AlignmentFileMissing`]. + /// Shows that the only fault is an absent alignment file. A sweep must skip such an alignment + /// and must not count a failure. See [`AppError::AlignmentFileMissing`]. pub fn is_missing_alignment_file(&self) -> bool { matches!(self, AppError::AlignmentFileMissing { .. }) } @@ -115,11 +128,11 @@ impl From for AppError { } impl AppError { - /// Whether this is a user-requested stop rather than a failure. + /// Shows that the user stopped the job. It does not show a fault. /// - /// Long jobs have to tell the two apart — reporting someone's own Cancel click as an error is - /// both wrong and alarming — and the distinction lives here so callers do not resort to - /// matching on message text. + /// A long job must know the difference between the two. To report the Cancel click of the user + /// as an error is incorrect, and it makes the user afraid. This method holds the difference, so + /// a caller does not compare the text of a message. pub fn is_cancelled(&self) -> bool { matches!( self, diff --git a/crates/navigator-app/src/export.rs b/crates/navigator-app/src/export.rs index 251655c4..7b457533 100644 --- a/crates/navigator-app/src/export.rs +++ b/crates/navigator-app/src/export.rs @@ -1,6 +1,9 @@ -//! Result exports (gap §6): pure formatters that turn a cached analysis result into a shareable -//! file body — TSV / HTML / BED. Kept free of I/O and `App` so they're trivially unit-testable; the -//! app layer loads the result and writes the returned `String` to the user-chosen path. +//! Result exports (gap §6). Each function here is a formatter. It changes a cached analysis result +//! into the body of a file that the user can share. The formats are TSV, HTML, and BED. +//! +//! These functions do no I/O and do not use `App`, so a unit test can call them directly. The app +//! layer reads the result, calls a formatter, and writes the `String` to the path that the user +//! chose. use navigator_analysis::coverage::CoverageResult; use navigator_analysis::haplo::CallState; @@ -13,12 +16,13 @@ use navigator_domain::reconciliation::DnaType; use crate::{Block, BranchReport, DescentReport, ProjectBlockTree}; -/// Minimal HTML text escaping for the small, controlled strings we embed (population names etc.). +/// A small HTML escape for the short strings that the app puts in a page. One example is the name +/// of a population. fn esc(s: &str) -> String { s.replace('&', "&").replace('<', "<").replace('>', ">") } -/// Shared inline stylesheet for the HTML exports (self-contained — no external assets). +/// The stylesheet that each HTML export holds. The page is complete and needs no other file. const HTML_STYLE: &str = "body{font-family:-apple-system,Segoe UI,Roboto,sans-serif;margin:2rem;color:#222}\ h1{font-size:1.3rem}h2{font-size:1rem;margin-top:1.5rem}\ table{border-collapse:collapse;margin-top:.5rem}\ @@ -29,8 +33,9 @@ th{background:#f3f3f3}td:first-child,th:first-child{text-align:left}\ // ---- coverage ---------------------------------------------------------------- -/// Coverage as TSV: a `#`-commented genome-wide metrics header, then a per-contig table (joining the -/// samtools-style depth stats with the GATK-style callable breakdown). +/// The coverage as TSV. The file starts with a header of genome-wide metrics, and each header line +/// starts with `#`. A table for each contig follows. The table holds the depth values in the +/// samtools form together with the callable values in the GATK form. pub fn coverage_tsv(cov: &CoverageResult) -> String { let mut out = String::new(); out.push_str("# DUNavigator coverage export\n"); @@ -83,7 +88,8 @@ pub fn coverage_tsv(cov: &CoverageResult) -> String { out } -/// Coverage as a self-contained HTML page (genome-wide summary + per-contig table). +/// The coverage as a complete HTML page. The page holds a genome-wide summary and a table for each +/// contig. pub fn coverage_html(cov: &CoverageResult, label: &str) -> String { let mut rows = String::new(); for s in &cov.contig_coverage_stats { @@ -169,8 +175,9 @@ pub fn read_metrics_tsv(m: &ReadMetrics) -> String { // ---- ancestry ---------------------------------------------------------------- -/// Ancestry as a single TSV table — super-population then fine-population rows distinguished by a -/// `level` column — under a `#`-commented metadata header. +/// The ancestry as one TSV table. The super-population rows come first, and the fine-population +/// rows follow. The `level` column shows the difference. A metadata header is above the table, and +/// each header line starts with `#`. pub fn ancestry_tsv(a: &AncestryResult) -> String { let mut out = String::new(); out.push_str("# DUNavigator ancestry export\n"); @@ -259,13 +266,18 @@ pub fn ancestry_html(a: &AncestryResult) -> String { // ---- mtDNA variants ---------------------------------------------------------- -/// mtDNA variants (vs rCRS) as TSV: position, compact notation, region, ref/alt, type. -/// The Y-DNA / mtDNA **descent report** (root→terminal lineage) as TSV: one row per defining SNP of -/// each node on the path, with the subject's call state and observed base. Mirrors the on-screen -/// per-node descent grid so it can be shared / diffed outside the app. -/// TSV for a [`BranchReport`] — one row per defining marker in the reported subtree, with the -/// sample's observed base + call state + evidence. Shareable for placement spot-checks / researcher -/// exchange. Missing evidence renders as `.` (VCF convention). +/// The mtDNA variants against rCRS as TSV. Each row holds the position, the short notation, the +/// region, the reference allele, the alternate allele, and the type. +/// +/// The Y-DNA or mtDNA **descent report** as TSV, from the root to the terminal lineage. The file +/// holds one row for each SNP that defines a node on the path. Each row also holds the call state +/// of the subject and the observed base. The file has the same content as the descent grid on the +/// screen, so a user can share it or compare it outside the app. +/// +/// TSV for a [`BranchReport`]. The file holds one row for each marker that defines a node in the +/// reported subtree. Each row holds the observed base of the sample, the call state, and the +/// evidence. A user can share the file to check a placement or to send it to a researcher. A row +/// with no evidence shows `.`, which is the VCF convention. pub fn branch_report_tsv(report: &BranchReport) -> String { let dna = match report.dna { DnaType::Y => "Y-DNA", @@ -348,13 +360,17 @@ pub fn descent_tsv(report: &DescentReport) -> String { // ---- project block tree ------------------------------------------------------ -/// The cohort block tree as TSV: one row per block, in the aggregate's own pre-order, with `depth` -/// carrying the shape. Candidate branches (inferred from shared private variants, not named in the -/// published tree) are marked in the `kind` column and named `candidate` rather than left blank, so -/// a reader of the file alone can't mistake one for a published haplogroup. +/// The cohort block tree as TSV. The file holds one row for each block, in the pre-order of the +/// aggregate. The `depth` column gives the shape of the tree. /// -/// Members are a comma-joined cell rather than one row each: the unit a researcher shares from this -/// view is the *branch*, and exploding it per member would bury the tree shape. +/// A candidate branch comes from private variants that members share. The published tree does not +/// name it. The `kind` column marks such a branch with the word `candidate`, and the column is +/// never blank. So a reader of the file alone can not mistake a candidate for a published +/// haplogroup. +/// +/// One cell holds all members, with a comma between them. The file does not use one row for each +/// member. A researcher shares the *branch* from this view. One row for each member hides the shape +/// of the tree. pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { let dna = match tree.dna { DnaType::Y => "Y-DNA", @@ -399,9 +415,9 @@ pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { names.join(","), )); } - // Per-carrier evidence for every candidate. A candidate is an inference, and a shared export - // that showed only "1 SNP, 3 members" would ask the reader to trust it — the depth and derived - // fraction behind each call are what let them judge it instead. + // The evidence of each carrier, for every candidate. A candidate is a deduction. An export + // that showed only "1 SNP, 3 members" asks the reader to trust it. The depth and the derived + // fraction behind each call let the reader judge it. let candidates: Vec<&Block> = tree .blocks .iter() @@ -427,8 +443,8 @@ pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { } } - // The members the tree does not account for belong in the same file — a shared export that - // silently covered only the placed fraction would misrepresent the cohort. + // The same file must hold the members that the tree does not cover. An export that held only + // the placed members, with no note, gives a false picture of the cohort. if !tree.unplaced.is_empty() { out.push_str("\n# not on this tree\nname\tterminal\treason\n"); for u in &tree.unplaced { @@ -442,8 +458,9 @@ pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { out } -/// The cohort block tree as a self-contained HTML page: the same rows, indented by depth so the -/// shape reads at a glance, with candidate branches called out. +/// The cohort block tree as a complete HTML page. The page holds the same rows. Each row has an +/// indent for its depth, so the reader sees the shape quickly. The page marks each candidate +/// branch. pub fn block_tree_html(tree: &ProjectBlockTree, project: &str) -> String { let dna = match tree.dna { DnaType::Y => "Y-DNA", @@ -530,8 +547,9 @@ pub fn mtdna_variants_tsv(variants: &[MtVariant]) -> String { // ---- IBD segments ------------------------------------------------------------ -/// IBD segments as TSV (`chromosome start end length_cm snp_count`), 1-based bp. The match -/// browser's "Export segments CSV" — a tab-delimited table for downstream analysis / sharing. +/// The IBD segments as TSV, with the columns `chromosome`, `start`, `end`, `length_cm`, and +/// `snp_count`. The positions are 1-based. The "Export segments CSV" button of the match browser +/// makes this file. A user can read the table into another tool or send it to a partner. pub fn ibd_segments_tsv(segments: &[IbdSegment]) -> String { let mut out = String::from("# DUNavigator IBD segments export\n"); out.push_str("chromosome\tstart_position\tend_position\tlength_cm\tsnp_count\thalf_identical\n"); @@ -593,10 +611,13 @@ fn lineage_html(lb: &LineageBrief, title: &str) -> String { s } -/// The subject brief as a self-contained "DNA Story" HTML document — the casual-reader report a user -/// can save or print. Mirrors the Simple-mode card stack. When an AI narration is provided (a cached -/// "Polish with AI" result), it leads the document as a clearly-labelled, additive section above the -/// structured facts. +/// The subject brief as a complete "DNA Story" HTML document. This report is for a reader who is +/// not a specialist, and the user can save it or print it. It has the same content as the card +/// stack of Simple mode. +/// +/// The caller can supply an AI narration, which is a cached result of "Polish with AI". That text +/// comes first in the document, above the structured facts, with a clear label. It adds to those +/// facts and does not replace them. pub fn subject_brief_html(b: &SubjectBrief, narration: Option<&crate::NarratedBrief>) -> String { let mut body = String::new(); body.push_str(&format!("

{} — Your DNA Story

\n", esc(&b.headline.name))); @@ -674,8 +695,9 @@ pub fn subject_brief_html(b: &SubjectBrief, narration: Option<&crate::NarratedBr body.push_str("

Neanderthal ancestry

\n"); body.push_str(&format!("

{}

\n", esc(&a.pattern))); body.push_str(&format!("

{}

\n", esc(&a.summary_phrase))); - // A count over what was assayed, never a "percent Neanderthal" (design S1/S7) — the export - // has to hold the same line as the UI or the two disagree about what was measured. + // Report a count of the markers that the test measured. Never report a "percent + // Neanderthal" figure. See design S1 and S7. The export must give the same statement as + // the UI. If not, the two disagree about the measurement. body.push_str(&format!( "

{} of {} marker copies

\n", a.total_copies, a.possible_copies @@ -844,7 +866,7 @@ mod tests { assert!(tsv .lines() .any(|l| l.starts_with("chr1\t500\t42\t480\t96.00\t30.10\t35.0\t58.0\t470\t10\t15\t0\t0\t5"))); - // HTML variant renders without panicking and includes the title. + // The HTML function completes with no panic, and the page holds the title. assert!(coverage_html(&cov, "KANE-0001").contains("Coverage — KANE-0001")); } diff --git a/crates/navigator-app/src/ibd_exchange.rs b/crates/navigator-app/src/ibd_exchange.rs index 805b9bec..ad6d8c81 100644 --- a/crates/navigator-app/src/ibd_exchange.rs +++ b/crates/navigator-app/src/ibd_exchange.rs @@ -5,9 +5,10 @@ use super::*; impl App { // ---- IBD Phase 2: encrypted edge-to-edge exchange (D1 substrate) ------- // - // The AppView brokers discovery/consent + relays opaque ciphertext (never decrypts). These - // wrap the `/api/v1/exchange/*` endpoints; the crypto (X25519/X3DH-lite/AES-GCM) lives in - // `navigator_sync::exchange`. All calls are device-key-signed (no per-call OAuth). + // The AppView is the broker for discovery and consent. It also relays opaque ciphertext, and + // it never decrypts that ciphertext. These methods wrap the `/api/v1/exchange/*` endpoints. + // The cryptography, which is X25519, X3DH-lite, and AES-GCM, is in + // `navigator_sync::exchange`. The device key signs each call, and no call uses OAuth. /// The signed-in account's X25519 identity key (load-or-generate), with its public half /// published to the AppView (`POST /exchange/key`, idempotent upsert) so partners can fetch it. @@ -24,8 +25,8 @@ impl App { Ok(ik) } - /// Fetch a peer's published X25519 public key (STANDARD base64), or `None` if they haven't - /// published one. Public read — no signature. + /// Read the X25519 public key that a peer published, in STANDARD base64. The method returns + /// `None` when the peer published no key. This read is public and needs no signature. pub async fn fetch_exchange_key(&self, did: &str) -> Result, AppError> { let url = self.appview_url("exchange/key"); let resp = self @@ -46,11 +47,15 @@ impl App { Ok(v.get("x25519_pub").and_then(|x| x.as_str()).map(str::to_string)) } - /// Open an exchange request to a specific partner DID — the direct counterpart to the - /// suggestion-mediated [`ibd_introduce`] (`POST /api/v1/exchange/request`). Generates an opaque - /// request URI, signs the canonical request message, and posts it. The partner discovers it via - /// [`exchange_incoming`] (symmetric-blind) and consents; on mutual consent a session opens. Returns - /// the request URI to track. `scope` carries an optional project scope (team-ACL-gated server-side). + /// Open an exchange request to one partner DID (`POST /api/v1/exchange/request`). This method + /// is the direct form of [`ibd_introduce`], which works through a suggestion. + /// + /// The method makes an opaque request URI, signs the canonical request message, and sends it. + /// The partner finds the request with [`exchange_incoming`], which is symmetric-blind, and then + /// agrees. After both parties agree, a session opens. + /// + /// The method returns the request URI, and the caller uses it to track the request. `scope` can + /// carry a project scope, and the server gates that scope with the team ACL. pub async fn exchange_request( &self, partner_did: &str, @@ -103,7 +108,8 @@ impl App { }) } - /// Poll for inbound (symmetric-blind) exchange requests awaiting this account's consent. + /// Poll for the exchange requests that arrived and that need the consent of this account. The + /// view is symmetric-blind. pub async fn exchange_incoming(&self) -> Result, AppError> { let v = self.exchange_get_poll("exchange/incoming", &[]).await?; Ok(v.get("items") @@ -169,8 +175,9 @@ impl App { .unwrap_or_default()) } - /// Relay an opaque ciphertext `blob` to `to_did` in a session. The signed hash binds the blob to - /// its routing (the broker stores ciphertext only). Returns the broker envelope id. + /// Relay an opaque ciphertext `blob` to `to_did` in a session. The signed hash binds the blob + /// to its route. The broker stores only ciphertext. The method returns the envelope id of the + /// broker. pub async fn exchange_relay(&self, session_id: &str, to_did: &str, seq: i32, blob: &str) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let dev = self.ensure_device_key().await?; @@ -225,11 +232,15 @@ impl App { self.appview_post("exchange/ack", body).await.map(|_| ()) } - /// Establish a shared session key for a consent-ready session: publish/load our identity key, - /// fetch the partner's, exchange ephemeral keys via the relay (handshake, seq 0), and derive the - /// X3DH-lite session key. Polls the relay up to ~15s for the partner's handshake. The returned - /// [`EstablishedSession`] then seals/opens payloads. (Live-only — needs a running AppView + the - /// partner edge online to complete the handshake.) + /// Make a shared session key for a session that both parties agreed to. + /// + /// The method publishes or reads our identity key, then reads the identity key of the partner. + /// The two edges exchange short-life keys through the relay, as a handshake at seq 0. The + /// method then derives the X3DH-lite session key. It polls the relay for up to 15 seconds for + /// the handshake of the partner. + /// + /// The [`EstablishedSession`] value then seals a payload and opens a payload. This method needs + /// a live AppView and a partner edge that is online, so a test can not run it offline. pub async fn open_exchange_session(&self, info: &ExchangeSessionInfo) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let ik = self.ensure_exchange_key().await?; @@ -241,7 +252,8 @@ impl App { let hs = exchange::Envelope::handshake(&ek).to_blob().map_err(AppError::Sync)?; self.exchange_relay(&info.session_id, &info.partner_did, 0, &hs).await?; - // Wait for the partner's handshake (seq 0 / a Handshake envelope), acking just it. + // Wait for the handshake of the partner, which is seq 0 in a Handshake envelope. + // Acknowledge only that envelope. let mut their_ek: Option = None; for _ in 0..15 { for env in self.exchange_relay_pull(&info.session_id).await? { @@ -290,8 +302,9 @@ impl App { .await } - /// Pull + decrypt + ack the data payloads waiting on an established session (returns plaintexts - /// in pull order). Non-data / undecryptable envelopes are left un-acked. + /// Read, decrypt, and acknowledge each data payload on an open session. The method returns the + /// plaintexts in the order of the read. It does not acknowledge an envelope that holds no data, + /// or an envelope that it can not decrypt. pub async fn exchange_receive(&self, session: &EstablishedSession) -> Result>, AppError> { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let mut out = Vec::new(); @@ -299,7 +312,7 @@ impl App { let Ok(parsed) = exchange::Envelope::from_blob(&env.blob) else { continue; }; - // AAD binds the sender's routing: from = the partner (sender), to = us. + // The AAD binds the route of the sender. `from` is the partner, and `to` is us. let aad = exchange::relay_aad(&session.session_id, &env.from_did, &did, env.seq); if let Ok(pt) = exchange::open(&session.key, &aad, &parsed) { out.push(pt); @@ -309,12 +322,19 @@ impl App { Ok(out) } - /// Run a **federated IBD exchange** over an established session (gap §4): send our IBD-panel - /// dosages, receive the partner's, detect IBD locally (both peers run the symmetric detector → - /// identical summary), then exchange + verify signed [`IbdAttestation`]s. `agreed` ⇒ the partner's - /// signature verified and both summary hashes match. Only panel dosages cross the wire (encrypted; - /// the broker never sees them). `my_source` supplies our dosages; the refs are opaque biosample - /// pointers carried in the attestation. Live-only — needs the partner edge online. + /// Do a **federated IBD exchange** on an open session (gap §4). + /// + /// The method sends the dosages of our IBD panel and receives the dosages of the partner. It + /// then finds the IBD segments on this machine. Both peers run the same symmetric detector, so + /// both get the same summary. The two edges then exchange signed [`IbdAttestation`] values and + /// check them. + /// + /// The `agreed` field is true when the signature of the partner is correct and the two summary + /// hashes are the same. + /// + /// Only the panel dosages cross the network. The code encrypts them, and the broker never sees + /// them. `my_source` gives our dosages. The refs are opaque pointers to a biosample, and the + /// attestation carries them. The method needs the partner edge online. pub async fn exchange_ibd( &self, session: &EstablishedSession, @@ -337,8 +357,9 @@ impl App { .await } - /// The dosage-level core of [`exchange_ibd`] — takes the panel dosages directly (e.g. from a - /// consensus profile, or synthetic vectors in tests) rather than resolving an [`IbdSource`]. + /// The core of [`exchange_ibd`] at the dosage level. This method takes the panel dosages + /// directly and does not read an [`IbdSource`]. The dosages can come from a consensus profile, + /// or from a test vector. pub async fn exchange_ibd_with_dosages( &self, session: &EstablishedSession, @@ -351,8 +372,9 @@ impl App { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let dev = self.ensure_device_key().await?; - // Fit the relay's 1 MiB envelope: decimate a large panel (both peers apply the same - // position-based rule, so the intersection is preserved). Detect on the decimated set we send. + // Make the data fit the 1 MiB envelope of the relay. For a large panel, the code removes + // sites. Both peers use the same rule, which depends on the position, so the two sets still + // intersect. The detector uses the smaller set that the code sends. let my_sites = decimate_for_exchange(my_sites); // 1. Send our dosages (the IBD panel is on CHM13 / hs1). @@ -363,7 +385,7 @@ impl App { self.exchange_send(session, 1, &dos.to_bytes().map_err(AppError::Import)?) .await?; - // 2. Receive the partner's dosages (buffering any attestation that arrives early). + // 2. Receive the dosages of the partner. Keep an attestation that arrives too early. let mut partner_sites: Option> = None; let mut partner_att: Option = None; for _ in 0..EXCHANGE_POLL_ROUNDS { @@ -382,7 +404,8 @@ impl App { let partner_sites = partner_sites .ok_or_else(|| AppError::AppView("partner IBD dosages not received (peer offline?)".into()))?; - // 3. Detect IBD locally (symmetric — the partner computes the same summary). + // 3. Find the IBD segments on this machine. The detector is symmetric, so the partner + // calculates the same summary. let comparison = detect_ibd_sites(&my_sites, &partner_sites, ReferenceBuild::Chm13v2, config); // 4. Sign our attestation over the computed summary. @@ -421,7 +444,7 @@ impl App { let partner_att = partner_att.ok_or_else(|| AppError::AppView("partner attestation not received (peer offline?)".into()))?; - // 7. Verify the partner's signature + summary-hash agreement. + // 7. Check the signature of the partner and compare the two summary hashes. let sig_ok = du_atproto::verify_did_key( &partner_att.signing_public_key, partner_att.canonical().as_bytes(), @@ -468,8 +491,8 @@ impl App { .map(|c| IbdSource::Chip(c.id))) } - /// The subject's IBD-panel dosages from its best source (panel-restricted — only the canonical IBD - /// sites, not the whole genome, so that's all that can leave the device). + /// The IBD-panel dosages of the subject, from its best source. The set holds only the + /// canonical IBD sites and not the full genome. So no other site can leave the device. pub async fn ibd_dosages_for_subject(&self, guid: SampleGuid) -> Result, AppError> { let source = self.best_ibd_source_for_subject(guid).await?.ok_or_else(|| { AppError::Import("no IBD-capable data for this subject (need an alignment or a chip profile)".into()) @@ -507,8 +530,9 @@ impl App { ) .await?; self.record_ibd_exchange(guid, session, request_uri, &result).await?; - // Advance the ledger before either publish: the comparison is done and persisted, so the - // conversation is complete whether or not the network steps below succeed. + // Advance the ledger before the two publish steps. The comparison is complete and in the + // store. So the conversation is complete, and a failure in the network steps below does + // not change that. self.mark_matching_exchanged(guid, session, request_uri).await?; // Best-effort: publish our attestation to the PDS (skipped for did:key; never fails the exchange). let _ = self.publish_ibd_attestation(&result.my_attestation).await; @@ -557,8 +581,10 @@ impl App { Ok(navigator_store::ibd_exchange::list_for_biosample(self.store.pool(), guid).await?) } - /// Publish a signed attestation to the PDS (the AppView indexes it via Jetstream). No-op for a - /// did:key local identity (self-certifying, no repo to write). Idempotent via a session-derived rkey. + /// Publish a signed attestation to the PDS. The AppView then indexes it through Jetstream. + /// + /// The method does nothing for a local did:key identity. Such an identity certifies itself and + /// has no repository to write to. The rkey comes from the session, so a second call is safe. pub async fn publish_ibd_attestation(&self, att: &IbdAttestation) -> Result<(), AppError> { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; if did.starts_with("did:key:") { @@ -578,18 +604,23 @@ impl App { Ok(()) } - /// Issue a device-key-signed `exchange-poll` GET to an `/api/v1/` endpoint, with `extra` - /// query params appended. Shared by incoming / pending / relay-pull — the exchange endpoints - /// all sign the same canonical poll string, so this is the only thing they add over - /// [`App::appview_get_signed`]. + /// Send an `exchange-poll` GET to an `/api/v1/` endpoint with a device-key signature, and + /// add the `extra` query parameters. + /// + /// Three callers use this method: the poll for requests that arrived, the poll for open + /// sessions, and the relay read. Each exchange endpoint signs the same canonical poll string. + /// That string is the only part that this method adds to [`App::appview_get_signed`]. async fn exchange_get_poll(&self, path: &str, extra: &[(&str, &str)]) -> Result { self.appview_get_signed(path, exchange::messages::poll, extra).await } - /// Enqueue the anchor records every child record references: the subject's biosample summary - /// and each of its sequence runs, at their **deterministic** rkeys so the at:// URIs resolve. - /// Idempotent — the outbox coalesces per `entity_ref` and re-publishing overwrites in place — so - /// child publishes call this freely to guarantee there's always a biosample to tie back to. + /// Put the anchor records in the queue. Each child record points to these anchors. The anchors + /// are the biosample summary of the subject and each of its sequence runs. They use **fixed** + /// rkeys, so the at:// URIs resolve. + /// + /// A second call is safe. The outbox joins the rows with the same `entity_ref`, and a second + /// publish replaces the record. So a child publish can always call this method, and a biosample + /// always exists for the child to point to. async fn ensure_subject_anchor(&self, did: &str, biosample_guid: SampleGuid) -> Result<(), AppError> { // Sequence runs first (the biosample record links to them). for run in self.list_sequence_runs(biosample_guid).await? { @@ -614,9 +645,10 @@ impl App { .await } - /// Publish the alignment's coverage summary to the signed-in account's PDS (with - /// refresh-on-expiry and retry/backoff via [`AsyncSync`]). Anchors the subject first so the - /// record's biosample/sequence-run refs resolve. + /// Publish the coverage summary of the alignment to the PDS of the active account. + /// [`AsyncSync`] refreshes an expired token and tries again after a failure, with a longer + /// delay each time. The method publishes the subject anchors first, so the biosample ref and + /// the sequence-run ref of the record resolve. pub async fn publish_coverage(&self, alignment_id: i64) -> Result<(), AppError> { let did = self.require_account()?; // auth check before touching the DB let guid = self.biosample_of_alignment(alignment_id).await?; @@ -626,25 +658,32 @@ impl App { "coverage", &format!("alignment:{alignment_id}"), NS_ALIGNMENT, - // Deterministic rkey → the idempotent put path (never a fresh create), so re-publishing - // or two concurrent drains converge on one record instead of duplicating. + // A fixed rkey selects the put path, which is safe to repeat. The code never calls + // create here. So a second publish, or two drains at the same time, give one record + // and not two. Some(&alignment_rkey(alignment_id)), value, ) .await } - /// Publish a subject's **consensus** ancestry estimate to the signed-in account's PDS — one - /// populationBreakdown record per method (ADMIXTURE / PCA_PROJECTION_GMM / FINE_ADMIXTURE / - /// G25_NMONTE), each linked to the biosample. Subject-level (the breakdown is computed from the - /// pooled autosomal consensus, not per alignment), so one authoritative record set per subject - /// rather than a conflicting set per sequencing run. The researcher opt-in act for the ancestry - /// section — anonymized population proportions only. + /// Publish the **consensus** ancestry estimate of a subject to the PDS of the active account. + /// + /// The method writes one populationBreakdown record for each method. The methods are ADMIXTURE, + /// PCA_PROJECTION_GMM, FINE_ADMIXTURE, and G25_NMONTE. Each record links to the biosample. + /// + /// The estimate belongs to the subject, not to one alignment, because the code calculates the + /// breakdown from the pooled autosomal consensus. So each subject has one record set with + /// authority. A set for each sequence run would give records that disagree. + /// + /// This is the action that a researcher opts in to for the ancestry section. Only anonymous + /// population proportions cross the network. pub async fn publish_ancestry(&self, biosample_guid: SampleGuid) -> Result<(), AppError> { let did = self.require_account()?; // auth check before touching the DB self.ensure_subject_anchor(&did, biosample_guid).await?; // the breakdown links back to it let biosample_ref = biosample_at_uri(&did, biosample_guid); - // One outbox row per method, keyed by subject+method so re-publishing coalesces per estimate. + // One outbox row for each method. The key is the subject and the method, so a second + // publish joins the rows of one estimate. for r in &self.consensus_ancestry_results(biosample_guid).await? { let value = serde_json::to_value(population_breakdown_record(r).with_biosample_ref(Some(biosample_ref.clone())))?; @@ -655,9 +694,10 @@ impl App { Ok(()) } - /// Publish the anonymized biosample summary (sex, haplogroups) **and its sequence runs** to the - /// signed-in account's PDS — the subject anchor every derived record ties back to. Deterministic - /// rkeys make it idempotent (a re-publish overwrites rather than duplicating). + /// Publish the anonymous biosample summary, which holds the sex and the haplogroups, **and its + /// sequence runs** to the PDS of the active account. These records are the subject anchor, and + /// each derived record points to them. The method uses a fixed rkey, so a second publish + /// replaces the records and does not add a copy. pub async fn publish_biosample(&self, biosample_guid: SampleGuid) -> Result<(), AppError> { let did = self.require_account()?; // auth check before touching the DB self.ensure_subject_anchor(&did, biosample_guid).await diff --git a/crates/navigator-app/src/matching.rs b/crates/navigator-app/src/matching.rs index e2520480..9f53e6b0 100644 --- a/crates/navigator-app/src/matching.rs +++ b/crates/navigator-app/src/matching.rs @@ -1,19 +1,22 @@ -//! `impl App` methods for the **matching ledger** — the durable state behind federated-IBD -//! discovery and consent. +//! `impl App` methods for the **matching ledger**. The ledger is the durable state behind +//! federated-IBD discovery and consent. //! -//! The pieces this coordinates already existed: the AppView's candidate engine -//! ([`App::ibd_suggestions`]), the blind introduction broker ([`App::ibd_introduce`]), the -//! consent round-trip and encrypted channel (`ibd_exchange.rs`), and the stored result -//! (`navigator_store::ibd_exchange`). What was missing is the thread between them — every -//! in-flight request lived in UI memory, so a restart forgot that we had asked anyone anything. +//! The parts that this module coordinates already existed. They are the candidate engine of the +//! AppView ([`App::ibd_suggestions`]) and the blind introduction broker +//! ([`App::ibd_introduce`]). They are also the consent messages with the encrypted channel +//! (`ibd_exchange.rs`), and the stored result (`navigator_store::ibd_exchange`). //! -//! [`App::refresh_matching`] is the single reconcile: it adopts what the broker reports, advances -//! rows the broker has moved on, and never overwrites a decision we made locally. +//! The connection between these parts was absent. Each open request stayed in the memory of the +//! UI. So after a restart, the app did not know that it had sent a request to anybody. +//! +//! [`App::refresh_matching`] is the one reconcile. It adopts the state that the broker reports. It +//! advances a row when the broker moves that row forward. It never writes over a decision that the +//! user made on this machine. use super::*; -/// Region tag an attestation is filed under, derived from the exchange purpose -/// (`ibd.ibd_discovery_index.match_region_type` is `AUTOSOMAL`/`X`/`Y`/`MT`). +/// The region tag of an attestation. The code takes the tag from the purpose of the exchange. The +/// field `ibd.ibd_discovery_index.match_region_type` holds `AUTOSOMAL`, `X`, `Y`, or `MT`. fn region_type_for(purpose: &str) -> &str { match purpose { "IBD_Y" => "Y", @@ -77,16 +80,18 @@ impl App { /// Reconcile the ledger with the broker, then return the full list. /// - /// Three passes, in order of increasing knowledge: - /// 1. `/exchange/incoming` — inbound requests we have not seen. Adopted with - /// `insert_if_absent`, so a request we already declined stays declined even though the - /// broker keeps listing it. - /// 2. `/exchange/pending` — mutual consent happened: the partner DID and session id are now - /// known. Advances anything not already terminal (a completed exchange stays completed). - /// 3. Stored results — a completed exchange marks its request `EXCHANGED`. + /// There are three passes. Each pass knows more than the pass before it. + /// 1. `/exchange/incoming` gives the requests that arrived and that the app did not see. The + /// app adopts each one with `insert_if_absent`. So a request that the user declined stays + /// declined, and the broker can continue to list it. + /// 2. `/exchange/pending` shows that both parties agreed. The partner DID and the session id + /// are now known. This pass advances each row that is not yet in a final state. A complete + /// exchange keeps its state. + /// 3. The stored results set the request of a complete exchange to `EXCHANGED`. /// - /// A pass that fails does not abort the others: a broker hiccup should degrade the view, not - /// empty it. The first error is returned once the local state is consistent. + /// A pass that fails does not stop the other passes. A short broker fault must make the view + /// less complete, but it must not empty the view. The method returns the first error after the + /// local state is consistent. pub async fn refresh_matching(&self) -> Result, AppError> { let mut first_err: Option = None; @@ -111,8 +116,9 @@ impl App { match self.exchange_pending().await { Ok(pending) => { for info in pending { - // An unknown session means the request was opened on another device (or the - // ledger predates it) — adopt it so the session is still runnable here. + // An unknown session shows that another device opened the request. The + // ledger can also be older than the request. Adopt the session, so the user + // can still run it on this machine. let existing = navigator_store::ibd_request::get(self.store.pool(), &info.request_uri).await?; let mut row = existing.unwrap_or_else(|| { new_row( @@ -158,11 +164,11 @@ impl App { } } - /// Ask to be introduced to a candidate and record the conversation. + /// Ask the broker for an introduction to a candidate, and record the conversation. /// - /// Carries both AppView sample handles into the ledger: `target_sample_guid` (ours) and - /// `suggested_sample_guid` (theirs) are the only two identifiers an attestation can be filed - /// under, and the suggestion is the one place we ever see them. + /// The method puts both AppView sample handles in the ledger. `target_sample_guid` is our + /// handle and `suggested_sample_guid` is the handle of the partner. An attestation can use only + /// these two identifiers. The suggestion is the one place where the app sees them. pub async fn request_introduction( &self, suggestion: &IbdSuggestion, @@ -182,9 +188,9 @@ impl App { self.matching_entry(&intro.request_uri).await } - /// Consent to (or decline) an inbound request, recording our decision durably. The decision is - /// written whatever the broker says next — re-polling must never resurrect a request we - /// turned down. + /// Agree to a request that arrived, or decline it, and write the decision to the store. The + /// app writes the decision, and a later report from the broker does not change it. A new poll + /// must never return a request that the user declined. pub async fn matching_consent( &self, request_uri: &str, @@ -236,9 +242,9 @@ impl App { Ok(()) } - /// Record the AppView sample handles for a conversation that did not come from a suggestion - /// (or whose suggestion predated the AppView returning our own handle). Without both, a - /// completed comparison cannot be attested. + /// Record the AppView sample handles of a conversation that has no suggestion. A suggestion + /// can also be older than the AppView change that added our own handle. The app needs both + /// handles. Without them, it can not attest a complete comparison. pub async fn set_matching_sample_refs( &self, request_uri: &str, @@ -259,8 +265,8 @@ impl App { Ok(()) } - /// Record that an exchange attempt failed, so the row reads as `FAILED` with the reason rather - /// than sitting at `READY` forever. + /// Record a failed try of an exchange. The row then shows `FAILED` with the reason. Without + /// this record, the row stays at `READY` for all time. pub async fn record_matching_failure(&self, request_uri: &str, err: &str) -> Result<(), AppError> { let Some(mut row) = navigator_store::ibd_request::get(self.store.pool(), request_uri).await? else { return Ok(()); @@ -272,15 +278,17 @@ impl App { Ok(()) } - /// Drop a conversation from the local ledger. The broker keeps its own record, so a still-live - /// request can reappear on the next refresh — this forgets, it does not cancel. + /// Remove a conversation from the local ledger. The broker keeps its own record. So an open + /// request can come back at the next refresh. This method removes a local row. It does not + /// cancel the request. pub async fn forget_matching_request(&self, request_uri: &str) -> Result<(), AppError> { navigator_store::ibd_request::delete(self.store.pool(), request_uri).await?; Ok(()) } - /// Mark a conversation complete once its exchange result is stored, adopting the request if the - /// ledger has never seen it (a session opened on another device, or one predating the ledger). + /// Mark a conversation as complete after the app stores the result of its exchange. If the + /// ledger has no row for the request, the method adds one. Another device can open a session, + /// and a session can also be older than the ledger. pub(crate) async fn mark_matching_exchanged( &self, guid: SampleGuid, @@ -309,8 +317,9 @@ impl App { .ok_or_else(|| AppError::AppView(format!("no matching request {request_uri}"))) } - /// Tell the AppView to stop suggesting a candidate (`POST /api/v1/ibd/dismiss`). The dismissal - /// is kept server-side across recomputes, so it survives without any local mirror. + /// Tell the AppView to remove a candidate from its suggestions (`POST /api/v1/ibd/dismiss`). + /// The server keeps this decision when it calculates the candidates again. So the app needs no + /// local copy of the decision. pub async fn ibd_dismiss(&self, suggested_sample_guid: &str) -> Result<(), AppError> { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let key = self.ensure_device_key().await?; @@ -326,13 +335,15 @@ impl App { Ok(()) } - /// Report a completed comparison to the AppView (`POST /api/v1/ibd/attest`) — the step that - /// turns a private, edge-computed match into a discovery signal. Only coarse totals travel: - /// two opaque sample handles, a region tag, cM and segment count. Never coordinates, never - /// genotypes. + /// Report a complete comparison to the AppView (`POST /api/v1/ibd/attest`). This step changes + /// a private match, which the device calculated, into a discovery signal. + /// + /// Only approximate totals cross the network. They are two opaque sample handles, a region tag, + /// the cM value, and the count of segments. A coordinate never crosses. A genotype never + /// crosses. /// - /// The signed `cm` is formatted `{:.1}` to match the AppView's own canonical string byte for - /// byte; a mismatch there fails signature verification, not parsing. + /// The code formats the signed `cm` value as `{:.1}`, which gives the same canonical string as + /// the AppView. A difference here fails the signature check. It does not fail the parser. pub async fn ibd_attest( &self, request_uri: &str, @@ -365,14 +376,15 @@ impl App { Ok(()) } - /// Attest a completed exchange if — and only if — it is attestable: both parties agreed on the - /// summary, and we know both AppView sample handles. + /// Attest a complete exchange, but only when the app can attest it. Two conditions apply. Both + /// parties must agree on the summary, and the app must know both AppView sample handles. + /// + /// Agreement is a condition because the AppView confirms an edge only when both parties report + /// a compatible total. The partner can dispute a comparison. A total from such a comparison + /// puts a claim on the discovery graph that our own run says is wrong. /// - /// Agreement is the gate because the AppView confirms an edge only when both parties report a - /// compatible total; filing a one-sided figure from a comparison our partner disputes would put - /// a claim on the discovery graph that our own run says is wrong. Handles are missing whenever - /// the conversation did not come from a suggestion (a direct request never names them), so this - /// is a no-op rather than an error. + /// The handles are absent when the conversation has no suggestion, because a direct request + /// never names them. In that case the method does nothing and gives no error. pub(crate) async fn attest_exchange_if_possible(&self, request_uri: &str) -> Result { let entry = self.matching_entry(request_uri).await?; let (Some(mine), Some(theirs)) = (entry.my_sample_ref.clone(), entry.partner_sample_ref.clone()) else { @@ -419,8 +431,9 @@ mod tests { assert_eq!(region_type_for("GENEALOGY_PII"), "AUTOSOMAL"); } - /// Cross-repo contract: these strings are signed, and the AppView rebuilds them byte for byte - /// (`du_db::ibd::messages`). A drift here fails as a signature rejection, not a parse error. + /// A contract between two repositories. The device key signs these strings, and the AppView + /// makes the same strings in `du_db::ibd::messages`. A change on one side only fails as a + /// rejected signature. It does not fail as a parse error. #[test] fn canonical_dismiss_and_attest_messages() { let did = "did:plc:abc123"; diff --git a/crates/navigator-app/src/recruitment.rs b/crates/navigator-app/src/recruitment.rs index 4a14b881..8ba1a7e8 100644 --- a/crates/navigator-app/src/recruitment.rs +++ b/crates/navigator-app/src/recruitment.rs @@ -1,9 +1,15 @@ -//! `impl App` methods for the AppView's signed recruitment Edge API (`/api/v1/recruitment/*`, -//! social roadmap 3c) — the **response** side: list the caller's open invitations and accept/decline -//! them. Campaign creation stays on the AppView web flow (it's gated to a group-project admin, which -//! the Navigator can't yet act as). Device-key-signed like the social/exchange clients; reuses the -//! shared [`appview_post`](App::appview_post) / [`appview_get_signed`](App::appview_get_signed) transport. Invitations -//! also arrive as SYSTEM notifications, so this pairs with the Community → Notifications surface. +//! `impl App` methods for the signed recruitment Edge API of the AppView +//! (`/api/v1/recruitment/*`, social roadmap 3c). +//! +//! This module is the **response** side. It lists the open invitations of the caller. It also +//! accepts or declines them. To make a campaign, the user must use the web flow of the AppView. +//! Only an administrator of a group project can make a campaign, and Navigator can not yet act as +//! one. +//! +//! The device key signs each call, as it does for the social client and the exchange client. This +//! module uses the shared [`appview_post`](App::appview_post) and +//! [`appview_get_signed`](App::appview_get_signed) transport. An invitation also arrives as a +//! SYSTEM notification. So this module works together with the Community → Notifications view. use super::*; use navigator_sync::recruitment::messages; diff --git a/crates/navigator-app/src/settings.rs b/crates/navigator-app/src/settings.rs index ae5e6a39..1191c0ba 100644 --- a/crates/navigator-app/src/settings.rs +++ b/crates/navigator-app/src/settings.rs @@ -1,9 +1,14 @@ //! Persisted application settings at `~/.decodingus/config/settings.json`. //! -//! These are consulted by the resolvers in [`crate`] **below** any environment variable (env wins → -//! settings → built-in default), so the Settings UI can change app behavior — AppView URL, Y-tree -//! provider, tree-cache TTL, theme — without env vars or a relaunch. The file is small; resolvers -//! re-read it per call (they run per-analysis, not in a hot loop), so edits apply immediately. +//! The resolvers in [`crate`] read these settings. An environment variable has priority over a +//! setting, and a setting has priority over the built-in default. +//! +//! So the Settings UI can change the behaviour of the app with no environment variable and no +//! restart. The user can change the AppView URL, the Y-tree provider, the TTL of the tree cache, +//! and the theme. +//! +//! The file is small, and a resolver reads it again at each call. A resolver runs once for each +//! analysis, not in a loop, so a change applies immediately. use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -13,10 +18,13 @@ pub struct AppSettings { /// Y-tree provider: `"decodingus"` or `"ftdna"`. #[serde(default, skip_serializing_if = "Option::is_none")] pub y_tree_provider: Option, - /// Prefer a trusted external caller (GATK4 GVCF / 1240K call set, imported via the sidecar fast - /// path) over Navigator's own genotyping. When on (the built-in default), an external Y/mt/ - /// autosomal call wins reconciliation and Navigator's internal caller does not re-walk that - /// alignment. `None` = the default (on); set `Some(false)` to always run the internal caller. + /// Use a trusted external caller before the internal caller of Navigator. The external caller + /// is a GATK4 GVCF or a 1240K call set, and the sidecar fast path imports it. + /// + /// When this setting is on, which is the built-in default, an external Y call, mt call, or + /// autosomal call wins the reconciliation. The internal caller does not walk that alignment + /// again. `None` means the default, which is on. `Some(false)` makes the internal caller always + /// run. #[serde(default, skip_serializing_if = "Option::is_none")] pub prefer_external_calls: Option, /// AppView base URL (tree API + sequencer-lab lookup). @@ -28,11 +36,12 @@ pub struct AppSettings { /// UI theme: `"dark"` or `"light"`. #[serde(default, skip_serializing_if = "Option::is_none")] pub theme: Option, - /// Ask before downloading large reference files. + /// Ask the user before a download of a large reference file. #[serde(default, skip_serializing_if = "Option::is_none")] pub prompt_before_download: Option, - /// UI scale (egui zoom factor) — raise it on a native-4K / HiDPI display where the OS reports a - /// 1.0 scale factor and the default text is tiny. `None` = 1.0. + /// The scale of the UI, which is the zoom factor of egui. Increase it on a 4K display or a + /// HiDPI display when the operating system reports a scale factor of 1.0 and the text is very + /// small. `None` means 1.0. #[serde(default, skip_serializing_if = "Option::is_none")] pub ui_scale: Option, /// Interface mode: `"simple"` (casual single-person briefs) or `"advanced"` (full power-user UI). @@ -48,25 +57,28 @@ pub struct AppSettings { /// Model id to request (as reported by `GET /models`), e.g. "llama-3.1-8b-instruct". #[serde(default, skip_serializing_if = "Option::is_none")] pub llm_model: Option, - /// Max response (completion) tokens to request. Reasoning models spend most of this on their - /// chain-of-thought, so it must be large enough for the thinking *and* the answer. `None` = default. + /// The maximum count of response tokens to request. A model that reasons uses most of these + /// tokens for its internal steps. So the value must be large enough for those steps and for the + /// answer. `None` means the default. #[serde(default, skip_serializing_if = "Option::is_none")] pub llm_max_tokens: Option, /// Check GitHub Releases for a newer installer at startup and notify. `None` = the built-in /// default (enabled); set `Some(false)` to opt out. Never auto-installs. #[serde(default, skip_serializing_if = "Option::is_none")] pub check_for_updates: Option, - /// A version the user asked not to be reminded about (the exact `latest_version` string). A - /// *newer* release than this still notifies. + /// A version that the user does not want a reminder about. The value is the exact + /// `latest_version` string. The app still gives a notification for a newer release. #[serde(default, skip_serializing_if = "Option::is_none")] pub skip_update_version: Option, - /// Last window inner size `[width, height]` in egui points, remembered across launches. `None` - /// until the first run persists it. On restore it is fitted to the current monitor (an over-large - /// remembered size — e.g. from a bigger display — is shrunk to fit). + /// The last inner size of the window as `[width, height]` in egui points. The app keeps this + /// value between launches. It is `None` until the first run writes it. + /// + /// At the next start, the app fits the size to the current monitor. A stored size can be too + /// large, for example from a larger display, and the app then makes it smaller. #[serde(default, skip_serializing_if = "Option::is_none")] pub window_size: Option<[f32; 2]>, - /// Last selected navigation view (`"dashboard"` / `"subjects"` / `"projects"` / `"community"`), so - /// the app reopens where it was left. + /// The navigation view that the user selected last. The value is `"dashboard"`, `"subjects"`, + /// `"projects"`, or `"community"`. The app opens this view at the next start. #[serde(default, skip_serializing_if = "Option::is_none")] pub last_nav: Option, /// Last focused subject (biosample GUID), restored once the subject list has loaded. @@ -77,27 +89,33 @@ pub struct AppSettings { #[serde(default, skip_serializing_if = "Option::is_none")] pub last_detail_tab: Option, - // ── Chromosome-painter (copying-LAI) calibration knobs (all `None` → the built-in default) ── - /// Reference-haplotype switch intensity per cM (copying recombination). Lower → longer copied - /// tracts → cleaner population calls / less drifted-isolate over-attraction. Default 0.1. + // ── Calibration values for the chromosome painter (copy-LAI). `None` = the built-in default ── + /// The switch intensity of a reference haplotype for each cM. This value models recombination + /// in the copy step. A lower value gives longer copied tracts. Longer tracts give a cleaner + /// population call, and they attract a drifted isolate less. The default is 0.1. #[serde(default, skip_serializing_if = "Option::is_none")] pub lai_recomb_per_cm: Option, - /// Per-population reference cap (haplotypes). Balances the panel so large 1000G samples don't - /// out-vote by count. Default 50. + /// The maximum count of reference haplotypes for each population. This limit balances the + /// panel, so a large 1000G sample does not win only by its count. The default is 50. #[serde(default, skip_serializing_if = "Option::is_none")] pub lai_max_ref_haps: Option, /// Global-composition gate: drop super-populations below this genome-wide fraction. Default 0.05. #[serde(default, skip_serializing_if = "Option::is_none")] pub lai_min_ancestry: Option, - /// Ancestry-segment switch intensity per cM (Viterbi smoothing). Lower → longer segments. Default 0.05. + /// The switch intensity of an ancestry segment for each cM, which the Viterbi step uses to + /// make the track smooth. A lower value gives longer segments. The default is 0.05. #[serde(default, skip_serializing_if = "Option::is_none")] pub lai_switch_per_cm: Option, - /// Minimum segment length in centiMorgans (shorter runs merge into the neighbour). Default 4.0. - /// In genetic distance, not sites, so the setting keeps its meaning when the panel's marker - /// density changes. + /// The minimum length of a segment in centiMorgans. The code joins a shorter run to its + /// neighbour. The default is 4.0. + /// + /// The unit is genetic distance, not a count of sites. So the value stays correct when the + /// marker density of the panel changes. #[serde(default, skip_serializing_if = "Option::is_none")] pub lai_min_segment_cm: Option, - /// Per-population size correction exponent (0 = off, 1 = full per-haplotype average). Default 0.5. + /// The exponent that corrects for the size of each population. A value of 0 turns the + /// correction off. A value of 1 gives the full average for each haplotype. The default is + /// 0.5. #[serde(default, skip_serializing_if = "Option::is_none")] pub lai_size_normalize: Option, /// Copy mismatch/mutation rate μ. Default 0.02. @@ -106,7 +124,8 @@ pub struct AppSettings { } impl AppSettings { - /// `~/.decodingus/config/settings.json` (honoring `NAVIGATOR_REFGENOME_DIR`, same base as the + /// `~/.decodingus/config/settings.json`. The path obeys `NAVIGATOR_REFGENOME_DIR` and uses the + /// same base as the /// reference-source overrides). pub fn path() -> PathBuf { navigator_refgenome::cache::base_dir() @@ -114,12 +133,16 @@ impl AppSettings { .join("settings.json") } - /// Load settings; a missing or unreadable/invalid file yields the empty default. + /// Read the settings. If the file is absent, unreadable, or invalid, the function gives the + /// empty default. + /// + /// The read goes through [`navigator_refgenome::cache::read_atomic`]. So a [`Self::save`] call + /// at the same time can not make the read fail. On Windows, no process can open the file for a + /// short time during a replace. /// - /// Reads through [`navigator_refgenome::cache::read_atomic`] so a concurrent [`Self::save`] - /// can't make the load fail: on Windows the file is briefly unopenable mid-replace, and this - /// returning the default would quietly discard the user's settings — including, since - /// `copying_lai_params()` reads them at paint time, their painter calibration. + /// Without this protection, the function returns the default and the app removes the settings + /// of the user without a message. The painter calibration is one of those settings, because + /// `copying_lai_params()` reads the settings at paint time. pub fn load() -> Self { navigator_refgenome::cache::read_atomic(&Self::path()) .ok() @@ -128,14 +151,17 @@ impl AppSettings { .unwrap_or_default() } - /// The `~/.decodingus` base directory (honoring `NAVIGATOR_REFGENOME_DIR`). + /// The `~/.decodingus` base directory. The path obeys `NAVIGATOR_REFGENOME_DIR`. pub fn cache_base_dir() -> PathBuf { navigator_refgenome::cache::base_dir() } - /// Persist to disk (creating the `config/` dir), pretty-printed. Written **atomically** (temp + - /// rename) — settings are saved from several UI paths that can overlap, and a non-atomic write - /// risks the same torn-file corruption seen on `reference_sources.json`. + /// Write the settings to disk in a readable layout, and make the `config/` directory if it + /// does not exist. + /// + /// The write is **atomic**. The function writes a temporary file and then renames it. Some UI + /// paths save the settings at the same time. A write that is not atomic can give a torn file, + /// as it did with `reference_sources.json`. pub fn save(&self) -> std::io::Result<()> { let json = serde_json::to_string_pretty(self).map_err(std::io::Error::other)?; navigator_refgenome::cache::atomic_write(&Self::path(), json.as_bytes()) diff --git a/crates/navigator-app/src/social.rs b/crates/navigator-app/src/social.rs index 4eaf8b2f..762940b9 100644 --- a/crates/navigator-app/src/social.rs +++ b/crates/navigator-app/src/social.rs @@ -1,12 +1,17 @@ -//! `impl App` methods for the AppView's signed social Edge API (`/api/v1/social/*`) — the -//! communication core the alpha/beta testers use to reach the team (support threads), read the -//! community feed (+ federated posts), and receive notifications. +//! `impl App` methods for the signed social Edge API of the AppView (`/api/v1/social/*`). //! -//! Every call is **device-key-signed** (no per-call OAuth), exactly like the IBD `exchange` client: -//! reads are a replay-guarded signed GET (`did`/`ts`/`sig` query); writes carry `did` + `signature` -//! (and `ts` where the canonical string includes it) in the JSON body. Canonical signing strings live -//! in [`navigator_sync::social::messages`] and mirror the AppView byte-for-byte. PII-free: only a DID, -//! a signature, and content the user chose to send crosses the wire. +//! This module is the communication core. A tester uses it for three tasks. The tester speaks to +//! the team in a support thread, reads the community feed with its federated posts, and receives a +//! notification. +//! +//! The device key signs **each call**, and no call uses OAuth. The IBD `exchange` client works in +//! the same way. A read is a signed GET with a replay guard, and its `did`, `ts`, and `sig` values +//! go on the query. A write puts `did` and `signature` in the JSON body. The write also puts `ts` +//! there when the canonical string holds a timestamp. +//! +//! [`navigator_sync::social::messages`] holds the canonical strings for a signature. These strings +//! are the same as the strings of the AppView. The module sends no personal data. Only a DID, a +//! signature, and the content that the user chose to send cross the network. use super::*; @@ -15,7 +20,7 @@ use navigator_sync::social::messages; /// One of the caller's support threads (team↔tester), as listed by `GET /social/threads`. #[derive(Debug, Clone, serde::Deserialize)] pub struct SocialThreadSummary { - /// Conversation id (UUID string) — the key for reading/replying. + /// The conversation id, as a UUID string. It is the key to read the thread and to reply. pub conversation_id: String, #[serde(default)] pub subject: Option, @@ -77,7 +82,8 @@ pub struct FeedItem { pub parent_post_id: Option, } -/// A PDS-federated community post mirrored into the feed (read-only — voting/reply/block stay native). +/// A federated community post from a PDS, copied into the feed. The user can only read it. A vote, +/// a reply, and a block stay in the native AppView records. #[derive(Debug, Clone, serde::Deserialize)] pub struct FederatedItem { #[serde(default)] @@ -190,9 +196,11 @@ impl App { self.appview_get_signed("social/feed", messages::poll, &[]).await } - /// Post to the community feed (optionally tagged with a `topic`, or as a reply to `parent`); - /// returns the new post id. A reputation gate maps to [`AppError`] (HTTP 403) — surface it as a - /// "not enough reputation yet" hint in the UI. + /// Send a post to the community feed and return the id of the new post. The caller can add a + /// `topic` tag, or make the post a reply to `parent`. + /// + /// A reputation gate gives HTTP 403, which becomes an [`AppError`]. Show it in the UI as a hint + /// that the user does not have enough reputation. pub async fn post_community( &self, content: &str, @@ -214,21 +222,28 @@ impl App { Ok(v.get("id").and_then(|x| x.as_str()).unwrap_or_default().to_string()) } - /// Publish a community post to the signed-in account's PDS as a federated - /// `com.decodingus.atmosphere.feed.post` record (roadmap 3b). The AppView mirrors it into the - /// community feed via its Jetstream consumer (the read-only "via Atmosphere" entries), so this - /// is the portable, federated counterpart to the AppView-native [`post_community`](Self::post_community). + /// Publish a community post to the PDS of the active account. The record is a federated + /// `com.decodingus.atmosphere.feed.post` record (roadmap 3b). + /// + /// The Jetstream consumer of the AppView copies the record into the community feed as a + /// read-only "Atmosphere" entry. So this method is the portable, federated form of + /// [`post_community`](Self::post_community), which writes a native AppView record. + /// + /// The record goes through the sync **outbox**, so the publish is durable. It continues after a + /// restart, and it tries again with a longer delay after a temporary failure or an offline + /// failure. + /// + /// Each post is a **separate** record. The outbox `entity_ref` is a new id, because the app only + /// appends a post and never joins two posts. A summary record for one entity behaves in a + /// different way. `rkey: None` lets the PDS choose the TID. /// - /// Durable: the record goes through the sync **outbox**, so the publish survives restart and - /// retries with backoff on a transient/offline failure. Each post is a **distinct** record — the - /// outbox `entity_ref` is a fresh id (posts are append-only, never coalesced like the per-entity - /// summary records), and `rkey: None` lets the PDS assign the TID. Federated posts are - /// deliberately **not** in `PUBLISHED_COLLECTIONS`: a PULL reconcile must never resurrect a post - /// the user deleted on their PDS. + /// A federated post is **not** in `PUBLISHED_COLLECTIONS`, by design. A PULL reconcile must never + /// return a post that the user deleted on their PDS. /// - /// Errors when signed out, and for a local `did:key` identity (self-certifying, no PDS repo to - /// write to) — the federated feed needs a real OAuth/PDS account. The UI gates the opt-in - /// accordingly and surfaces the error as a hint. + /// The method fails when no account is active. It also fails for a local `did:key` identity, + /// because that identity certifies itself and has no PDS repository to write to. The federated + /// feed needs an OAuth account with a PDS. The UI gates the option and shows the error as a + /// hint. pub async fn publish_feed_post(&self, content: &str, topic: Option<&str>) -> Result<(), AppError> { let did = self.require_account()?; if did.starts_with("did:key:") { @@ -251,7 +266,8 @@ impl App { .await } - /// Mark one notification read (`id = Some`) or all (`id = None`); returns how many were marked. + /// Mark one notification as read with `id = Some`, or mark all of them with `id = None`. The + /// method returns the count of the notifications that it marked. pub async fn mark_notification_read(&self, id: Option<&str>) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let dev = self.ensure_device_key().await?; @@ -337,7 +353,7 @@ mod tests { assert!(v.get("createdAt").and_then(|c| c.as_str()).is_some()); assert!(v.get("meta").is_none() && v.get("reply").is_none()); - // A blank topic is omitted entirely. + // The code removes a blank topic. let v2 = crate::feed_post_record("no topic", Some(" "), None); assert!(v2.get("topic").is_none()); } diff --git a/crates/navigator-app/src/sync.rs b/crates/navigator-app/src/sync.rs index b8031df3..c848d5aa 100644 --- a/crates/navigator-app/src/sync.rs +++ b/crates/navigator-app/src/sync.rs @@ -5,10 +5,14 @@ use super::*; impl App { // ---- sync durability: outbox enqueue + drain (gap §5) ------------------- - /// Enqueue a built record for publishing to the signed-in account's PDS. The publish becomes - /// durable: it survives restart and retries automatically (with backoff) on a transient/offline - /// failure instead of being lost. Re-enqueuing the same `entity_ref` coalesces (newest wins). - /// Errors [`AppError::NotAuthenticated`] when signed out (we need the destination DID). + /// Put a record in the queue for a publish to the PDS of the active account. + /// + /// The publish is then durable. It continues after a restart, and it tries again with a longer + /// delay after a temporary failure or an offline failure. The app does not lose it. + /// + /// A second entry for the same `entity_ref` replaces the first entry, and the newest record + /// wins. The method returns [`AppError::NotAuthenticated`] when no account is active, because + /// the queue needs the DID of the destination. pub(crate) async fn enqueue_publish( &self, kind: &str, @@ -30,8 +34,8 @@ impl App { Ok(()) } - /// Pending (not-yet-published) outbox rows for the signed-in account — drives the UI's - /// "N pending" indicator. `0` when signed out. + /// The count of outbox rows for the active account that the app did not publish. The UI shows + /// this count in its "N pending" indicator. The value is `0` when no account is active. pub async fn outbox_pending_count(&self) -> Result { let Some(did) = self.current_account() else { return Ok(0); @@ -39,7 +43,8 @@ impl App { Ok(sync_outbox::pending_count(self.store.pool(), &did).await?) } - /// All non-completed outbox rows (PENDING + FAILED) for the signed-in account — a sync detail view. + /// Each outbox row for the active account that is not complete. The rows have the state + /// PENDING or FAILED. The sync detail view shows them. pub async fn outbox_entries(&self) -> Result, AppError> { let Some(did) = self.current_account() else { return Ok(Vec::new()); @@ -47,7 +52,8 @@ impl App { Ok(sync_outbox::list(self.store.pool(), &did).await?) } - /// Recent publish outcomes (success/failure) for the signed-in account — the audit trail. + /// The recent results of a publish for the active account. A result is a success or a failure. + /// Together the results are the audit trail. pub async fn sync_history(&self, limit: i64) -> Result, AppError> { let Some(did) = self.current_account() else { return Ok(Vec::new()); @@ -55,16 +61,24 @@ impl App { Ok(sync_history::recent(self.store.pool(), &did, limit).await?) } - /// Prune orphaned alignment (coverage-summary) records from the signed-in account's PDS repo — - /// the duplicates left by the pre-deterministic-rkey `create` race (two records for one - /// alignment, only one tracked in `sync_state`). Lists every alignment record in the repo and - /// removes any whose rkey is **not accounted for** — i.e. neither a `sync_state`-tracked rkey nor - /// the deterministic `aln-{id}` of a live local alignment. With `apply == false` it's a dry run - /// (reports what it would delete, touches nothing). Returns the outcome. + /// Remove orphan alignment records, which hold a coverage summary, from the PDS repository of + /// the active account. + /// + /// These orphans are duplicates. An earlier `create` call chose the record key itself, and two + /// calls could race. The repository then held two records for one alignment, and `sync_state` + /// held only one of them. + /// + /// The method lists each alignment record in the repository. It removes a record when the rkey + /// of that record has no source. A rkey has a source when `sync_state` holds it, or when it is + /// the `aln-{id}` key of a local alignment that still exists. + /// + /// With `apply == false`, the method only reports the records that it would delete and changes + /// nothing. The method returns the result. pub async fn prune_orphan_alignments(&self, apply: bool) -> Result { let did = self.require_account()?; - // Accounted-for rkeys: everything tracked in sync_state for the alignment collection, plus - // the deterministic key for every live local alignment (so a not-yet-drained one isn't culled). + // The rkeys with a source. These are each rkey in sync_state for the alignment + // collection, and the fixed key of each local alignment that still exists. The second group + // keeps a row that the app did not yet publish. let mut keep: std::collections::HashSet = sync_state::list_for_collection(self.store.pool(), &did, NS_ALIGNMENT) .await? @@ -104,10 +118,15 @@ impl App { Ok(report) } - /// Attempt to publish the ready outbox rows for the signed-in account. Each success is logged to - /// history and its row removed; a transient failure reschedules the row with exponential backoff - /// and stops the batch (we're likely offline); a non-transient failure marks the row `FAILED`. - /// A no-op (and `Ok`) when signed out. Safe to call repeatedly (periodically + after a publish). + /// Try to publish the ready outbox rows for the active account. + /// + /// After a success, the method writes a history row and removes the outbox row. After a + /// temporary failure, it sets a later time for the row and stops the batch, because the machine + /// is probably offline. The delay becomes longer after each try. After a permanent failure, it + /// sets the row to `FAILED`. + /// + /// The method does nothing and returns `Ok` when no account is active. The caller can call it + /// many times, at an interval and after a publish. pub async fn drain_outbox(&self) -> Result { let Some(did) = self.current_account() else { return Ok(DrainOutcome::default()); @@ -122,8 +141,8 @@ impl App { let batch = sync_outbox::ready(self.store.pool(), &did, &now.to_rfc3339(), OUTBOX_BATCH).await?; for entry in batch { let value: serde_json::Value = serde_json::from_str(&entry.payload)?; - // Idempotency: if we've published this entity before, update the PDS-assigned record in - // place (putRecord at the kept rkey) instead of creating a duplicate. + // If the app published this entity before, change the record that the PDS holds. Use + // putRecord at the rkey that the app kept. Do not make a duplicate record. let known = sync_state::get(self.store.pool(), &did, &entry.entity_ref).await?; let result = match (&known, &entry.rkey) { (Some(ss), _) => engine.push_put(&entry.collection, &ss.rkey, value).await, @@ -152,7 +171,8 @@ impl App { outcome.published.push((entry.kind.clone(), rref.uri)); } Err(e) if e.is_transient() => { - // Offline / 5xx / timeout: back off and stop — the rest of the batch will wait too. + // The machine is offline, or the server gave a 5xx or a timeout. Wait, and + // stop. The other rows of the batch also wait. let next = now + chrono::Duration::seconds(backoff_secs(attempt)); sync_outbox::reschedule( self.store.pool(), @@ -180,7 +200,7 @@ impl App { Ok(outcome) } - /// Append a sync-history row for a finished push attempt. + /// Add a sync-history row for a push that is complete. async fn log_history( &self, entry: &sync_outbox::OutboxEntry, @@ -204,16 +224,22 @@ impl App { Ok(()) } - /// **PULL reconcile** (gap §5-p2): fetch the account's own records from the PDS and reconcile - /// against what we published (`sync_state`), last-write-wins / remote-authoritative. For records - /// we recognise (via the kept rkey) that changed on the PDS, apply remote→local where the data - /// model allows (today: a biosample's sex / center) and re-track the CID. Records missing remotely - /// are flagged for re-publish; remote records with no local mapping are counted (the fed records - /// are PII-free *summaries* and carry no local guid, so they can't reconstruct a local entity). + /// Do a **PULL reconcile** (gap §5-p2). The method reads the records of the account from the + /// PDS and compares them with the records that the app published, which `sync_state` holds. The + /// policy is last-write-wins, and the remote copy has authority. + /// + /// The app knows a record by the rkey that it kept. When such a record changed on the PDS, the + /// method applies the remote values to the local record where the data model permits it. Today + /// this is the sex and the center of a biosample. The method then stores the new CID. + /// + /// The method marks a record for a new publish when the PDS no longer holds it. It counts a + /// remote record that has no local record. A federated record is a summary with no personal + /// data. It holds no local guid, so the app can not make a local entity from it. pub async fn pull_sync(&self) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; if did.starts_with("did:key:") { - // A local did:key identity has no PDS repo — PULL/publish need a real OAuth (did:plc) account. + // A local did:key identity has no PDS repository. A PULL and a publish need an OAuth + // account with a did:plc identity. return Err(AppError::Import( "PDS sync needs a signed-in PDS account — the local did:key identity has no PDS repo".into(), )); @@ -266,9 +292,12 @@ impl App { Ok(out) } - /// Apply a remote record onto local state. Only the editable, locally-authoritative bits the - /// PII-free fed record carries can be applied; derived-summary collections are recomputed locally, - /// so they're tracked but not overwritten. + /// Apply a remote record to the local state. + /// + /// The method applies only the values that the user can edit and that the local store owns. A + /// federated record has no personal data and carries only those values. The app calculates a + /// derived summary again on this machine. So the app tracks such a collection but does not + /// write to it. pub(crate) async fn apply_remote( &self, collection: &str, @@ -338,23 +367,31 @@ impl App { Ok(()) } - /// Load (or, on first use, generate + publish) this installation's Ed25519 **device key** - /// — the signing key that authenticates Edge→AppView calls (federated IBD and, later, the - /// whole signed surface). The key seed lives in the OS keychain scoped to the signed-in - /// DID; its public half is published once to the user's PDS as a - /// [`DEVICE_KEY_COLLECTION`] record so the AppView (which ingests it via Jetstream) can - /// verify our signatures. Idempotent: the record is keyed by its own `did:key`, so a - /// re-publish overwrites rather than duplicates, and an already-present record is left - /// alone. Errors [`AppError::NotAuthenticated`] when signed out. + /// Read the Ed25519 **device key** of this installation. At the first use, the method makes + /// the key and publishes it. + /// + /// This key signs a call from the edge to the AppView. Federated IBD uses it today, and later + /// the full signed surface will use it. The OS keychain holds the seed of the key under the + /// active DID. + /// + /// The method publishes the public half of the key one time to the PDS of the user, as a + /// [`DEVICE_KEY_COLLECTION`] record. The AppView reads that record through Jetstream, and it + /// can then check our signatures. /// - /// This does *not* wait for ingest — the signed AppView calls absorb the 403→200 lag with - /// bounded retries (see the IBD client). + /// A second call is safe. The record key is the `did:key` value itself. So a second publish + /// replaces the record and does not add a duplicate, and the method does not change a record + /// that already exists. The method returns [`AppError::NotAuthenticated`] when no account is + /// active. + /// + /// The method does *not* wait for the AppView to read the record. A signed call to the AppView + /// absorbs the delay from 403 to 200 with a limited count of tries. See the IBD client. pub async fn ensure_device_key(&self) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let key = DeviceKey::load_or_generate(KEYCHAIN_SERVICE, &did)?; - // A local did:key identity self-certifies (the AppView verifies the signature against the DID - // itself), so there is no PDS record to publish — and no OAuth session to do it with. + // A local did:key identity certifies itself, because the AppView checks the signature + // against the DID. So there is no PDS record to publish. There is also no OAuth session + // for such a publish. if did.starts_with("did:key:") { return Ok(key); } @@ -376,14 +413,16 @@ impl App { Ok(key) } - /// Federated IBD — **Step 1**: fetch this account's pseudonymous match suggestions from + /// Federated IBD, **Step 1**. Fetch this account's pseudonymous match suggestions from /// the AppView (`GET /api/v1/ibd/suggestions`). /// - /// The AppView mines our already-published `fed.*` records into a top-K candidate list; - /// no genotypes leave the device here. The call is authenticated by signing - /// `"ibd-poll\n\n"` with the device key (registered on first use). A 403 right - /// after first-time registration means the AppView hasn't ingested the device-key record - /// yet, so it's retried with exponential backoff. + /// The AppView reads the `fed.*` records that we published and makes a list of the best + /// candidates. No genotype leaves the device in this step. + /// + /// To authenticate the call, the device key signs `"ibd-poll\n\n"`. The app registers + /// that key at its first use. A 403 response directly after the first registration shows that + /// the AppView did not yet read the device-key record. The client then tries again, and the + /// delay becomes longer after each try. pub async fn ibd_suggestions(&self) -> Result, AppError> { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let key = self.ensure_device_key().await?; @@ -393,8 +432,8 @@ impl App { loop { let ts = Utc::now().timestamp().to_string(); let sig = key.sign(&format!("ibd-poll\n{did}\n{ts}")); - // reqwest URL-encodes query values, so the STANDARD-base64 sig (`+` `/` `=`) is - // safely escaped. + // reqwest applies URL encoding to a query value. So it escapes the `+`, `/`, and `=` + // characters of a STANDARD base64 signature. let resp = self .auth .http @@ -417,21 +456,24 @@ impl App { } } - /// Federated IBD — **Step 2**: request an introduction to a suggested candidate + /// Federated IBD, **Step 2**. Request an introduction to a suggested candidate /// (`POST /api/v1/ibd/introduce`). /// - /// Signs `"ibd-introduce\n\n"` and posts - /// `{ did, suggestedSampleGuid, signature }`. Returns the AppView's `request_uri` and - /// status (`PENDING`). This endpoint only opens the request — it exchanges no genetic data. - /// The downstream consent round-trip and encrypted segment exchange run over the separate edge - /// channel in `ibd_exchange` once both parties consent. + /// The method signs `"ibd-introduce\n\n"` and sends + /// `{ did, suggestedSampleGuid, signature }`. It returns the `request_uri` of the AppView and + /// the status, which is `PENDING`. + /// + /// This endpoint only opens the request. It exchanges no genetic data. After both parties + /// agree, the consent messages and the encrypted segment exchange use the separate edge channel + /// in `ibd_exchange`. pub async fn ibd_introduce(&self, suggested_sample_guid: &str) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let key = self.ensure_device_key().await?; let ts = Utc::now().timestamp(); let sig = key.sign_fresh(ts, &format!("ibd-introduce\n{did}\n{suggested_sample_guid}")); - // The AppView's IntroduceBody deserializes plain snake_case (no serde rename), and - // parses the guid as a UUID — send it verbatim from the suggestion. + // The IntroduceBody type of the AppView reads plain snake_case names and has no serde + // rename. It parses the guid as a UUID. Send the guid exactly as the suggestion gives + // it. let body = serde_json::json!({ "did": did, "suggested_sample_guid": suggested_sample_guid, diff --git a/crates/navigator-app/src/sync_reconcile.rs b/crates/navigator-app/src/sync_reconcile.rs index 8b37312e..0e97bd00 100644 --- a/crates/navigator-app/src/sync_reconcile.rs +++ b/crates/navigator-app/src/sync_reconcile.rs @@ -1,10 +1,15 @@ -//! Pure PULL reconcile planner (gap §5-p2). Given what we last published (`sync_state` rows, each with -//! the PDS CID + payload fingerprint at push time) and the records currently on the PDS, decide what to -//! do per record — with **no I/O**, so it's exhaustively unit-tested. The app executes the plan. +//! The PULL reconcile planner (gap §5-p2). This module has no I/O, so a unit test can cover every +//! case. The app does the plan that this module makes. //! -//! Policy: **last-write-wins, remote authoritative on divergence** (the confirmed §5-p2 decision). A -//! record that changed on the PDS since our push is applied locally; if our local copy *also* changed -//! since the push (we can detect that via the payload hash), it's still applied but flagged a conflict. +//! The planner reads two inputs. The first input is the record of our last publish. That record is +//! the `sync_state` rows, and each row holds the PDS CID and the payload fingerprint at the time of +//! the push. The second input is the set of records that the PDS holds now. The planner then +//! decides the action for each record. +//! +//! The policy is **last-write-wins**, and the remote copy has authority when the two copies differ. +//! This was the §5-p2 decision. The app applies a record that changed on the PDS after our push. If +//! our local copy also changed after the push, the app still applies the remote record, but it +//! marks a conflict. The payload hash shows a local change. use navigator_store::sync_state::StoredSyncState; use navigator_sync::RemoteRecord; @@ -12,25 +17,32 @@ use navigator_sync::RemoteRecord; /// One reconcile decision for a record. #[derive(Debug, Clone, PartialEq)] pub enum ReconcileAction { - /// Remote matches what we published and local is unchanged — nothing to do. + /// The remote record is the same as our published record, and the local record did not change. + /// There is no action. InSync { entity_ref: String }, - /// Remote changed since our push (or we have a local-only edit) — apply remote→local. `conflict` - /// ⇒ local *also* changed since the push (both diverged; remote wins, logged). + /// The remote record changed after our push, or there is a local edit. Apply the remote record + /// to the local record. `conflict` shows that the local record also changed after the push. + /// Both copies changed, the remote copy wins, and the app writes a log entry. ApplyRemote { entity_ref: String, collection: String, remote: RemoteRecord, conflict: bool, }, - /// Local was published but the record is gone on the PDS — re-publish our copy. + /// The app published the local record, but the PDS no longer holds it. Publish our copy + /// again. RePush { entity_ref: String }, - /// A record exists on the PDS we have no local sync-state for — adopt it locally. + /// The PDS holds a record, and we have no local sync-state row for it. Add the record to the + /// local store. AdoptRemote { collection: String, remote: RemoteRecord }, } -/// Plan the reconcile for one collection. `local` pairs each published entity with its *current* local -/// payload hash (`None` = not recomputed / assume clean); compared to the stored push-time hash to tell -/// whether local changed. `remote` is the PDS's current records for the same collection. +/// Make the reconcile plan for one collection. +/// +/// `local` gives the current local payload hash of each published entity. A value of `None` means +/// that the app did not calculate the hash again, and the planner treats the record as clean. The +/// planner compares this hash with the hash from the time of the push. A difference shows a local +/// change. `remote` is the set of records that the PDS holds now for the same collection. pub fn plan(local: &[(StoredSyncState, Option)], remote: &[RemoteRecord]) -> Vec { use std::collections::HashSet; let mut actions = Vec::new(); @@ -63,7 +75,7 @@ pub fn plan(local: &[(StoredSyncState, Option)], remote: &[RemoteRecord] } } None => { - // We published it but it's gone on the PDS — re-publish. + // The app published the record, but the PDS no longer holds it. Publish again. actions.push(ReconcileAction::RePush { entity_ref: ss.entity_ref.clone(), }); diff --git a/crates/navigator-app/src/update.rs b/crates/navigator-app/src/update.rs index efb5c31e..57cf8f7c 100644 --- a/crates/navigator-app/src/update.rs +++ b/crates/navigator-app/src/update.rs @@ -1,38 +1,46 @@ -//! Checking GitHub Releases for a newer installer, and *notifying* the user — never auto-updating. +//! A check of GitHub Releases for a newer installer. This module only *tells* the user. It never +//! updates the app without a command from the user. //! -//! Installers are published to the GitHub Releases of `JamesKane/decodingus-navigator` (`v*` tags; -//! Alpha/Beta/RC builds are marked *prerelease*). This module fetches that list, finds the highest -//! version, compares it against the running build ([`BUILD_VERSION`]), and — if it's -//! newer and the user hasn't chosen to skip it — returns an [`UpdateInfo`] pointing at the release -//! page and the platform-appropriate installer asset. The UI turns that into a dismissible prompt; -//! downloading/installing is entirely the user's choice. +//! The team publishes each installer to the GitHub Releases of `JamesKane/decodingus-navigator`, +//! under a `v*` tag. An Alpha build, a Beta build, and an RC build each have the *prerelease* +//! mark. +//! +//! This module reads that list and finds the highest version. It then compares that version with +//! the version of this build, which is [`BUILD_VERSION`]. The module returns an [`UpdateInfo`] +//! value when the published version is newer and the user did not choose to skip it. The value +//! points to the release page and to the correct installer for the platform. +//! +//! The UI shows this information as a prompt that the user can close. Only the user can start the +//! download and the installation. use serde::Deserialize; use crate::error::AppError; use crate::settings::AppSettings; -/// The GitHub Releases API for the app repo. We list releases (rather than `/releases/latest`, which -/// excludes prereleases) so Alpha/Beta builds are considered too. +/// The GitHub Releases API for the repository of the app. The code lists all releases. It does not +/// use `/releases/latest`, because that endpoint hides a prerelease. So the check also sees an +/// Alpha build and a Beta build. const RELEASES_URL: &str = "https://api.github.com/repos/JamesKane/decodingus-navigator/releases"; -/// The version this build calls itself when comparing against published releases. +/// The version name of this build. The code compares this name with the published releases. /// -/// `CARGO_PKG_VERSION` alone is not usable for that comparison, and the reason is easy to miss: -/// the workspace version is a bare `0.1.0` while every shipped tag is `v0.1.0-alpha.N`. Under -/// SemVer a release outranks its own prereleases, so the running build always looked *newer* than -/// every alpha on offer and the check returned "up to date" every single time. Sixteen alphas were -/// published without one user ever being notified. +/// `CARGO_PKG_VERSION` alone does not work for that comparison. The reason is easy to miss. The +/// workspace version is a plain `0.1.0`, but each tag that the team ships is `v0.1.0-alpha.N`. /// -/// The workspace version deliberately stays numeric — the Windows installer formats want -/// `x.y.z` — so the full version is injected at package time instead, from the tag being built. -/// A developer build has no such tag and falls back, which is correct: it should not claim to be -/// any particular release. +/// In SemVer, a release has a higher rank than its own prereleases. So this build always looked +/// *newer* than each alpha on the server, and the check always answered "up to date". The team +/// published sixteen alphas, and no user received a notification. /// -/// Note for anyone testing this locally: `option_env!` is read at compile time and Cargo does not -/// treat the variable as an input, so changing it does not by itself trigger a rebuild — touch this -/// file, or build clean, or you will keep measuring the previous value. CI builds from a fresh -/// checkout, so the packaged artifact is never affected. +/// The workspace version stays numeric by design, because a Windows installer format needs `x.y.z`. +/// So the package step puts the full version here, from the tag of that build. A build on a +/// developer machine has no tag and uses the default value. That result is correct, because such a +/// build must not claim to be a release. +/// +/// A note for a local test. Rust reads `option_env!` at compile time, and Cargo does not treat the +/// variable as an input. So a change to the variable does not start a rebuild. Touch this file, or +/// build clean. If not, you continue to see the value from the earlier build. CI always builds from +/// a new checkout, so this problem never reaches a packaged artifact. pub(crate) const BUILD_VERSION: &str = match option_env!("NAVIGATOR_RELEASE_VERSION") { Some(v) => v, None => env!("CARGO_PKG_VERSION"), @@ -41,13 +49,14 @@ pub(crate) const BUILD_VERSION: &str = match option_env!("NAVIGATOR_RELEASE_VERS /// A newer installer is available. Serialized so it can cross the worker `Command`/`Event` channel. #[derive(Debug, Clone, serde::Serialize)] pub struct UpdateInfo { - /// The running build's version ([`BUILD_VERSION`]). + /// The version of this build ([`BUILD_VERSION`]). pub current_version: String, - /// The newest published version (tag, without a leading `v`). + /// The newest published version. The value is the tag with no `v` at the start. pub latest_version: String, /// The release's display name (falls back to the tag). pub name: String, - /// The GitHub release page — always present, used as the fallback download link. + /// The GitHub release page. This value is always present, and the UI uses it as the second + /// download link. pub release_url: String, /// The direct download URL for this platform's installer asset, if one matched. pub download_url: Option, @@ -84,18 +93,23 @@ struct GhAsset { } impl crate::App { - /// Check GitHub Releases for a newer installer than the running build. Returns `Ok(None)` when - /// already current (or the newest release is one the user chose to skip); `Ok(Some(info))` when - /// a newer version is available. Network/parse failures are surfaced as [`AppError::Update`] — - /// callers treat a failed check as non-fatal. + /// Check GitHub Releases for an installer that is newer than this build. + /// + /// The method returns `Ok(None)` when this build is current. It also returns `Ok(None)` when + /// the user chose to skip the newest release. It returns `Ok(Some(info))` when a newer version + /// exists. + /// + /// A network fault or a parse fault becomes an [`AppError::Update`] value. A caller must + /// continue after a failed check. pub async fn check_for_update(&self) -> Result, AppError> { let current_str = BUILD_VERSION; let current = Version::parse(current_str) .ok_or_else(|| AppError::Update(format!("unparseable build version {current_str}")))?; let releases = fetch_releases().await?; - // Highest version among non-draft releases (prereleases included, so Alpha→Alpha upgrades - // are offered). `max_by` needs the parsed version; releases with a non-version tag are skipped. + // Find the highest version among the releases that are not a draft. The set holds each + // prerelease, so the check offers an upgrade from one Alpha to the next Alpha. `max_by` + // needs the parsed version. The code skips a release when its tag is not a version. let best = releases .into_iter() .filter(|r| !r.draft) @@ -110,15 +124,17 @@ impl crate::App { } let latest_version = rel.tag_name.trim_start_matches(['v', 'V']).to_string(); - // Honor a "skip this version" choice — but a version *newer* than the skipped one still - // notifies (the skip is keyed to the exact version string). + // Obey a "skip this version" choice from the user. But a version that is *newer* than the + // skipped version still gives a notification, because the skip holds one exact version + // string. if AppSettings::load().skip_update_version.as_deref() == Some(latest_version.as_str()) { return Ok(None); } Ok(Some(UpdateInfo { - // Trimmed the same way `latest_version` is, so the UI's "current › latest" reads - // consistently — the injected value is a tag and carries a leading `v`. + // Remove the same characters that the code removes from `latest_version`. The + // "current > latest" text in the UI is then consistent. The build step supplies a tag, + // and a tag starts with `v`. current_version: current_str.trim_start_matches(['v', 'V']).to_string(), latest_version, name: rel.name.clone().unwrap_or_else(|| rel.tag_name.clone()), @@ -152,9 +168,13 @@ async fn fetch_releases() -> Result, AppError> { .map_err(|e| AppError::Update(e.to_string())) } -/// Pick the installer asset for the current platform from a release's assets. macOS ships a single -/// universal2 `.dmg`; Windows an NSIS `*-setup.exe`; Linux an `.AppImage` / `.deb` per-arch. Returns -/// the first name-matching asset (preferring one whose name carries this arch or "universal"). +/// Select the installer asset for this platform from the assets of a release. +/// +/// macOS has one universal2 `.dmg` file. Windows has an NSIS `*-setup.exe` file. Linux has an +/// `.AppImage` file and a `.deb` file for each architecture. +/// +/// The function returns the first asset with a name that matches. It selects an asset whose name +/// holds this architecture or the word "universal" before it selects another asset. fn pick_installer_asset(assets: &[GhAsset]) -> Option { let exts: &[&str] = if cfg!(target_os = "macos") { &[".dmg"] @@ -182,9 +202,11 @@ fn pick_installer_asset(assets: &[GhAsset]) -> Option { .map(|a| a.browser_download_url.clone()) } -/// A minimal `MAJOR.MINOR.PATCH[-prerelease]` version. Ordered so a release outranks its own -/// prereleases (`0.2.0` > `0.2.0-alpha.1`) and higher numbers win — sufficient for our `vX.Y.Z` -/// release tags; we deliberately don't implement full SemVer build-metadata precedence. +/// A small `MAJOR.MINOR.PATCH[-prerelease]` version. +/// +/// The order puts a release above its own prereleases, so `0.2.0` is above `0.2.0-alpha.1`. A +/// higher number also wins. This order is enough for a `vX.Y.Z` release tag. The code does not +/// implement the full SemVer rule for build metadata, by design. #[derive(Debug, PartialEq, Eq)] struct Version { nums: (u64, u64, u64), @@ -227,11 +249,12 @@ impl Ord for Version { /// Compare two prerelease strings dot-part by dot-part, numerically where both parts are numbers. /// -/// A plain string compare is wrong the moment a counter reaches two digits: `"alpha.9"` sorts -/// *above* `"alpha.16"`, because `'9' > '1'`. That is not hypothetical here — the project reached -/// `alpha.16` with this comparator in place, and picking the highest of the published tags returned -/// `alpha.9`. SemVer's own rule is the fix: numeric identifiers compare numerically, and a numeric -/// identifier ranks below an alphanumeric one. +/// A plain string compare is wrong when a counter reaches two digits. `"alpha.9"` then sorts +/// *above* `"alpha.16"`, because the character `9` is above the character `1`. +/// +/// This fault occurred in this project. The version reached `alpha.16` with the old comparator, and +/// a search for the highest published tag returned `alpha.9`. The SemVer rule is the correction. A +/// numeric part compares as a number, and a numeric part ranks below a part with letters. fn compare_prerelease(a: &str, b: &str) -> std::cmp::Ordering { use std::cmp::Ordering; @@ -293,9 +316,9 @@ mod tests { assert_eq!(v("0.1.0"), v("v0.1.0")); } - /// The bug that made the whole feature inert: a build calling itself `0.1.0` outranks every - /// `0.1.0-alpha.N` tag, so the newest alpha never looked newer and no user was ever notified. - /// A released build has to identify itself with its own prerelease to be comparable. + /// The fault that made the full feature inert. A build with the name `0.1.0` ranks above each + /// `0.1.0-alpha.N` tag. So the newest alpha never looked newer, and no user received a + /// notification. A release build must use its own prerelease name, or the comparison fails. #[test] fn a_release_build_compares_against_the_alphas_it_shipped_beside() { // What the bare workspace version did. @@ -307,8 +330,8 @@ mod tests { assert!(v("v0.1.0-alpha.17") > v("0.1.0-alpha.16")); } - /// Whatever this build calls itself, it must at least be parseable — otherwise the check fails - /// with "unparseable build version" rather than doing anything useful. + /// The parser must accept the name of this build. If not, the check stops with the message + /// "unparseable build version" and does no useful work. #[test] fn the_build_version_is_a_version() { assert!( @@ -328,8 +351,9 @@ mod tests { #[test] fn picks_platform_asset() { - // Include an asset for every platform so the test is meaningful on any CI runner (macOS, - // Windows, and Linux — where the picker looks for .AppImage/.deb). + // Add an asset for each platform, so the test is useful on any CI runner. The runners are + // macOS, Windows, and Linux. On Linux the code looks for a .AppImage file or a .deb + // file. let assets = vec![ GhAsset { name: "DUNavigator_0.2.0_universal.dmg".into(), @@ -359,8 +383,10 @@ mod tests { } else if cfg!(target_os = "windows") { assert_eq!(picked.as_deref(), Some("https://example/exe")); } else { - // Linux: an .AppImage or .deb (the picker prefers an arch/universal-tagged asset — the - // x86_64 AppImage on an x86_64 runner — else the first extension match). + // On Linux, the code selects a .AppImage file or a .deb file. It first looks for an + // asset with this architecture or the word "universal" in its name. One example is the + // x86_64 AppImage on an x86_64 runner. If it finds none, it takes the first file with + // a correct extension. assert!(matches!( picked.as_deref(), Some("https://example/appimage") | Some("https://example/deb") From 15ae0f99b96123cc23578a129592e82202df98d3 Mon Sep 17 00:00:00 2001 From: James Kane Date: Tue, 18 Aug 2026 11:59:43 -0500 Subject: [PATCH 05/33] docs(ste): navigator-app, ten more files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit brief, ftdna_import, import_profiles, maintenance, publish, the three test and example files, and the last of the small ones. Twenty-three of thirty-three files are now at zero; the crate goes 3,893 to 3,096. Two checker corrections came out of this batch, both of which had been inflating the count: The em-dash rule judged raw text, so cargo's `--` argument separator inside a fenced shell block — `cargo test … -- --ignored` — read as an em-dash aside. It now judges the code-stripped text like every other rule. That alone removed several hundred false positives across the repository. `genotyping array` is a Technical Name and was being flagged as an `-ing` form. The dictionary now declares the `-ing` Technical Names in their own section rather than leaving them implicit in the checker, so the two stay in step. The prose keeps the reasoning and loses the compression, which is the intended trade. The FTDNA matcher still explains why a genetic distance of 3 to 11 over 100 markers is normal inside a single-haplogroup project and why only a near-exact haplotype identifies the same person. The publish gate still explains why a WGS-labelled Y-only extract must not file a coverage record. Both now say it in sentences a second-language reader can parse. `cargo check -p navigator-app --all-targets` passes. Co-Authored-By: Claude Opus 5 (1M context) --- .../navigator-app/examples/blocktree_check.rs | 17 +- crates/navigator-app/examples/realign_wgs.rs | 27 +- crates/navigator-app/src/brief.rs | 186 ++++++----- crates/navigator-app/src/ftdna_import.rs | 229 +++++++++----- crates/navigator-app/src/import_profiles.rs | 294 ++++++++++++------ crates/navigator-app/src/maintenance.rs | 191 +++++++----- crates/navigator-app/src/publish.rs | 226 +++++++++----- .../tests/mastervar_autosomal_real.rs | 27 +- documents/STE-dictionary.md | 13 +- scripts/ste-check.py | 6 +- 10 files changed, 764 insertions(+), 452 deletions(-) diff --git a/crates/navigator-app/examples/blocktree_check.rs b/crates/navigator-app/examples/blocktree_check.rs index 5a53b503..2a86ef8e 100644 --- a/crates/navigator-app/examples/blocktree_check.rs +++ b/crates/navigator-app/examples/blocktree_check.rs @@ -1,6 +1,8 @@ -//! Throwaway validation for the project block tree: run `App::project_block_tree` against the live -//! workspace and print the result as an indented tree. Proves the real data path — tree fetch, name -//! index, induced subtree, collapse — on an actual multi-thousand-member cohort. +//! A temporary check for the project block tree. It calls `App::project_block_tree` on the live +//! workspace and prints the result as a tree with an indent for each level. +//! +//! The check covers the real data path on a cohort with some thousands of members. That path is the +//! tree fetch, the name index, the induced subtree, and the collapse step. //! //! ```bash //! cargo run -p navigator-app --example blocktree_check -- @@ -71,8 +73,10 @@ async fn main() -> Result<(), Box> { ); } - // Split the unplaced: "no placement at all" is expected (STR-only kits), but "has a terminal - // this tree does not carry" is provider/build skew worth naming. + // Separate the two groups of subjects with no place in the tree. A subject with no placement + // is normal, because an STR-only kit has none. The second group has a terminal node that this + // tree does not hold. That group shows a difference between the provider and the build, and + // the report must give its count. let (skew, unplaced_none): (Vec<_>, Vec<_>) = tree.unplaced.iter().partition(|u| u.terminal.is_some()); println!( "unplaced: {} with no Y placement · {} with a terminal absent from this tree", @@ -89,7 +93,8 @@ async fn main() -> Result<(), Box> { }); } - // Pre-order with depth as indent is exactly how the aggregate is ordered, so this prints itself. + // The aggregate is already in pre-order, and the depth gives the indent. So the code prints + // the rows in the order that it receives them. for b in tree.blocks.iter().take(60) { let indent = " ".repeat(b.depth); let folded = if b.collapsed.is_empty() { diff --git a/crates/navigator-app/examples/realign_wgs.rs b/crates/navigator-app/examples/realign_wgs.rs index ccb9feb0..3160d77c 100644 --- a/crates/navigator-app/examples/realign_wgs.rs +++ b/crates/navigator-app/examples/realign_wgs.rs @@ -1,9 +1,11 @@ //! Drive a whole-genome realignment headlessly, for the phase 5 WGS-scale validation. //! -//! The GUI can start this job, but a run measured in hours should not depend on a window staying -//! open — and the validation wants a timestamped log of where the time went, which the progress -//! cards do not keep. This is the same `App::realign_alignment` the UI calls, with the stage -//! reports printed instead of drawn. +//! The GUI can start this job. But a run of many hours must not depend on an open window. The +//! validation also needs a log with a timestamp for each stage, and the progress cards do not keep +//! one. +//! +//! This example calls the same `App::realign_alignment` function that the UI calls. It prints each +//! stage report, and the UI draws it. //! //! ```bash //! cargo run --release -p navigator-app --example realign_wgs -- @@ -13,8 +15,8 @@ //! # to find; a run that is killed outright leaves them regardless //! ``` //! -//! Ctrl-C cancels through the job's own token rather than killing the process, so the scratch -//! directory — hundreds of GB at WGS scale — is still cleaned up on the way out. +//! Ctrl-C stops the job through the cancel token of that job. It does not stop the process. So the +//! example still removes the scratch directory, which holds hundreds of GB for a WGS sample. use std::path::PathBuf; use std::sync::Mutex; @@ -34,9 +36,10 @@ async fn main() -> Result<(), Box> { .map(PathBuf::from) .unwrap_or_else(|_| home.join(".decodingus/references/chm13v2.0.fa")); let scratch_root = std::env::var("SCRATCH").ok().map(PathBuf::from); - // `PRESET` overrides the technology inference, which refuses any test type it does not know - // rather than guessing — correct for the app, but it puts real vendor products (`Y_ELITE`) - // out of reach of a smoke test. + // `PRESET` replaces the value that the code deduces from the technology. That code refuses a + // test type that it does not know, and it never makes an estimate. This behaviour is correct + // for the app. But it also puts a real vendor product, such as `Y_ELITE`, out of the reach of + // a quick test. let preset = match std::env::var("PRESET") { Ok(p) => Some(Preset::parse(&p).map_err(|e| format!("PRESET={p}: {e}"))?), Err(_) => None, @@ -113,9 +116,9 @@ async fn main() -> Result<(), Box> { match app.realign_alignment(alignment_id, params, cancel, progress).await { Ok(outcome) => { - // A resumed run did not necessarily run the stage that counts a given figure, and the - // earlier attempt may have been killed before it wrote one down. "not measured" is the - // honest rendering; a zero here would read as a result. + // A run that continues an earlier run does not always do the stage that counts a + // figure. The earlier run can also stop before it writes that figure. So the report + // shows "not measured". A zero value here looks like a result. let count = |n: Option| n.map(|n| n.to_string()).unwrap_or_else(|| "not measured".into()); println!( "\ndone in {:.1} min\n alignment #{} at {}\n reads written: {}\n duplicates marked: {}\n source unmapped reads (had a chance to place): {}", diff --git a/crates/navigator-app/src/brief.rs b/crates/navigator-app/src/brief.rs index 55936821..c454d626 100644 --- a/crates/navigator-app/src/brief.rs +++ b/crates/navigator-app/src/brief.rs @@ -1,12 +1,16 @@ -//! Composition of a casual-reader [`SubjectBrief`]: pull the existing analysis signals for one -//! subject, load the narrative reference pack, and assemble the render-ready model via the pure -//! templating in `navigator_domain::brief`. +//! This module makes a [`SubjectBrief`] for a reader who is not a specialist. //! -//! The reference pack is loaded with **graceful fallback** (decided 2026-06-22): a bundled seed is -//! the always-available floor; a CDN-hosted pack refreshes/augments it when reachable; a stale cache -//! covers a failed refresh. A brief is never blocked by a missing pack — sections degrade to the -//! structured facts the analysis already provides, and [`SubjectBrief::pack_status`] records how -//! fresh the narrative is. +//! It reads the analysis signals of one subject, reads the narrative reference pack, and builds the +//! model that the UI draws. The template code in `navigator_domain::brief` does the last step, and +//! that code is pure. +//! +//! The module reads the reference pack in three steps, and each step has a fallback. The team +//! decided this on 2026-06-22. The app always holds a seed pack, which is the lowest step. A pack +//! on the CDN refreshes and extends the seed when the network permits it. An old cache covers a +//! failed refresh. +//! +//! An absent pack never stops a brief. Each section then falls back to the structured facts of the +//! analysis. [`SubjectBrief::pack_status`] records the age of the narrative. use crate::{decodingus_appview_url, App, AppError}; use navigator_domain::ancestry::AncestryResult; @@ -20,14 +24,16 @@ use navigator_domain::reconciliation::{CompatibilityLevel, Consensus, DnaType}; use navigator_domain::testtype::{self, TargetType}; use navigator_refgenome::cache as refgenome_cache; -/// The bundled seed pack — the offline floor. Authored in `assets/brief-pack.seed.json`. +/// The seed pack in the application bundle. It is the lowest step, and it works offline. The file +/// is `assets/brief-pack.seed.json`. const SEED_PACK: &str = include_str!("../assets/brief-pack.seed.json"); /// Default CDN location of the refreshable reference pack. Override with `NAVIGATOR_BRIEF_PACK_URL`. /// A 404 / unreachable host falls back gracefully to the cache, then the bundled seed. const DEFAULT_BRIEF_PACK_URL: &str = "https://assets.decodingus.org/briefs/brief-pack.json"; -/// How long a downloaded pack is trusted before a refresh is attempted (days). +/// The count of days that the app trusts a downloaded pack. After this time, the app tries a +/// refresh. const BRIEF_PACK_TTL_DAYS: u64 = 7; fn brief_pack_url() -> String { @@ -53,12 +59,14 @@ pub(crate) fn cache_is_fresh(path: &std::path::Path, ttl_days: u64) -> bool { .unwrap_or(false) } -/// How long a per-haplogroup enrichment record is trusted before a refresh is attempted (days). +/// The count of days that the app trusts the extra record of one haplogroup. After this time, the +/// app tries a refresh. const HAPLO_ENRICH_TTL_DAYS: u64 = 30; -/// Live haplogroup content fetched from the AppView, cached per (dna-type, name). `found = false` is -/// a negative-cache marker (the endpoint answered but had nothing) so a definitively-absent -/// haplogroup is not re-requested every rebuild. +/// Haplogroup content from the AppView. The cache key is the DNA type together with the name. +/// +/// A `found = false` value marks an absent record. The endpoint answered, but it held nothing. So +/// the app does not request that haplogroup again at each rebuild. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] struct HaploEnrichment { found: bool, @@ -73,7 +81,7 @@ struct HaploEnrichment { } impl HaploEnrichment { - /// Does this carry any narrative/age content worth folding in? + /// Shows that this record holds narrative content or age content for the brief. fn has_content(&self) -> bool { self.found && (self.formed_ybp.is_some() || self.origin.is_some() || self.story.is_some()) } @@ -99,9 +107,13 @@ fn haplo_enrich_cache_path(dna_type: DnaType, name: &str) -> std::path::PathBuf } impl App { - /// Build the plain-language brief for one subject. Pulls the consensus haplogroups, the best - /// alignment's coverage, and the run's test type, and joins them to the reference pack. Always - /// returns a brief (degrading per-section); only a store error propagates. + /// Build the plain-language brief for one subject. + /// + /// The method reads the consensus haplogroups, the coverage of the best alignment, and the test + /// type of the run. It then joins these values to the reference pack. + /// + /// The method always returns a brief. A section with no data falls back to a simpler form. Only + /// a store error stops the method. pub async fn subject_brief(&self, biosample_guid: SampleGuid) -> Result { let bio = navigator_store::biosample::get(self.store.pool(), biosample_guid) .await? @@ -120,14 +132,17 @@ impl App { let test_code = run.as_ref().map(|r| r.test_type.clone()); let (pack, pack_status) = self.load_brief_pack().await; - // The brief is prose written for the reader, so it is built in *their* language. Resolved - // here rather than passed in because every consumer wants the same thing: the UI renders it, - // the HTML export writes it to a file the user keeps, and the local-LLM prompt hands it to a - // model that should answer in the language the user is reading. + // The brief is prose for the reader, so the code builds it in the language of that + // reader. The code finds the language here, and the caller does not supply it, because + // each caller needs the same language. The UI draws the brief. The HTML export writes it + // to a file that the user keeps. The local-LLM prompt gives it to a model, and that model + // must answer in the language that the user reads. let lang = i18n::load_lang().unwrap_or(Lang::En); - // Consensus lineages (None when not placed yet, or N/A for the test). Each terminal is - // enriched best-effort from the live haplogroup endpoint (cached); pack values stand offline. + // The consensus lineages. A value is None when the app did not place the subject, or when + // the test does not cover that lineage. The code tries to add content for each terminal + // node from the haplogroup endpoint, and it caches the result. Offline, the pack values + // apply. let cons_y = self.haplogroup_consensus(biosample_guid, DnaType::Y).await?; let cons_mt = self.haplogroup_consensus(biosample_guid, DnaType::Mt).await?; let mut enriched = false; @@ -157,11 +172,14 @@ impl App { .await .ok() .flatten(); - // Deep (ancient) components. Reading *only* `ANCIENT_ADMIXTURE` is also what keeps a - // stale `PCA_PROJECTION_GMM` / `G25_NMONTE` row — persisted by the build whose - // fabricated numbers prompted this rebuild — from resurfacing in the brief, the - // DNA-story HTML export, or the LLM facts. Absent when the three ancient sources - // can't express the sample's ancestry: no card beats a wrong card. + // The deep, or ancient, components. The code reads *only* `ANCIENT_ADMIXTURE`. + // An older build wrote incorrect numbers to a `PCA_PROJECTION_GMM` row and a + // `G25_NMONTE` row, and that fault caused this rebuild. A read of only one source + // keeps those old rows out of three places. They are the brief, the DNA-story HTML + // export, and the facts for the LLM. + // + // The value is absent when the three ancient sources can not express the ancestry + // of the sample. No card is better than a wrong card. let ancient = if crate::ANCIENT_ANCESTRY_ENABLED { self.consensus_ancestry(biosample_guid, navigator_analysis::ancestry::ANCIENT_ADMIXTURE) .await @@ -175,8 +193,10 @@ impl App { _ => None, }; - // Runs-of-homozygosity (relatedness / endogamy). Read-only: only surfaced when it is already - // been computed and cached (the brief must stay cheap — ROH computation is on-demand). + // The runs of homozygosity, which show relatedness and endogamy. The code only reads + // them. It shows a value only when an earlier run calculated it and wrote it to the cache. + // The brief must stay fast, and the ROH calculation runs only at the request of the + // user. let roh = self.cached_roh(biosample_guid).await?.map(|r| { brief::roh_brief( lang, @@ -188,8 +208,9 @@ impl App { ) }); - // Archaic (Neanderthal) markers — same contract as ROH: read-only, surfaced only once the - // count has been computed and cached, so the brief stays cheap. + // The archaic markers for Neanderthal. The rule is the same as the rule for ROH. The code + // only reads them, and it shows a value only after an earlier run calculated the count and + // wrote it to the cache. The brief must stay fast. let archaic = self.cached_archaic(biosample_guid).await?.map(|a| { brief::archaic_brief( lang, @@ -217,7 +238,8 @@ impl App { summary: headline_summary(lang, &bio.donor_identifier, paternal.as_ref(), maternal.as_ref()), }; - // Computed before the brief is assembled, because `paternal` is moved into it. + // The code calculates this value first, because the next step moves `paternal` into the + // brief. let realign_offer = self .realign_offer(biosample_guid, paternal.is_some(), default_aln.map(|(_, aln)| aln)) .await?; @@ -240,27 +262,32 @@ impl App { }) } - /// Whether to offer this subject a realignment, and which alignment to offer. + /// Shows whether to offer a realignment to this subject, and which alignment to offer. /// - /// Every condition here exists to avoid proposing hours of work that would change nothing: + /// Each condition below stops an offer of many hours of work that would change nothing. /// - /// - **A paternal line to improve.** Realignment buys Y-chromosome discovery and essentially - /// nothing else — ancestry, IBD and the autosomes already handle GRCh37/38 and give the same - /// answer either way. With no Y placed there is no payoff, so no offer. - /// - **Reads to re-map.** A chip or VCF-only subject has no alignment; an alignment row without - /// a file can not be read. - /// - **No CHM13 alignment already.** The offer claims part of their paternal line can not - /// currently be read; for someone who already has data on the complete assembly, by any route, - /// that claim is simply false — even if some older file of theirs has never been realigned. - /// - **Reads the job would actually act on** — not already on CHM13, not itself a realignment, - /// and not already realigned once. Those three are not re-implemented here: they are - /// [`crate::realign::realignable_for_subject`], the same rule the batch count and the job - /// itself use. Offering work the job would then refuse is worse than not offering it. + /// - **The subject has a paternal line to improve.** A realignment gives discovery on the Y + /// chromosome and almost nothing more. The ancestry, the IBD, and the autosomes already work + /// on GRCh37 and GRCh38, and they give the same answer on CHM13. With no Y placement there is + /// no gain, so the app makes no offer. + /// - **The subject has reads to map again.** A subject with only a chip or a VCF has no + /// alignment. An alignment row with no file has no reads. + /// - **The subject has no CHM13 alignment.** The offer states that the app can not read part of + /// the paternal line of the user. That statement is false for a user who already has data on + /// the complete assembly, by any route. An older file of that user can still be without a + /// realignment, and the statement stays false. + /// - **The job would act on the reads.** Three tests apply. The alignment must not be on + /// CHM13. It must not be a realignment. It must not have a realignment already. /// - /// Among qualifying alignments it prefers the subject's default — the widest, then deepest test, - /// per [`Self::default_alignment_for_subject`] — and otherwise takes the first. That matters for - /// someone holding both a whole genome and a Y-only test: the whole genome is the one whose - /// realignment answers more. + /// This code does not repeat those tests. [`crate::realign::realignable_for_subject`] holds + /// them, and the batch count and the job call the same function. An offer of work that the + /// job then refuses is worse than no offer. + /// + /// Among the alignments that pass, the code selects the default alignment of the subject. That + /// is the test with the largest breadth, and then the largest depth, from + /// [`Self::default_alignment_for_subject`]. If there is none, the code takes the first + /// alignment. This choice matters for a user with a whole genome and a Y-only test. The + /// realignment of the whole genome answers more questions. async fn realign_offer( &self, biosample_guid: SampleGuid, @@ -273,12 +300,16 @@ impl App { let alignments = navigator_store::alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; - // Already reading this person's Y against the complete assembly — however they got there. - // The per-alignment rule below would still find, say, an old GRCh37 file nobody has - // realigned, and the *job* would indeed act on it; but the offer's promise is about the - // subject's paternal line rather than about one file, and for this reader that promise is - // already kept. Observed on a donor holding four CHM13 alignments and being told their - // father's line had nowhere to be read from. + // The app already reads the Y chromosome of this person against the complete assembly. + // The route to that state does not matter. + // + // The rule below, which looks at one alignment, can still find an old GRCh37 file with no + // realignment. The *job* would act on that file. But the offer makes a promise about the + // paternal line of the subject, not about one file. For this reader the app already keeps + // that promise. + // + // A donor with four CHM13 alignments saw this fault. The app told that donor that it could + // read nothing of the line of their father. if alignments .iter() .any(|a| crate::realign::is_target_build(&a.reference_build)) @@ -317,7 +348,7 @@ impl App { .ok() .and_then(|s| serde_json::from_str(&s).ok()); - // Fresh cache → use it without touching the network. + // The cache is new enough. Use it, and do not use the network. if let Some(cp) = &cached { if cache_is_fresh(&cache_path, BRIEF_PACK_TTL_DAYS) { pack.merge(cp.clone()); @@ -325,7 +356,8 @@ impl App { } } - // Stale / absent → try a refresh, falling back to the stale cache (then the seed). + // The cache is old or absent. Try a refresh. If the refresh fails, use the old cache, and + // then the seed pack. let url = brief_pack_url(); let fetched: Result = async { let resp = self @@ -368,11 +400,16 @@ impl App { (pack, status) } - /// Best-effort live enrichment for one haplogroup: cache-first (30-day TTL), else a short-timeout - /// `GET {appview}/api/v1/haplogroup/{name}`. A definitive answer (200 / 404) is cached — including - /// "not found" — so it is not re-requested each rebuild; a transient network error is *not* cached, - /// so enrichment self-heals once connectivity returns. Returns content only when there is something - /// worth folding in (an age or narrative). + /// Read the extra content for one haplogroup, if the network permits it. + /// + /// The method reads the cache first, and the cache entry is valid for 30 days. If the cache has + /// no entry, the method sends `GET {appview}/api/v1/haplogroup/{name}` with a short timeout. + /// + /// The method caches a definite answer, which is a 200 response or a 404 response. It caches + /// the "not found" result also, so it does not send the request again at each rebuild. It does + /// not cache a temporary network error, so the content appears after the network returns. + /// + /// The method returns content only when that content has an age or a narrative. async fn enrich_haplogroup(&self, name: &str, dna_type: DnaType) -> Option { if name.trim().is_empty() { return None; @@ -453,8 +490,9 @@ fn parse_haplo_enrichment(body: &str) -> HaploEnrichment { } } -/// Assemble a lineage section from the consensus + pack content, overlaying live `enrich`ment when -/// present (it wins over pack values for age/origin/story). `is_paternal` only chooses the lookup. +/// Build a lineage section from the consensus and the pack content. The `enrich` value, when it is +/// present, replaces the age, the origin, and the story of the pack. `is_paternal` only selects the +/// lookup. fn build_lineage( lang: Lang, kind: LineageKind, @@ -542,18 +580,20 @@ fn build_ancestry( .iter() .filter(|c| c.percentage >= 0.5) .map(|c| { - // Pack content (by code, then display name) supplies an optional friendlier name - // and the explanation — so a bare code like "ANF" reads as "Anatolian Farmer". + // The pack content gives a clearer name and the explanation. The code looks + // up the pack by the code first, and then by the display name. So a plain code + // such as "ANF" becomes "Anatolian Farmer". let direct = pack .population(&c.population_code) .or_else(|| pack.population(&c.population_name)); let name = direct .and_then(|p| p.name.clone()) .unwrap_or_else(|| c.population_name.clone()); - // The model's reference set mixes ancient and *modern* populations; the modern - // ones (e.g. Colombian/Puerto Rican standing in for Native American ancestry) - // rarely have their own blurb, so fall back to the continental description rather - // than leaving real non-European signal unexplained. + // The reference set of the model holds ancient populations and *modern* + // populations. A modern population usually has no text of its own. One example + // is a Colombian or Puerto Rican population, which represents Native American + // ancestry. So the code uses the continental description. Without it, a real + // signal from outside Europe has no explanation. let blurb = direct.and_then(|p| p.blurb.clone()).or_else(|| { population_super(&c.population_code) .map(population_name) diff --git a/crates/navigator-app/src/ftdna_import.rs b/crates/navigator-app/src/ftdna_import.rs index df39997c..7adece22 100644 --- a/crates/navigator-app/src/ftdna_import.rs +++ b/crates/navigator-app/src/ftdna_import.rs @@ -1,11 +1,17 @@ -//! FTDNA project import — the matching/dedup engine + two-phase plan/commit (design §5/§6). +//! The FTDNA project import. This module holds the engine that matches a kit to a subject and +//! finds a duplicate. It also holds the two steps of the import, which are the plan and the commit. +//! See design §5 and §6. //! -//! Phase 1 scope (roster + ancestry, the spine): parse the batch CSVs, join by kit number, match -//! each kit against the workspace, and produce a reviewable **plan** (dry-run, no writes). A separate -//! commit step applies the plan with the admin's resolutions for fuzzy candidates. +//! Phase 1 covers the roster and the ancestry, which are the base of the import. The module does +//! four steps. It parses the batch CSV files, joins them by the kit number, matches each kit +//! against the workspace, and makes a **plan** for the administrator. This phase writes nothing. //! -//! Deep per-member data (Big Y / mtDNA / Family Finder) and the wide Y-STR chart are layered on by -//! later slices; this module only wires identity + MDKA + membership. +//! A separate commit step applies the plan. That step uses the decisions of the administrator for +//! each candidate that the engine is not sure about. +//! +//! A later change adds the deep data of each member, which is Big Y, mtDNA, and Family Finder. It +//! also adds the wide Y-STR chart. This module only connects the identity, the MDKA rows, and the +//! membership. use std::collections::BTreeMap; use std::path::PathBuf; @@ -28,10 +34,11 @@ struct CatalogSample { accession: Option, } -/// Tuning for the matching engine. +/// The values that control the engine that matches a kit to a subject. #[derive(Debug, Clone)] pub struct FtdnaImportOptions { - /// Minimum fuzzy score (0..1) for a workspace Subject to be offered as a merge candidate. + /// The lowest score, from 0 to 1, that lets the engine offer a subject as a merge candidate. + /// This score is not exact, and the engine calculates it from the name. pub fuzzy_threshold: f32, } @@ -62,8 +69,9 @@ pub struct FuzzyCandidate { pub reasons: Vec, } -/// How the matcher proposes to handle a kit. Auto-merge is locked for an exact vendor-id hit; fuzzy -/// hits are queued for the admin (never auto-merged). +/// The action that the engine proposes for a kit. The engine merges without a question only for an +/// exact match on the vendor id. For a match that is not exact, it adds the kit to a list, and the +/// administrator decides. The engine never merges such a kit on its own. #[derive(Debug, Clone)] pub enum MatchKind { /// No workspace match → create a new Subject. @@ -91,8 +99,10 @@ pub struct FtdnaPlanRow { pub input: FtdnaSubjectInput, } -/// Recognized-input + scan counts for the review header — so a missing/misclassified file (e.g. no -/// roster) is immediately visible rather than silently producing all-orphan rows. +/// The counts of the input files that the code recognized, and of the rows that it read. The review +/// header shows these counts. So the administrator sees an absent file, or a file with the wrong +/// class, at once. One example is an import with no roster. Without these counts, such an import +/// gives rows with no subject and no message. #[derive(Debug, Clone, Default)] pub struct FtdnaPlanStats { /// Roster rows parsed from `Member_Information`. @@ -149,7 +159,7 @@ pub enum FtdnaResolution { /// What the commit did. #[derive(Debug, Clone, Default)] pub struct FtdnaImportSummary { - /// The project the kits were imported into (resolved/created at commit). + /// The project that received the kits. The commit step finds this project or makes it. pub project_id: i64, pub created: usize, pub merged: usize, @@ -164,8 +174,11 @@ pub struct FtdnaImportSummary { pub errors: Vec, } -/// A Subject's imported genealogy bundle: vendor ids, FTDNA member labels, and MDKA rows. PII — -/// for local display only (never federated). Empty when nothing was imported for the Subject. +/// The genealogy data that the app imported for one subject. It holds the vendor ids, the FTDNA +/// member labels, and the MDKA rows. +/// +/// This data is personal. The app shows it on this machine only, and it never sends it to the +/// network. The value is empty when the app imported nothing for the subject. #[derive(Debug, Clone, Default)] pub struct FtdnaGenealogy { pub external_ids: Vec, @@ -174,7 +187,7 @@ pub struct FtdnaGenealogy { } impl FtdnaGenealogy { - /// Nothing imported → the detail card can be skipped. + /// Shows that the app imported nothing. The UI can then skip the detail card. pub fn is_empty(&self) -> bool { self.external_ids.is_empty() && self.member.is_none() && self.mdka.is_empty() } @@ -191,10 +204,15 @@ impl App { }) } - /// Attach a vendor id (kit number) to a Subject from the subject editor. Rejects a blank - /// source/id, and refuses to bind a `(source, external_id)` that already belongs to a *different* - /// Subject (the `(source, external_id)` uniqueness is the dedup anchor — never silently re-point - /// it; the caller resolves the conflict). Idempotent for the same Subject. + /// Add a vendor id, which is a kit number, to a subject from the subject editor. + /// + /// The method refuses a blank source and a blank id. It also refuses a `(source, external_id)` + /// pair that belongs to a *different* subject. + /// + /// That pair is unique, and the app uses it to find a duplicate donor. The method must never + /// move the pair to another subject without a message. The caller resolves such a conflict. + /// + /// A second call for the same subject is safe. pub async fn add_external_id( &self, guid: SampleGuid, @@ -218,7 +236,8 @@ impl App { /// Detach a vendor id (by row id) from a Subject. pub async fn delete_external_id(&self, id: i64) -> Result<(), AppError> { - // Recover the owning subject before the row is gone, so we can refresh its published record. + // Read the subject of this row before the code deletes the row. The app then refreshes + // the published record of that subject. let guid = external_id::get(self.store.pool(), id).await?.map(|e| e.biosample_guid); external_id::delete(self.store.pool(), id).await?; if let Some(guid) = guid { @@ -227,13 +246,22 @@ impl App { Ok(()) } - /// Backfill public-catalog external ids (`IGSR`/`HGDP`/INSDC) derivable from each subject's local - /// provenance ([`navigator_domain::identity::catalog_ids_from_provenance`]) — so bulk-imported - /// public datasets publish ids that match their existing AppView catalog rows. Deterministic and - /// network-free; a friendly-name-only sample contributes nothing. Idempotent (skips ids already - /// present); a `(namespace, value)` already owned by a *different* subject is counted as a - /// conflict and left untouched (never silently re-pointed). `apply == false` is a dry run. - /// Adds via the store directly (no per-id re-publish); re-publish the affected subjects after. + /// Add the public-catalog external ids that the code can derive from the local provenance of + /// each subject. The namespaces are `IGSR`, `HGDP`, and INSDC, and + /// [`navigator_domain::identity::catalog_ids_from_provenance`] derives them. + /// + /// A public dataset that a user imported in bulk then publishes ids that match its rows in the + /// catalog of the AppView. + /// + /// The method is deterministic and uses no network. A sample with only a friendly name gives no + /// id. A second call is safe, because the method skips an id that already exists. + /// + /// The method counts a `(namespace, value)` pair that belongs to a *different* subject as a + /// conflict, and it changes nothing. It never moves such a pair without a message. + /// `apply == false` makes the method report the changes and write nothing. + /// + /// The method writes to the store directly, and it does not publish a record for each id. + /// Publish the subjects that changed after the method completes. pub async fn backfill_catalog_ids( &self, project_id: Option, @@ -274,7 +302,8 @@ impl App { if row.biosample_guid == b.guid { out.ids_added += 1; } else { - // (namespace,value) already belongs to another subject — a dup import; leave it. + // This (namespace, value) pair belongs to another subject. The user + // imported the same data twice. Change nothing. out.conflicts += 1; } } @@ -283,10 +312,14 @@ impl App { Ok(out) } - /// Fetch one public-catalog sample record from the AppView samples API (`/api/v1/samples/{alias}`, - /// public read) by its alias (= our `donor_identifier`). `Ok(None)` for a 404 (alias unknown to - /// the catalog — expected while server-side corrections are pending). The authoritative - /// `accession` it returns is the datum our local `sample_accession` lacks. + /// Read one public-catalog sample record from the samples API of the AppView. The path is + /// `/api/v1/samples/{alias}`, the read is public, and the alias is our `donor_identifier`. + /// + /// A 404 response gives `Ok(None)`, which means that the catalog does not know the alias. That + /// result is normal while a correction on the server is not complete. + /// + /// The `accession` value in the response has authority. Our local `sample_accession` field does + /// not hold it. async fn fetch_catalog_sample(&self, base: &str, alias: &str) -> Result, AppError> { let url = format!("{}/api/v1/samples/{alias}", base.trim_end_matches('/')); let resp = self @@ -309,14 +342,24 @@ impl App { Ok(Some(s)) } - /// Resolve each subject against the AppView samples API and attach, **in one pass**, the full set - /// of public-catalog ids: the catalog *name* id (IGSR/HGDP, derived from the donor id) **and** the - /// authoritative INSDC *accession* the API returns (`SAMN…` → BIOSAMPLE, `ERS…` → ENA, `SRS…` → - /// SRA) — plus correcting the local `sample_accession` placeholder. A superset of - /// [`backfill_catalog_ids`](Self::backfill_catalog_ids) (the offline name-only path); use that one - /// when the API is unavailable. By default only subjects whose `donor_identifier` looks like a - /// catalog alias (IGSR/HGDP) are queried (`all` overrides), to avoid hammering the API with - /// friendly-name 404s. `apply == false` is a dry run. `limit` caps how many are queried. + /// Look up each subject in the samples API of the AppView and add the full set of + /// public-catalog ids **in one pass**. + /// + /// The set holds two kinds of id. The first is the catalog *name* id, which is an IGSR id or an + /// HGDP id, and the code derives it from the donor id. The second is the INSDC *accession* that + /// the API returns, and that value has authority. A `SAMN` prefix gives BIOSAMPLE, an `ERS` + /// prefix gives ENA, and an `SRS` prefix gives SRA. The method also corrects the local + /// `sample_accession` field, which holds a temporary value. + /// + /// This method does more than [`backfill_catalog_ids`](Self::backfill_catalog_ids), which uses + /// the name only and needs no network. Use that method when the API is not available. + /// + /// By default the method queries only a subject whose `donor_identifier` looks like a catalog + /// alias, which is an IGSR alias or an HGDP alias. The `all` option removes that limit. The + /// default stops many 404 responses for a friendly name. + /// + /// `apply == false` makes the method write nothing. `limit` sets the maximum count of + /// queries. pub async fn backfill_accessions( &self, project_id: Option, @@ -356,8 +399,9 @@ impl App { }; out.resolved += 1; let fetched_acc = sample.accession.as_deref().map(str::trim).filter(|a| !a.is_empty()); - // One pass: the catalog *name* id (from the donor id) + the authoritative INSDC *accession* - // (from the API, when it is a real one) — the union of both sources via the shared helper. + // One pass gives both ids. The catalog *name* id comes from the donor id. The INSDC + // *accession* comes from the API, when the API holds a real one. The shared helper + // joins the two sources. let ids = navigator_domain::identity::catalog_ids_from_provenance(&b.donor_identifier, fetched_acc); if ids.is_empty() { continue; @@ -403,11 +447,15 @@ impl App { Ok(out) } - /// Re-publish a subject's biosample anchor after its identifier set changed, so the AppView's - /// mirror (which full-replaces `external_ids`) honors the add/remove. Deterministic rkey → the - /// re-publish overwrites in place. **Only for a subject already federated** and while signed in — - /// signed out, or a never-published subject, is a no-op (we do not newly federate a donor just - /// because a local id was attached). + /// Publish the biosample anchor of a subject again, after the set of identifiers of that + /// subject changed. The mirror of the AppView replaces the full `external_ids` field, so it + /// then holds each addition and each removal. + /// + /// The method uses a fixed rkey, so the second publish replaces the record. + /// + /// The method acts **only for a subject that the app already published**, and only while an + /// account is active. With no active account, or for a subject that the app never published, it + /// does nothing. A new local id must not put a donor on the network for the first time. async fn republish_biosample_ids(&self, guid: SampleGuid) -> Result<(), AppError> { let Some(did) = self.current_account() else { return Ok(()); @@ -421,8 +469,9 @@ impl App { self.publish_biosample(guid).await } - /// Insert or update a Subject's MDKA for one lineage from the subject editor (one row per - /// lineage; stamps `updated_at`). Pass a `source` of `MANUAL` for hand-entered rows. + /// Insert or change the MDKA of a subject for one lineage, from the subject editor. There is + /// one row for each lineage, and the method sets `updated_at`. Give a `source` of `MANUAL` for + /// a row that the user typed. pub async fn upsert_mdka(&self, guid: SampleGuid, mdka: NewMdka) -> Result<(), AppError> { let now = Utc::now().to_rfc3339(); mdka::upsert(self.store.pool(), guid, &mdka, &now).await?; @@ -485,8 +534,8 @@ impl App { ystr: ystr.len(), scanned_subjects: 0, }; - // A roster was provided iff there are member rows — only then is "orphan" (data without a - // roster row) a meaningful flag. + // The import holds a roster only when it holds member rows. The "orphan" mark applies + // only in that case. An orphan is data with no roster row. let roster_provided = !members.is_empty(); // Join by kit number (BTreeMap → stable, kit-sorted plan). @@ -530,7 +579,8 @@ impl App { label: display_label(&kit, &input), kit_number: kit, y_terminal, - // Orphan only when a roster was provided but this kit is not in it. + // Mark the kit as an orphan only when the import holds a roster and that roster + // does not name the kit. in_roster: !roster_provided || roster.contains(&input.kit_number), ystr_count: input.ystr_markers.len(), kind, @@ -545,8 +595,9 @@ impl App { }) } - /// Apply a plan. `resolutions` carries the admin's choice for each fuzzy (`NeedsConfirm`) kit; - /// an unresolved fuzzy row defaults to **New** (conservative — never silently merge). + /// Apply a plan. `resolutions` holds the decision of the administrator for each kit with the + /// `NeedsConfirm` mark. A kit with that mark and no decision becomes a **New** subject. This + /// default is the safe one, because the method must never merge a kit without a decision. pub async fn commit_ftdna_import( &self, plan: &FtdnaImportPlan, @@ -555,7 +606,7 @@ impl App { let mut summary = FtdnaImportSummary::default(); let now = Utc::now().to_rfc3339(); - // Resolve the target project, creating it now if the plan targeted a new one. + // Find the target project. Make the project now when the plan names a new one. let project_id = match plan.project_id { Some(id) => id, None => { @@ -629,7 +680,8 @@ impl App { } }; - // Vendor identity (idempotent; never steals a conflicting id). + // The vendor identity. A second call is safe, and the code never moves an id that + // belongs to another subject. external_id::add(pool, guid, IdSource::FTDNA, &input.kit_number).await?; // FTDNA-reported member labels. @@ -648,7 +700,8 @@ impl App { ) .await?; - // MDKA from paternal (Y) + maternal (Mt) ancestry, when there is anything worth storing. + // The MDKA rows from the paternal (Y) ancestry and the maternal (Mt) ancestry. The code + // writes a row only when the ancestry holds a value. let mut wrote = 0; if let Some(m) = input.paternal.as_ref().and_then(|a| mdka_from(a, Lineage::Y)) { mdka::upsert(pool, guid, &m, now).await?; @@ -667,9 +720,10 @@ impl App { .map(subgroup_role); biosample_project::add(pool, guid, project_id, role.as_deref(), now).await?; - // Y-STR profile from the wide overview (Phase 2). Attached only when CREATING a new Subject; - // on a merge the existing Subject already carries its own data sources, so we add the FTDNA - // identity/membership/MDKA metadata above but skip duplicating the Y-STR profile. + // The Y-STR profile from the wide overview, which is Phase 2. The code adds the profile + // only when it makes a new subject. On a merge, the subject already holds its own data + // sources. So the code adds the FTDNA identity, the membership, and the MDKA data above, + // and it does not add a second Y-STR profile. let wrote_str = !input.ystr_markers.is_empty() && target.is_none(); if wrote_str { str_profile::create( @@ -699,9 +753,12 @@ impl App { Ok(external_id::list_for(self.store.pool(), guid).await?) } - /// Reverse of [`external_ids`]: the Subject bound to a `(source, external_id)` vendor id, if any. - /// This is the exact-match dedup anchor (design §5.1) — e.g. resolve an FTDNA kit number to the - /// biosample it was imported under. Returns `None` when the id is unknown to the workspace. + /// The reverse of [`external_ids`]. The method returns the subject of a + /// `(source, external_id)` vendor id, when one exists. + /// + /// This lookup is the exact-match anchor that finds a duplicate donor, in design §5.1. One use + /// is to find the biosample of an FTDNA kit number. The method returns `None` when the + /// workspace does not hold the id. pub async fn find_biosample_by_external_id( &self, source: &str, @@ -724,14 +781,17 @@ impl App { Ok(mdka::list_for(self.store.pool(), guid).await?) } - /// Project ids a Subject belongs to (via the M:N membership table). + /// The ids of the projects that hold this subject. The method reads the M:N membership + /// table. pub async fn project_membership_ids(&self, guid: SampleGuid) -> Result, AppError> { Ok(biosample_project::list_projects_for(self.store.pool(), guid).await?) } - /// Autocluster a project's members by Y-STR and propagate SNP branches to STR-only members - /// (the project clustering view). Branch per member = its FTDNA-reported terminal SNP; markers = - /// the merged Y-STR profiles. The O(n²) compute runs on a blocking thread. + /// Group the members of a project by their Y-STR values, and copy an SNP branch to a member + /// that has only STR values. The project cluster view shows this result. + /// + /// The branch of a member is the terminal SNP that FTDNA reports for it. The markers are the + /// merged Y-STR profiles. The calculation costs O(n²), so it runs on its own thread. pub async fn cluster_project_ystr( &self, project_id: i64, @@ -839,10 +899,15 @@ impl App { reasons.push(format!("same Y terminal {ex}")); } } - // Y-STR genetic distance — a SAME-PERSON signal only at (near-)zero GD over many markers. - // A loose GD threshold floods inside a single-haplogroup project, where every member is - // related and within-project distances of GD 3–11 over 100 markers are normal. Only an - // exact (or off-by-one) haplotype uniquely identifies the same person, not a clade cousin. + // The genetic distance of the Y-STR values. This distance shows the SAME PERSON only + // when it is zero, or almost zero, across many markers. + // + // A high limit gives many false results in a project with one haplogroup. Each member + // of such a project is a relative, and a distance of 3 to 11 across 100 markers is + // normal there. + // + // Only an exact haplotype, or a haplotype with one difference, names the same person. A + // larger distance names a cousin in the same clade. if !input.ystr_markers.is_empty() && !e.ystr.is_empty() { let (diff, compared) = navigator_domain::strprofile::str_distance(&input.ystr_markers, &e.ystr); if compared >= 67 && diff <= 1 { @@ -882,8 +947,9 @@ impl App { struct ExistingSubject { guid: SampleGuid, donor_identifier: String, - /// Terminal SNP of the subject's computed Y consensus (may be an ISOGG long-form label that - /// does not reduce to an SNP — then Y-STR is the reliable signal). + /// The terminal SNP of the Y consensus that the app calculated for the subject. The value can + /// be a long ISOGG label with no SNP inside it. In that case the Y-STR values are the signal + /// that the code can trust. y_terminal: Option, /// The subject's merged Y-STR markers (across all imported profiles), for genetic-distance match. ystr: Vec, @@ -899,8 +965,9 @@ fn empty_input(kit: &str) -> FtdnaSubjectInput { } } -/// The terminal SNP token of a haplogroup label or clade path: the last segment after splitting on -/// `>` (clade) or `-` (haplogroup prefix). `"R-FGC29071"` and `"CTS4466>S1115>FGC29071"` → `FGC29071`. +/// The terminal SNP token of a haplogroup label or a clade path. The function splits the text on +/// `>` for a clade, or on `-` for a haplogroup prefix, and returns the last part. Both +/// `"R-FGC29071"` and `"CTS4466>S1115>FGC29071"` give `FGC29071`. fn terminal_snp(label: &str) -> Option { let t = label.rsplit(['>', '-']).next()?.trim(); (!t.is_empty()).then(|| t.to_string()) @@ -929,7 +996,8 @@ fn clean_name(name: Option<&str>) -> Option { } } -/// Build an MDKA payload from an ancestry row, or `None` if it carries nothing worth storing. +/// Build an MDKA value from an ancestry row. The function returns `None` when the row holds no +/// data for the store. fn mdka_from(a: &AncestryRow, lineage: Lineage) -> Option { if a.ancestor_name.is_none() && a.origin_place.is_none() && a.country.is_none() && a.latitude.is_none() { return None; @@ -957,13 +1025,14 @@ fn panel_name_for_count(n: usize) -> String { } } -/// The clade `Sub Group` value as a membership role: keep it compact (the terminal segment), dropping -/// the leading sort number. +/// The `Sub Group` value of a clade, as a membership role. The function keeps the last part only, +/// and it removes the sort number at the start. fn subgroup_role(sub_group: &str) -> String { terminal_snp(sub_group).unwrap_or_else(|| sub_group.trim().to_string()) } -/// Jaccard overlap of lowercased word tokens (len ≥ 2) — a cheap name-similarity proxy in `0..=1`. +/// The Jaccard overlap of the word tokens, in lower case, with a length of 2 or more. The value is +/// a fast measurement of the similarity of two names, from 0 to 1. fn name_similarity(a: &str, b: &str) -> f32 { let toks = |s: &str| -> std::collections::HashSet { s.split(|c: char| !c.is_ascii_alphanumeric()) diff --git a/crates/navigator-app/src/import_profiles.rs b/crates/navigator-app/src/import_profiles.rs index 50683edc..ee6ce6ee 100644 --- a/crates/navigator-app/src/import_profiles.rs +++ b/crates/navigator-app/src/import_profiles.rs @@ -2,10 +2,15 @@ //! 2026-06 simplification round; `use super::*` reaches the crate-root types + free helpers. use super::*; -/// Process-wide memo of the parsed Y-SNP dictionary. Now that [`YsnpDictionary`] prefers the full -/// ~2M-row catalog, parsing it per resolve/annotate call (`y_snp_names_at` runs on every Y-SNP-table -/// view) would re-read ~200 MB each time; this parses once and reuses it. Keyed by the resolved -/// dictionary file's path + signature (mtime:size), so a refreshed dictionary is picked up. +/// One copy of the parsed Y-SNP dictionary for the full process. +/// +/// [`YsnpDictionary`] now selects the full catalog, which holds about 2 million rows. A parse of +/// that file at each call would read about 200 MB each time. The `y_snp_names_at` function runs at +/// each view of the Y-SNP table, so those calls are frequent. This value holds the result of one +/// parse. +/// +/// The key is the path of the dictionary file together with its signature, which is the mtime and +/// the size. So the code reads a new dictionary after the user replaces the file. type YsnpMemo = Mutex)>>; static YSNP_MEMO: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -47,9 +52,10 @@ impl App { ) -> Result { let text = std::fs::read_to_string(csv_path)?; let markers = strprofile::parse_csv(&text).map_err(AppError::Import)?; - // Merge into an existing same-panel profile rather than creating a duplicate — e.g. a Big Y - // CUSTOM (700/500) panel re-imported after the FTDNA project import already made one. Union - // the markers, the freshly-imported value winning on a conflict. + // Add the markers to a profile of the same panel, when one exists. Do not make a second + // profile. One example is a Big Y CUSTOM panel, of 700 or 500 markers, that the user + // imports after the FTDNA project import made the profile. The code joins the two marker + // sets. On a conflict, the value from the new import wins. if let Some(existing) = str_profile::find_by_panel(self.store.pool(), biosample_guid, panel_name).await? { let mut merged = existing.markers.clone(); for m in markers { @@ -84,10 +90,15 @@ impl App { // ---- SNP variants ------------------------------------------------------ - /// Import a subject's SNP variant calls from a file. `.vcf` is parsed as a VCF (reusing - /// the shared column parser); `.csv`/`.tsv` as a `contig,position,ref,alt[,rsid][,gt]` - /// table (a YSEQ/Sanger panel export fits this). Indels/symbolic alleles are dropped - /// (SNP-only). `source_type` sets the concordance weight (Sanger = gold standard). + /// Import the SNP variant calls of a subject from a file. + /// + /// The code parses a `.vcf` file as a VCF, with the shared column parser. It parses a `.csv` + /// file or a `.tsv` file as a `contig,position,ref,alt[,rsid][,gt]` table. A YSEQ panel export + /// and a Sanger panel export have that shape. + /// + /// The code keeps only SNPs. It removes each indel and each symbolic allele. `source_type` sets + /// the weight of the source in the concordance calculation, and a Sanger source has the highest + /// weight. pub async fn import_variants_from_file( &self, biosample_guid: SampleGuid, @@ -98,8 +109,9 @@ impl App { .file_name() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| "variants".into()); - // Match `.vcf`, plus bgzipped/gzipped `.vcf.gz` / `.vcf.bgz` (extension() alone sees only - // the trailing `.gz`, which would mis-route a compressed VCF to the CSV branch). + // Match `.vcf`, and also `.vcf.gz` and `.vcf.bgz` from bgzip or gzip. The `extension()` + // function reads only the last `.gz` part, and that value would send a compressed VCF to + // the CSV branch. let is_vcf = path .file_name() .and_then(|n| n.to_str()) @@ -107,9 +119,10 @@ impl App { .is_some_and(|n| n.ends_with(".vcf") || n.ends_with(".vcf.gz") || n.ends_with(".vcf.bgz")); let calls = if is_vcf { - // Genotype-aware: a vendor VCF (FTDNA Big Y / YSEQ) reports reference sites too, so only - // the genotype-selected ALT is kept (see parse_vcf_subject_snps). Sites-only VCFs keep - // every listed variant. Handles a bgzipped `.vcf.gz` transparently. + // The parser reads the genotype. A vendor VCF from FTDNA Big Y or YSEQ also reports + // a reference site. So the code keeps only the ALT value that the genotype selects. + // See parse_vcf_subject_snps. For a VCF with sites only, the code keeps each listed + // variant. The parser also reads a `.vcf.gz` file from bgzip. parse_vcf_subject_snps(path)? } else { let text = std::fs::read_to_string(path)?; @@ -119,10 +132,15 @@ impl App { return Err(AppError::Import("no SNP variants found in file".into())); } - // Vendor-aware tagging for VCFs: recognize FTDNA Big Y / Y Elite / YSEQ / mtFull from the - // header + filename + sibling readme, and record the vendor label, a meaningful SourceType, - // and the reference build (feeds Y/mt placement liftover). A generic VCF keeps the caller's - // label/source_type. CSV imports are unchanged. + // Find the vendor of a VCF and mark the record. The code recognizes FTDNA Big Y, Y Elite, + // YSEQ, and mtFull. It reads the header, the file name, and a readme file in the same + // directory. + // + // The code then records the vendor label, a correct SourceType, and the reference build. + // The Y placement and the mt placement use that build for the liftover. + // + // A VCF with no vendor keeps the label and the source_type of the caller. A CSV import does + // not change. let (source_label, source_type, reference_build) = if is_vcf { let (meta, contigs) = peek_vcf_header(path); let vendor = @@ -147,15 +165,20 @@ impl App { source_type, reference_build, calls, - // Recorded so the VCF can be re-read to genotype at tree positions (the role - // `alignment.bam_path` plays for a CRAM) — see `App::vset_base_calls`. + // The code records this path, so it can read the VCF again and genotype at the + // positions of the tree. The `alignment.bam_path` field has the same role for a CRAM. + // See `App::vset_base_calls`. source_path: Some(path.to_string_lossy().into_owned()), }; let set = variant_set::create(self.store.pool(), &new).await?; - // Place a vendor Y-NGS VCF (FTDNA Big Y / YSEQ / Full Genomes / …) on import so it lands a - // Y haplogroup without a manual Refresh — the VCF *is* the called Y-SNP set. Best-effort: an - // offline tree or an autosomal/mt-only VCF just leaves the calls (no chrY → no-op). + // Place a vendor Y-NGS VCF at the import, so the subject gets a Y haplogroup and the user + // does not press Refresh. Such a VCF comes from FTDNA Big Y, YSEQ, Full Genomes, or a + // similar test, and the file *is* the set of Y-SNP calls. + // + // The step is optional. The cache can hold no tree, and a VCF can hold only autosomal + // data or mt data. In each case the code writes the calls and does nothing more. A file + // with no chrY data gives no placement. let has_chr_y = set .calls .iter() @@ -168,13 +191,21 @@ impl App { Ok(set) } - /// Import a CompleteGenomics **masterVar** whole-genome variant table (`var-*-ASM.tsv[.bz2]`, - /// the old CG sequencing service's `cgatools` output). The file is streamed and decompressed - /// off-thread ([`navigator_analysis::mastervar`]) into SNP calls — each diploid het becomes a - /// `0/1`, a homozygous/haploid call a `1/1` / `1`, indels and `ref`/`no-call` spans dropped - /// (SNP-only, matching the VCF/CSV importer). Stored as a `WgsShortRead` set on GRCh37 (CG's - /// only build; chrM = rCRS), then Y-placed on import like a vendor Y-NGS VCF. mtDNA falls out - /// via the multi-source mt consensus (a non-chip set's chrM feeds `mt_source_calls`). + /// Import a CompleteGenomics **masterVar** whole-genome variant table. The file name is + /// `var-*-ASM.tsv` or `var-*-ASM.tsv.bz2`, and the `cgatools` program of the old CG sequencing + /// service wrote it. + /// + /// [`navigator_analysis::mastervar`] reads and decompresses the file on another thread, and it + /// makes SNP calls. A diploid heterozygous call becomes `0/1`. A homozygous call becomes `1/1`, + /// and a haploid call becomes `1`. The code removes each indel, each `ref` span, and each + /// `no-call` span. It keeps only SNPs, as the VCF importer and the CSV importer do. + /// + /// The code stores the result as a `WgsShortRead` set on GRCh37, which is the only build of CG. + /// The chrM contig uses rCRS. The code then places the Y haplogroup at the import, as it does + /// for a vendor Y-NGS VCF. + /// + /// The mtDNA result comes from the mt consensus of many sources. The chrM data of a set that is + /// not a chip feeds `mt_source_calls`. pub async fn import_mastervar_from_file( &self, biosample_guid: SampleGuid, @@ -219,12 +250,17 @@ impl App { Ok(set) } - /// Import an FTDNA Big Y CSV variant report (Named or Private Variants) — the data a project - /// admin gets when their access tier exposes the browser CSVs but not the BAM/CRAM/VCF. The - /// rows are GRCh38 chrY derived-allele calls, so they are stored as a `TargetedNgs` variant set - /// on GRCh38 (FTDNA's native Y-tree build) and placed via the vendor path on import — the Named - /// report lands a Y haplogroup directly (positions match the tree, no liftover). Private - /// Variants are stored too (novel loci, off-tree) for the record. + /// Import a Big Y CSV variant report from FTDNA. The report is the Named report or the Private + /// Variants report. A project administrator receives these files when the access level gives + /// the browser CSV files but no BAM file, CRAM file, or VCF file. + /// + /// Each row is a derived-allele call on chrY in GRCh38. So the code stores the rows as a + /// `TargetedNgs` variant set on GRCh38, which is the native build of the Y tree of FTDNA. The + /// code then places the subject with the vendor path at the import. + /// + /// The Named report gives a Y haplogroup directly, because its positions match the tree and + /// need no liftover. The code also stores the Private Variants. Those loci are new and are not + /// on the tree, and the store keeps them as a record. pub async fn import_ftdna_csv_variants( &self, biosample_guid: SampleGuid, @@ -241,15 +277,17 @@ impl App { source_path: Some(path.to_string_lossy().into_owned()), }; let set = variant_set::create(self.store.pool(), &new).await?; - // Place Y from the vendor (non-Chip) sets — the Named report carries the tree-defining SNPs. + // Place the Y haplogroup from the vendor sets, which are the sets that are not a chip. + // The Named report holds the SNPs that define a node of the tree. if let Err(e) = self.assign_y_vendor_vcfs(biosample_guid).await { eprintln!("FTDNA CSV Y placement deferred ({e})"); } Ok(set) } - /// Add a manually-entered variant set — paste `contig,position,ref,alt` rows (e.g. - /// Sanger/YSEQ confirmations). `source_type` sets the weight (Sanger = 1.0). + /// Add a variant set that the user typed. The user pastes `contig,position,ref,alt` rows. One + /// example is a set of confirmations from Sanger or YSEQ. `source_type` sets the weight, and a + /// Sanger source has the weight 1.0. pub async fn add_variants( &self, biosample_guid: SampleGuid, @@ -269,8 +307,9 @@ impl App { Ok(variant_set::create(self.store.pool(), &new).await?) } - /// The build to emit a subject's BISDNA calls on: the first of its alignments whose - /// reference build maps to a dictionary key, else `"hs1"` (the project default). + /// The build for the BISDNA calls of a subject. The method takes the first alignment whose + /// reference build has a dictionary key. If there is none, it returns `"hs1"`, which is the + /// default of the project. pub(crate) async fn bisdna_target_build(&self, biosample_guid: SampleGuid) -> String { if let Ok(aligns) = alignment::list_for_biosample(self.store.pool(), biosample_guid).await { for a in &aligns { @@ -282,13 +321,19 @@ impl App { "hs1".to_string() } - /// Annotate position-only Y variants with the catalogued Y-SNP **name** at that site, for the two - /// Y-SNP tables (multi-source variant profile + private-Y union). Resolves the subject's Y build - /// key (CHM13→`hs1`, else GRCh38/GRCh37 — same rule as the BISDNA importer), loads the Y-SNP - /// dictionary (the full catalog, memoized), and returns `position → canonical name` for the - /// requested positions only. Best-effort: a missing dictionary yields an empty map (not an error), - /// so the tables simply show no extra names. Looking a position up against the wrong build just - /// misses — there are no false labels, only possibly-absent ones. + /// Add the catalogued Y-SNP **name** to each Y variant that has a position and no name. Two + /// tables use this map: the variant profile with many sources, and the union of the private-Y + /// sets. + /// + /// The method finds the Y build key of the subject. A CHM13 build gives `hs1`, and the other + /// builds give GRCh38 or GRCh37. The BISDNA importer uses the same rule. + /// + /// The method then reads the Y-SNP dictionary, which is the full catalog and stays in memory. + /// It returns a map from a position to a canonical name, for the requested positions only. + /// + /// The step is optional. An absent dictionary gives an empty map and no error, and the tables + /// then show no extra name. A lookup against the wrong build finds nothing. So the table can + /// hold an absent name, but it never holds a wrong name. pub async fn y_snp_names_at( &self, biosample_guid: SampleGuid, @@ -309,14 +354,23 @@ impl App { Ok(names) } - /// Ensure a Y-SNP dictionary is present, downloading the full catalog (`dictionary.tsv`, - /// ~208 MB) from the asset release on first use — it is too big and too volatile (~weekly YBrowse - /// refresh) to bundle in the installer. No-op when a dictionary (the chromo2 panel or the full - /// catalog) is already installed, or the user pointed `NAVIGATOR_YSNP_DIR` at one. The download - /// is verified against a small published manifest (`ysnp_manifest.json`, the ancestry - /// [`AssetManifest`](navigator_analysis::manifest::AssetManifest) shape) so a rebuild is a - /// re-publish, not a client change. Best-effort — the caller then loads, degrading clearly if the - /// dictionary is still absent. Publish with `packaging/publish-assets.sh ysnp`. + /// Make sure that a Y-SNP dictionary is on the machine. At the first use, the method downloads + /// the full catalog, `dictionary.tsv`, which is about 208 MB. + /// + /// The installer does not hold that file. The file is too large, and YBrowse refreshes it about + /// once each week. + /// + /// The method does nothing when the machine already holds a dictionary. That dictionary can be + /// the chromo2 panel or the full catalog. It also does nothing when `NAVIGATOR_YSNP_DIR` points + /// to one. + /// + /// The method checks the download against a small published manifest, + /// `ysnp_manifest.json`. That file has the shape of the ancestry + /// [`AssetManifest`](navigator_analysis::manifest::AssetManifest). So a rebuild of the catalog + /// is a new publish and not a change to the client. + /// + /// The step is optional. The caller then reads the dictionary, and it reports the state clearly + /// when the file is still absent. Publish the file with `packaging/publish-assets.sh ysnp`. pub async fn ensure_ysnp_dictionary(&self) -> Result<(), AppError> { const YSNP_ASSET_BASE: &str = "https://github.com/JamesKane/decodingus-navigator/releases/download/assets-ysnp"; @@ -349,8 +403,9 @@ impl App { &mut noop, ) .await?; - // Verify the streamed digest against the manifest (no 208 MB re-read). A manifest without an - // entry passes through advisory, matching `AssetManifest::verify`. + // Check the digest from the stream against the manifest. The code does not read the 208 MB + // file again. A manifest with no entry for the file gives a warning only, as + // `AssetManifest::verify` does. if let Some(entry) = manifest.assets.get("dictionary.tsv") { if !got.eq_ignore_ascii_case(&entry.sha256) { let _ = std::fs::remove_file(&dest); @@ -363,12 +418,19 @@ impl App { Ok(()) } - /// Import a BISDNA chromo2 Y-SNP export. Each named marker is resolved to a locus via the - /// Y-SNP dictionary on `build` (when `None`, the subject's alignment build, else `"hs1"`). - /// Only **positive** (derived) calls become variant calls: a negative is not a variant. - /// `no_call`, back-mutated, and dictionary-unresolved markers are tallied but not emitted. - /// The genotype is a QC cross-check only — the file's verdict (independent of the Illumina - /// TOP strand) decides derived/ancestral. Stored as a `Chip`-weighted [`VariantSet`]. + /// Import a chromo2 Y-SNP export from BISDNA. + /// + /// The Y-SNP dictionary changes each marker name into a locus on `build`. When `build` is + /// `None`, the method uses the alignment build of the subject, and then `"hs1"`. + /// + /// Only a **positive**, or derived, call becomes a variant call. A negative call is not a + /// variant. The method counts a `no_call` marker, a back-mutated marker, and a marker that the + /// dictionary does not hold. It writes none of those three. + /// + /// The genotype is a quality cross-check only. The verdict in the file decides between derived + /// and ancestral, and that verdict does not depend on the Illumina TOP strand. + /// + /// The method stores the result as a [`VariantSet`] with the `Chip` weight. pub async fn import_bisdna_from_file( &self, biosample_guid: SampleGuid, @@ -404,10 +466,13 @@ impl App { .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| "BISDNA".into()); - // Also record an array QC summary so the chromo2 chip appears under Data Sources → - // Chip / Array Profiles (the placeable per-SNP calls live in the variant set below; a - // genotyping array legitimately has both a QC/provenance summary and its calls). BISDNA - // is a Y-only haploid panel: every called marker is a Y marker, heterozygosity is n/a. + // Also write a quality summary for the array. The chromo2 chip then appears under + // Data Sources, in the Chip and Array Profiles list. The variant set below holds the SNP + // calls that the code can place. An array correctly has both a quality summary with its + // provenance and a set of calls. + // + // BISDNA is a haploid Y panel. Each called marker is a Y marker, and heterozygosity does + // not apply. let total = calls.len() as i64; let called = total - outcome.no_call as i64; let chip = NewChipProfile { @@ -442,10 +507,13 @@ impl App { }; let variant_set = variant_set::create(self.store.pool(), &new).await?; - // Compute the Y haplogroup on import (best-effort; an offline tree just leaves the calls), - // mirroring the array path in `import_chip_profile_from_csv`. Without this a chromo2/BISDNA - // panel imports its calls but never auto-places — it has no cached alignment genotypes, so - // `rebuild-signatures` can't place it later either, leaving the subject's Y at . + // Calculate the Y haplogroup at the import. The step is optional, and with no tree the + // code writes the calls only. The array path in `import_chip_profile_from_csv` does the + // same. + // + // Without this step, a chromo2 panel from BISDNA imports its calls and never gets a + // placement. Such a panel has no alignment genotypes in the cache, so `rebuild-signatures` + // can not place it later either. The Y value of the subject then stays at . if derived_calls > 0 { if let Err(e) = self.assign_y_bisdna(biosample_guid, Some(&build)).await { eprintln!("BISDNA Y placement deferred ({e})"); @@ -473,17 +541,26 @@ impl App { // ---- chip / array profiles --------------------------------------------- - /// Import a genotyping-array raw-data export (CSV/TSV) and store its QC summary. - /// `provider` overrides vendor detection when given; `chip_version` is optional. - /// Import a genotyping-array raw-data export and (1) store its QC summary as a [`ChipProfile`], - /// (2) store the haploid Y/MT genotype rows as a `Chip`-source [`VariantSet`], and (3) - /// best-effort place the Y (and, where present, mtDNA) haplogroup on import — the consumer-array - /// counterpart to BISDNA's chromo2 path. 23andMe carries both Y and MT rows; AncestryDNA carries - /// Y but no usable mtDNA. The stored observed bases flow through the same - /// [`assign_y_bisdna`](Self::assign_y_bisdna) / [`assign_mt_chip`](Self::assign_mt_chip) + - /// `assemble_assignment_robust` placement as BISDNA, with plus-strand reconciliation to the tree. - /// Placement is best-effort: an unreachable tree (offline) leaves the calls stored for a later - /// manual "Assign … (panel)" — it does not fail the import. + /// Import the raw-data export of a genotyping array, as a CSV file or a TSV file. + /// + /// The method does three things. It writes the quality summary as a [`ChipProfile`]. It writes + /// the haploid Y rows and MT rows as a [`VariantSet`] with the `Chip` source. It then tries to + /// place the Y haplogroup, and the mtDNA haplogroup when the file holds one. + /// + /// This method is the consumer-array form of the chromo2 path of BISDNA. A 23andMe file holds + /// both Y rows and MT rows. An AncestryDNA file holds Y rows and no mtDNA rows that the app can + /// use. + /// + /// The stored bases go through the same placement as BISDNA. That path is + /// [`assign_y_bisdna`](Self::assign_y_bisdna) or [`assign_mt_chip`](Self::assign_mt_chip), + /// followed by `assemble_assignment_robust`, and it reconciles each call to the plus strand of + /// the tree. + /// + /// The placement is optional. With no network, the code stores the calls, and the user can + /// press "Assign … (panel)" later. A failed placement does not fail the import. + /// + /// `provider` replaces the vendor that the code finds, when the caller gives it. `chip_version` + /// is optional. pub async fn import_chip_profile_from_csv( &self, biosample_guid: SampleGuid, @@ -514,10 +591,12 @@ impl App { }; let profile = chip_profile::create(self.store.pool(), &new).await?; - // Pull the haploid Y/MT genotype rows and store them as Chip-source variant calls so the - // haplogroup placement (and later re-placement) has them without re-reading the file. The - // observed allele goes in both `reference` and `alternate` (we do not know the ancestral); - // the placement reads `alternate`. + // Read the haploid Y rows and MT rows, and store them as variant calls with the Chip + // source. The haplogroup placement then has them, and a later placement also has them, + // with no second read of the file. + // + // The observed allele goes in `reference` and in `alternate`, because the app does not know + // the ancestral allele. The placement reads `alternate`. let haplo = chipprofile::haplo_calls(&text); if !haplo.is_empty() { let build = chipprofile::detect_build(&text); @@ -556,8 +635,9 @@ impl App { eprintln!("chip Y placement deferred ({e})"); } } - // AncestryDNA's stray MT rows are not a usable mtDNA panel — only place mtDNA when the - // array carries a real MT marker set (23andMe has thousands; the threshold filters noise). + // The few MT rows of an AncestryDNA file are not an mtDNA panel that the app can use. + // Place mtDNA only when the array holds a true MT marker set. A 23andMe file holds + // some thousands of such markers, and the limit below removes the noise. const MIN_MT_CALLS: usize = 20; if mt_count >= MIN_MT_CALLS { if let Err(e) = self.assign_mt_chip(biosample_guid).await { @@ -595,8 +675,9 @@ impl App { }; let seq = mtdna_store::create(self.store.pool(), &new).await?; - // Derive rCRS-relative variants and persist them, so an mtDNA FASTA yields a variant set on - // import (not only on the on-demand "show mutations" view) — like a chip/VCF import does. + // Derive the variants against rCRS and write them to the store. An mtDNA FASTA then gives + // a variant set at the import. Before this step, the set appeared only in the "show + // mutations" view. A chip import and a VCF import behave in the same way. let derived = navigator_analysis::mtvariants::derive(navigator_analysis::mtvariants::rcrs(), &seq.sequence); if !derived.is_empty() { let label = mt_vendor_label(seq.source_file_name.as_deref(), seq.defline.as_deref()); @@ -609,7 +690,8 @@ impl App { alternate: v.alternate.to_string(), rs_id: None, genotype: None, - // Derived from an rCRS diff, not a source VCF — no evidence to carry. + // These calls come from a comparison with rCRS and not from a source VCF. So + // there is no evidence to store. evidence: Default::default(), }) .collect(); @@ -626,9 +708,10 @@ impl App { let _ = variant_set::create(self.store.pool(), &set).await; } - // Haplogroup placement is intentionally NOT run here: it needs the mt haplotree (network), - // and coupling a deterministic import to a network fetch is what the alignment import - // deliberately avoids too. The mtDNA tab's "Assign mtDNA haplogroup" places it on demand. + // This method does NOT place the haplogroup, by design. A placement needs the mt + // haplotree, and a read of that tree needs the network. An import must stay deterministic, + // so it must not depend on the network. The alignment import follows the same rule. The + // user presses "Assign mtDNA haplogroup" on the mtDNA tab to place the subject. Ok(seq) } @@ -637,12 +720,17 @@ impl App { Ok(mtdna_store::list_for_biosample(self.store.pool(), biosample_guid).await?) } - /// Derive mtDNA variants for a stored sequence by comparing it to an rCRS reference - /// FASTA, and save them as a variant set (contig `rCRS`) so they appear alongside the - /// subject's other variants. The reference is validated as an mtDNA FASTA. - /// The mtDNA mutation list for a stored sequence: variants relative to the **bundled** rCRS - /// (NC_012920.1), via banded alignment — substitutions, insertions, and deletions in standard - /// mtDNA notation. On-demand (one ~16.5 kb alignment), not stored. The classic mtDNA result. + /// Derive the mtDNA variants of a stored sequence. The method compares that sequence with an + /// rCRS reference FASTA and writes the result as a variant set on the contig `rCRS`. The + /// variants then appear with the other variants of the subject. The method checks that the + /// reference file is an mtDNA FASTA. + /// + /// The mutation list holds the variants against the **bundled** rCRS sequence, NC_012920.1. A + /// banded alignment gives them. The list holds substitutions, insertions, and deletions, in the + /// standard mtDNA notation. + /// + /// The method runs at the request of the user, and it does one alignment of about 16.5 kb. It + /// stores nothing. This list is the classic mtDNA result. pub async fn mtdna_variants(&self, mtdna_id: i64) -> Result, AppError> { let seq = mtdna_store::get(self.store.pool(), mtdna_id) .await? diff --git a/crates/navigator-app/src/maintenance.rs b/crates/navigator-app/src/maintenance.rs index 3789c4fb..9e49a92c 100644 --- a/crates/navigator-app/src/maintenance.rs +++ b/crates/navigator-app/src/maintenance.rs @@ -1,14 +1,19 @@ -//! Workspace **chores** — the periodic batch jobs that keep a workspace current. +//! Workspace **chores**. A chore is a batch job that keeps a workspace current, and the user runs +//! it from time to time. //! -//! These existed only as CLI subcommands (`private-y --project`, `rebuild-signatures --stale-tree`, -//! `publish-origins`), each noted in its own design doc as wanting a GUI trigger, and each noted as -//! wanting *the same answer* rather than a third bespoke button. This module is that answer: the -//! chores are named, surveyed and driven from one place, so the CLI and the GUI run identical code -//! and a fourth chore is a table entry rather than a new surface. +//! Each chore was a CLI subcommand only. They are `private-y --project`, +//! `rebuild-signatures --stale-tree`, and `publish-origins`. The design document of each chore +//! asked for a trigger in the GUI. Each document also asked for *one* answer, not a third separate +//! button. //! -//! **Surveying is deliberately on demand.** Two of the three cost real work to measure — one walks -//! every alignment, another fetches and parses a multi-MB haplotree — so nothing here runs off a -//! render path. The UI asks once, when a user asks it to. +//! This module is that answer. It names each chore, surveys them, and runs them from one place. +//! The CLI and the GUI then run the same code. A fourth chore is a new row in a table, and not a +//! new screen. +//! +//! **The survey runs only at the request of the user, by design.** Two of the three chores cost +//! real work to measure. One walks each alignment, and another reads and parses a haplotree of many +//! MB. So no code here runs during a paint. The UI asks one time, when the user presses the +//! button. use super::*; use crate::fastpath::{chr_m_gvcf_for_alignment, chr_y_gvcf_for_alignment}; @@ -17,11 +22,12 @@ use crate::fastpath::{chr_m_gvcf_for_alignment, chr_y_gvcf_for_alignment}; /// cohort views need, re-place anything a new tree invalidated, then publish. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] pub enum Chore { - /// Compute and cache each subject's private-Y bucket. Nothing cross-subject — shared unnamed - /// variants, candidate branches — can mean anything until this has been walked once. + /// Calculate the private-Y group of each subject and write it to the cache. A result that + /// covers more than one subject is not correct before this walk runs one time. Such results are + /// the unnamed variants that subjects share, and the candidate branches. PrivateY, - /// Re-place subjects whose calls were scored against a superseded haplotree, or whose derived - /// consensus names a branch this tree no longer carries. + /// Place a subject again when an old haplotree scored its calls. Also place a subject again + /// when its consensus names a branch that this tree no longer holds. StaleTree, /// Publish MDKA ancestral origins for the subjects the consent predicate allows. PublishOrigins, @@ -46,8 +52,9 @@ pub struct ChoreSurvey { pub chore: Chore, /// Items the chore would act on. pub due: usize, - /// Items it would consider — `due` of `total` is what makes "0 due" readable as "nothing to - /// do" rather than "nothing found". + /// The items that the chore would examine. The pair `due` of `total` lets the user read "0 + /// due" as "there is no work". Without `total`, that text can also mean "the survey found + /// nothing". pub total: usize, /// Why the chore can not run at all (not signed in, no tree). `Some` disables it. pub blocked: Option, @@ -59,26 +66,32 @@ pub struct ChoreOutcome { pub done: usize, pub skipped: usize, pub failed: usize, - /// One line for the status bar — chore-specific, since "12 done" means different things. + /// One line of text for the status bar. Each chore writes its own text, because "12 done" + /// refers to a different item in each chore. pub summary: String, } -/// What [`App::replace_against_current_tree`] did for one subject. Per-alignment call failures are -/// counted rather than raised: an alignment whose file is gone is a superseded vendor download, not -/// a reason to leave the subject un-replaced. +/// The work that [`App::replace_against_current_tree`] did for one subject. +/// +/// The code counts a call failure of one alignment. It does not stop on that failure. An alignment +/// with an absent file is an old vendor download. It is not a reason to leave the subject in its +/// earlier place. #[derive(Debug, Clone, Copy, Default, PartialEq)] pub struct TreeReplace { pub calls_replaced: usize, pub calls_failed: usize, - /// Alignments skipped because their file is gone. Kept apart from `calls_failed` so a workspace - /// whose vendor downloads have been cleaned out does not report a wall of errors for the one - /// outcome that is expected and harmless. + /// The count of alignments that the chore skipped, because the file of each one is absent. + /// + /// This count is separate from `calls_failed`. A user can remove the old vendor downloads of a + /// workspace. That workspace must not then report many errors for a result that is normal and + /// harmless. pub calls_skipped: usize, pub profiles_rebuilt: usize, } impl TreeReplace { - /// Tally one per-alignment call, sorting "its file is gone" out of the failures. + /// Count the call of one alignment. Put an absent file in its own group, and not with the + /// failures. fn record(&mut self, outcome: Result<(), AppError>) { match outcome { Ok(()) => self.calls_replaced += 1, @@ -88,21 +101,23 @@ impl TreeReplace { } } -/// Per-subject result of a private-Y refresh. +/// The result of a private-Y refresh, for one subject. #[derive(Debug, Clone, Copy, Default, PartialEq)] pub struct PrivateYRefresh { pub computed: usize, pub skipped: usize, pub failed: usize, - /// Alignments whose file is gone — a superseded vendor download, not a computation failure. - /// Counted apart so a missing file never reads as an error. + /// The count of alignments with an absent file. Such a file is an old vendor download. It is + /// not a fault in the calculation. The count is separate, so an absent file never looks like an + /// error. pub missing_file: usize, pub novel: usize, } impl App { - /// Survey every chore. Runs the real selectors, so it costs what the chores cost to *decide* - /// (not to do) — call it from a button, never from a paint. + /// Survey each chore. The method runs the real selectors. So it costs the work that a chore + /// needs to *decide*, and not the work to do that chore. Call this method from a button. Never + /// call it during a paint. pub async fn maintenance_survey(&self) -> Result, AppError> { let mut out = Vec::with_capacity(Chore::ALL.len()); @@ -121,8 +136,9 @@ impl App { blocked: None, }); - // Stale placements: two independent symptoms, unioned. Neither subsumes the other — a - // consensus is derived and persisted separately, so it rots while every call beneath it + // An old placement has two separate symptoms, and the code joins the two sets. One set + // does not hold the other. The app derives a consensus and stores it separately. So a + // consensus becomes old while each call below it // stays current. let stale = match self.stale_tree_targets(false).await { Ok(v) => ChoreSurvey { @@ -131,7 +147,7 @@ impl App { total: self.list_all_biosamples().await?.len(), blocked: None, }, - // No tree cached yet, or it could not be parsed: report the reason rather than zero. + // The cache holds no tree, or the parser refused it. Report the reason, not zero. Err(e) => ChoreSurvey { chore: Chore::StaleTree, due: 0, @@ -141,7 +157,7 @@ impl App { }; out.push(stale); - // Publishing needs an account: without one there is nowhere to publish to. + // A publish needs an account. Without an account there is no destination. let publishable = mdka::publishable(self.store.pool(), Lineage::Y.as_str()).await?.len(); out.push(ChoreSurvey { chore: Chore::PublishOrigins, @@ -153,11 +169,15 @@ impl App { Ok(out) } - /// Subjects due for re-placement: those whose *source calls* carry another tree's fingerprint, - /// unioned with those whose *derived consensus* names a branch this tree does not carry. + /// The subjects that need a new placement. The method joins two sets. + /// + /// In the first set, the *source calls* of a subject carry the fingerprint of another tree. In + /// the second set, the *derived consensus* of a subject names a branch that this tree does not + /// hold. /// - /// `include_unknown` also takes calls that predate the fingerprint field — 80% of them, mostly - /// BAM re-walks — so it is opt-in rather than the default. + /// `include_unknown` adds the calls that are older than the fingerprint field. Those calls are + /// 80% of the total, and most of them are BAM walks. So the user must select this option, and + /// it is not the default. pub async fn stale_tree_targets(&self, include_unknown: bool) -> Result, AppError> { let by_fingerprint = self.subjects_placed_against_another_tree(include_unknown).await?; let off_tree = self.subjects_labelled_off_tree().await?; @@ -168,41 +188,51 @@ impl App { Ok(v) } - /// Re-place one subject against the current tree: its per-alignment Y/mt calls first, then the - /// pooled profiles built from them. + /// Place one subject against the current tree again. The method scores the Y calls and the mt + /// calls of each alignment first. It then builds the pooled profiles from those calls. + /// + /// The step for each alignment was the half that was absent. A rebuild of only the profiles + /// refreshes `consensus_profile` and changes no `haplogroup_call` row. So a call with a name + /// from an older tree stays after each sweep. + /// + /// Subject `1087` showed this fault. Its `aln:903` row read `CP086569.2:27785335 G->A`, and its + /// current `aln:864` row read `R-BY66248`. The Y card then reported "sources diverge below + /// root". /// - /// The per-alignment step is the half that was missing. Rebuilding only the profiles refreshes - /// `consensus_profile` while leaving every `haplogroup_call` row untouched, so a call carrying a - /// name from an older tree survives every sweep — `1087` held `aln:903` reading - /// `CP086569.2:27785335 G->A` against a current `aln:864` of `R-BY66248`, which is what the Y - /// card reported as "sources diverge below root". Worse, the sweep selects subjects *by* those - /// call fingerprints ([`Self::stale_tree_targets`]), so a subject it never corrected stayed due - /// forever and the count never fell. + /// The fault was worse than one wrong card. The sweep selects a subject *by* those call + /// fingerprints, in [`Self::stale_tree_targets`]. So a subject that the sweep never corrected + /// stayed in the list for all time, and the count never became smaller. /// - /// Order matters: the calls are the profile's input, so re-placing them after the build would - /// leave the profile a version behind. Each step is best-effort and independent — an alignment - /// whose file is gone must not stop the rest of the subject from being brought current. + /// The order of the two steps matters. The calls are the input of the profile. A new placement + /// after the build leaves the profile one version behind. /// - /// Re-scoring is guarded by the alignment's own fingerprint (file hash + tree hash), so a - /// subject already current costs a fingerprint comparison rather than a walk. When the tree - /// genuinely changed, doing that work *is* the chore. + /// Each step is independent, and a failure in one step does not stop the others. An alignment + /// with an absent file must not stop the work on the rest of the subject. + /// + /// The fingerprint of the alignment, which is the file hash with the tree hash, guards the new + /// score. So a subject that is already current costs one comparison and not a walk. When the + /// tree did change, that walk *is* the chore. pub async fn replace_against_current_tree(&self, guid: SampleGuid) -> Result { let mut r = TreeReplace::default(); for aln in self.list_alignments_for_biosample(guid).await.unwrap_or_default() { - // Fast-path (sidecar GVCF) calls first, and only then the CRAM-walk assignment. + // Do the fast-path calls, which come from a sidecar GVCF, first. Do the CRAM-walk + // assignment after them. + // + // These rows have `external` provenance. Here "external" names the caller of the + // pipeline. It does not name an authority that the app only relays. + // `assign_y_from_gvcf` places the calls of the GVCF against *our* tree. So the stored + // name is only as current as the tree in the cache on the day of the import. Subject + // `altai363p` carried `chrY:5216846A>C [Node721]` from such an import. // - // These are the `external`-provenance rows, but "external" here means the pipeline's - // caller, not an authority we merely relay: `assign_y_from_gvcf` places the GVCF's - // calls against *our* tree, so the stored name is only as current as the tree cached - // the day it was imported — `altai363p` carried `chrY:5216846A>C [Node721]` from one. - // They are ours to re-derive. Skipping them would leave the very rows the internal - // assignment defers to (see `has_preferred_external_call`) as the only stale ones left, - // which is the case that prompted this. - // Prefer the recorded paths; fall back to locating the GVCFs beside the alignment. - // The fallback is what makes this work on the existing corpus at all — every alignment - // imported before the paths were recorded has none, so keying solely on the record - // would have fixed only future imports and left the subjects that prompted this - // permanently stale. + // The app must derive these rows again. Without this step, the internal assignment + // defers to those rows, in `has_preferred_external_call`, and they stay the only old + // rows in the workspace. That case caused this code. + // + // Use the recorded paths first. If there are none, look for the GVCF files beside the + // alignment. This second step is necessary for the corpus that exists today. Each + // alignment from before the recorded-path change has no path. A key on the record + // alone corrects only a future import, and it leaves the subjects that caused this + // change old for all time. let recorded = self.recorded_sidecars(aln.id).await.ok().flatten(); let y_gvcf = recorded .as_ref() @@ -212,8 +242,9 @@ impl App { .as_ref() .and_then(|s| s.chr_m_gvcf.clone()) .or_else(|| chr_m_gvcf_for_alignment(&aln)); - // A GVCF that has since gone (superseded vendor download, unmounted volume) is skipped - // rather than counted against the subject. + // The code skips a GVCF that is no longer on disk. The cause is an old vendor + // download or a volume that the user removed. The code does not count it against the + // subject. for outcome in [ match y_gvcf.filter(|p| p.is_file()) { Some(p) => Some(self.assign_y_from_gvcf(aln.id, &p).await.map(|_| ())), @@ -229,9 +260,9 @@ impl App { { r.record(outcome); } - // The CRAM-walk assignments. An alignment whose file has been removed since import - // reports `AlignmentFileMissing` here and is skipped — the sidecar calls above may still - // have re-placed the subject perfectly well without it. + // The CRAM-walk assignments. An alignment with a file that the user removed after + // the import gives `AlignmentFileMissing` here, and the code skips it. The sidecar + // calls above can still place the subject correctly without that alignment. for outcome in [ self.assign_y_haplogroup(aln.id).await.map(|_| ()), self.assign_mtdna_haplogroup_from_alignment(aln.id).await.map(|_| ()), @@ -239,21 +270,23 @@ impl App { r.record(outcome); } } - // Both arms rebuild regardless: a source-less lineage just yields an empty profile, and a - // subject can be stale on one arm while current on the other. + // The code rebuilds both lineages in each case. A lineage with no source gives an empty + // profile. A subject can also be old on one lineage and current on the other. self.build_y_profile(guid).await?; self.build_mt_profile(guid).await?; r.profiles_rebuilt = 2; Ok(r) } - /// Refresh one subject's private-Y: every alignment, or — when there is none — every - /// non-chip variant set. + /// Refresh the private-Y of one subject. The method uses each alignment. When the subject has + /// no alignment, it uses each variant set that is not a chip. + /// + /// The variant-set path is necessary, and it is not only for a clean design. Most members of a + /// real Y project have no alignment. Before this path, those members had no private-Y, and that + /// gap made each candidate branch inert. /// - /// The variant-set arm is not a fallback for tidiness: most members of a real Y project have no - /// alignment at all, and until it existed they had no private-Y, which is what made candidate - /// branches inert. Lifted out of the CLI so the GUI runs the same code rather than a second - /// implementation that drifts. + /// This code came out of the CLI, so the GUI runs the same code. A second copy would become + /// different over time. pub async fn refresh_private_y(&self, guid: SampleGuid, force: bool) -> Result { let mut r = PrivateYRefresh::default(); let alignments = self.list_alignments_for_biosample(guid).await.unwrap_or_default(); @@ -276,8 +309,8 @@ impl App { return Ok(r); } for a in &alignments { - // A row whose file is gone is not a computation failure — reporting it as one buries - // the real errors. + // A row with an absent file is not a fault in the calculation. A report of it as a + // fault hides the real errors. if !a.bam_path.as_deref().is_some_and(|p| std::path::Path::new(p).exists()) { r.missing_file += 1; continue; diff --git a/crates/navigator-app/src/publish.rs b/crates/navigator-app/src/publish.rs index d26a7d4c..2e848285 100644 --- a/crates/navigator-app/src/publish.rs +++ b/crates/navigator-app/src/publish.rs @@ -5,22 +5,29 @@ use super::*; impl App { // ---- publish ----------------------------------------------------------- - /// Build the alignment (coverage) record JSON for an alignment — the shared - /// `com.decodingus.atmosphere.alignment` contract the AppView ingests (floats as strings). - /// Links back to the subject's biosample + sequence-run records via their deterministic at:// - /// URIs in `did`'s repo, so the AppView can tie this coverage summary to its subject. + /// Build the JSON of the alignment record, which holds the coverage. The record follows the + /// shared `com.decodingus.atmosphere.alignment` contract that the AppView reads, and each float + /// is a string. + /// + /// The record links to the biosample record and the sequence-run records of the subject. It + /// uses their fixed at:// URIs in the repository of `did`. So the AppView can join this + /// coverage summary to its subject. pub(crate) async fn coverage_record(&self, did: &str, alignment_id: i64) -> Result { let cov = self .cached_coverage(alignment_id) .await? .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("coverage for alignment {alignment_id}"))))?; let aln = self.alignment_or_err(alignment_id).await?; - // A whole-genome-labeled alignment whose reads are actually Y-scoped — a chrY-only extract, - // or a Y test (Big Y / Y Elite) that came in mislabeled WGS — must not publish a coverage - // summary. The AppView files an `alignment` record under whole-genome statistics, so its - // near-zero autosomal depth and callable footprint would skew the aggregate WGS coverage - // distributions. Genuine Y-targeted tests are exempt: their test type is published, so the - // AppView cohorts their Y coverage separately from WGS. + // An alignment can carry the label of a whole genome while its reads cover only the Y + // chromosome. The file can be a chrY extract, or a Y test such as Big Y or Y Elite with a + // wrong WGS label. The app must not publish a coverage summary for such an alignment. + // + // The AppView files an `alignment` record under the statistics of a whole genome. The + // autosomal depth and the callable area of these files are almost zero. Those values move + // the WGS coverage distribution of the full cohort. + // + // A true Y test does not have this problem. The app publishes its test type, so the AppView + // puts its Y coverage in a group that is separate from WGS. let is_wgs = matches!( sequence_run::get(self.store.pool(), aln.sequence_run_id) .await? @@ -62,20 +69,24 @@ impl App { Ok(serde_json::to_value(&record)?) } - /// The subject's persisted **consensus** ancestry estimates ([`CONSENSUS_SOURCE_ID`]) — one per - /// method (ADMIXTURE / FINE_ADMIXTURE), newest-first. Ancestry is estimated from the pooled - /// autosomal consensus (not per alignment), so this is the subject's authoritative breakdown. - /// Empty until the consensus ancestry has been estimated. + /// The stored **consensus** ancestry estimates of the subject ([`CONSENSUS_SOURCE_ID`]). There + /// is one estimate for each method, which is ADMIXTURE or FINE_ADMIXTURE, and the newest comes + /// first. + /// + /// The code estimates the ancestry from the pooled autosomal consensus, and not from one + /// alignment. So this list is the breakdown of the subject with authority. The list is empty + /// until an estimate runs. /// - /// Two filters guard what leaves the machine, because federating a wrong breakdown is far worse - /// than showing one locally — a PDS record outlives the bug that produced it. + /// Two filters control what leaves the machine. A wrong breakdown on the network is much worse + /// than a wrong breakdown on the screen. A PDS record stays after the app corrects the fault. /// - /// * `RETIRED_METHODS` are **never** published, flag or no flag. The PCA-centroid ancient - /// estimators produced fabricated breakdowns; the estimators are gone, but rows they persisted - /// still sit in databases written before the rebuild. This makes sure a build that can no - /// longer *produce* those numbers can't *publish* them either. - /// * The current ancient method (`ANCIENT_ADMIXTURE`) is published only while ancient ancestry - /// is enabled ([`crate::ANCIENT_ANCESTRY_ENABLED`]), keeping that flag a true kill switch. + /// * The app **never** publishes a method in `RETIRED_METHODS`, with a flag or without one. The + /// ancient estimators that used a PCA centroid gave incorrect breakdowns. Those estimators are + /// gone, but a database from before the rebuild still holds their rows. This filter makes sure + /// that a build which can no longer *make* those numbers can not *publish* them. + /// * The app publishes the current ancient method, `ANCIENT_ADMIXTURE`, only while ancient + /// ancestry is on, in [`crate::ANCIENT_ANCESTRY_ENABLED`]. That flag is then a true switch to + /// stop the feature. pub(crate) async fn consensus_ancestry_results( &self, biosample_guid: SampleGuid, @@ -92,9 +103,11 @@ impl App { .collect()) } - /// The populationBreakdown record JSON for each consensus ancestry estimate of a subject (one - /// per method), linked to the biosample — the shared `com.decodingus.atmosphere.populationBreakdown` - /// contract the AppView ingests (floats as strings). Empty if none computed. + /// The JSON of a populationBreakdown record for each consensus ancestry estimate of a subject. + /// There is one record for each method, and each record links to the biosample. + /// + /// The record follows the shared `com.decodingus.atmosphere.populationBreakdown` contract that + /// the AppView reads, and each float is a string. The list is empty when no estimate exists. async fn consensus_ancestry_records( &self, did: &str, @@ -111,8 +124,9 @@ impl App { .collect() } - /// Build the anonymized biosample record JSON — sex, center, and best-effort Y/mt - /// haplogroup calls. Donor identifiers / accession / description are never carried. + /// Build the JSON of the anonymous biosample record. It holds the sex, the center, and the Y + /// and mt haplogroup calls when they exist. The record never holds a donor identifier, an + /// accession, or a description. pub(crate) async fn biosample_record( &self, did: &str, @@ -124,10 +138,15 @@ impl App { let y = self.consensus_haplogroup(biosample_guid, DnaType::Y).await?; let mt = self.consensus_haplogroup(biosample_guid, DnaType::Mt).await?; let runs = self.list_sequence_runs(biosample_guid).await?; - // External identifiers (vendor kits + public catalog ids), a pure field rename onto the wire - // shape. Published plaintext — the AppView keeps vendor ids off every public surface via its - // `is_public` namespace policy; catalog ids (PGP/IGSR/ENA…) are already public. This is the - // deterministic dedup anchor the AppView keys a re-published donor on. + // The external identifiers, which are the vendor kits and the public catalog ids. This + // step only renames the fields for the wire format. + // + // The app publishes these values as plaintext. The `is_public` namespace policy of the + // AppView keeps a vendor id off each public screen. A catalog id from PGP, IGSR, or ENA is + // already public. + // + // These identifiers are the fixed anchor that the AppView uses to find a duplicate when a + // user publishes the same donor again. let external_ids = self .external_ids(biosample_guid) .await? @@ -145,11 +164,16 @@ impl App { Ok(serde_json::to_value(&record)?) } - /// Build a sequence-run characterization record JSON (platform/instrument/test — no files). - /// `instrument_id` (the sequencer serial inferred from read names) is published so the AppView - /// can grow its crowd-sourced instrument→lab map (`fed.sequencerun.instrument_id` → the - /// `instrument_observation`→proposal→accept consensus). It identifies the physical sequencer, - /// not the donor — no PII, consistent with the anonymized fed-record posture. + /// Build the JSON of a sequence-run record. It holds the platform, the instrument, and the + /// test. It holds no file. + /// + /// The app publishes `instrument_id`, which is the serial number of the sequencer that the code + /// deduces from the read names. The AppView uses that value to build its map from an instrument + /// to a laboratory. The path is `fed.sequencerun.instrument_id`, then an + /// `instrument_observation`, then a proposal, and then the accepted consensus. + /// + /// The value names the physical sequencer. It does not name the donor. It holds no personal + /// data, and it follows the rule for each anonymous federated record. pub(crate) async fn sequence_run_record( &self, did: &str, @@ -167,18 +191,26 @@ impl App { run.mean_insert_size, Utc::now().to_rfc3339(), ) - // Publish the known lab so the AppView can display it (and learn the instrument→lab map — - // many serials, e.g. PacBio, are not in its dataset). See [`SequenceRun::sequencing_facility`]. + // Publish the laboratory when the app knows it. The AppView then shows it, and the + // AppView also learns the map from an instrument to a laboratory. Its dataset holds no + // entry for many serial numbers, such as the PacBio numbers. See + // [`SequenceRun::sequencing_facility`]. .with_facility(run.sequencing_facility.clone()) - // Exact sequenced yield + read chemistry back the standardized DTC test label the AppView - // renders/groups by (`du_domain::testprofile`). Both `Option`al — older records omit them. + // The exact yield and the read chemistry support the standard DTC test label. The AppView + // draws that label and groups by it, in `du_domain::testprofile`. Both fields are + // `Option`, because an older record holds neither. .with_read_profile(run.total_bases, run.read_type.clone()); Ok(serde_json::to_value(&record)?) } - /// Best-effort consensus haplogroup for a subject arm, for the federated biosample record: - /// manual override > genome-level placed terminal > per-run label reconciliation (all via - /// [`haplogroup_consensus`](Self::haplogroup_consensus)). `None` when nothing has been called. + /// The consensus haplogroup of one lineage of a subject, for the federated biosample record. + /// + /// The method takes the first value that exists, in this order. First, a manual value from the + /// user. Second, the terminal node of the genome-level placement. Third, the reconciled label + /// of each run. + /// + /// [`haplogroup_consensus`](Self::haplogroup_consensus) gives all three values. The method + /// returns `None` when no call exists. async fn consensus_haplogroup( &self, biosample_guid: SampleGuid, @@ -192,22 +224,30 @@ impl App { /// Build the private-variants record JSON to publish for an alignment/contig. /// - /// **chrY** publishes only the *filtered, publishable* private-Y set — the whole-chrY de-novo - /// calls after backbone subtraction, callable masking, structural-region filtering, and the - /// strict novel-marker [`PublishGate`] — tagged as unverified singleton candidates. It never - /// publishes the raw de-novo flood (CHM13's Y is haplogroup J; an R sample's J-vs-R divergence - /// plus paralog mismaps would otherwise drown AppView curators in non-viable SNPs). + /// For **chrY**, the method publishes only the private-Y set that passed each filter. /// - /// Other contigs (chrM) publish their raw de-novo calls — a small, well-behaved rCRS-relative - /// set that needs no tree-relative filtering. + /// That set holds the de-novo calls across chrY after four steps. The code removes the backbone + /// variants, applies the callable mask, removes the structural regions, and applies the strict + /// novel-marker [`PublishGate`]. Each published variant carries the mark of a single unverified + /// candidate. + /// + /// The method never publishes the full de-novo set for chrY. The Y chromosome of CHM13 belongs + /// to haplogroup J. Take a sample in haplogroup R. The difference between J and R, and the + /// reads that map to the wrong paralog, give many SNPs that no curator can use. + /// + /// For another contig, such as chrM, the method publishes the raw de-novo calls. That set is + /// small, it behaves well, and it is relative to rCRS. It needs no filter against a tree. pub(crate) async fn variants_record(&self, alignment_id: i64, contig: &str) -> Result { let variants = if navigator_analysis::contig::is_chr_y(contig) { let bucket = self.private_y_variants_self_masked(alignment_id).await?; - // QC gate: if the filtered novel count is implausibly high (contamination / low coverage / - // reference-build mismatch — e.g. a GRCh38 alignment, whose chrY reference is far noisier - // and whose shared-lineage variants the hs1-native tree can't fully resolve), the whole - // set is suspect. Publish nothing rather than flood curators with candidates from a sample - // we have already flagged; the variants still show in the in-app DISPLAY under the banner. + // A quality gate. A count of new variants that is too high shows a problem with the + // sample. The causes are contamination, low coverage, and a wrong reference build. A + // GRCh38 alignment is one example: its chrY reference holds more noise, and the + // hs1-native tree can not resolve each of its shared-lineage variants. + // + // In that case the full set is doubtful, and the app publishes nothing. A curator must + // not receive many candidates from a sample that the app already marked. The app still + // shows those variants on the screen, below the warning banner. if let Some(warn) = bucket.qc_banner() { eprintln!("private-variants publish skipped for alignment {alignment_id}: {warn}"); Vec::new() @@ -259,9 +299,11 @@ impl App { Ok(client.create_record(NS_ALIGNMENT, value, None).await?) } - /// Publish a subject's **consensus** ancestry estimates (one populationBreakdown per method) - /// using an explicit `client` (the testable core; production callers use - /// [`publish_ancestry`](Self::publish_ancestry)). Returns a ref per record. + /// Publish the **consensus** ancestry estimates of a subject with the `client` that the caller + /// gives. The method writes one populationBreakdown record for each method. + /// + /// This function is the core, and a test can call it directly. In the app, callers use + /// [`publish_ancestry`](Self::publish_ancestry). The method returns one ref for each record. pub async fn publish_ancestry_with( &self, client: &PdsClient, @@ -298,11 +340,14 @@ impl App { .await?) } - /// Build an ancestral-origin record for one MDKA row, or `None` when it must not be published. + /// Build an ancestral-origin record for one MDKA row. The method returns `None` when the app + /// must not publish that row. /// - /// Every field gate lives in [`AncestralOriginRecord::build`] — this only supplies the join - /// keys. The lineage is mapped to the AppView's `Y_DNA`/`MT_DNA` spelling; `Auto` is not - /// published at all, having no tree to hang from. + /// [`AncestralOriginRecord::build`] holds the gate for each field. This method only gives the + /// join keys. + /// + /// The method changes the lineage name to the form that the AppView uses, which is `Y_DNA` or + /// `MT_DNA`. It never publishes an `Auto` lineage, because that lineage has no tree. pub(crate) async fn ancestral_origin_record( &self, did: &str, @@ -343,11 +388,12 @@ impl App { .map_err(AppError::from) } - /// Publish one MDKA's ancestral origin using an explicit `client`. `Ok(None)` means the row was - /// refused by a gate — a normal outcome, not an error. + /// Publish the ancestral origin of one MDKA with the `client` that the caller gives. A result + /// of `Ok(None)` shows that a gate refused the row. That result is normal and is not an error. /// - /// Uses `putRecord` at a deterministic rkey, so correcting an MDKA and re-running overwrites - /// that ancestor's record rather than accumulating duplicates of the same man. + /// The method calls `putRecord` at a fixed rkey. So a user can correct an MDKA and run the + /// method again, and the new record replaces the record of that ancestor. The repository does + /// not collect duplicates of one man. pub async fn publish_ancestral_origin_with( &self, client: &PdsClient, @@ -364,15 +410,19 @@ impl App { /// Publish the ancestral origins of every subject this workspace may publish for. /// - /// Enqueues rather than posting directly: the outbox retries, survives being offline, and maps - /// a deterministic rkey onto `putRecord`, so re-running after correcting an MDKA overwrites - /// that ancestor's record instead of accumulating duplicates. That makes the batch resumable by - /// construction — running it twice is a no-op on the AppView. + /// The method puts each record in the outbox. It does not send a record directly. The outbox + /// tries again after a failure, it continues after an offline period, and it maps a fixed rkey + /// onto `putRecord`. + /// + /// So a user can correct an MDKA and run the batch again, and the new record replaces the + /// record of that ancestor. The repository does not collect duplicates. For this reason the + /// batch is always safe to run again, and a second run changes nothing on the AppView. + /// + /// With `dry_run`, the method builds each record and applies each gate, but it adds nothing to + /// the outbox. A user can then read the counts before any genealogy leaves the machine. /// - /// `dry_run` builds and gates everything but enqueues nothing, so the counts can be inspected - /// before any genealogy leaves the machine. The consent predicate is in - /// [`navigator_store::mdka::publishable`]; the field gates are in - /// [`AncestralOriginRecord::build`]. + /// [`navigator_store::mdka::publishable`] holds the consent test. + /// [`AncestralOriginRecord::build`] holds the gate for each field. pub async fn publish_ancestral_origins( &self, lineage: Lineage, @@ -389,8 +439,8 @@ impl App { report.refused += 1; continue; }; - // What the gates actually let through, so a dry run reports coverage rather than a - // bare total. + // The count of rows that pass each gate. A dry run then reports the coverage, and not + // only a total. if value.get("originPlace").is_some() { report.with_place += 1; } else if value.get("originCountry").is_some() { @@ -425,10 +475,14 @@ impl App { } } -/// Fold a [`CoverageResult`]'s two per-contig views (samtools-style stats + -/// callable-state counts) into the shared lexicon's `contigs[]`, paired by contig -/// name — the same join `export::coverage_tsv` uses. Contigs present in the stats -/// but missing callable counts (should not happen) fall back to zeros. +/// Join the two views that a [`CoverageResult`] holds for each contig, and write the result to the +/// `contigs[]` field of the shared lexicon. +/// +/// The two views are the statistics in the samtools form and the counts of each callable state. The +/// key of the join is the contig name. `export::coverage_tsv` uses the same join. +/// +/// A contig can appear in the statistics with no callable count. That state must not occur, and the +/// function then writes zeros. fn contig_metrics(cov: &CoverageResult) -> Vec { cov.contig_coverage_stats .iter() @@ -474,8 +528,8 @@ mod tests { } } - /// chrY carrying millions of reads, autosomes/chrX only a trace of mismapped ones — the - /// Y-only-extract shape. + /// The shape of a chrY extract. The chrY contig holds millions of reads. Each autosome and the + /// X chromosome hold only a few reads that map to the wrong place. fn y_scoped_coverage() -> CoverageResult { CoverageResult { contig_coverage_stats: vec![cstat("chrY", 3_000_000), cstat("chr1", 30), cstat("chrX", 12)], @@ -498,8 +552,8 @@ mod tests { .id } - /// A WGS-labeled but Y-scoped alignment must not publish a coverage summary — it would poison - /// the AppView's whole-genome statistics. + /// An alignment with a WGS label that holds only Y reads must not publish a coverage summary. + /// Such a summary makes the whole-genome statistics of the AppView incorrect. #[tokio::test] async fn wgs_y_scoped_coverage_is_withheld() { let app = App::new(Store::open_in_memory().await.unwrap()); @@ -511,8 +565,8 @@ mod tests { assert!(matches!(err, AppError::Conflict(_)), "expected Conflict, got {err:?}"); } - /// A Y-targeted test (Big Y) with the *same* Y-scoped shape publishes normally — its Y coverage - /// is expected and the AppView cohorts it apart from WGS. + /// A Y test, such as Big Y, has the *same* shape and publishes as usual. Its Y coverage is + /// correct, and the AppView puts it in a group that is separate from WGS. #[tokio::test] async fn y_targeted_coverage_still_publishes() { let app = App::new(Store::open_in_memory().await.unwrap()); diff --git a/crates/navigator-app/tests/mastervar_autosomal_real.rs b/crates/navigator-app/tests/mastervar_autosomal_real.rs index 8a63cec0..641e7ba2 100644 --- a/crates/navigator-app/tests/mastervar_autosomal_real.rs +++ b/crates/navigator-app/tests/mastervar_autosomal_real.rs @@ -1,11 +1,19 @@ -//! Isolated end-to-end check: a real CompleteGenomics masterVar → autosomal consensus → ancestry, -//! entirely on an **in-memory** workspace (nothing touches the user's `~/.decodingus/navigator-rs.db`). -//! The read-only ancestry / IBD-panel assets are still read from `~/.decodingus/ancestry` (or the -//! `NAVIGATOR_*` overrides), so this only runs where those assets are installed. +//! A separate end-to-end check. It reads a real CompleteGenomics masterVar file, makes the +//! autosomal consensus, and then estimates the ancestry. //! -//! Ignored by default (needs the local dump + assets). Run: -//! MASTERVAR_TSV=/path/to/var-GS00253-DNA_A01_200_37-ASM.tsv.bz2 \ -//! cargo test -p navigator-app --test mastervar_autosomal_real -- --ignored --nocapture +//! The test uses an **in-memory** workspace. It does not touch the `~/.decodingus/navigator-rs.db` +//! file of the user. +//! +//! The test still reads the ancestry assets and the IBD-panel assets from `~/.decodingus/ancestry`, +//! or from the path in a `NAVIGATOR_*` variable. It reads and does not write them. So the test runs +//! only on a machine that holds those assets. +//! +//! The test has the `ignore` mark, because it needs the local dump and the assets. To run it: +//! +//! ```bash +//! MASTERVAR_TSV=/path/to/var-GS00253-DNA_A01_200_37-ASM.tsv.bz2 \ +//! cargo test -p navigator-app --test mastervar_autosomal_real -- --ignored --nocapture +//! ``` use std::path::Path; use std::time::Instant; @@ -83,8 +91,9 @@ async fn mastervar_feeds_autosomal_and_ancestry() { } } } - // Do not fail the whole check if an ancestry asset is absent — the autosomal consensus (the - // thing this PR wires up) already proved the masterVar feeds the pipeline. + // An absent ancestry asset must not fail the full check. The autosomal consensus is the + // part that this change adds, and it already shows that the masterVar file reaches the + // pipeline. Err(e) => println!("[{:>7.1?}] ancestry skipped: {e}", t.elapsed()), } } diff --git a/documents/STE-dictionary.md b/documents/STE-dictionary.md index fd1b2fcf..7ff4189c 100644 --- a/documents/STE-dictionary.md +++ b/documents/STE-dictionary.md @@ -32,8 +32,9 @@ A Technical Name is a noun. It can be a compound noun. It cannot be a verb. ### Genetics and sequence data alignment · allele · ancestry · admixture · autosome · base · biosample · build · call · caller · -chromosome · consensus · contig · coverage · depth · donor · genome · genotype · haplogroup · -haplotype · indel · kit · lineage · marker · panel · pedigree · ploidy · position · read · +chromosome · consensus · contig · coverage · depth · donor · genome · genotype · +genotyping array · haplogroup · haplotype · indel · kit · lineage · marker · panel · pedigree · +ploidy · position · read · read metrics · reference · reference genome · region · segment · sequence · sequence run · sex · signature · site · subject · variant · Y-STR @@ -51,6 +52,14 @@ query · realignment · record · row · schema · store · table · workspace AppView · attestation · consent · device key · DID · exchange · handle · IBD · PDS · record key · session · signature · suggestion · token +### Technical Names that end in `-ing` + +STE 2 forbids an `-ing` form as a verb or an adjective. These are declared nouns, so they are +permitted: genotyping array · mapping · sequencing · painting · matching · encoding · decoding · +indexing · logging · polling · signing · setting · heading · listing · ordering · padding · casing · +tracking · caching · processing · operating system · pacing · sampling · scaling · streaming · +spilling · phasing · binning · masking · trimming · clipping · calling · sorting · merging + ## Technical Verbs STE permits a project to declare Technical Verbs when no approved verb has the meaning. Use these diff --git a/scripts/ste-check.py b/scripts/ste-check.py index 8b645a79..439700ef 100755 --- a/scripts/ste-check.py +++ b/scripts/ste-check.py @@ -126,7 +126,7 @@ def _technical_names(): "processing", "pending", "missing", "remaining", "existing", "following", "corresponding", "underlying", "according", "including", # Technical Names from documents/STE-dictionary.md that end in -ing. - "operating", "pacing", "sampling", "scaling", "streaming", "spilling", "phasing", + "operating", "genotyping", "pacing", "sampling", "scaling", "streaming", "spilling", "phasing", "binning", "masking", "trimming", "clipping", "calling", "sorting", "merging", "reading", "writing", "counting", "timing", "build", "backing", } @@ -218,7 +218,9 @@ def analyse(items, kind): for idiom in IDIOMS: if re.search(rf"\b{re.escape(idiom.lower())}\b", low): v["STE8 idiom/metaphor/informal"].append((ln, idiom)) - if "—" in ptext or " -- " in ptext: + # Judge the code-stripped text: a fenced shell block puts cargo's `--` argument separator + # in the paragraph, and that is not an em-dash aside. + if "—" in clean or " -- " in clean: v["STE6 em-dash aside"].append((ln, "")) return v From 19cc8c4e05b56cebf8a55b239fa99f7b59bb557f Mon Sep 17 00:00:00 2001 From: James Kane Date: Tue, 18 Aug 2026 15:32:54 -0500 Subject: [PATCH 06/33] docs(ste): navigator-app, the scientific rationale files blocktree, commands, llm, realign. Twenty-seven of thirty-three files at zero; the crate goes 3,096 to 2,694. blocktree.rs is the file this standard was most likely to damage, and the one worth reading to judge the result. Almost every constant in it is a threshold defended by a measurement, and the defence is the comment. All of it survives: - why six "novel" calls inside 32 bp are one misaligned read and not six mutations, and that this shape produced most of the first candidate branches on the CTS4466 cohort; - why three branches inside a 567 bp window at 56.83 Mb are one repeat unit mis-mapping rather than three lineage events, and why a kilobase is the right scale; - why five positions carried by all 111 private-Y donors are reference-vs-population differences that a bundled blocklist cannot anticipate; - why the frequency rule abstains below a donor count instead of throwing away every genuine branch in a four-donor cohort. Each now reads as short declarative sentences instead of one dense paragraph with three subordinate clauses. Longer on the page, and a reader parsing English as a second language can follow the argument. Two more Technical Names declared, because the checker was flagging real domain terms: `genotyping array` and `reasoning model`. The dictionary now carries a Local LLM section and lists the `-ing` Technical Names explicitly. `cargo check -p navigator-app --all-targets` passes. Co-Authored-By: Claude Opus 5 (1M context) --- crates/navigator-app/src/blocktree.rs | 348 +++++++++++++++----------- crates/navigator-app/src/commands.rs | 271 ++++++++++++-------- crates/navigator-app/src/llm.rs | 238 ++++++++++++------ crates/navigator-app/src/queries.rs | 59 +++-- crates/navigator-app/src/realign.rs | 192 +++++++------- documents/STE-dictionary.md | 8 +- scripts/ste-check.py | 2 +- 7 files changed, 684 insertions(+), 434 deletions(-) diff --git a/crates/navigator-app/src/blocktree.rs b/crates/navigator-app/src/blocktree.rs index 2de5d04a..d7b59b24 100644 --- a/crates/navigator-app/src/blocktree.rs +++ b/crates/navigator-app/src/blocktree.rs @@ -1,36 +1,46 @@ -//! Project **block tree** — the cohort counterpart to the per-subject descent report. +//! The project **block tree**. It is the cohort form of the descent report of one subject. //! -//! Given a project, build the induced subtree of the haplotree spanning its members' terminal -//! haplogroups: every branch that any member lies on, each carrying the run of defining SNPs that -//! are phylogenetically equivalent on it (a *block*), with members hanging off their own terminal. -//! This is the FTDNA "Block Tree" surface, over placements Navigator already computed. +//! For a project, this module builds the part of the haplotree that covers the terminal haplogroups +//! of its members. That subtree holds each branch that a member lies on. //! -//! Two rules shape everything here: +//! Each branch carries a *block*, which is the run of SNPs that define it and that the tree treats +//! as equivalent. Each member appears below its own terminal branch. //! -//! - **It reads placements, never re-places.** Terminals come from `haplogroup_terminals` — the same -//! reconciliation the subjects table and project report use. Nothing here can move a subject. -//! - **A member that can't be placed is reported, not dropped** ([`UnplacedMember`]). On a multi-lab -//! cohort provider/build skew is expected; silently omitting those members would make the tree -//! look like it accounts for the whole project when it does not. +//! This surface is the FTDNA "Block Tree", and it uses the placements that Navigator already made. //! -//! Design: `documents/design/project-block-tree.md`. +//! Two rules control this module: +//! +//! - **It reads a placement, and it never makes one.** Each terminal comes from +//! `haplogroup_terminals`. The subjects table and the project report use the same +//! reconciliation. No code here can move a subject. +//! - **The report names a member with no placement** ([`UnplacedMember`]). It does not remove that +//! member. In a cohort from many laboratories, a difference between providers and builds is +//! normal. Without those members, the tree looks complete when it is not. +//! +//! The design is in `documents/design/project-block-tree.md`. use std::collections::BTreeSet; use super::*; -/// Collapse a run of member-less single-child branches only when it is at least this long. A lone -/// intermediate branch is worth naming; a run of two or more is noise between the splits the cohort -/// actually resolves. +/// The minimum length of a run of branches that the code joins into one. Each branch in the run has +/// one child and no member. +/// +/// One branch between two splits has a name that the reader needs. A run of two or more such +/// branches only fills space between the splits that the cohort resolves. pub const COLLAPSE_MIN_RUN: usize = 2; impl App { - /// Build the [`ProjectBlockTree`] for `project_id`. + /// Build the [`ProjectBlockTree`] of `project_id`. + /// + /// The method returns `Ok(None)` only when the project has no member. + /// + /// A project with no placed member still gives a tree. That tree has an empty `blocks` list, + /// and it holds each member in `unplaced`. This answer is a useful one, because it tells the + /// user that the app placed nothing yet. /// - /// `Ok(None)` only when the project has no members at all. A project whose members are all - /// unplaced still yields a tree — empty `blocks`, everyone in `unplaced` — because that is a - /// meaningful answer ("nothing here is placed yet"), and it costs no tree fetch: the multi-MB - /// download + parse is skipped entirely when no member has a terminal. + /// That answer also costs nothing. When no member has a terminal, the method does not download + /// the tree and does not parse it. That document is many MB. pub async fn project_block_tree( &self, project_id: i64, @@ -41,11 +51,12 @@ impl App { return Ok(None); } - // One bulk reconciliation for the whole workspace rather than a query per member — the same - // call `project_report` and `project_str_overview` make. + // One reconciliation covers the full workspace. The code does not send one query for each + // member. `project_report` and `project_str_overview` call the same function. let terminals = self.haplogroup_terminals().await?; - // (guid, display name, terminal) per member, for the requested lineage. + // The guid, the display name, and the terminal of each member, for the lineage that the + // caller requested. let wanted: Vec<(SampleGuid, String, Option)> = members .iter() .map(|b| { @@ -57,9 +68,9 @@ impl App { }) .collect(); - // Cheap first, as `descent_report` does: with nothing placed there is no tree to draw, and - // fetching + parsing a multi-MB document to discover that is pure waste. Common on a - // freshly imported project. + // Do the fast test first, as `descent_report` does. With no placement there is no tree to + // draw. A download and a parse of a document of many MB, only to learn that, is work with + // no result. This state is common in a project that a user imported a moment ago. if wanted.iter().all(|(_, _, t)| t.is_none()) { let mut unplaced: Vec = wanted .into_iter() @@ -70,8 +81,9 @@ impl App { dna, blocks: Vec::new(), unplaced, - // The *configured* provider: no tree was fetched, so no runtime fallback happened - // either. `build_key` stays empty for the same reason — nothing was parsed. + // This value is the provider in the settings. The code downloaded no tree, so it + // also used no second provider. `build_key` stays empty for the same reason, + // because the code parsed nothing. provider: match y_tree_provider() { YTreeProvider::DecodingUs => "decodingus".to_string(), YTreeProvider::Ftdna => "ftdna".to_string(), @@ -82,9 +94,9 @@ impl App { })); } - // Provider and build key are whatever the fetch actually resolved to, not what was - // configured: the mtDNA path falls back to FTDNA at runtime when the DecodingUs tree can't be - // remapped, and its loci are rCRS either way — not the Y coordinate space. + // The provider and the build key come from the download, and not from the settings. The + // mtDNA path uses the FTDNA tree when the code can not remap the DecodingUs tree. The loci + // of that path are rCRS loci in each case, and they are not in the Y coordinate space. let (tree, provider, build_key) = match dna { DnaType::Y => match y_tree_provider() { YTreeProvider::DecodingUs => { @@ -97,7 +109,8 @@ impl App { YTreeProvider::Ftdna => { let json = self.fetch_ftdna_y_tree().await?; let tree = navigator_analysis::haplo::parse_ftdna_json(&json).map_err(AppError::Import)?; - // The FTDNA Y tree is published on GRCh38, whatever the members are aligned to. + // FTDNA publishes its Y tree on GRCh38. The build of each member does not + // change that. (tree, "ftdna", "GRCh38") } }, @@ -107,12 +120,12 @@ impl App { } }; - // One name index for the whole cohort. The per-subject path scans the node map linearly for - // its single terminal, which would be quadratic here. + // One name index covers the full cohort. The path for one subject reads the node map from + // start to end to find its terminal. Here that method would cost O(n²). let index = navigator_analysis::haplo::name_index(&tree); - // Resolve placement first, so the private-Y load below covers only members who actually - // appear on the tree — on a real cohort that is a fraction of the roster (243 of 1881 on - // R1b-CTS4466Plus), and an unplaced member's private variants can't be drawn anywhere. + // Find the placements first. The private-Y read below then covers only the members that + // appear on the tree. In a real cohort that group is small: 243 of 1,881 members on + // R1b-CTS4466Plus. The view can draw no private variant of a member with no placement. let mut placed: Vec<(SampleGuid, String, i64)> = Vec::new(); let mut unplaced = Vec::new(); for (guid, name, terminal) in wanted { @@ -162,9 +175,9 @@ impl App { } roll_up_subtree_members(&mut blocks); let blocks = collapse_blocks(blocks, COLLAPSE_MIN_RUN); - // Candidates go in *after* the collapse: they are leaves with members, so they could never - // be absorbed, and inserting them earlier would only make the collapse reason about - // synthetic nodes. + // The code adds each candidate *after* the collapse step. A candidate is a leaf with + // members, so the collapse can never absorb it. An earlier insert only makes the collapse + // examine nodes that the code made. let (blocks, candidate_conflicts, candidate_recurrent) = insert_candidate_branches(blocks, &private); unplaced.sort_by(|a, b| (&a.name, a.guid.0).cmp(&(&b.name, b.guid.0))); @@ -179,13 +192,18 @@ impl App { })) } - /// The one coordinate space the cohort's tree is parsed under: the **modal** DecodingUs build key - /// across the members' alignments, falling back to `hs1`. + /// The one coordinate space for the tree of the cohort. It is the most frequent DecodingUs + /// build key across the alignments of the members. The default is `hs1`. /// - /// A cohort spans builds, so there is no per-subject answer as there is in `descent_report`. - /// Picking one is safe because node names and topology are build-independent — only the loci - /// *positions* are, and the aggregate carries the key so the view can say which it means. Ties - /// break on the key name, so the choice does not depend on map iteration order. + /// A cohort holds more than one build, so there is no answer for one subject, as there is in + /// `descent_report`. + /// + /// One choice is safe, because the node names and the shape of the tree do not depend on the + /// build. Only the *positions* of the loci depend on it. The aggregate carries the key, so the + /// view can name the space that it shows. + /// + /// Two keys with the same count give the key that sorts first. So the choice does not depend on + /// the order of a map. async fn project_build_key(&self, members: &[Biosample]) -> &'static str { let guids: Vec = members.iter().map(|b| b.guid).collect(); let Ok(alns) = alignment::list_for_biosamples(self.store.pool(), &guids).await else { @@ -201,10 +219,10 @@ impl App { } } -/// Fill `subtree_members` — members at or below each block. +/// Fill `subtree_members`, which holds the members at each block and below it. /// -/// `blocks` is in pre-order, so walking it **backwards** visits every child before its parent and one -/// pass suffices. +/// The `blocks` list is in pre-order. So a read from the end to the start reaches each child before +/// its parent, and one pass is enough. fn roll_up_subtree_members(blocks: &mut [Block]) { let mut from_children: HashMap = HashMap::with_capacity(blocks.len()); for i in (0..blocks.len()).rev() { @@ -216,8 +234,8 @@ fn roll_up_subtree_members(blocks: &mut [Block]) { } } -/// Synthesize a [`Locus`] standing for a private (unnamed) variant, so a candidate branch's shared -/// variants render through exactly the same path as a named branch's defining SNPs. +/// Make a [`Locus`] value for a private variant, which has no name. The view then draws the shared +/// variants of a candidate branch through the same code as the SNPs of a named branch. fn private_locus(v: &PrivateVariant) -> Locus { Locus { position: v.position, @@ -227,14 +245,19 @@ fn private_locus(v: &PrivateVariant) -> Locus { } } -/// The positions a subject carries as **high-confidence new-branch candidates**: novel (not in the -/// tree at all) *and* in unique sequence. +/// The positions that a subject carries as **new-branch candidates with high confidence**. Such a +/// position is new, so the tree does not hold it, and it is in unique sequence. +/// +/// The function removes a *known* variant that is off the path, by design. That variant supports a +/// finer branch that already exists. It asks a question about the placement, and not about a new +/// branch. +/// +/// The function also removes a call in a structural region. A palindrome and an amplicon on chrY +/// hold paralogs. Two men with the "same" call in such a region share a mapping artefact more often +/// than they share an ancestor. /// -/// Off-path-*known* variants are excluded on purpose — those support an existing finer branch, which -/// is a placement question, not a new one. Structural-region calls are excluded because chrY -/// palindromes and amplicons are paralog-prone: two men "sharing" a call there are far more likely to -/// share a mapping artefact than an ancestor. Sharing noise would manufacture branches, which is the -/// one failure mode this feature must not have. +/// Shared noise makes a branch that does not exist. That result is the one fault that this feature +/// must not produce. fn candidate_positions(bucket: &PrivateBucket) -> BTreeSet { let novel: BTreeSet = bucket .variants @@ -245,18 +268,20 @@ fn candidate_positions(bucket: &PrivateBucket) -> BTreeSet { drop_clustered(&novel) } -/// How far apart two novel calls must be to count as independent mutations. +/// The minimum distance between two new calls that come from separate mutations. /// -/// Real Y mutations are scattered across megabases; a handful of "novel" calls within tens of bases -/// is one misaligned read smearing several false SNVs, which is why the GVCF path already imposes a -/// depth floor for the same reason. On the CTS4466 cohort the first candidate branches were built -/// almost entirely from such clusters — six positions inside 32 bp, gaps of 5–8 bp. +/// A real Y mutation is far from the next one, at a distance of megabases. A group of "new" calls +/// inside tens of bases comes from one read that the mapper placed wrongly, and that read gives +/// some false SNVs. The GVCF path applies a depth limit for the same reason. +/// +/// In the CTS4466 cohort, almost every first candidate branch came from such a group. One group +/// held six positions inside 32 bp, with gaps of 5 bp to 8 bp. const CANDIDATE_MIN_SEPARATION_BP: i64 = 100; -/// Drop every position that has another candidate within [`CANDIDATE_MIN_SEPARATION_BP`]. +/// Remove each position that has another candidate inside [`CANDIDATE_MIN_SEPARATION_BP`]. /// -/// The whole cluster goes, not the extras: when several calls share one mapping event there is no -/// basis for electing one of them the real mutation. +/// The function removes the full group, and not only the extra positions. When some calls come from +/// one mapping event, no rule can select one of them as the real mutation. fn drop_clustered(positions: &BTreeSet) -> BTreeSet { let ordered: Vec = positions.iter().copied().collect(); ordered @@ -271,22 +296,27 @@ fn drop_clustered(positions: &BTreeSet) -> BTreeSet { .collect() } -/// Share of the cohort's private-Y-bearing members above which a position is treated as -/// **population-shared** rather than private. +/// The share of the members with private-Y data above which the code treats a position as +/// **shared by the population**, and not as private. +/// +/// A variant that most of a cohort carries did not start on one branch of that cohort. On +/// R1b-CTS4466Plus, *all* 111 donors with private-Y data carried five positions. Those positions +/// are differences between the reference and the population. They are real, but they are not +/// private. /// -/// A variant carried by most of a cohort did not arise on one branch of it. On R1b-CTS4466Plus five -/// positions were carried by *all* 111 donors with private-Y — those are reference-vs-population -/// differences, real but not private, and the bundled cohort-shared blocklist (derived from a -/// 3,352-sample CHM13 cohort that predates this collection) does not list them. Deriving the -/// exclusion from the cohort in hand catches what a bundled list can not anticipate. +/// The blocklist in the application bundle does not name them. That list comes from a CHM13 cohort +/// of 3,352 samples, and that cohort is older than this collection. A rule that reads the cohort in +/// the workspace finds what a fixed list can not. const COHORT_SHARED_FRACTION: f64 = 0.25; -/// Donors required before the frequency rule engages at all. +/// The count of donors that the frequency rule needs before it applies. +/// +/// The rule examines what a *large* population shares. A candidate branch needs two carriers, by +/// definition. So in a small cohort, two carriers are already a large share. With four donors, each +/// true branch goes past a limit of 25% and the code removes it. /// -/// The rule reasons about what a *large* population shares. A candidate branch needs two carriers by -/// definition, so in a small cohort two carriers are already a large share — at four donors, every -/// genuine branch would exceed a 25% ceiling and be thrown away. Below this many donors there is no -/// population to argue from, so the rule abstains rather than guessing. +/// Below this count of donors there is no population for the rule to examine. So the rule does +/// nothing, and it makes no estimate. const COHORT_SHARED_MIN_DONORS: usize = 20; /// Positions carried by more than [`COHORT_SHARED_FRACTION`] of the members that have private-Y. @@ -313,15 +343,18 @@ fn population_shared_positions(blocks: &[Block], private: &HashMap) -> BTreeSet { let defining: Vec = candidate_defining_positions(blocks, private).into_iter().collect(); defining @@ -361,12 +395,14 @@ fn clustered_candidate_positions(blocks: &[Block], private: &HashMap) -> BTreeSet { let mut blocks_per_position: HashMap> = HashMap::new(); for block in blocks { @@ -393,26 +429,32 @@ fn recurrent_positions(blocks: &[Block], private: &HashMap, private: &HashMap, ) -> (Vec, usize, usize) { - // Computed across all blocks before any group is accepted — a position defining branches under - // two parents is disqualified everywhere, not just wherever it happens to be seen second. + // The code calculates this set across each block, before it accepts any group. A position + // that defines a branch below two parents fails everywhere. It does not fail only at the + // second place where the code reads it. let recurrent: BTreeSet = recurrent_positions(&blocks, private) .into_iter() .chain(population_shared_positions(&blocks, private)) @@ -427,7 +469,8 @@ pub(crate) fn insert_candidate_branches( out.push(block); continue; } - // position → the members of *this block* carrying it, keyed by index into `block.members`. + // A map from a position to the members of *this block* that hold it. The key of a member + // is its index in `block.members`. let mut carriers: HashMap> = HashMap::new(); for (i, m) in block.members.iter().enumerate() { let Some(bucket) = private.get(&m.guid) else { continue }; @@ -451,8 +494,9 @@ pub(crate) fn insert_candidate_branches( continue; } - // Largest first, so a broader branch is accepted before the finer ones nested inside it. - // Ties broken deterministically: more shared variants, then lowest position. + // Take the largest group first, so the code accepts a wide branch before the finer + // branches inside it. Two groups of the same size compare on the count of shared variants, + // and then on the lowest position. So the order is always the same. let mut ordered: Vec<(BTreeSet, Vec)> = groups.into_iter().collect(); for (_, positions) in &mut ordered { positions.sort_unstable(); @@ -490,7 +534,7 @@ pub(crate) fn insert_candidate_branches( .map(|(_, (_, _, id))| *id) }; - // Each member goes to the smallest accepted set containing it. + // Each member goes to the smallest accepted set that holds it. let owner: HashMap = (0..block.members.len()) .filter_map(|m| { accepted @@ -598,15 +642,18 @@ pub(crate) fn insert_candidate_branches( /// Fold runs of **member-less single-child** branches into the branch below them, when the run is at /// least `min_run` long. /// -/// An induced subtree over a deep haplotree is mostly such chains: intermediate branches no member -/// sits on that split nothing within this cohort. Merging them is not a display trick — within this -/// cohort those branches genuinely are one undivided block, so the absorbed loci join the survivor's -/// own (root-most first) and the absorbed names are kept in [`Block::collapsed`]. +/// A subtree over a deep haplotree holds mostly such chains. Those are the branches that no member +/// sits on, and that divide nothing inside this cohort. /// -/// `subtree_members` survives untouched: an absorbed node has no members of its own and exactly one -/// child, so its count already equals the survivor's. +/// The join is not only a change to the display. Inside this cohort those branches are one block +/// that nothing divides. So the loci of an absorbed branch go to the branch that stays, with the +/// loci nearest the root first. [`Block::collapsed`] keeps the names of the absorbed branches. /// -/// Pure — no tree, no I/O — so it is unit-testable on a hand-built `Vec`. +/// The function does not change `subtree_members`. An absorbed node has no member of its own and +/// has one child. So its count is already the count of the branch that stays. +/// +/// The function is pure. It reads no tree and does no I/O, so a unit test can call it with a +/// `Vec` that the test builds. pub(crate) fn collapse_blocks(blocks: Vec, min_run: usize) -> Vec { if blocks.is_empty() || min_run == 0 { return blocks; @@ -626,8 +673,9 @@ pub(crate) fn collapse_blocks(blocks: Vec, min_run: usize) -> Vec let mut by_id: HashMap = blocks.iter().map(|b| (b.node_id, b.clone())).collect(); let mut absorbed: HashSet = HashSet::new(); - // `blocks` is pre-order, so a run's root-most node is always reached first. A node whose parent - // is itself absorbable is therefore mid-run and already handled by the run's head. + // The `blocks` list is in pre-order, so the code always reaches the node of a run that is + // nearest the root first. A node whose parent the code can also absorb is in the middle of a + // run. So the head of that run already covers it. for b in &blocks { if !absorbable(b) { continue; @@ -671,8 +719,9 @@ pub(crate) fn collapse_blocks(blocks: Vec, min_run: usize) -> Vec target.parent = head_parent; } - // Re-emit in the original pre-order minus the absorbed nodes, depth recomputed against the - // surviving parents. Rewiring only ever moves a node *up* to an ancestor, so pre-order holds. + // Write the list again in the first pre-order, with no absorbed node. The code calculates each + // depth against the parents that stay. A change to a link only moves a node *up* to an + // ancestor, so the list stays in pre-order. let mut depth: HashMap = HashMap::new(); let mut out = Vec::with_capacity(blocks.len() - absorbed.len()); for b in &blocks { @@ -764,10 +813,12 @@ mod tests { /// └─> D(smith) /// ``` /// - /// A run of exactly two member-less single-child branches (`A`, `B`) above a placed one. `D` - /// keeps `root` a branch point, so the run's head is `A` — otherwise `root` would be absorbable - /// too and the whole spine would fold (which is correct, and is what - /// `collapse_stops_a_run_at_a_placed_branch` covers). + /// A run of two branches, `A` and `B`, above a branch with a member. Each branch in the run + /// has one child and no member. + /// + /// The member `D` keeps `root` a point where the tree divides. So the head of the run is `A`. + /// Without `D`, the code could also absorb `root`, and the full line would become one block. + /// That result is correct, and `collapse_stops_a_run_at_a_placed_branch` covers it. fn chain() -> Vec { let mut b = vec![ block(1, "root", 0, &[]), @@ -814,7 +865,7 @@ mod tests { #[test] fn collapse_respects_min_run() { - // With a threshold of 3, the run of 2 is left alone. + // With a limit of 3, the code does not change the run of 2. let out = collapse_blocks(chain(), 3); let names: Vec<&str> = out.iter().map(|b| b.name.as_str()).collect(); assert_eq!(names, vec!["root", "A", "B", "C", "D"]); @@ -897,7 +948,8 @@ mod tests { assert!(cand.name.is_empty(), "the view localizes a candidate's label"); assert_eq!(cand.parent, Some(1)); assert_eq!(cand.depth, 1); - // Both shared positions are equivalent on this branch — one block, two loci. + // The two shared positions are equivalent on this branch. They give one block with two + // loci. let mut pos: Vec = cand.loci.iter().map(|l| l.position).collect(); pos.sort_unstable(); assert_eq!(pos, vec![100_000, 200_000]); @@ -926,8 +978,9 @@ mod tests { #[test] fn structural_region_and_off_path_calls_never_form_a_branch() { let b = block(1, "R-X", 0, &["kane", "smith"]); - // Both men "share" a palindrome call and a known off-path SNP. Neither is evidence of a - // shared ancestor — the first is a paralog artefact, the second an existing branch. + // The two men "share" a call in a palindrome and a known SNP that is off the path. + // Neither call shows a shared ancestor. The first is a paralog artefact. The second marks + // a branch that already exists. let shared = PrivateBucket { terminal: "R-X".into(), variants: vec![ @@ -943,7 +996,8 @@ mod tests { #[test] fn nested_sharing_nests_the_candidate_branches() { let b = block(1, "R-X", 0, &["a", "b", "c"]); - // All three share 100; a and b also share 200 — a finer branch inside the broader one. + // Each of the three members shares position 100. Members a and b also share position 200, + // which gives a finer branch inside the wider one. let p = privates( &b, &[ @@ -971,7 +1025,7 @@ mod tests { let mut fine_names: Vec<&str> = fine.members.iter().map(|m| m.name.as_str()).collect(); fine_names.sort_unstable(); assert_eq!(fine_names, vec!["a", "b"]); - // Pre-order: a parent is emitted before its child. + // The list is in pre-order, so a parent comes before its child. let pos = |id: i64| out.iter().position(|x| x.node_id == id).unwrap(); assert!(pos(broad.node_id) < pos(fine.node_id)); } @@ -979,8 +1033,9 @@ mod tests { #[test] fn overlapping_non_nested_sharing_is_counted_as_a_conflict_not_forced() { let b = block(1, "R-X", 0, &["a", "b", "c"]); - // {a,b} share 100; {b,c} share 200. Neither set contains the other, so they can not both be - // branches of one tree — the smaller-ranked one is dropped and counted. + // The set {a,b} shares position 100, and the set {b,c} shares position 200. Neither set + // holds the other. So one tree can not hold both as a branch. The code removes the set with + // the lower rank and counts it. let p = privates( &b, &[bucket(&[100_000]), bucket(&[100_000, 200_000]), bucket(&[200_000])], @@ -997,8 +1052,8 @@ mod tests { #[test] fn a_member_with_no_computed_private_y_is_simply_not_grouped() { let b = block(1, "R-X", 0, &["kane", "smith"]); - // Only `kane` has a bucket at all — `smith` was never analyzed, which is not the same as - // having no private variants. + // Only `kane` has a bucket. No analysis ran for `smith`, and that state is not the same + // as a subject with no private variant. let p: HashMap = [(b.members[0].guid, bucket(&[100]))].into_iter().collect(); let (out, _, _) = insert_candidate_branches(vec![b], &p); assert_eq!(out.len(), 1); @@ -1009,8 +1064,9 @@ mod tests { #[test] fn clustered_calls_are_dropped_whole() { - // Six "novel" calls inside 32 bp is one misaligned read, not six mutations — the shape that - // produced most of the first candidate branches on the CTS4466 cohort. + // Six "new" calls inside 32 bp come from one read that the mapper placed wrongly. They are + // not six mutations. This shape produced most of the first candidate branches on the + // CTS4466 cohort. let cluster = bucket(&[16342231, 16342238, 16342245, 16342253, 16342258, 16342263]); assert!( candidate_positions(&cluster).is_empty(), @@ -1025,9 +1081,9 @@ mod tests { #[test] fn a_position_defining_branches_under_two_parents_is_rejected() { - // 11311865 was shared by two members under one block *and* two under another. A variant that - // arose twice can not mark a new branch, and the laminar check can't see it — it reasons - // inside a single block. + // Two members below one block shared position 11311865, and two members below another + // block also shared it. A variant that occurred two times can not mark a new branch. The + // laminar test can not find this state, because it examines one block only. let mut left = block(1, "R-A", 0, &["a", "b"]); left.subtree_members = 2; let mut right = block(2, "R-B", 0, &["c", "d"]); @@ -1068,9 +1124,9 @@ mod tests { #[test] fn candidates_clustering_across_the_cohort_are_all_rejected() { - // The 56.83 Mb case: three branches with *different* member sets inside 567 bp. Three - // independent lineage events in under a kilobase is not a thing; one repeat unit - // mis-mapping across several donors is. + // The case at 56.83 Mb. Three branches with *different* member sets are inside 567 bp. + // Three separate lineage events inside one kilobase do not occur. One repeat unit that the + // mapper places wrongly across some donors does occur. let mut a = block(1, "R-A", 0, &["a", "b"]); a.subtree_members = 2; let mut b = block(2, "R-B", 0, &["c", "d"]); @@ -1108,8 +1164,8 @@ mod tests { // ---- export --------------------------------------------------------------- - /// A two-block tree with one candidate branch and one unplaced member — enough to exercise - /// every column the export has to get right. + /// A tree with two blocks, one candidate branch, and one member with no placement. This shape + /// covers each column that the export must write correctly. fn exportable() -> ProjectBlockTree { let mut named = block(1, "R-X", 0, &["kane"]); named.subtree_members = 3; diff --git a/crates/navigator-app/src/commands.rs b/crates/navigator-app/src/commands.rs index e84e14a6..d4140d32 100644 --- a/crates/navigator-app/src/commands.rs +++ b/crates/navigator-app/src/commands.rs @@ -34,15 +34,22 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("project {id}")))) } - /// Delete a project, detaching its members first. Subjects are first-class and shared across - /// projects, so deleting the grouping keeps the subjects — it only removes their membership in - /// this project (and clears the legacy home column for subjects homed here). + /// Delete a project, and remove its members from it first. + /// + /// A subject is an independent record, and many projects can hold the same subject. So a delete + /// of the project keeps each subject. It removes only the membership of that subject in this + /// project. It also clears the old home column of a subject whose home is this project. pub async fn delete_project(&self, id: i64) -> Result<(), AppError> { - // A project is only a grouping — subjects are first-class and shared across projects. Deleting - // it detaches its members (drops the M:N memberships + clears the legacy home column for - // subjects homed here) and removes the project; the subjects themselves remain in the - // workspace. This lets a mis-targeted import be undone: delete the project and re-import - // cleanly, rather than being stuck because "N subjects still belong to it". + // A project is only a group. A subject is an independent record, and many projects can + // hold the same subject. + // + // A delete does three steps. It removes each membership from the M:N table. It clears the + // old home column of a subject whose home is this project. It then removes the project. + // Each subject stays in the workspace. + // + // So a user can undo an import that went to the wrong project. The user deletes the project + // and imports again. Without this behaviour, the message "N subjects still belong to it" + // stops the user. biosample_project::remove_all_for_project(self.store.pool(), id).await?; biosample::clear_home_project(self.store.pool(), id).await?; if !project::delete(self.store.pool(), id).await? { @@ -51,9 +58,9 @@ impl App { Ok(()) } - /// Register a biosample, assigning its stable `SampleGuid` here (identity is an - /// app-layer decision, not the UI's). Verifies the target project exists first so - /// the caller gets a clear `NotFound` rather than a raw foreign-key error. + /// Add a biosample and give it a stable `SampleGuid` here. The app layer decides the identity, + /// and the UI does not. The method checks that the target project exists first. So the caller + /// receives a clear `NotFound` error and not a raw foreign-key error. pub async fn add_biosample( &self, project_id: Option, @@ -76,8 +83,9 @@ impl App { Ok(b) } - /// Update a subject's editable fields (identity, accession, description, center, sex). - /// Empty strings are normalized to NULL. Returns the updated record. + /// Change the fields of a subject that the user can edit. They are the identity, the + /// accession, the description, the center, and the sex. The method changes an empty string to + /// NULL. It returns the new record. pub async fn update_biosample( &self, guid: SampleGuid, @@ -111,7 +119,8 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("biosample {}", guid.0)))) } - /// Assign a subject to a project (validating the project exists). `None` clears it. + /// Add a subject to a project. The method checks that the project exists. A value of `None` + /// removes the subject from its project. pub async fn add_biosample_to_project(&self, guid: SampleGuid, project_id: Option) -> Result<(), AppError> { if let Some(pid) = project_id { if project::get(self.store.pool(), pid).await?.is_none() { @@ -124,9 +133,11 @@ impl App { Ok(()) } - /// Delete a subject. Refused (with a clear message) when it still has dependent data — - /// sequencing runs or any imported profile — so the user removes data first rather than - /// silently orphaning rows. + /// Delete a subject. + /// + /// The method refuses, and gives a clear message, when the subject still has data. That data is + /// a sequence run or an imported profile. So the user removes the data first. Without this + /// guard, the delete leaves rows with no subject and gives no message. pub async fn delete_biosample(&self, guid: SampleGuid) -> Result<(), AppError> { let runs = self.list_sequence_runs(guid).await?.len(); let strs = self.list_str_profiles(guid).await?.len(); @@ -140,9 +151,10 @@ impl App { {variants} variant-set, {chips} chip, {mt} mtDNA record(s) — remove its data first" ))); } - // The guard above ensures no runs/profiles remain; sweep any derived-only orphans - // (stale haplogroup/consensus/reconciliation/ancestry/IBD rows from an earlier - // incomplete delete) so removing the subject can never leave dangling rows. + // The guard above makes sure that no run and no profile stays. This step then removes + // each derived row with no owner. Such a row is an old haplogroup, consensus, + // reconciliation, ancestry, or IBD row from a delete that did not complete. So the delete + // of the subject can never leave a row behind. biosample::clear_data(self.store.pool(), guid).await?; if !biosample::delete(self.store.pool(), guid).await? { return Err(AppError::Store(StoreError::NotFound(format!("biosample {}", guid.0)))); @@ -157,10 +169,14 @@ impl App { Ok(created) } - /// A Y-targeted test (Big Y, Targeted Y, a Y-SNP pack, …) or any Y-STR profile is definitive - /// evidence of a male subject. Set the biosample's sex to "Male" when such data is present and - /// it is not already recorded as male. Best-effort and idempotent — safe to call after any run - /// or STR-profile import (it re-derives the verdict from the stored data each time). + /// A Y test or a Y-STR profile is proof that the subject is male. A Y test is a Big Y test, a + /// Targeted Y test, or a Y-SNP pack. + /// + /// The method sets the sex of the biosample to "Male" when such data exists and the record does + /// not already hold that value. + /// + /// The step is optional, and a second call is safe. Call it after any run import or STR-profile + /// import. It reads the stored data and decides again at each call. pub(crate) async fn assign_male_for_y_evidence(&self, guid: SampleGuid) -> Result<(), AppError> { use navigator_domain::testtype::{by_code, TargetType}; let has_y_test = self @@ -186,9 +202,9 @@ impl App { Ok(alignment::create(self.store.pool(), &aln).await?) } - /// Update a sequence run's descriptive fields (test type required; platform defaults to - /// "UNKNOWN" when blank; instrument/layout optional). Read metrics are preserved. Returns - /// the updated record. + /// Change the descriptive fields of a sequence run. The test type is necessary. A blank + /// platform becomes "UNKNOWN". The instrument and the layout are optional. The method keeps the + /// read metrics and returns the new record. pub async fn update_sequence_run( &self, id: i64, @@ -223,8 +239,9 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("sequence run {id}")))) } - /// Update an alignment's descriptive fields (reference build + aligner required; variant - /// caller optional). File paths are managed by import/probe. Returns the updated record. + /// Change the descriptive fields of an alignment. The reference build and the aligner are + /// necessary, and the variant caller is optional. The import step and the probe step control + /// the file paths. The method returns the new record. pub async fn update_alignment( &self, id: i64, @@ -245,16 +262,18 @@ impl App { self.alignment_or_err(id).await } - /// An alignment by id, or `None` if there is no such row. + /// The alignment with this id, or `None` when the store holds no such row. /// - /// Public because provenance made alignments something callers ask about directly — the UI - /// needs the row to say "realigned to hs1 from alignment #N" rather than just listing files. + /// This method is public because a caller now asks about one alignment directly. The provenance + /// feature caused that change. The UI needs the row to write "realigned to hs1 from alignment + /// #N". Before, it only listed the files. pub async fn alignment(&self, id: i64) -> Result, AppError> { Ok(alignment::get(self.store.pool(), id).await?) } - /// Fetch an alignment by id, mapping a missing row to a `NotFound` error. The standard way - /// the analysis/query methods resolve an `alignment_id` before touching its BAM/CRAM. + /// Read the alignment with this id, and change an absent row into a `NotFound` error. Each + /// analysis method and each query method uses this method to resolve an `alignment_id` before + /// it opens the BAM file or the CRAM file. pub(crate) async fn alignment_or_err(&self, id: i64) -> Result { alignment::get(self.store.pool(), id) .await? @@ -264,17 +283,23 @@ impl App { /// The alignment's BAM/CRAM path, confirmed to still resolve on disk. The standard way a read /// path turns an [`Alignment`] into a path to open. /// - /// Both checks belong together and belong *early*. A recorded path that no longer resolves is - /// routine in a long-lived workspace — vendor downloads get cleaned out, volumes get unmounted — - /// but nothing checked for it, so the failure surfaced as a bare `No such file or directory` - /// from inside the reader, after the caller had already fetched a multi-MB haplotree. Worse, one - /// caller read that io error as *the tree* being unavailable and fell back to the FTDNA tree, - /// which then failed on the same absent file: a misleading log line, a wasted download, and a - /// silent change of tree provider, all from a deleted BAM. + /// The two checks belong together, and they belong *early*. + /// + /// A recorded path that no longer points to a file is normal in a workspace with a long life. A + /// user removes old vendor downloads, and a user disconnects a volume. + /// + /// No code checked for that state. So the fault appeared as a plain `No such file or directory` + /// error from inside the reader. By that point the caller had already downloaded a haplotree of + /// many MB. /// - /// The existence check races anything that deletes the file a microsecond later; that is fine. - /// It is here to name the common case correctly and cheaply, not to make opening infallible — - /// callers still handle a read error from the open itself. + /// The result was worse than one unclear message. One caller read that io error as an absent + /// *tree*. It then used the FTDNA tree, and that read failed on the same absent file. One + /// deleted BAM file gave an incorrect log line, a download with no purpose, and a change of + /// tree provider with no message. + /// + /// Another process can delete the file directly after this check. That result is acceptable. + /// The check names the common case correctly and at a low cost. It does not make the open + /// operation safe, and each caller still handles a read error from that operation. pub(crate) fn alignment_file(aln: &Alignment) -> Result { let path = aln.bam_path.clone().ok_or(AppError::MissingPaths(aln.id))?; let p = PathBuf::from(&path); @@ -287,8 +312,9 @@ impl App { /// Delete a sequence run and everything beneath it (its alignments + cached analysis /// artifacts). This is how a mistaken BAM/CRAM import is undone. pub async fn delete_sequence_run(&self, id: i64) -> Result<(), AppError> { - // Capture the run's subject + alignments before the cascade so we can purge any derived - // haplogroup/consensus data keyed on those alignments (it would otherwise go stale). + // Read the subject and the alignments of the run before the cascade. The code then + // removes each derived haplogroup row and consensus row with a key on those alignments. + // Without this step, those rows stay and become incorrect. let biosample = sequence_run::get(self.store.pool(), id) .await? .map(|r| r.biosample_guid); @@ -306,10 +332,15 @@ impl App { Ok(()) } - /// Merge `secondary` sequence run into `primary` (both must belong to `biosample_guid`): - /// reparent the secondary run's alignments onto the primary, then delete the now-empty secondary - /// (its analysis artifacts travel with the alignments — they are alignment-keyed). Destructive + - /// irreversible. Returns the number of alignments moved. + /// Join the `secondary` sequence run to the `primary` run. Both runs must belong to + /// `biosample_guid`. + /// + /// The method moves each alignment of the secondary run to the primary run. It then deletes the + /// secondary run, which is now empty. Each analysis artifact moves with its alignment, because + /// the key of an artifact is the alignment. + /// + /// This method destroys data, and the user can not undo it. It returns the count of the + /// alignments that it moved. pub async fn merge_sequence_runs( &self, biosample_guid: SampleGuid, @@ -335,14 +366,17 @@ impl App { count += 1; } } - // The secondary is now empty; delete it (cascade is a no-op for alignments — already moved). + // The secondary run is now empty, so delete it. The cascade does nothing for the + // alignments, because the code already moved them. sequence_run::delete(self.store.pool(), secondary).await?; Ok(count) } - /// Delete a single alignment and its cached analysis artifacts (the parent run is kept). + /// Delete one alignment and each analysis artifact in its cache. The method keeps the parent + /// run. pub async fn delete_alignment(&self, id: i64) -> Result<(), AppError> { - // Resolve the subject (via run) before deleting, to purge derived haplogroup/consensus data. + // Find the subject through the run before the delete. The code then removes each derived + // haplogroup row and consensus row. let biosample = match alignment::get(self.store.pool(), id).await? { Some(a) => sequence_run::get(self.store.pool(), a.sequence_run_id) .await? @@ -358,53 +392,73 @@ impl App { Ok(()) } - /// Remove derived data keyed on now-deleted alignments: each alignment's Y + mt haplogroup calls - /// (`aln:` / `aln::mt`), and the subject's genome-level consensus profiles + painting - /// (Y/mt/Auto), which were pooled from sources that may no longer exist. The consensus is - /// recomputable on demand; clearing it makes the displayed haplogroup fall back to reconciling the - /// remaining cached calls (or nothing), rather than showing a stale placement. A user manual - /// override is left intact. + /// Remove the derived data whose key is an alignment that the app deleted. + /// + /// That data is the Y haplogroup call and the mt haplogroup call of each alignment, which use + /// the keys `aln:` and `aln::mt`. It is also the genome-level consensus profiles and + /// the painting of the subject, for Y, mt, and Auto. The app pooled those results from sources + /// that can now be absent. + /// + /// The app can calculate a consensus again at any time. After this method clears it, the + /// displayed haplogroup comes from the cached calls that remain, or from nothing. Without this + /// step, the app shows an old placement. + /// + /// The method keeps a value that the user set. async fn purge_alignment_derived(&self, biosample: SampleGuid, alignment_ids: &[i64]) -> Result<(), AppError> { let pool = self.store.pool(); for &aln in alignment_ids { haplogroup_call::delete_one(pool, biosample, DnaType::Y, &format!("aln:{aln}")).await?; haplogroup_call::delete_one(pool, biosample, DnaType::Mt, &format!("aln:{aln}:mt")).await?; - // The per-alignment ancestry estimates die with the alignment. + // The ancestry estimate of each alignment goes with that alignment. ancestry_result::delete_for_alignment(pool, aln).await?; } for dna in ["Y", "Mt", "Auto"] { consensus_profile::delete(pool, biosample, dna).await?; } - // Every signature-keyed cache, from the one list — this used to name three of the four by - // hand and leave the Tier-B archaic segments behind, still keyed to a deleted alignment. + // Each signature-keyed cache, from the one list. Before this list, the code named three + // of the four caches by hand. It left the Tier-B archaic segments in the store, with a key + // on an alignment that the app had deleted. for cache in sig_cache::ALL { cache.delete(pool, biosample).await?; } - // The audit log describes the consensus we just wiped; clear it so deleting the last run - // can't leave a stale RUN_RECORDED history pointing at gone alignments. It is re-appended - // when the consensus is next rebuilt from any remaining calls. + // The audit log describes the consensus that this method removed. Clear the log also. + // Without that step, a delete of the last run leaves an old RUN_RECORDED entry that names + // absent alignments. The app writes the log again at the next rebuild of the consensus, + // from the calls that remain. recon_store::clear_audit(pool, biosample, DnaType::Y).await?; recon_store::clear_audit(pool, biosample, DnaType::Mt).await?; Ok(()) } - /// Reset a subject's analysis: clear **all** sequencing + derived/imported data (runs, - /// alignments, cached artifacts, Y/mt haplogroups + consensus + reconciliation, ancestry, IBD - /// results, and chip/STR/variant/mtDNA profiles) while keeping the subject itself — its - /// identity (name/sex/center), vendor IDs, project memberships, and MDKA genealogy. The - /// recovery tool for a botched import: clears orphaned/garbage rows so the subject can be - /// re-imported cleanly. Atomic ([`biosample::clear_data`] runs in one transaction). + /// Reset the analysis of a subject. The method clears **all** sequence data and each derived + /// or imported result. + /// + /// It removes the runs, the alignments, and the cached artifacts. It removes the Y and mt + /// haplogroups with their consensus and reconciliation rows. It also removes the ancestry, the + /// IBD results, and the chip, STR, variant, and mtDNA profiles. + /// + /// It keeps the subject. It also keeps the identity of that subject, which is the name, the + /// sex, and the center. It keeps the vendor IDs, the project memberships, and the MDKA + /// genealogy. + /// + /// This method is the recovery tool for an import that went wrong. It removes each row with no + /// owner, so the user can import the subject again. The work is atomic, because + /// [`biosample::clear_data`] runs in one transaction. pub async fn clear_biosample_data(&self, guid: SampleGuid) -> Result<(), AppError> { biosample::clear_data(self.store.pool(), guid).await?; - // Imported external autosomal call-set dosages live in their own table (outside the - // biosample cascade) — drop them too so a cleared subject starts truly empty. + // The dosages of an imported external autosomal call set are in their own table, outside + // the cascade of the biosample. Remove them also, so a subject that the user clears holds + // nothing. navigator_store::external_panel_dosage::delete_for_biosample(self.store.pool(), guid).await?; Ok(()) } - /// Reset only the subject's haplogroup placement (calls + consensus + override/audit, Y & mt), - /// keeping coverage/ancestry/imported data. Drops a stale legacy lineage so re-analysis re-places - /// it; the placement repopulates on the next full analysis (WGS) or re-import (vendor data). + /// Reset only the haplogroup placement of the subject. That placement is the calls, the + /// consensus, and the override and audit rows, for Y and for mt. + /// + /// The method keeps the coverage, the ancestry, and the imported data. It removes an old + /// lineage, so the next analysis places the subject again. The placement returns at the next + /// full analysis of a WGS sample, or at the next import of vendor data. pub async fn clear_haplogroup_data(&self, guid: SampleGuid) -> Result<(), AppError> { biosample::clear_haplogroup_data(self.store.pool(), guid).await?; Ok(()) @@ -457,10 +511,13 @@ impl App { .await } - /// Like [`save_analysis`] but stamps provenance: `source` (`navigator-walk` | - /// `pipeline-sidecar`) and `completeness` (`full` | `partial`). The fast-path sidecar - /// ingest uses this so the manual deep pass can tell a sidecar/partial result apart from a - /// full walk and upgrade it rather than skip it. + /// The same work as [`save_analysis`], but the method also writes the provenance. The + /// provenance is `source`, which is `navigator-walk` or `pipeline-sidecar`, and `completeness`, + /// which is `full` or `partial`. + /// + /// The fast-path sidecar import uses this method. The manual deep pass can then see the + /// difference between a partial sidecar result and a full walk. It replaces the partial result + /// and does not skip it. pub async fn save_analysis_with_provenance( &self, alignment_id: i64, @@ -488,12 +545,18 @@ impl App { .await?) } - /// Like [`save_analysis_with_provenance`] but refuses to **downgrade** an existing artifact: - /// if a result is already stored for this `(kind, version)` whose completeness is at least the - /// incoming one (e.g. a full `navigator-walk` scan vs an incoming `partial` sidecar), the - /// existing artifact is kept untouched. The fast-path sidecar ingest uses this so re-importing - /// a project folder can't clobber real deep scans with lite sidecar stats. Returns whether the - /// write actually happened (`false` = kept the existing, equal-or-fuller result). + /// The same work as [`save_analysis_with_provenance`], but the method never replaces a better + /// artifact with a worse one. + /// + /// The store can already hold a result for this `(kind, version)` pair. When the completeness + /// of that result is the same as the new one, or higher, the method keeps the stored artifact. + /// One example is a full `navigator-walk` scan against a new `partial` sidecar result. + /// + /// The fast-path sidecar import uses this method. So a second import of a project folder can + /// not replace a real deep scan with the smaller statistics of a sidecar. + /// + /// The method returns `true` when it wrote the artifact. It returns `false` when it kept the + /// stored result, which was the same or better. pub async fn save_analysis_no_downgrade( &self, alignment_id: i64, @@ -513,10 +576,15 @@ impl App { Ok(true) } - /// Persist a marker that a Navigator walk failed for this alignment (e.g. an undecodable / - /// corrupt CRAM). Stored as the `error`/`"1"` artifact so the project report can surface a - /// "Failed" cell instead of a silent blank; cleared by [`clear_analysis_error`] on the next - /// successful walk. Best-effort — a failure to record the marker is swallowed (it is diagnostic). + /// Write a mark that shows a failed Navigator walk for this alignment. One cause is a CRAM + /// file that the reader can not decode. + /// + /// The store holds the mark as the `error` artifact with the value `"1"`. The project report + /// then shows a "Failed" cell and not an empty cell. [`clear_analysis_error`] removes the mark + /// after the next good walk. + /// + /// The step is optional. The code hides a failure to write the mark, because the mark is only a + /// diagnostic. pub async fn record_analysis_error(&self, alignment_id: i64, step: &str, message: &str) { let mut message = message.to_string(); message.truncate(500); // keep the payload small; the head carries the cause @@ -543,17 +611,23 @@ impl App { } } - /// The alignment's source-file signature (`mtime:size`) for cache staleness. `None` when the - /// alignment / its path is gone or unstattable — then the cache is trusted (nothing to - /// recompute against). Cheap: a metadata stat, no file read (content hashing is the separate, - /// deferred federation-identity path). + /// The signature of the source file of the alignment, as `mtime:size`. The code uses it to + /// find an old cache entry. + /// + /// The value is `None` when the alignment is absent, when its path is absent, or when the + /// operating system can not read the metadata. The code then trusts the cache, because it has + /// no value to compare. + /// + /// The call is fast. It reads the metadata and does not read the file. The content hash is a + /// separate path for the federation identity, and it runs later. async fn bam_source_sig(&self, alignment_id: i64) -> Option { let aln = alignment::get(self.store.pool(), alignment_id).await.ok().flatten()?; file_signature(Path::new(&aln.bam_path?)) } - /// `(source, completeness)` of a cached artifact, defaulting `None` columns to - /// `("navigator-walk", "full")` (pre-provenance rows). `None` when no artifact exists. + /// The `(source, completeness)` pair of a cached artifact. A `None` column becomes + /// `("navigator-walk", "full")`, because a row from before the provenance change holds no + /// value. The method returns `None` when no artifact exists. pub async fn analysis_provenance( &self, alignment_id: i64, @@ -579,8 +653,9 @@ impl App { ) -> Result, AppError> { match artifact::get(self.store.pool(), alignment_id, kind, algorithm_version).await? { Some(a) => { - // Treat a cached result as a miss when the source file changed since it was computed - // (BAM-mtime invalidation) — the caller then recomputes + re-stamps it. + // Treat a cached result as absent when the source file changed after the + // calculation. The mtime of the BAM file shows that change. The caller then + // calculates the result again and writes a new signature. let current = self.bam_source_sig(alignment_id).await; if !artifact_is_fresh(a.source_sig.as_deref(), current.as_deref()) { return Ok(None); diff --git a/crates/navigator-app/src/llm.rs b/crates/navigator-app/src/llm.rs index c950e10f..b6bf7e97 100644 --- a/crates/navigator-app/src/llm.rs +++ b/crates/navigator-app/src/llm.rs @@ -1,10 +1,16 @@ -//! Local-LLM client (OpenAI-compatible): configuration + resolvers, health/model discovery (M0), -//! and brief narration via chat completions (M1). +//! The client for a local LLM, in the OpenAI-compatible form. //! -//! The entire feature is **local-only** by design (see `documents/design/local-llm-integration.md`): -//! Navigator is a *client* of a model server the user runs (LM Studio / Ollama / llama.cpp). There is -//! no hosted-provider path and no API key. The transport is the OpenAI Chat Completions wire format, -//! the common denominator across local runtimes, spoken over the app's existing `reqwest` client. +//! This module holds the configuration and its resolvers. It also holds the health check with the +//! model list (M0), and the narration of a brief through chat completions (M1). +//! +//! The full feature is **local only**, by design. See +//! `documents/design/local-llm-integration.md`. +//! +//! Navigator is a *client* of a model server that the user runs. That server is LM Studio, Ollama, +//! or llama.cpp. There is no path to a hosted provider, and there is no API key. +//! +//! The transport is the OpenAI Chat Completions wire format, which each local runtime accepts. The +//! module sends each request with the `reqwest` client of the app. use crate::{App, AppError, AppSettings}; use navigator_domain::brief::SubjectBrief; @@ -16,19 +22,24 @@ use navigator_domain::results_context::{ use navigator_refgenome::cache as refgenome_cache; use serde::{Deserialize, Serialize}; -/// LM Studio's default OpenAI-compatible base URL — the happy-path local server. +/// The default OpenAI-compatible base URL of LM Studio. That server is the one that most users +/// run. pub const DEFAULT_LLM_BASE_URL: &str = "http://localhost:1234/v1"; -/// Default max response tokens. Generous so a reasoning model has room for its full chain-of-thought -/// plus the answer (a small cap is consumed entirely by reasoning and `content` comes back empty). -/// It is a ceiling, not a target — non-reasoning models stop well before it. +/// The default maximum count of response tokens. +/// +/// The value is large, so a model that reasons has space for each internal step and for the answer. +/// With a small value, the internal steps use every token and the `content` field comes back empty. +/// +/// The value is a limit and not a target. A model that does not reason stops long before it. pub const DEFAULT_LLM_MAX_TOKENS: u32 = 8192; /// Resolved local-LLM configuration (env → settings → default, like the other resolvers). #[derive(Debug, Clone, PartialEq, Eq)] pub struct LlmConfig { pub enabled: bool, - /// Base URL including the OpenAI-compatible path prefix (e.g. `.../v1`), no trailing slash. + /// The base URL with the OpenAI-compatible path prefix, such as `.../v1`. It must not end with + /// a slash. pub base_url: String, /// Model id to request, or `None` to let the server use its single loaded model. pub model: Option, @@ -61,7 +72,7 @@ fn resolve_max_tokens(env: Option, settings: Option) -> u32 { .unwrap_or(DEFAULT_LLM_MAX_TOKENS) } -/// The configured local-LLM settings, honoring `NAVIGATOR_LLM_*` over the persisted values. +/// The settings of the local LLM. A `NAVIGATOR_LLM_*` variable has priority over a stored value. pub fn llm_config() -> LlmConfig { let s = AppSettings::load(); LlmConfig { @@ -72,13 +83,17 @@ pub fn llm_config() -> LlmConfig { } } -/// Is `base_url`'s host a loopback address? Drives the Settings warning when a user points the client -/// at a non-local server (results would leave the machine). Conservative: anything we can't confirm -/// is loopback is treated as remote. +/// Shows whether the host of `base_url` is a loopback address. +/// +/// The Settings screen uses this result. It gives a warning when the user points the client at a +/// server on another machine. The results then cross the network. +/// +/// The test is careful. It treats each host that it can not confirm as a loopback address as a +/// remote host. pub fn is_loopback_url(base_url: &str) -> bool { let after_scheme = base_url.split_once("://").map(|(_, r)| r).unwrap_or(base_url); let authority = after_scheme.split(['/', '?', '#']).next().unwrap_or(""); - // Strip an IPv6 bracket or a trailing :port to get the bare host. + // Remove an IPv6 bracket, and remove a `:port` at the end, to get the host alone. let host = if let Some(rest) = authority.strip_prefix('[') { rest.split(']').next().unwrap_or("") } else { @@ -106,7 +121,8 @@ struct ChatMessage { content: String, } -/// One prior turn of an "ask my results" conversation, carried by the UI and replayed as context. +/// One earlier turn of an "ask my results" conversation. The UI holds it and sends it again as +/// context. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ChatTurn { pub from_user: bool, @@ -120,9 +136,13 @@ struct ChatRequest { temperature: f32, max_tokens: u32, stream: bool, - /// llama.cpp/LM Studio Jinja-template kwargs. We pass `{"enable_thinking": false}` so reasoning - /// models (Gemma 4, Qwen, DeepSeek-R1) never emit a thinking channel — saving the tokens/latency - /// `strip_reasoning` would otherwise discard. Skipped from the body when unset. + /// The Jinja-template arguments of llama.cpp and LM Studio. + /// + /// The app sends `{"enable_thinking": false}`. A model that reasons, such as Gemma 4, Qwen, or + /// DeepSeek-R1, then writes no internal channel. This setting saves the tokens and the time + /// that `strip_reasoning` would remove. + /// + /// The app leaves this field out of the body when it holds no value. #[serde(skip_serializing_if = "Option::is_none")] chat_template_kwargs: Option, } @@ -130,7 +150,8 @@ struct ChatRequest { /// One parsed SSE `data:` chunk from a streamed chat completion. struct StreamDelta { content: Option, - /// `"stop"` | `"length"` | … — `"length"` on a reasoning model means it never reached the answer. + /// The reason that the model stopped, such as `"stop"` or `"length"`. A value of `"length"` + /// from a model that reasons shows that the model never reached the answer. finish_reason: Option, model: Option, } @@ -156,18 +177,21 @@ fn parse_stream_event(data: &str) -> Option { }) } -/// Strip a reasoning model's chain-of-thought from the visible answer: drop any `…` -/// blocks (some servers inline reasoning in `content` this way) and anything before a closing -/// ``. Returns the trimmed final answer. +/// Remove the internal steps of a reasoning model from the answer that the user sees. +/// +/// The function removes each `…` block, because some servers put those steps in +/// `content`. It also removes each character before the last `` tag. It returns the final +/// answer with no space at the start and no space at the end. fn strip_reasoning(content: &str) -> String { - // Reasoning models emit a leading `…` block then the answer; keep only what - // follows the last close tag. + // A reasoning model writes a `…` block first, and the answer after it. Keep + // only the text after the last close tag. let after = match content.rfind("") { Some(pos) => &content[pos + "".len()..], None => content, }; - // A remaining opening tag means reasoning was truncated with no answer (it ran out of budget) → - // drop it so the result is empty and the caller reports the reasoning-ran-out error. + // An open tag with no close tag shows that the model stopped in the middle of its internal + // steps, with no answer. It used each available token. Remove the text, so the result is empty + // and the caller reports that fault. let answer = match after.find("") { Some(open) => &after[..open], None => after, @@ -175,8 +199,9 @@ fn strip_reasoning(content: &str) -> String { answer.trim().to_string() } -/// An AI-assisted narration of a [`SubjectBrief`], with the model that produced it (for labelling). -/// Always rendered *alongside* the structured cards, never instead of them. +/// A narration of a [`SubjectBrief`] from an AI model, with the name of that model for the label. +/// The UI always shows this text *beside* the structured cards. It never shows the text in place of +/// those cards. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct NarratedBrief { pub prose: String, @@ -201,9 +226,12 @@ impl App { self.llm_models_at(&cfg.base_url).await } - /// The on-disk cached narration for a brief, if one exists for the currently-configured model — - /// a **no-network** lookup (only when a model is explicitly set) used to fold the AI story into - /// the exported "DNA Story" without triggering generation. + /// The narration of a brief from the disk cache, when the cache holds one for the current + /// model. + /// + /// This lookup uses **no network**, and it runs only when the settings name a model. The + /// exported "DNA Story" uses it to add the AI text. The lookup never starts a new + /// generation. pub fn cached_narration(&self, brief: &SubjectBrief) -> Option { let cfg = llm_config(); let model = cfg.model?; // explicit only — avoid resolving the loaded model (a network call) @@ -215,9 +243,13 @@ impl App { .and_then(|s| serde_json::from_str(&s).ok()) } - /// Health check + model discovery against an explicit base URL — used by the Settings - /// "Test connection" button so the user can verify a URL *before* saving it. `GET {base}/models` - /// (the OpenAI-compatible discovery endpoint). Errors are plain-language for the UI. + /// Check the health of a server at an explicit base URL, and list its models. + /// + /// The "Test connection" button of the Settings screen calls this method. So the user can check + /// a URL *before* the app stores it. + /// + /// The method sends `GET {base}/models`, which is the OpenAI-compatible endpoint for a model + /// list. Each error message is in plain language for the UI. pub async fn llm_models_at(&self, base_url: &str) -> Result, AppError> { let base = base_url.trim().trim_end_matches('/'); if base.is_empty() { @@ -245,17 +277,26 @@ impl App { Ok(parsed.data.into_iter().map(|m| m.id).collect()) } - /// Build the subject's brief and narrate it via the local model (M1 entry point used by the UI). + /// Build the brief of the subject, and give it to the local model for a narration. The UI calls + /// this method, which is the entry point of M1. pub async fn narrate_subject(&self, guid: SampleGuid) -> Result { let brief = self.subject_brief(guid).await?; self.narrate_brief(&brief).await } - /// Ask the local model to rewrite the brief's facts as casual-reader prose. Grounded by - /// [`llm_prompt`] (facts-only, no health, preserve uncertainty); cached on disk keyed by the - /// fact sheet + model (so changing inputs or model regenerates, and re-opening is free). Returns - /// `Err(AppError::Llm)` on disabled / unreachable / bad / unsafe output — the UI then keeps the - /// deterministic brief unchanged. Never the *only* output the user sees. + /// Ask the local model to write the facts of the brief as prose for a reader who is not a + /// specialist. + /// + /// [`llm_prompt`] gives the model its instructions. Those instructions allow facts only. They + /// do not allow health content, and they keep each statement of uncertainty. + /// + /// The disk cache holds the result. The key is the fact sheet with the model. So a change to the + /// input or to the model gives a new narration, and a second view of the same brief costs + /// nothing. + /// + /// The method returns `Err(AppError::Llm)` when the feature is off, when the server does not + /// answer, or when the output is bad or unsafe. The UI then shows the fixed brief with no + /// change. This text is never the *only* output that the user sees. pub async fn narrate_brief(&self, brief: &SubjectBrief) -> Result { self.narrate_brief_streaming(brief, |_| {}).await } @@ -290,11 +331,15 @@ impl App { .await } - /// Explain a single result signal (M5 per-tab "Explain this") in plain language, streaming the - /// prose to `on_chunk`. Grounded in only that signal's curated section (see - /// [`navigator_domain::results_context::signal_section`]); cached and health-guarded like brief - /// narration. `Err` when the assistant is off / unreachable / the subject has nothing for that - /// signal — the UI then just does not show an explanation. + /// Explain one result signal in plain language, for the "Explain this" button of M5 on each + /// tab. The method sends the prose to `on_chunk` as the model writes it. + /// + /// The model receives only the section for that signal. See + /// [`navigator_domain::results_context::signal_section`]. The cache and the health guard work as + /// they do for the narration of a brief. + /// + /// The method returns `Err` when the assistant is off, when the server does not answer, or when + /// the subject has no data for that signal. The UI then shows no explanation. pub async fn narrate_signal_streaming( &self, guid: SampleGuid, @@ -315,12 +360,18 @@ impl App { .await } - /// Shared cached-streaming narration core for [`narrate_brief_streaming`] and - /// [`narrate_signal_streaming`]: cache-first (a hit replays the cached prose through `on_chunk`), - /// else stream a completion, reject health-straying output, and cache. The cache key is - /// `model + system + facts`, so changing the prompt, the facts, or the model regenerates rather - /// than serving a stale narration written under the old instructions. `subdir` separates brief - /// vs per-signal caches under `briefs/`. + /// The shared core that [`narrate_brief_streaming`] and [`narrate_signal_streaming`] call. + /// + /// The method reads the cache first. On a hit, it sends the stored prose through `on_chunk`. If + /// there is no hit, it reads a completion from the model, refuses output with health content, + /// and writes the result to the cache. + /// + /// The cache key is `model + system + facts`. So a change to the prompt, to the facts, or to the + /// model gives a new narration. The method never returns text that an older set of instructions + /// produced. + /// + /// `subdir` keeps the cache of a brief separate from the cache of each signal, below + /// `briefs/`. async fn run_cached_narration( &self, cfg: &LlmConfig, @@ -374,10 +425,17 @@ impl App { Ok(result) } - /// Answer one "ask my results" question, grounded in the subject's brief (M2). The brief's fact - /// sheet is the only source of facts; chat `history` is replayed for continuity. Off / unreachable - /// / bad output → `Err`. A clearly health/medical question (in or out) is met with a fixed - /// ancestry-only deflection instead of a model answer. + /// Answer one "ask my results" question from the brief of the subject (M2). + /// + /// The fact sheet of the brief is the only source of facts. The method sends the chat `history` + /// again, so the conversation continues. + /// + /// The method returns `Err` when the feature is off, when the server does not answer, or when + /// the output is bad. + /// + /// A clear medical question receives a fixed answer about the ancestry limits of the app, and + /// not an answer from the model. The method applies that rule to the question and to the + /// answer. pub async fn answer_question( &self, guid: SampleGuid, @@ -387,8 +445,9 @@ impl App { self.answer_question_streaming(guid, history, question, |_| {}).await } - /// Streaming variant of [`answer_question`]: visible answer text is sent to `on_chunk` as it - /// arrives (the final return value is authoritative). The scope-guard deflections do not stream. + /// The stream form of [`answer_question`]. The method sends each part of the answer to + /// `on_chunk` as it arrives, and the return value at the end has authority. A fixed answer from + /// the scope guard does not stream. pub async fn answer_question_streaming( &self, guid: SampleGuid, @@ -400,7 +459,8 @@ impl App { if !cfg.enabled { return Err(AppError::Llm("The AI assistant is turned off.".into())); } - // Incoming scope guard: do not even ask the model a medical question. + // The scope guard for the question. The app must not send a medical question to the + // model. if llm_prompt::mentions_health(&question) { return Ok(llm_prompt::health_deflection().to_string()); } @@ -434,24 +494,31 @@ impl App { let (answer, _) = self .chat_complete_streaming(&cfg, &model, messages, &mut on_chunk) .await?; - // Outgoing scope guard: a strayed answer is replaced by the deflection, not shown. + // The scope guard for the answer. The app replaces an answer that leaves the permitted + // subjects with the fixed text, and it never shows that answer. if llm_prompt::mentions_health(&answer) { return Ok(llm_prompt::health_deflection().to_string()); } Ok(answer) } - /// Assemble the broader grounding context for the M4 chat: the subject brief plus curated, - /// summary-level facts for the other signals (genetic sex, Y-STR panels, private-Y variants, - /// mtDNA mutations, IBD matches). Every signal is best-effort — a missing or un-run one is simply - /// omitted, so the chat grounds in whatever the subject actually has without erroring out. + /// Build the wider context for the M4 chat. That context is the brief of the subject and a + /// summary of each other signal. + /// + /// The signals are the genetic sex, the Y-STR panels, the private-Y variants, the mtDNA + /// mutations, and the IBD matches. + /// + /// Each signal is optional. The method leaves out a signal that is absent, and a signal that no + /// analysis produced. So the chat uses the data that the subject has, and the method gives no + /// error. pub async fn results_context(&self, guid: SampleGuid) -> Result { use navigator_analysis::mtvariants::MtRegion; use navigator_analysis::sex::{Confidence, InferredSex}; let brief = self.subject_brief(guid).await?; - // Genetic sex (needs an alignment; only a definite call is grounded). + // The genetic sex. This value needs an alignment, and the context holds only a definite + // call. let sex = match self.default_alignment_for_subject(guid).await? { Some((_, aln_id)) => self.cached_sex(aln_id).await?.and_then(|r| { let label = match r.inferred_sex { @@ -472,8 +539,9 @@ impl App { None => None, }; - // Y-STR panels — name + marker count only (never the raw values: token cost, no answerable - // gain, and they are lineage patterns not facts to recite). + // The Y-STR panels, as a name and a count of markers. The context never holds the values + // themselves. Those values cost many tokens, they answer no question, and they are patterns + // of a lineage and not facts for the model to repeat. let ystr: Vec = self .list_str_profiles(guid) .await @@ -485,7 +553,7 @@ impl App { }) .collect(); - // Private Y variants — the PrivateBucket confidence split. + // The private Y variants, as the confidence groups of the PrivateBucket type. let private_y = self.donor_private_y(guid).await?.map(|b| PrivateYFact { novel_unique: b.novel_in_unique_sequence(), off_path: b.off_path(), @@ -553,8 +621,9 @@ impl App { }) } - /// Resolve a concrete model id for a request — `cfg.model` if set, else the server's single - /// loaded model (servers like Ollama require a name). + /// Find the model id for a request. The method uses `cfg.model` when the settings hold it. If + /// not, it uses the one model that the server holds in memory. A server such as Ollama needs a + /// name. async fn resolve_model_id(&self, cfg: &LlmConfig) -> Result { match cfg.model.clone() { Some(m) => Ok(m), @@ -567,11 +636,15 @@ impl App { } } - /// Stream a chat completion (`stream: true`), forwarding visible answer text to `on_chunk` as it - /// arrives and returning the final `(answer, model)` with reasoning stripped. Reasoning is - /// suppressed live: only the post-`` answer streams (so a thinking model shows nothing - /// until it starts answering). Uses `Response::chunk()` (no extra dependency). Shared by - /// narration and Q&A; callers apply their own health guard. + /// Read a chat completion as a stream, with `stream: true`. The method sends each part of the + /// answer to `on_chunk` as it arrives. It returns the final `(answer, model)` pair, and it + /// removes the internal steps of the model from that answer. + /// + /// The method also removes those steps during the stream. Only the text after `` + /// reaches `on_chunk`. So a reasoning model shows nothing until it starts the answer. + /// + /// The method uses `Response::chunk()` and needs no other crate. The narration path and the + /// question path both call it, and each caller applies its own health guard. async fn chat_complete_streaming( &self, cfg: &LlmConfig, @@ -585,8 +658,9 @@ impl App { temperature: 0.4, max_tokens: cfg.max_tokens, stream: true, - // Grounded "explain my results" never needs chain-of-thought — disable it at the server - // so Gemma 4 et al. do not waste tokens/latency on a reasoning channel we'd discard. + // An "explain my results" answer never needs the internal steps of the model. Turn + // them off at the server. A model such as Gemma 4 then does not use tokens and time on + // a channel that the app removes. chat_template_kwargs: Some(serde_json::json!({ "enable_thinking": false })), }; let url = format!("{}/chat/completions", cfg.base_url.trim_end_matches('/')); @@ -595,7 +669,8 @@ impl App { .http .post(&url) .json(&req) - // Generous — a reasoning model on a large local model can take minutes to think + answer. + // The timeout is long. A large reasoning model on this machine can need some minutes + // for its internal steps and the answer. .timeout(std::time::Duration::from_secs(300)) .send() .await @@ -691,9 +766,10 @@ mod tests { fn strip_reasoning_keeps_final_answer() { assert_eq!(strip_reasoning("pondering…The answer."), "The answer."); assert_eq!(strip_reasoning(" plain answer "), "plain answer"); - // Reasoning closed, then a stray re-open with no answer. + // The model closed its internal steps, then opened a second block and gave no answer. assert_eq!(strip_reasoning("aanswer b"), "answer"); - // Unterminated reasoning (hit the cap) → nothing usable. + // The internal steps have no close tag, because the model used each available token. The + // result holds nothing that the app can use. assert_eq!(strip_reasoning("still thinking"), ""); } diff --git a/crates/navigator-app/src/queries.rs b/crates/navigator-app/src/queries.rs index 5c53c3c7..43f51515 100644 --- a/crates/navigator-app/src/queries.rs +++ b/crates/navigator-app/src/queries.rs @@ -2,20 +2,25 @@ //! 2026-06 simplification round; `use super::*` reaches the crate-root types + free helpers. use super::*; -/// Every analysis artifact of a set of alignments, pre-loaded and indexed for the report builders. +/// Each analysis artifact of a set of alignments. The code reads them in advance and indexes them +/// for the report builders. /// -/// Reading one cached result through [`App::load_analysis`] costs two queries — the artifact, then -/// the `alignment` row it needs to stat the BAM for staleness — plus that stat. A project -/// report reads five kinds per alignment for every member, so the per-cell form meant thousands of -/// round-trips to open one tab. This loads them all in a single `IN` query and stats each BAM once. +/// A read of one cached result through [`App::load_analysis`] costs two queries and one stat call. +/// The queries read the artifact and then the `alignment` row, and the code needs that row to stat +/// the BAM file. /// -/// The staleness rule is the same one [`App::load_analysis`] applies: a cached payload is a miss -/// when the source file's `mtime:size` has changed since it was computed. +/// A project report reads five kinds of artifact for each alignment of each member. So the earlier +/// form, one read for each cell, sent thousands of queries to open one tab. This type reads each +/// artifact in one `IN` query and stats each BAM file one time. +/// +/// The rule for an old result is the rule of [`App::load_analysis`]. A cached payload is absent when +/// the `mtime:size` value of the source file changed after the calculation. struct AlignmentArtifacts { /// `(alignment id, kind, algorithm version)` → the stored artifact. by_key: HashMap<(i64, String, String), AnalysisArtifact>, - /// Current source signature per alignment (`None` when the file is gone or unstattable, which - /// means "trust the cache" — there is nothing to recompute against). + /// The current source signature of each alignment. A value of `None` means that the file is + /// absent, or that the operating system can not read its metadata. The code then trusts the + /// cache, because it has no value to compare. sigs: HashMap>, } @@ -27,7 +32,8 @@ impl AlignmentArtifacts { .into_iter() .map(|a| ((a.alignment_id, a.kind.clone(), a.algorithm_version.clone()), a)) .collect(); - // One stat per alignment, from the row we already hold — not one per artifact read. + // One stat call for each alignment, from the row that the code already holds. The code + // does not make one stat call for each artifact. let sigs = alignments .iter() .map(|a| { @@ -38,13 +44,14 @@ impl AlignmentArtifacts { Ok(Self { by_key, sigs }) } - /// The stored artifact, with no staleness check — the equivalent of a bare `artifact::get`. + /// The stored artifact, with no check for an old result. The method does the same work as a + /// plain `artifact::get` call. fn raw(&self, alignment_id: i64, kind: &str, version: &str) -> Option<&AnalysisArtifact> { self.by_key.get(&(alignment_id, kind.to_string(), version.to_string())) } - /// The decoded payload, or `None` when absent, stale, or undecodable — matching - /// [`App::load_analysis`]. + /// The decoded payload. The method returns `None` when the artifact is absent, when it is out + /// of date, or when the decoder refuses it. [`App::load_analysis`] behaves in the same way. fn fresh(&self, alignment_id: i64, kind: &str, version: &str) -> Option { let a = self.raw(alignment_id, kind, version)?; let current = self.sigs.get(&alignment_id).and_then(|s| s.as_deref()); @@ -68,7 +75,8 @@ impl AlignmentArtifacts { impl App { // ---- queries ----------------------------------------------------------- - /// Biosamples belonging to a project (M:N membership ∪ legacy home column). + /// The biosamples of a project. The set is the union of the M:N memberships and the old home + /// column. pub async fn list_biosamples(&self, project_id: i64) -> Result, AppError> { Ok(biosample::list_members_for_project(self.store.pool(), project_id).await?) } @@ -78,10 +86,14 @@ impl App { Ok(biosample::list_all(self.store.pool()).await?) } - /// Bulk per-subject analysis status for the Subjects list, in one query (mirrors - /// [`haplogroup_terminals`](Self::haplogroup_terminals)). A subject is `Complete` once every - /// alignment it owns has a full `coverage` artifact at the current version; otherwise `Pending`. - /// Subjects with no alignments are omitted (the list shows no status for them). + /// The analysis status of each subject for the Subjects list, in one query. The method has the + /// same shape as [`haplogroup_terminals`](Self::haplogroup_terminals). + /// + /// A subject is `Complete` when each of its alignments has a full `coverage` artifact at the + /// current version. If not, the subject is `Pending`. + /// + /// The result holds no subject with no alignment, and the list then shows no status for such a + /// subject. pub async fn subject_analysis_status(&self) -> Result, AppError> { let census = artifact::analyzed_census(self.store.pool(), "coverage", coverage::COVERAGE_VERSION).await?; Ok(census @@ -100,10 +112,13 @@ impl App { /// Sequence runs for a biosample. pub async fn list_sequence_runs(&self, biosample_guid: SampleGuid) -> Result, AppError> { let mut runs = sequence_run::list_for_biosample(self.store.pool(), biosample_guid).await?; - // One-time backfill: runs analyzed before read stats were mirrored onto the run carry no - // `total_reads` (and older imports no `library_layout`). Recover them from a cached - // `read_metrics` artifact on any of the run's alignments and persist, so the card shows - // library stats + PE/SE without a re-walk. + // A backfill that runs one time. A run that the app analyzed before it copied the read + // statistics to the run row holds no `total_reads` value. An older import also holds no + // `library_layout` value. + // + // The code reads those values from a cached `read_metrics` artifact on any alignment of the + // run, and writes them to the run row. The card then shows the library statistics and the + // PE or SE value with no second walk. for run in &mut runs { if run.total_reads.is_some() && run.library_layout.is_some() && run.total_bases.is_some() { continue; diff --git a/crates/navigator-app/src/realign.rs b/crates/navigator-app/src/realign.rs index 358c95da..1f4ecf46 100644 --- a/crates/navigator-app/src/realign.rs +++ b/crates/navigator-app/src/realign.rs @@ -1,22 +1,23 @@ -//! Registering a realigned alignment — stage D of the realignment module. +//! This module registers a realigned alignment. It is stage D of the realignment module. //! -//! Stage C leaves a sorted, duplicate-marked CRAM on disk. That file is not yet an *alignment* as -//! far as the workspace is concerned; this is where it becomes one, so every existing analysis can -//! run against it without knowing it was produced rather than imported. +//! Stage C writes a CRAM file to disk. The code sorts that file and marks its duplicates. The +//! workspace does not yet hold it as an *alignment*. This module makes it one. Each analysis can +//! then run against it, and no analysis needs to know that the app made the file. //! -//! ## Additive, always +//! ## This module only adds //! -//! The realigned row is inserted under the **same `SequenceRun`** as its source — the same physical -//! library, only mapped differently — and the source is left exactly as it was. That mirrors how -//! the sidecar fast path stayed additive, and it is the property that makes realignment safe to -//! offer: nothing a user already had can be lost by trying it, and a realigned alignment can be -//! deleted and rebuilt from a source that never changed. +//! The code inserts the realigned row under the **same `SequenceRun`** as its source. That run is +//! the same physical library with a different map. The code does not change the source row. +//! +//! The sidecar fast path behaves in the same way, and this behaviour makes a realignment safe to +//! offer. A user can not lose data when they try it. A user can also delete a realigned alignment +//! and build it again from a source that did not change. //! //! ## The reference is part of the file //! -//! A CRAM can not be read without the reference it was compressed against, so `reference_path` is -//! recorded on the row rather than resolved by convention later. An alignment whose reference has -//! moved is unreadable, and the row is the only place that knows which one it was. +//! A reader can not open a CRAM file without the reference that compressed it. So the row holds +//! `reference_path`, and no later code finds that path by a rule. A reader can not open an +//! alignment whose reference moved, and the row is the only record of that reference. use std::path::{Path, PathBuf}; @@ -27,24 +28,26 @@ use navigator_store::alignment; use crate::error::AppError; use crate::{sha256_file_async, App}; -/// What produced a derived alignment, recorded in `Alignment::derivation`. +/// The process that made a derived alignment. `Alignment::derivation` holds this value. +/// +/// The format is `realign:-`. So the row shows *that* the app realigned the +/// alignment, and it also shows *how*. /// -/// Spelled `realign:-` so the row says both *that* it was realigned and *how* — -/// a re-run under a different backend or preset is a different artifact, and support questions -/// start with which one produced a given file. +/// A second run with another backend, or another preset, gives a different file. A support question +/// starts with the process that made the file. pub fn derivation_tag(backend: &str, preset: &str) -> String { format!("realign:{backend}-{preset}") } impl App { - /// Register the output of a realignment as a new alignment derived from `source_id`. + /// Add the output of a realignment as a new alignment that comes from `source_id`. /// - /// `cram` is stage C's output and `reference` the FASTA it was compressed against — both are - /// required, because a CRAM without its reference is unreadable rather than merely awkward. + /// `cram` is the output of stage C. `reference` is the FASTA file that compressed it. The + /// method needs both values. Without its reference, no reader can open a CRAM file. /// - /// `backend` and `preset` are taken separately rather than as a ready-made derivation string - /// so the recorded format stays authoritative here; a caller can not invent its own spelling - /// that later queries then fail to recognise. + /// The method takes `backend` and `preset` as two values, and not as one derivation string. + /// This module then controls the format. A caller can not write its own format, which a later + /// query would not recognize. pub async fn register_realigned_alignment( &self, source_id: i64, @@ -56,9 +59,9 @@ impl App { ) -> Result { let source = self.alignment_or_err(source_id).await?; - // Realigning to the build a sample is already on produces a second copy of the same - // information at hours of cost. The design calls for refusing it; doing so here rather - // than in the UI means the CLI and any future caller are covered by the same rule. + // A realignment to the build that a sample already uses gives a second copy of the same + // data, and it costs many hours. The design refuses that work. This check is in the app + // layer and not in the UI. So the CLI and each later caller follow the same rule. if builds_match(&source.reference_build, reference_build) { return Err(AppError::Import(format!( "alignment #{source_id} is already on {}; realigning it to the same build would \ @@ -74,13 +77,14 @@ impl App { }); } - // Hash now rather than lazily. The file was just written, so it is in the page cache and - // this is nearly free; leaving it for the first analysis would pay the read twice. + // Calculate the hash now, and not at the first use. The code wrote the file a moment ago, + // so the page cache holds it and the read costs almost nothing. A hash at the first + // analysis reads the file a second time. let content_sha256 = sha256_file_async(cram.to_path_buf()).await?; let created = self .record_alignment(NewAlignment { - // The same library — only the mapping changed. + // The library is the same. Only the mapping changed. sequence_run_id: source.sequence_run_id, reference_build: reference_build.to_string(), aligner: backend.to_string(), @@ -97,10 +101,10 @@ impl App { Ok(created) } - /// The alignments derived from `source_id`, if any. + /// The alignments that come from `source_id`, when the workspace holds any. /// - /// The UI asks this before offering "Realign": a sample that has already been realigned should - /// be told so rather than silently given a second copy. + /// The UI calls this method before it offers the "Realign" action. The app must tell a user + /// that a sample already has a realignment. It must not make a second copy with no message. pub async fn derived_alignments(&self, source_id: i64) -> Result, AppError> { let source = self.alignment_or_err(source_id).await?; let siblings = alignment::list_for_run(self.store.pool(), source.sequence_run_id).await?; @@ -110,12 +114,13 @@ impl App { .collect()) } - /// The subject an alignment belongs to, via its sequencing run. + /// The subject of an alignment. The method finds it through the sequence run. + /// + /// The UI needs this value to name the subject of a realignment. /// - /// The UI needs this to say whose realignment is running. Without it the running card matched on - /// alignment id alone, and a page showing subject A during a job on subject B told A their - /// genome was being rebuilt — the ownership question has to be answered where the mapping from - /// alignment to subject actually lives. + /// Before this method, the progress card compared the alignment id alone. A page that showed + /// subject A during a job on subject B then told subject A that the app rebuilt their genome. + /// The code that maps an alignment to a subject must answer this question. pub async fn subject_of_alignment(&self, id: i64) -> Result, AppError> { let Some(aln) = alignment::get(self.store.pool(), id).await? else { return Ok(None); @@ -127,7 +132,7 @@ impl App { ) } - /// The alignment `id` was derived from, or `None` when it is an original. + /// The alignment that gave `id`, or `None` when `id` is an original alignment. pub async fn derivation_source(&self, id: i64) -> Result, AppError> { let aln = self.alignment_or_err(id).await?; match aln.derived_from_alignment_id { @@ -136,16 +141,20 @@ impl App { } } - /// The alignments in `project_id` that a realignment to `target_build` would actually act on. + /// The alignments in `project_id` that a realignment to `target_build` would act on. /// - /// Computed up front rather than discovered while running, because the honest thing to tell - /// someone before a job measured in *days* is how many samples it covers. Skips what would be - /// refused anyway — anything already on the target build, anything already realigned, and - /// anything with no file to read — so the count is the real one rather than an upper bound. + /// The method finds them before the job starts. It does not find them during the job. A job of + /// this size can run for *days*, and the app must tell the user how many samples it covers. + /// + /// The method skips each alignment that the job would refuse. Those are an alignment on the + /// target build, an alignment with a realignment already, and an alignment with no file. So the + /// count is the true count and not a maximum. pub async fn realignable_in_project(&self, project_id: i64, target_build: &str) -> Result, AppError> { - // One query for the whole project rather than one per member — the same idiom - // `project_report` uses on this very tab. Measured on a 2,504-member project: 2.7 ms for the - // grouped query against 17.7 ms for the per-member loop, and this runs twice per batch. + // One query covers the full project. The code does not send one query for each member. + // `project_report` uses the same method on this tab. + // + // A measurement on a project with 2,504 members gave 2.7 ms for the grouped query and + // 17.7 ms for the loop. This code runs two times in each batch. let guids: Vec<_> = self .list_biosamples(project_id) .await? @@ -154,9 +163,9 @@ impl App { .collect(); let rows = navigator_store::alignment::list_for_biosamples(self.store.pool(), &guids).await?; - // Grouped by subject, not flattened: the rule's "already realigned" condition asks whether - // anything *in that subject's own set* was derived from a given alignment, so it has to see - // one subject's alignments at a time. + // The code groups the rows by subject and does not make one flat list. The "already + // realigned" condition asks whether an alignment *in the set of that subject* comes from a + // given alignment. So the rule must read the alignments of one subject together. let mut by_subject: std::collections::HashMap<_, Vec> = std::collections::HashMap::new(); for (guid, alignment) in rows { by_subject.entry(guid).or_default().push(alignment); @@ -171,20 +180,22 @@ impl App { Ok(out) } - /// The cached FASTA for `build`, if it has already been fetched. + /// The FASTA file for `build` in the cache, when the app already downloaded it. /// - /// Public because the realignment job needs the reference *before* it starts: mapping to a - /// build whose FASTA is not cached would otherwise stall at the index stage while gigabytes - /// download, with nothing on screen explaining the wait. + /// This method is public because the realignment job needs the reference *before* it starts. + /// Without the check, a map to a build with no FASTA file in the cache stops at the index + /// stage. The download of some GB then runs, and the screen gives the user no reason for the + /// wait. pub fn cached_reference_path(&self, build: &str) -> Option { self.gateway.cached_reference(build) } - /// Where stage C should write, for a realignment of `source_id` to `build`. + /// The path where stage C writes, for a realignment of `source_id` to `build`. /// - /// Derived files live beside the workspace rather than next to the vendor's original: the - /// source directory may be read-only, on removable media, or somewhere the user does not - /// expect Navigator to write tens of GB. + /// A derived file goes beside the workspace. It does not go beside the original file of the + /// vendor. There are three reasons. A source directory can refuse a write. It can be on + /// removable media. It can also be in a place where the user does not expect tens of GB from + /// Navigator. pub fn realigned_output_path(&self, source_id: i64, build: &str) -> PathBuf { navigator_domain::paths::decodingus_dir() .join("realigned") @@ -192,45 +203,52 @@ impl App { } } -/// The build realignment targets when nothing says otherwise — the complete assembly, which is the -/// only reason the module exists. +/// The default target build of a realignment. It is the complete assembly, and that assembly is the +/// only reason for this module. pub const DEFAULT_TARGET_BUILD: &str = "chm13v2.0"; -/// Whether `build` is the realignment target — the complete assembly. +/// Shows whether `build` is the target of a realignment, which is the complete assembly. /// -/// `pub` so the UI can ask the question rather than spelling out its own comparison. Both Advanced -/// realign cards used to do the latter, with `eq_ignore_ascii_case` and no trim, which is a subtly -/// different rule from the one the job enforces. +/// The function is `pub`, so the UI calls it and writes no comparison of its own. Both realign +/// cards of Advanced mode wrote their own comparison before. They used `eq_ignore_ascii_case` and +/// removed no space, and that rule is not the rule of the job. pub fn is_target_build(build: &str) -> bool { builds_match(build, DEFAULT_TARGET_BUILD) } -/// Which of one subject's `alignments` a realignment to `target_build` would actually act on. +/// The alignments of one subject that a realignment to `target_build` would act on. +/// +/// The function takes the full list and not one alignment. Two of the four conditions are about the +/// *set*. The function skips an alignment when another item in the list comes from it. The output +/// keeps the order of the input. /// -/// Takes the whole list rather than one alignment because two of the four conditions are about the -/// *set*: an alignment is skipped if something in the list was already derived from it. Ordered as -/// the input is. +/// The project-wide count and the single-subject offer of Simple mode both call this function. So +/// the two can not become different. /// -/// Shared by the project-wide count and the Simple-mode single-subject offer, so that the two -/// can not drift apart. If they disagreed, a user would be told a batch covers a sample it then -/// silently skips — or be offered four hours of work that the job itself would refuse. +/// A difference between them has two effects. The app can tell a user that a batch covers a sample, +/// and then skip that sample with no message. The app can also offer four hours of work that the +/// job then refuses. pub(crate) fn realignable_for_subject(alignments: &[Alignment], target_build: &str) -> Vec { alignments .iter() .filter(|a| a.bam_path.is_some() && !a.is_derived()) .filter(|a| !builds_match(&a.reference_build, target_build)) - // Already realigned by an earlier run: its output is sitting in this same list. + // An earlier run already realigned this alignment, and its output is in this same + // list. .filter(|a| !alignments.iter().any(|d| d.derived_from_alignment_id == Some(a.id))) .map(|a| a.id) .collect() } -/// Whether two build names refer to the same reference for this purpose. +/// Shows whether two build names name the same reference, for this purpose. +/// +/// The function removes the space at each end and then compares the two stored strings. It ignores +/// the case of a letter. /// -/// Compared case-insensitively on the recorded strings, after trimming. Deliberately *not* -/// normalised through a `canonical_build`: `chm13v2.0` and `chm13v2.0_maskedY_rCRS` share -/// coordinates but differ in chrM and in PAR masking, so realigning between them is a real -/// operation rather than a no-op. +/// The function does *not* pass the names through `canonical_build`, by design. The builds +/// `chm13v2.0` and `chm13v2.0_maskedY_rCRS` use the same coordinates. But their chrM contigs are +/// different, and their PAR masks are different. So a realignment between the two does real +/// work. fn builds_match(a: &str, b: &str) -> bool { a.trim().eq_ignore_ascii_case(b.trim()) } @@ -252,8 +270,9 @@ mod tests { assert!(!builds_match("GRCh38", "chm13v2.0")); } - /// The masked variant shares CHM13's coordinates but differs in chrM and PAR masking, so - /// moving between them is a real realignment and must not be refused as a no-op. + /// The masked build uses the CHM13 coordinates. But its chrM contig and its PAR mask are + /// different. So a realignment between the two does real work, and the app must not refuse + /// it. #[test] fn the_masked_chm13_variant_is_a_different_build() { assert!(!builds_match("chm13v2.0", "chm13v2.0_maskedY_rCRS")); @@ -289,15 +308,17 @@ mod tests { assert!(realignable_for_subject(&[aln(1, "GRCh38", true, Some(9))], "chm13v2.0").is_empty()); } - /// The set-level condition: once its realigned output sits beside it, the source is done. Without - /// this the offer would reappear after a four-hour job and invite an identical second one. + /// The condition on the set. When the realigned output is beside the source, the work on that + /// source is complete. Without this rule, the offer returns after a four-hour job, and the user + /// starts the same job again. #[test] fn a_source_that_has_already_been_realigned_is_not_offered_again() { let list = [aln(1, "GRCh38", true, None), aln(2, "chm13v2.0", true, Some(1))]; assert!(realignable_for_subject(&list, "chm13v2.0").is_empty()); } - /// A subject holding several originals gets each of them, in input order — the caller picks. + /// For a subject with more than one original alignment, the function returns each of them, in + /// the order of the input. The caller then selects one. #[test] fn every_qualifying_original_is_returned_in_order() { let list = [ @@ -308,8 +329,9 @@ mod tests { assert_eq!(realignable_for_subject(&list, "chm13v2.0"), vec![5, 8]); } - /// The masked CHM13 variant is a different build, so it is still worth realigning to plain - /// CHM13 — the same distinction `builds_match` draws above, reaching the rule that uses it. + /// The masked CHM13 build is a different build. So a realignment to plain CHM13 still gives a + /// result. `builds_match` above makes the same distinction, and this test covers the rule that + /// calls it. #[test] fn the_masked_variant_is_still_realignable_to_plain_chm13() { let list = [aln(1, "chm13v2.0_maskedY_rCRS", true, None)]; @@ -321,8 +343,8 @@ mod tests { assert!(is_target_build("chm13v2.0")); assert!(is_target_build(" CHM13v2.0 ")); assert!(!is_target_build("GRCh38")); - // The masked variant is a different reference, so a subject holding only that one has not - // yet had their Y read against plain CHM13. + // The masked build is a different reference. So the app did not yet read the Y chromosome + // of a subject with only that build against plain CHM13. assert!(!is_target_build("chm13v2.0_maskedY_rCRS")); } } diff --git a/documents/STE-dictionary.md b/documents/STE-dictionary.md index 7ff4189c..16021cc8 100644 --- a/documents/STE-dictionary.md +++ b/documents/STE-dictionary.md @@ -47,6 +47,11 @@ BAM · BED · CRAM · FASTA · gVCF · index · JSON · masterVar · sidecar · app · artifact · cache · command · event · liftover · migration · outbox · profile · project · query · realignment · record · row · schema · store · table · workspace · worker +### Local LLM + +chat completion · context · fact sheet · model · model server · narration · prompt · +reasoning model · token + ### Federation AppView · attestation · consent · device key · DID · exchange · handle · IBD · PDS · record key · @@ -58,7 +63,8 @@ STE 2 forbids an `-ing` form as a verb or an adjective. These are declared nouns permitted: genotyping array · mapping · sequencing · painting · matching · encoding · decoding · indexing · logging · polling · signing · setting · heading · listing · ordering · padding · casing · tracking · caching · processing · operating system · pacing · sampling · scaling · streaming · -spilling · phasing · binning · masking · trimming · clipping · calling · sorting · merging +spilling · phasing · binning · masking · trimming · clipping · calling · sorting · merging · +reasoning model · streaming ## Technical Verbs diff --git a/scripts/ste-check.py b/scripts/ste-check.py index 439700ef..d7b46c01 100755 --- a/scripts/ste-check.py +++ b/scripts/ste-check.py @@ -126,7 +126,7 @@ def _technical_names(): "processing", "pending", "missing", "remaining", "existing", "following", "corresponding", "underlying", "according", "including", # Technical Names from documents/STE-dictionary.md that end in -ing. - "operating", "genotyping", "pacing", "sampling", "scaling", "streaming", "spilling", "phasing", + "operating", "genotyping", "reasoning", "pacing", "sampling", "scaling", "streaming", "spilling", "phasing", "binning", "masking", "trimming", "clipping", "calling", "sorting", "merging", "reading", "writing", "counting", "timing", "build", "backing", } From b88b34b86b50d6caf04eb044ae97f9daad1c1a52 Mon Sep 17 00:00:00 2001 From: James Kane Date: Wed, 19 Aug 2026 07:05:27 -0500 Subject: [PATCH 07/33] docs(ste): navigator-app, the pipeline and import files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit app.rs (tests), commands, fastpath, import_unified, queries, realign_job. Thirty of thirty-three files at zero; the crate goes 2,694 to 1,710. Only analysis.rs, lib.rs and haplogroup.rs remain. These are the files where a comment is a post-mortem, and the conversion is worth reading for that reason. Every finding survives: - why the BGZF end-of-file marker cannot prove a file is complete — noodles finishes its stream from `Drop`, so a cancelled merge left 13.2 GB of an expected 30 GB wearing a valid marker, and the next run marked duplicates on a truncated alignment; - how a 59 GB `mapped.bam`, four hours of revert and mapping, was destroyed by that marker bug combined with a resumed run deleting an artifact it had not written; - why scratch is sized from the source's uncompressed volume and not its file size: 17 GB of CRAM is ~70 GB of BAM-equivalent data, and the file-size estimate told a user a 200 GB job needed 69; - why a Y-only extract must be forced male — the chrX/autosome ratio reads it as female, which silently disabled the entire Y pipeline; - why `min_dp 4` removes misaligned-read clusters without touching real private SNVs. The prose is longer and flatter. The reasoning is intact, and a reader parsing English as a second language can now follow it a sentence at a time. `cargo check -p navigator-app --all-targets` passes. Co-Authored-By: Claude Opus 5 (1M context) --- crates/navigator-app/src/analysis.rs | 379 +++++++---- crates/navigator-app/src/fastpath.rs | 653 ++++++++++++------- crates/navigator-app/src/import_unified.rs | 692 ++++++++++++++------- crates/navigator-app/src/queries.rs | 470 ++++++++------ crates/navigator-app/src/realign_job.rs | 533 +++++++++------- crates/navigator-app/tests/app.rs | 534 ++++++++++------ 6 files changed, 2055 insertions(+), 1206 deletions(-) diff --git a/crates/navigator-app/src/analysis.rs b/crates/navigator-app/src/analysis.rs index 8c2c0691..f618706b 100644 --- a/crates/navigator-app/src/analysis.rs +++ b/crates/navigator-app/src/analysis.rs @@ -6,9 +6,10 @@ use navigator_analysis::{contig, CancelToken}; impl App { // ---- analysis (compute + persist) -------------------------------------- - /// Run the coverage + callable walker on an alignment's BAM and persist the result - /// as a versioned `coverage` artifact. The blocking noodles I/O runs on a blocking - /// thread so the async runtime is not stalled. + /// Run the coverage walker and the callable walker on the BAM file of an alignment. The method + /// writes the result as a `coverage` artifact with a version. + /// + /// The I/O of noodles blocks, so it runs on its own thread. The async runtime then continues. pub async fn run_coverage( &self, alignment_id: i64, @@ -34,17 +35,20 @@ impl App { .await } - /// Run coverage using the alignment's own stored BAM/reference paths, then persist. - /// Errors if the alignment is unknown or has no paths recorded. + /// Calculate the coverage with the BAM path and the reference path of the alignment, and then + /// write the result. The method fails when the store holds no such alignment, and when that + /// alignment holds no path. pub async fn run_coverage_for_alignment(&self, alignment_id: i64) -> Result { self.run_coverage_for_alignment_with_progress(alignment_id, |_, _| {}, CancelToken::none()) .await } - /// Like [`run_coverage_for_alignment`], reporting `progress(contigs_done, contigs_total)` as - /// the whole-genome pass walks each contig (the slow step — minutes on a real WGS BAM — so a - /// progress bar can advance instead of sitting frozen). The callback runs on the blocking - /// thread. + /// The same work as [`run_coverage_for_alignment`], with a progress report. The method calls + /// `progress(contigs_done, contigs_total)` as the whole-genome pass reads each contig. + /// + /// That pass is the slow step, and it needs some minutes on a real WGS BAM file. So a progress + /// bar can move, and the app does not look stopped. The callback runs on the thread that + /// blocks. pub async fn run_coverage_for_alignment_with_progress( &self, alignment_id: i64, @@ -53,8 +57,9 @@ impl App { ) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let bam = Self::alignment_file(&aln)?; - // The reference is not asked for at import — resolve the alignment's build via the gateway - // (cached, else download) when no FASTA was stored. + // The import step does not ask the user for the reference. When the alignment holds no + // FASTA path, the gateway finds the build of that alignment. It reads the cache, and it + // downloads the file when the cache holds none. let reference = match aln.reference_path { Some(p) => PathBuf::from(p), None => { @@ -63,8 +68,9 @@ impl App { .await? } }; - // For a targeted test (Big Y, etc.) restrict the walk to the target chromosome(s) so the - // headline depth reflects the target rather than being diluted to ~0 by the empty genome. + // For a targeted test, such as Big Y, read only the target chromosomes. The depth that the + // app reports then describes the target. Across the full genome, most contigs hold no read, + // and they make that value almost zero. let allowlist = self.coverage_target_allowlist(alignment_id).await?; let mut params = CallableLociParams::default(); let result = tokio::task::spawn_blocking(move || { @@ -92,21 +98,29 @@ impl App { Ok(result) } - /// The coverage contig allowlist for a targeted test, or `None` (whole genome) for WGS/autosomal. - /// A Y-targeted test (FTDNA Big Y, Y Elite, …) walks chrY only — plus chrM so the "has mtDNA - /// reads" signal survives for the few Big Ys that retained mitochondrial reads (the UI hides the - /// mtDNA sections when chrM has none). An mtDNA-targeted test walks chrM only. Build-agnostic - /// (both `chr`-prefixed and bare contig names are listed). + /// The list of contigs that the coverage walk reads, for a targeted test. The method returns + /// `None` for a WGS test and an autosomal test, and the walk then reads the full genome. + /// + /// A Y test, such as FTDNA Big Y or Y Elite, reads chrY. It also reads chrM, so the signal "this + /// test holds mtDNA reads" survives. A few Big Y files hold mitochondrial reads, and the UI + /// hides the mtDNA sections when chrM holds none. + /// + /// An mtDNA test reads chrM only. + /// + /// The list does not depend on the build. It holds each contig name with the `chr` prefix and + /// each name without it. async fn coverage_target_allowlist(&self, alignment_id: i64) -> Result>, AppError> { use navigator_domain::testtype::TargetType; let aln = self.alignment_or_err(alignment_id).await?; let Some(run) = sequence_run::get(self.store.pool(), aln.sequence_run_id).await? else { return Ok(None); }; - // `target_of` (not bare `by_code`) so a stored human label like "Big Y" — which a bulk - // import / --test-type override writes instead of BIG_Y_500/700 — still scopes the walk to - // chrY+chrM. Otherwise coverage walks the whole genome, which on a targeted multi-reference - // CRAM is the ~1-hour batch-analysis stall. + // The code calls `target_of` and not `by_code`. So a label that a person wrote, such as + // "Big Y", still limits the walk to chrY and chrM. A bulk import writes such a label, and + // the `--test-type` option also writes one, in place of BIG_Y_500 or BIG_Y_700. + // + // Without this call, the walk reads the full genome. On a targeted CRAM file with many + // references, that walk is the stop of about one hour in a batch analysis. let contigs: &[&str] = match navigator_domain::testtype::target_of(&run.test_type) { Some(TargetType::YChromosome) => &["chrY", "Y", "chrM", "chrMT", "M", "MT"], Some(TargetType::MtDna) => &["chrM", "chrMT", "M", "MT"], @@ -115,10 +129,14 @@ impl App { Ok(Some(contigs.iter().map(|s| s.to_string()).collect())) } - /// Whether a cached coverage result was computed at the right scope for the alignment's test. - /// A targeted test (Big Y, mtFull) must cover only its target contig(s); a whole-genome cached - /// result for it is stale — the headline depth was diluted across the empty genome — and must - /// be recomputed. Whole-genome tests (no allowlist) are always in scope. + /// Shows whether a cached coverage result covers the correct contigs for the test of this + /// alignment. + /// + /// A targeted test, such as Big Y or mtFull, must cover its target contigs only. A cached + /// whole-genome result for such a test is wrong, because the depth is small across the contigs + /// with no read. The app must calculate that result again. + /// + /// A whole-genome test has no list of contigs, and its result is always correct. pub(crate) async fn coverage_is_correctly_scoped( &self, alignment_id: i64, @@ -130,9 +148,12 @@ impl App { } } - /// Cached coverage for analysis reuse: the stored result, but only when it was computed at the - /// right scope for the test (see [`Self::coverage_is_correctly_scoped`]). A stale whole-genome - /// result for a targeted test reads as a cache miss so the caller recomputes it correctly. + /// The cached coverage result, for a later analysis. The method returns the stored result only + /// when that result covers the correct contigs for the test. See + /// [`Self::coverage_is_correctly_scoped`]. + /// + /// A whole-genome result for a targeted test is wrong. The method then returns nothing, and the + /// caller calculates the correct result. pub async fn cached_coverage_for_analysis(&self, alignment_id: i64) -> Result, AppError> { match self.cached_coverage(alignment_id).await? { Some(cov) if self.coverage_is_correctly_scoped(alignment_id, &cov).await? => Ok(Some(cov)), @@ -140,9 +161,11 @@ impl App { } } - /// Infer biological sex from the alignment's chrX:autosome read-density ratio, persisting - /// the result as a `sex` artifact. Cheap (BAI fast-path for BAM). `reference` is used only - /// for CRAM decode. + /// Find the biological sex from the ratio between the read density of chrX and the read density + /// of the autosomes. The method writes the result as a `sex` artifact. + /// + /// The step is fast, because a BAM file has a BAI index. The code uses `reference` only to + /// decode a CRAM file. pub async fn run_sex(&self, alignment_id: i64) -> Result { let (bam, reference) = self.alignment_paths(alignment_id).await?; let result = @@ -153,9 +176,11 @@ impl App { Ok(result) } - /// Write the inferred sex back to the biosample when the user did not provide one, so it - /// shows in the subjects table + header instead of "Unknown". No-op for Unknown sex or - /// when the biosample already carries a sex. + /// Write the sex that the code found to the biosample, when the user gave none. The subjects + /// table and the header then show that value in place of "Unknown". + /// + /// The method does nothing when the code found no sex, and when the biosample already holds + /// one. pub(crate) async fn write_back_inferred_sex( &self, alignment_id: i64, @@ -200,11 +225,14 @@ impl App { Ok(result) } - /// Mirror an alignment's library-level read stats onto its owning sequence run (`total_reads`, - /// `mean_read_length`, `mean_insert_size`) so the Data Sources run card shows them without - /// re-walking. Best-effort: a missing alignment/run is ignored. When a run has several - /// alignments the last write wins — these are per-library properties, so any pass is - /// representative. + /// Copy the library-level read statistics of an alignment to its sequence run. Those values are + /// `total_reads`, `mean_read_length`, and `mean_insert_size`. The run card of the Data Sources + /// tab then shows them, and the app reads no file again. + /// + /// The step is optional, and the method ignores an absent alignment and an absent run. + /// + /// When a run holds more than one alignment, the last write wins. These values describe the + /// library, so each alignment gives the same answer. pub(crate) async fn write_back_read_stats( &self, alignment_id: i64, @@ -242,51 +270,65 @@ impl App { self.load_analysis(alignment_id, "read_metrics", "1").await } - /// Scratch directory for alignments copied off a slow/removable volume (see [`localize`]). - /// Entries are owned by a [`LocalAlignment`] and removed when the last holder drops. + /// The scratch directory for an alignment that the code copied from a slow volume or a + /// removable volume. See [`localize`]. A [`LocalAlignment`] value owns each entry, and the code + /// removes that entry after the last holder drops it. pub(crate) fn align_cache_dir() -> std::path::PathBuf { navigator_refgenome::cache::base_dir().join("cache").join("aln") } - /// If `remote` lives on a slow/removable volume (a `/Volumes/…` mount), copy it — and its `.crai` - /// / `.bai` index — into the local cache and return the *local* path; otherwise return `remote` - /// unchanged. The analysis walkers do random-access record iteration (region seeks, per-read - /// decode), which is pathologically slow over a network/USB mount even though a plain sequential - /// **copy** of the same file is fast — so we pay one fast bulk copy up front and let every - /// subsequent pass read from local disk. The copy is reused across a subject's passes and cleared - /// per subject by [`clear_align_cache`]. A copy failure falls back to the remote path (slow, but - /// still works). + /// Copy `remote` to the local cache and return the *local* path, when that file sits on a slow + /// volume or a removable volume. Such a volume has a `/Volumes/…` mount point. The method also + /// copies the `.crai` index or the `.bai` index. For any other path, it returns `remote` with no + /// change. + /// + /// An analysis walker reads records at random positions. It seeks to a region, and it decodes + /// each read. That access is very slow over a network mount or a USB mount. A plain sequential + /// **copy** of the same file is fast. + /// + /// So the code pays for one fast copy first, and each later pass reads from the local disk. The + /// passes of one subject share that copy, and [`clear_align_cache`] removes it for each subject. + /// + /// A failed copy gives the remote path. The analysis is then slow, and it still works. pub(crate) async fn localize(&self, remote: &Path) -> LocalAlignment { if std::env::var_os("NAVIGATOR_NO_LOCALIZE").is_some() || !is_removable_volume(remote) { return LocalAlignment::borrowed(remote); } let local = Self::align_cache_dir().join(local_cache_name(remote)); - // Serialize the cache-check-then-copy per destination. The worker `tokio::spawn`s every - // command, so a batch walk and a per-alignment command genuinely overlap on one alignment; - // without this both miss the cache and both copy, writing a second full 40 GB pull over the - // network for nothing, after which the loser of the rename reads from the remote anyway. + // Order the two steps, the cache test and the copy, for each destination. The worker calls + // `tokio::spawn` for each command. So a batch walk and a command for one alignment do + // overlap on the same alignment. + // + // Without this lock, both find no cache entry and both copy. The second copy reads another + // 40 GB over the network for no result. The task that loses the rename then reads the + // remote file. + // + // The code holds the lock across the copy. So it must not take that lock a second time. No + // call of `localize` can run while another call is open *on the same path in the same + // task*. // - // The gate is held across the copy, so it must not be taken re-entrantly — no `localize` - // may be called while another is outstanding *on the same path in the same task*. The three - // call sites are sequential today (`debug_y_calls` awaits `base_calls` to completion before - // localizing itself); keep it that way. + // The three call sites run in sequence today. `debug_y_calls` waits for `base_calls` to + // complete before it localizes its own file. Keep that order. let gate = copy_gate(&local); let _copying = gate.lock().await; - // Size the remote once: it decides both whether an existing copy can be trusted and whether - // the one we make arrived whole. + // Read the size of the remote file one time. That value answers two questions. It shows + // whether the code can trust a copy that exists, and it shows whether the new copy is + // complete. let remote_len = tokio::fs::metadata(remote).await.ok().map(|m| m.len()); - // Another holder is already using this copy — share it and bump the count. + // Another holder already uses this copy. Share it, and add one to the count. if LocalAlignment::retain(&local, remote_len) { return LocalAlignment::owned(local); } let (remote_owned, local2) = (remote.to_path_buf(), local.clone()); match tokio::task::spawn_blocking(move || copy_with_index(&remote_owned, &local2, remote_len)).await { - // Registering can still fail if the copy was removed in the gap (a concurrent holder - // finishing and dropping to zero). Returning an `owned` handle to a missing path would - // fail the walk with a confusing ENOENT and then "clean up" a file that is not there. + // This step can still fail, because another holder can remove the copy in the time + // between the two steps. That holder completes its work and drops the count to zero. + // + // An `owned` handle to an absent path fails the walk with an ENOENT error that no user + // can read. The code then also tries to remove a file that is not there. Ok(Ok(())) if LocalAlignment::retain(&local, remote_len) => LocalAlignment::owned(local), Ok(Ok(())) => { eprintln!( @@ -306,21 +348,32 @@ impl App { } } - /// Run the unified quality-metrics walker — coverage + callable, read-level QC metrics, and - /// sex inference in **one pass** over the alignment's BAM/CRAM (vs. the separate passes - /// `run_coverage` + `run_read_metrics` + `run_sex` cost: 2 reads for BAM, 3 for CRAM). All - /// three sub-results are persisted under their existing artifact keys (`coverage`/ - /// `COVERAGE_VERSION`, `read_metrics`/`"1"`, `sex`/`"1"`), so `cached_coverage`/ - /// `cached_read_metrics`/`cached_sex` and the SV step's reuse logic keep working unchanged. + /// Run the unified quality-metrics walker. It makes **one pass** over the BAM file or the CRAM + /// file of the alignment. That pass gives three results: the coverage with the callable regions, + /// the quality metrics of each read, and the sex. + /// + /// The separate calls `run_coverage`, `run_read_metrics`, and `run_sex` cost more. They read a + /// BAM file two times, and a CRAM file three times. + /// + /// The method writes each of the three results under its existing artifact key. Those keys are + /// `coverage` with `COVERAGE_VERSION`, `read_metrics` with `"1"`, and `sex` with `"1"`. + /// + /// So `cached_coverage`, `cached_read_metrics`, `cached_sex`, and the reuse rule of the SV step + /// each work with no change. pub async fn run_unified_metrics(&self, alignment_id: i64) -> Result { self.run_unified_metrics_with_progress(alignment_id, |_, _| {}, CancelToken::none()) .await } - /// Like [`run_unified_metrics`], reporting `progress(contigs_done, contigs_total)` as the - /// (slow) whole-genome coverage portion finalizes each contig. Uses the per-contig parallel - /// walker (falling back to a sequential pass for CRAM / unindexed BAM); the callback is - /// `Fn + Sync` because it is invoked concurrently from the fan-out's worker threads. + /// The same work as [`run_unified_metrics`], with a progress report. The method calls + /// `progress(contigs_done, contigs_total)` as the whole-genome coverage step completes each + /// contig. That step is the slow one. + /// + /// The method uses the parallel walker, which works on each contig at the same time. For a CRAM + /// file, and for a BAM file with no index, it reads the file from start to end instead. + /// + /// The callback is `Fn + Sync`, because the worker threads of the parallel walker call it at the + /// same time. pub async fn run_unified_metrics_with_progress( &self, alignment_id: i64, @@ -329,13 +382,16 @@ impl App { ) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let run_id = aln.sequence_run_id; - // Copy off a slow/removable volume to local disk first — the walker's random-access record - // iteration is far slower over a network/USB mount than a one-shot bulk copy. - // Held for the whole walk: dropping it removes the local copy. + // Copy the file from a slow volume or a removable volume to the local disk first. The + // walker reads records at random positions, and that access is much slower over a network + // mount or a USB mount than one bulk copy. + // + // The code holds this value for the full walk. A drop of it removes the local copy. let bam = self.localize(&Self::alignment_file(&aln)?).await; let bam = bam.path().to_path_buf(); - // The walker requires a reference (CRAM decode + reference-N detection); resolve the - // build via the gateway when no FASTA was stored at import. + // The walker needs a reference. It decodes the CRAM file with that reference, and it finds + // each N base of the reference. When the import stored no FASTA path, the gateway finds the + // build. let reference = match aln.reference_path { Some(p) => PathBuf::from(p), None => { @@ -344,9 +400,12 @@ impl App { .await? } }; - // Restrict a targeted test (Big Y, mtFull) to its target contig(s), exactly like the - // standalone coverage walker — otherwise the headline depth is diluted across the empty - // genome (a Big Y reads as ~0.2× instead of ~50× on chrY). WGS keeps the whole-genome walk. + // Limit a targeted test, such as Big Y or mtFull, to its target contigs. The separate + // coverage walker uses the same rule. + // + // Across the full genome, most contigs hold no read, and they make the depth small. A Big Y + // test then reads as about 0.2x, and its true depth on chrY is about 50x. A WGS test keeps + // the whole-genome walk. let allowlist = self.coverage_target_allowlist(alignment_id).await?; let mut params = CallableLociParams::default(); let result = tokio::task::spawn_blocking(move || { @@ -376,11 +435,16 @@ impl App { self.save_analysis(alignment_id, "read_metrics", "1", &result.read_metrics) .await?; self.write_back_read_stats(alignment_id, &result.read_metrics).await?; - // Sex: a Y-targeted test (Big Y, Y Elite, …) sequences the donor's Y chromosome — he is male - // by definition. The chrX/autosome ratio the inference needs is not present in a chrY-scoped - // walk, and is unreliable even whole-genome (a Big Y's off-target chrX ≈ autosome ≈ 0.4× - // reads as *female*). So force Male for a Y-targeted test, overriding the inference + any - // prior auto-assignment; WGS / mt-targeted keep the walk's result. + // The sex. A Y test, such as Big Y or Y Elite, reads the Y chromosome of the donor. So that + // donor is male, by definition. + // + // A walk of chrY alone holds no ratio between chrX and the autosomes, and the code needs + // that ratio. The ratio is also wrong across the full genome. In a Big Y file, chrX and the + // autosomes each hold about 0.4x, and the code then reads the donor as *female*. + // + // So the code writes Male for a Y test. That value replaces the result of the ratio, and it + // replaces a value from an earlier run. A WGS test and an mt test keep the result of the + // walk. let y_targeted = matches!( sequence_run::get(self.store.pool(), run_id) .await? @@ -388,12 +452,18 @@ impl App { .and_then(|r| navigator_domain::testtype::target_of(&r.test_type)), Some(navigator_domain::testtype::TargetType::YChromosome) ); - // A Y-scoped alignment reads as male the same way a Y-targeted test does — chrY carries - // essentially all the reads while the autosomes hold only a few dozen mismapped ones (a - // Y-only extract, e.g. GRCh38 chrY reads realigned to hs1, or a Y-Elite/Big Y capture that - // came in mislabeled WGS). The ratio walk can then read it as *female*, which silently - // disables the whole Y pipeline (assign_y_haplogroup skips females before it ever fetches - // the tree). Detect it from the per-contig read counts and force male, exactly like a Y test. + // An alignment with reads on chrY only is male, as a Y test is. Its chrY contig holds + // almost each read, and its autosomes hold a few reads that the mapper placed wrongly. + // + // Two files have that shape. One is a chrY extract, such as GRCh38 chrY reads that the app + // realigned to hs1. The other is a Y Elite capture, or a Big Y capture, that arrived with a + // WGS label. + // + // The ratio can read such a file as *female*. That value stops the full Y pipeline with no + // message, because `assign_y_haplogroup` skips a female subject before it reads the tree. + // + // So the code finds this shape from the read count of each contig, and it writes Male, as it + // does for a Y test. let y_scoped = navigator_analysis::sex::is_y_scoped( result .coverage @@ -416,8 +486,9 @@ impl App { if let Some(sex) = &sex { self.save_analysis(alignment_id, "sex", "1", sex).await?; if male_by_scope { - // Definitive (Y test / Y-scoped ⇒ male): override any prior auto-inferred sex — - // including a stale false "Female" — rather than write-if-empty. + // This value is definite: a Y test, or an alignment with reads on chrY only, is + // male. So the code replaces a sex from an earlier run, and that set holds a wrong + // "Female" value. It does not only write into an empty field. if let Ok(guid) = self.biosample_of_alignment(alignment_id).await { biosample::set_sex(self.store.pool(), guid, "Male").await?; } @@ -436,15 +507,19 @@ impl App { alignment_id: i64, cancel: CancelToken, ) -> Result { - // Resume: a fresh cached SV result (source unchanged) is reused rather than recomputed. + // A new SV result in the cache, from a source file that did not change, is correct. The + // code uses that result and calculates nothing. if let Some(c) = self.cached_sv(alignment_id).await? { return Ok(c); } let aln = self.alignment_or_err(alignment_id).await?; let reference_build = aln.reference_build.clone(); - // Resolve the reference for decode (see alignment_reference_for_decode): required for a CRAM, - // None for a BAM. SV never consults reference *bases* — but decoding a CRAM record does, so - // the walker needs it too, not just the header-lengths probe. + // Find the reference for the decoder. See alignment_reference_for_decode. A CRAM file needs + // it, and a BAM file uses None. + // + // The SV step reads no reference *base*. But a decode of a CRAM record does read one. So the + // walker also needs the reference, and not only the step that reads the contig lengths from + // the header. let (bam, reference) = self.alignment_reference_for_decode(alignment_id).await?; let cov = match self.cached_coverage(alignment_id).await? { @@ -508,11 +583,16 @@ impl App { p.exists().then_some(p) } - /// Genotype short tandem repeats on `contig` from the alignment, via the enclosing-read caller - /// over the HipSTR reference tracts (haploid for chrY/chrM, diploid elsewhere). Persisted as a - /// `str:{contig}` artifact (so it is cached + source-invalidated like other analyses). Errors if - /// no STR reference is configured for the alignment's build (the tracts are build-specific — - /// CHM13/GRCh37 need their own reference or liftover, not yet wired). + /// Genotype the short tandem repeats on `contig` from the alignment. The caller reads each + /// record that covers a full tract, and it uses the HipSTR reference tracts. It calls chrY and + /// chrM as haploid, and each other contig as diploid. + /// + /// The method writes the result as a `str:{contig}` artifact. So the cache holds it, and a + /// change to the source file makes it invalid, as it does for another analysis. + /// + /// The method fails when no STR reference exists for the build of the alignment. The tracts + /// belong to one build. CHM13 and GRCh37 each need their own reference, or a liftover, and no + /// code does that work yet. pub async fn run_str_calls( &self, alignment_id: i64, @@ -533,8 +613,11 @@ impl App { // Resolve the reference for decode (see alignment_reference_for_decode): required for a CRAM, // None for a BAM. STR region-genotyping reads the alignment; it does not consult reference bases. let (bam, reference) = self.alignment_reference_for_decode(alignment_id).await?; - // chrY / chrM are haploid (one allele); autosomes + chrX (in a female) are diploid. We - // genotype chrY/chrM haploid and everything else diploid — sex-aware chrX is a refinement. + // A cell holds one copy of chrY and one copy of chrM, so each has one allele. It holds two + // copies of each autosome, and a female cell holds two copies of chrX. + // + // So the code calls chrY and chrM as haploid, and each other contig as diploid. A rule for + // chrX that reads the sex is a later improvement. let ploidy: u8 = if contig::is_haploid(&contig) { 1 } else { 2 }; let params = navigator_analysis::strcaller::StrCallerParams::default(); let genos = tokio::task::spawn_blocking(move || { @@ -553,16 +636,23 @@ impl App { Ok(genos) } - /// Compare the STR markers called from sequence (mapped to the FTDNA convention via the - /// corpus-calibrated [`navigator_analysis::strmarker`] table) against the subject's imported - /// vendor Y-STR profile — the By-Panel concordance view. One row per marker present in either - /// source: the called value + its calibration status, the imported value, and whether they agree. - /// `contig` is typically `chrY`. Reuses the cached `str:{contig}` calls. + /// Compare the STR markers from the sequence data with the vendor Y-STR profile that the user + /// imported. The By-Panel view shows this comparison. + /// + /// The [`navigator_analysis::strmarker`] table changes each called value to the FTDNA + /// convention. A corpus of real kits calibrated that table. + /// + /// The result holds one row for each marker in either source. A row holds the called value with + /// its calibration state, the imported value, and a flag that shows whether the two agree. + /// + /// The `contig` value is usually `chrY`. The method reads the `str:{contig}` calls from the + /// cache. pub async fn str_concordance(&self, alignment_id: i64, contig: String) -> Result, AppError> { use navigator_analysis::strmarker::{called_markers_build, normalize_marker, MarkerStatus, StrBuild}; - // The FTDNA convention offset is build-dependent for a few markers (the CHM13 liftover shifted - // some tract boundaries) — select the offsets for this alignment's build. + // For a few markers, the offset of the FTDNA convention changes with the build. The CHM13 + // liftover moved the boundary of some tracts. So the code reads the offsets of the build of + // this alignment. let build = alignment::get(self.store.pool(), alignment_id) .await? .map(|a| StrBuild::from_build_str(&a.reference_build)) @@ -625,11 +715,18 @@ impl App { Ok(out) } - /// Pick the subject's best STR-capable alignment and run the Y-STR concordance on chrY — the - /// entry point the UI calls. "STR-capable" = an alignment whose reference build has a HipSTR - /// reference present ([`str_reference_path`](Self::str_reference_path)); highest mean coverage - /// wins. A CRAM needs no stored reference here — [`run_str_calls`](Self::run_str_calls) resolves - /// it for decode. Errors with guidance when none qualifies (no HipSTR reference / no alignment). + /// Select the best alignment of the subject for STR work, and compare the Y-STR markers on + /// chrY. The UI calls this method. + /// + /// An alignment can do STR work when a HipSTR reference exists for its build. See + /// [`str_reference_path`](Self::str_reference_path). Among those alignments, the one with the + /// highest mean coverage wins. + /// + /// A CRAM file needs no stored reference here, because + /// [`run_str_calls`](Self::run_str_calls) finds one for the decoder. + /// + /// The method fails with a hint when no alignment passes. The two causes are an absent HipSTR + /// reference and a subject with no alignment. pub async fn str_concordance_for_subject( &self, biosample_guid: SampleGuid, @@ -640,7 +737,8 @@ impl App { if Self::str_reference_path(&a.reference_build).is_none() { continue; // no HipSTR reference for this build } - // A CRAM with no stored reference is fine — run_str_calls resolves it via the gateway. + // A CRAM file with no stored reference is acceptable, because run_str_calls finds one + // through the gateway. let cov = self .cached_coverage(a.id) .await @@ -705,9 +803,13 @@ impl App { .await } - /// Whole-contig **de-novo diploid** SNV calling (het 0/1 + hom-alt 1/1) on `contig`, cached per - /// alignment+contig. Reuses the alignment's BAM + reference (resolved from the build). Returns - /// [`SiteGenotype`]s in position order — feed to [`Self::diploid_vcf`]. + /// Call the **de-novo diploid** SNVs across the full `contig`. The caller writes a heterozygous + /// call as 0/1 and a homozygous alternate call as 1/1. The cache key is the alignment with the + /// contig. + /// + /// The method reads the BAM file of the alignment and its reference, which the code finds from + /// the build. It returns the [`SiteGenotype`] values in the order of their positions. Give them + /// to [`Self::diploid_vcf`]. pub async fn run_diploid_calls( &self, alignment_id: i64, @@ -734,8 +836,9 @@ impl App { Ok(calls) } - /// A diploid VCF (VCFv4.2, `GT:AD:DP:GQ:PL`) of the de-novo diploid SNV calls for `contig` - /// (computing + caching them if needed). The sample column is `aln`. + /// A diploid VCF file of the de-novo diploid SNV calls of `contig`. The file uses VCFv4.2, and + /// its format field is `GT:AD:DP:GQ:PL`. The method calculates those calls and writes them to + /// the cache when the cache holds none. The sample column is `aln`. pub async fn diploid_vcf( &self, alignment_id: i64, @@ -749,11 +852,16 @@ impl App { )) } - /// A **whole-genome** diploid VCF: de-novo SNV + indel calls over the diploid primary - /// chromosomes (1–22, X) of the alignment, per-contig cached. chrY and chrM are **excluded** — - /// they are haploid, so the diploid (het 0/1) model is wrong for them; their variants come from - /// the haploid caller and the Y/mt haplogroup + mtDNA-mutation features. Heavy (a real WGS - /// calling pass); the caller runs it off the UI thread (the export path). + /// A **whole-genome** diploid VCF file. It holds the de-novo SNV calls and indel calls across + /// the diploid primary chromosomes of the alignment, which are 1 to 22 and X. The cache holds + /// the result of each contig. + /// + /// The file holds **no** chrY data and **no** chrM data. A cell holds one copy of each, so the + /// diploid model, with its 0/1 calls, is wrong for them. Their variants come from the haploid + /// caller, and from the Y and mt haplogroup features with the mtDNA mutation list. + /// + /// This method is a full WGS calling pass, and it costs much. The caller runs it away from the + /// UI thread, on the export path. pub async fn diploid_vcf_genome(&self, alignment_id: i64, cancel: CancelToken) -> Result { let (bam, reference) = self.alignment_bam_reference(alignment_id).await?; let contigs = @@ -771,10 +879,15 @@ impl App { )) } - /// The subject's alignments on the **dominant reference build** (the build the most alignments - /// share, compared on the canonical build so `chm13v2`/`hs1` agree). The consensus diploid - /// genotype pools only same-build alignments — de-novo variant coordinates can't be merged - /// across builds by position without genome-wide liftover (out of scope). `None` if no alignments. + /// The alignments of the subject on the **most frequent reference build**. That build is the one + /// that the most alignments use, and the code compares the canonical build, so `chm13v2` and + /// `hs1` are the same build here. + /// + /// The consensus diploid genotype pools the alignments of one build only. The position of a + /// de-novo variant does not compare across two builds, and a join by position needs a liftover + /// of the full genome. That work is not in this feature. + /// + /// The method returns `None` when the subject has no alignment. pub(crate) async fn consensus_diploid_alignments(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alns = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; if alns.is_empty() { diff --git a/crates/navigator-app/src/fastpath.rs b/crates/navigator-app/src/fastpath.rs index ce97f3e6..ed45e9c0 100644 --- a/crates/navigator-app/src/fastpath.rs +++ b/crates/navigator-app/src/fastpath.rs @@ -2,10 +2,15 @@ //! 2026-06 simplification round; `use super::*` reaches the crate-root types + free helpers. use super::*; -/// Load a bundled chrY position mask/blocklist BED for a build (best-effort). `env_var` overrides the -/// path; otherwise the seeded `/masks/..bed`, trying the gzipped -/// `.bed.gz` first (how the bundled assets ship) then a plain `.bed`. Returns `None` if absent, -/// unparseable, or empty — so a missing cohort asset simply skips that filter rather than blocking. +/// Read a chrY position mask, or blocklist, from a BED file in the application bundle, for one +/// build. The step is optional. +/// +/// The variable `env_var` gives the path when the user sets it. If not, the code reads +/// `/masks/..bed`. It tries the `.bed.gz` file first, because the +/// bundle holds that form, and then a plain `.bed` file. +/// +/// The function returns `None` when the file is absent, when the parser refuses it, and when it +/// holds no row. An absent cohort asset then removes that filter, and it stops no work. fn load_y_position_bed(env_var: &str, stem: &str, build_token: &str) -> Option { let candidates: Vec = if let Ok(p) = std::env::var(env_var) { vec![PathBuf::from(p)] @@ -21,20 +26,26 @@ fn load_y_position_bed(env_var: &str, stem: &str, build_token: &str) -> Option.chrY.g.vcf.gz` next to the CRAM; a -/// per-run pipeline emits `gatk4/chrY.g.vcf.gz`, whose name has no sample prefix at all. Matching -/// only the dotted suffix missed every file of the second kind — and since finding the GVCF is what -/// lets placement skip decoding the CRAM, missing it silently turns a seconds-long read into a -/// minutes-long whole-chromosome walk. +/// The function reads the directory of the alignment first, where the ytree layout puts a +/// `*.chrY.g.vcf.gz` file. It then reads the known directories of each caller, where the usual name +/// is a plain `chrY.g.vcf.gz`. +/// +/// Both names matter. The flat ytree layout writes `.chrY.g.vcf.gz` beside the CRAM file. A +/// pipeline that works on one run writes `gatk4/chrY.g.vcf.gz`, and that name holds no sample +/// prefix. +/// +/// An earlier version matched the dotted name only, and it found no file of the second kind. The +/// GVCF file is what lets the placement skip the CRAM decode. So a search that fails changes a read +/// of some seconds into a walk of the full chromosome, which needs some minutes. fn gvcf_beside_alignment(aln: &Alignment, contig_token: &str) -> Option { let dotted = format!(".{contig_token}.g.vcf.gz"); let bare = format!("{contig_token}.g.vcf.gz"); @@ -52,9 +63,11 @@ fn gvcf_beside_alignment(aln: &Alignment, contig_token: &str) -> Option scan(dir).or_else(|| CALLER_SUBDIRS.iter().find_map(|sub| scan(&dir.join(sub)))) } -/// Locate a per-sample chrY GVCF for an alignment: the `NAVIGATOR_Y_GVCF` path override, else -/// [`gvcf_beside_alignment`]. `None` when absent — the private-Y path then falls back to the pileup -/// caller, and placement to a full CRAM walk. +/// Find the chrY GVCF file of one sample, for an alignment. The `NAVIGATOR_Y_GVCF` variable gives +/// the path when the user sets it. If not, the code calls [`gvcf_beside_alignment`]. +/// +/// The function returns `None` when it finds no file. The private-Y path then uses the pileup +/// caller, and the placement walks the full CRAM file. pub(crate) fn chr_y_gvcf_for_alignment(aln: &Alignment) -> Option { if let Ok(p) = std::env::var("NAVIGATOR_Y_GVCF") { let p = PathBuf::from(p); @@ -65,8 +78,9 @@ pub(crate) fn chr_y_gvcf_for_alignment(aln: &Alignment) -> Option { gvcf_beside_alignment(aln, "chry") } -/// Locate a per-sample chrM GVCF for an alignment: the `NAVIGATOR_M_GVCF` path override, else -/// [`gvcf_beside_alignment`]. The mtDNA counterpart to [`chr_y_gvcf_for_alignment`]. +/// Find the chrM GVCF file of one sample, for an alignment. The `NAVIGATOR_M_GVCF` variable gives +/// the path when the user sets it. If not, the code calls [`gvcf_beside_alignment`]. This function +/// is the mtDNA form of [`chr_y_gvcf_for_alignment`]. pub(crate) fn chr_m_gvcf_for_alignment(aln: &Alignment) -> Option { if let Ok(p) = std::env::var("NAVIGATOR_M_GVCF") { let p = PathBuf::from(p); @@ -77,9 +91,12 @@ pub(crate) fn chr_m_gvcf_for_alignment(aln: &Alignment) -> Option { gvcf_beside_alignment(aln, "chrm") } -/// The bundled-mask filename token for an alignment's reference build, or `None` when no chrY masks -/// ship for it. CHM13 masks are native (hs1); the GRCh38 masks are lifted from them (CrossMap -/// hs1→hg38). GRCh37 has no masks yet (bare-`Y` contig naming + no lifted set). +/// The token in the mask file name for the reference build of an alignment. The function returns +/// `None` when the bundle holds no chrY mask for that build. +/// +/// The CHM13 masks are native, on hs1. CrossMap moves them from hs1 to hg38, and those files are the +/// GRCh38 masks. There is no mask for GRCh37 yet. That build names its contig `Y`, and no code moved +/// the masks to it. fn y_mask_build_token(build: &str) -> Option<&'static str> { match canonical_build(build) { Some(ReferenceBuild::Chm13v2 | ReferenceBuild::Chm13v2MaskedRcrs) => Some("chm13v2"), @@ -95,10 +112,12 @@ type YRegionsHandle = std::sync::Arc, ) -> Result, AppError> { let aln = self.alignment_or_err(alignment_id).await?; - // The reference is required: a GVCF hom-ref site means "the sample's base == the - // reference base" — and the reference (e.g. CHM13 = HG002/J1 Y) is itself deep in the - // tree, so its base there is often the *derived* allele, not the ancestral. We read the - // reference base at every callable tree position (exactly what call_bases_at observes). + // The method needs the reference. A hom-ref site in a GVCF file states that the base of + // the sample equals the base of the reference. + // + // The reference is itself deep in the tree. The CHM13 reference holds the Y chromosome of + // HG002, which is in haplogroup J1. So its base at a tree position is often the *derived* + // allele and not the ancestral one. + // + // So the code reads the reference base at each callable tree position, which is the value + // that `call_bases_at` also reads. let reference = match aln.reference_path { Some(p) => PathBuf::from(p), None => { @@ -153,8 +177,9 @@ impl App { let ref_base = self.reference_bases(&reference, contig, &called.callable).await?; Ok(gvcf::assemble_calls(&called, &ref_base)) } - // Lifted: read the GVCF at each lifted contig + the reference bases there, then map - // observations back to tree positions (reverse-complementing minus-strand lifts). + // The code moved the positions. It reads the GVCF file at each new contig, and it + // reads the reference bases there. It then maps each observation back to a tree + // position. For a position on the minus strand, it takes the reverse complement. Some(lifted) => { let mut by_contig: HashMap> = HashMap::new(); for lp in &lifted { @@ -178,9 +203,13 @@ impl App { } } - /// Reference genome bases (uppercase A/C/G/T) at `positions` on `contig`. Reads the contig - /// sequence once off-thread; positions are 1-based. Non-ACGT / out-of-range positions are - /// omitted. Used by the GVCF fast path to resolve hom-ref tree sites to the actual base. + /// The reference genome bases at each of the `positions` on `contig`. Each base is an upper-case + /// A, C, G, or T. + /// + /// The method reads the contig sequence one time, on another thread. Each position is 1-based. + /// The result holds no position with another base, and no position outside the contig. + /// + /// The GVCF fast path calls this method. It needs the real base at a hom-ref tree site. async fn reference_bases( &self, reference: &Path, @@ -212,19 +241,27 @@ impl App { Ok(map) } - /// Fingerprint of a GVCF-sourced placement: the GVCF's content hash ⊕ the tree's hash. - /// Distinct from the CRAM-based [`Self::y_score_fingerprint`] (`gv:` vs `f:` prefix) so a - /// later deep analyze can tell the call came from a sidecar (phase: deep-pass skip logic). + /// The fingerprint of a placement that came from a GVCF file. The value joins the content hash + /// of that file with the hash of the tree. + /// + /// The value is different from the fingerprint of [`Self::y_score_fingerprint`], which the CRAM + /// path writes. This one starts with `gv:`, and that one starts with `f:`. So a later deep + /// analysis can see that the call came from a sidecar file, and it can then skip a step. async fn gvcf_fingerprint(&self, gvcf: &Path, tree_json: &str, tag: &str) -> Result { let h = sha256_file_async(gvcf.to_path_buf()).await?; Ok(format!("gv:{}|{}:{}", &h[..16], tag, &sha256_str(tree_json)[..16])) } - /// Assign a Y haplogroup from a precomputed chrY GVCF — no CRAM walk. Places against the - /// DecodingUs tree at the alignment's native build (liftover-free), records the call under - /// the same source key as the CRAM path (`aln:{id}`) with a `gv:`-prefixed fingerprint. - /// Errors if the build has no DecodingUs coordinates or the tree is unreachable; the caller - /// (`ingest_sidecars`) treats that as "leave Y for the deep pass". + /// Assign a Y haplogroup from a chrY GVCF file that another tool made. The method walks no CRAM + /// file. + /// + /// It places the sample against the DecodingUs tree, at the native build of the alignment, and + /// it needs no liftover. It writes the call under the same source key as the CRAM path, which is + /// `aln:{id}`, with a fingerprint that starts with `gv:`. + /// + /// The method fails when the DecodingUs tree has no coordinates for that build, and when it can + /// not read the tree. The caller is `ingest_sidecars`, and it then leaves the Y haplogroup for + /// the deep pass. pub async fn assign_y_from_gvcf(&self, alignment_id: i64, gvcf: &Path) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let build_key = decodingus_build_key(&aln.reference_build).ok_or_else(|| { @@ -236,13 +273,19 @@ impl App { let tree_json = self.fetch_decodingus_y_tree().await?; let tree = navigator_analysis::haplo::parse_decodingus_json(&tree_json, build_key).map_err(AppError::Import)?; let calls = self.gvcf_base_calls(alignment_id, "chrY", gvcf, &tree, None).await?; - // Robust (proportional-top) selection, not the strict alignment-tuned guard. A - // joint-genotyped GVCF gives confident calls that include a few stray ancestral - // contradictions on the deep backbone (recurrent sites, the CHM13=J1 reference, joint - // hard-filters); strict `path_admissible` then vetoes the genuine deep lineage and - // drops to a shallow node (HG00096 → A1b instead of its true R1b1a1b1a1a, which `score` - // ranks top at 344/364). This is the same confident-but-sparse-contradiction regime as - // BISDNA chip data — see [`assemble_assignment_robust`]. + // Use the proportional-top selection, and not the strict guard that the alignment path + // needs. + // + // A GVCF file from a joint genotype step gives confident calls. A few of those calls + // contradict the deep backbone with an ancestral state. The causes are a recurrent site, + // the CHM13 reference, which is in haplogroup J1, and the hard filters of the joint step. + // + // The strict `path_admissible` rule then refuses the true deep lineage and takes a node + // near the root. Sample HG00096 gave A1b, and its true terminal is R1b1a1b1a1a. The `score` + // function ranks that terminal first, at 344 of 364. + // + // The data has the same shape as BISDNA chip data: confident, with a few contradictions. + // See [`assemble_assignment_robust`]. let assignment = assemble_assignment_robust(&tree, &calls); if let Ok(bio) = self.biosample_of_alignment(alignment_id).await { let fp = self.gvcf_fingerprint(gvcf, &tree_json, "yt").await.ok(); @@ -260,10 +303,15 @@ impl App { Ok(assignment) } - /// Assign an mtDNA haplogroup from a precomputed chrM GVCF — no CRAM walk. Places against - /// the FTDNA mt tree; on CHM13 the tree's rCRS positions are lifted onto `chrM` (the cheap - /// self-generated rCRS↔chrM map), on GRCh38 they are read directly. Recorded under the CRAM - /// path's mt source key (`aln:{id}:mt`) with a `gv:`-prefixed fingerprint. + /// Assign an mtDNA haplogroup from a chrM GVCF file that another tool made. The method walks no + /// CRAM file. + /// + /// It places the sample against the mt tree of FTDNA. On GRCh38, it reads the rCRS positions of + /// that tree directly. On CHM13, it moves those positions onto the `chrM` contig. The code makes + /// that map itself, at a low cost. + /// + /// The method writes the call under the mt source key of the CRAM path, which is `aln:{id}:mt`, + /// with a fingerprint that starts with `gv:`. pub async fn assign_mt_from_gvcf(&self, alignment_id: i64, gvcf: &Path) -> Result { let tree_json = self.fetch_ftdna_mt_tree().await?; let tree = navigator_analysis::haplo::parse_ftdna_json(&tree_json).map_err(AppError::Import)?; @@ -271,8 +319,8 @@ impl App { let calls = self .gvcf_base_calls(alignment_id, "chrM", gvcf, &tree, source_build) .await?; - // Robust selection, as for Y (see assign_y_from_gvcf) — the GVCF's confident calls fit - // the proportional-top regime better than the strict alignment guard. + // Use the proportional-top selection, as the Y path does. See assign_y_from_gvcf. The + // confident calls of a GVCF file fit that rule better than the strict alignment guard. let assignment = assemble_assignment_robust(&tree, &calls); if let Ok(bio) = self.biosample_of_alignment(alignment_id).await { let fp = self.gvcf_fingerprint(gvcf, &tree_json, "mt").await.ok(); @@ -290,11 +338,14 @@ impl App { Ok(assignment) } - /// The sidecar paths this alignment was ingested from, as recorded by [`Self::ingest_sidecars`]. + /// The sidecar paths that this alignment came from. [`Self::ingest_sidecars`] writes them. + /// + /// The method reads the value and does not compare the mtime of the source. This value records + /// *what the app used*. It is not a derived result, so a change to the CRAM file does not make + /// it wrong. /// - /// Read directly, with no source-mtime freshness check: this is a record of *what was used*, - /// not a derived result that a changed CRAM invalidates. `None` for an alignment that never - /// went through the fast path (imported before this was recorded, or with no sidecars at all). + /// The method returns `None` for an alignment that never used the fast path. Such an alignment + /// arrived before this record existed, or it had no sidecar file. pub async fn recorded_sidecars(&self, alignment_id: i64) -> Result, AppError> { match artifact::get(self.store.pool(), alignment_id, SIDECARS_KIND, SIDECARS_VERSION).await? { Some(a) => Ok(serde_json::from_str(&a.payload).ok()), @@ -302,11 +353,17 @@ impl App { } } - /// Fast-path ingest of a sample's pipeline sidecars onto one alignment: place Y + mt from - /// the GVCFs, and fill sex / read-metrics / lite-coverage from the text sidecars — all - /// without touching the CRAM. Each step is independent and best-effort: a failure is - /// recorded in the returned report and the rest proceed (a missing/!matching sidecar just - /// leaves that result for the deep pass). Returns what it managed to fill. + /// Read the pipeline sidecar files of a sample onto one alignment, on the fast path. + /// + /// The method places the Y haplogroup and the mt haplogroup from the GVCF files. It fills the + /// sex, the read metrics, and a small coverage result from the text sidecar files. It reads no + /// CRAM file. + /// + /// Each step is independent, and each one is optional. A failure goes into the report that the + /// method returns, and the other steps continue. An absent sidecar file, or one that does not + /// match, leaves that result for the deep pass. + /// + /// The method returns the values that it filled. pub async fn ingest_sidecars( &self, alignment_id: i64, @@ -314,12 +371,17 @@ impl App { ) -> Result { let mut out = SidecarIngest::default(); - // Record which files this alignment was ingested from, before using them. Discovery is a - // directory scan done once at import, so without this the fast path is a one-shot: a Y - // placement made from a GVCF against the tree of the day could never be re-derived, and the - // resulting `haplogroup_call` row outlived every tree it was placed against. See - // `App::replace_against_current_tree`, which replays this. Best-effort — a workspace that - // can not record the paths should still get the ingest. + // Record the files that this alignment came from, before the code reads them. + // + // The code finds those files with one directory scan, at the import. Without this record, + // the fast path runs one time only. A Y placement from a GVCF file, against the tree of + // that day, could never run again. The `haplogroup_call` row then stayed after each later + // tree. + // + // `App::replace_against_current_tree` reads this record and runs the placement again. + // + // The step is optional. A workspace that can not write the paths must still receive the + // data. let _ = self .save_analysis_with_provenance( alignment_id, @@ -350,16 +412,20 @@ impl App { Err(e) => out.errors.push(format!("sex: {e}")), } } - // Read metrics: richest source wins — samtools `stats` (full, with histograms) > Picard - // AlignmentSummaryMetrics > samtools `flagstat` (counts only). + // The read metrics. The source with the most data wins. The order is samtools `stats`, + // which holds the full data with each histogram, then Picard AlignmentSummaryMetrics, then + // samtools `flagstat`, which holds counts only. match self.ingest_read_metrics(alignment_id, sidecars).await { Ok(true) => out.read_metrics = true, Ok(false) => {} Err(e) => out.errors.push(format!("read metrics: {e}")), } - // Coverage: samtools `coverage` gives per-contig stats; Picard CollectWgsMetrics gives the - // genome-wide depth distribution (median/sd/MAD, exclusion fractions, pct_Nx). Use whichever - // are present, overlaying the distribution onto the per-contig breakdown. + // The coverage. The samtools `coverage` output holds the statistics of each contig. The + // Picard CollectWgsMetrics output holds the depth distribution of the full genome. That + // distribution is the median, the sd, the MAD, the fractions that the tool excluded, and + // the pct_Nx values. + // + // Use each file that exists, and write the distribution onto the table of contigs. if sidecars.coverage.is_some() || sidecars.wgs_metrics.is_some() { match self.ingest_coverage_sidecar(alignment_id, sidecars).await { Ok(wrote) => out.lite_coverage = wrote, @@ -418,7 +484,8 @@ impl App { } else { return Ok(false); }; - // Do not downgrade a full deep walk on reimport — keep it if it is already equal-or-fuller. + // A second import must not replace a full deep walk with a smaller result. Keep the stored + // result when it is the same or better. let wrote = self .save_analysis_no_downgrade( alignment_id, @@ -432,8 +499,11 @@ impl App { Ok(wrote) } - /// Ingest lite coverage from the sidecar(s). Returns whether it was written (`false` = an - /// equal-or-fuller coverage artifact already exists, e.g. a deep walk on reimport). + /// Read the small coverage result from the sidecar files. The method returns `true` when it + /// wrote the result. + /// + /// It returns `false` when the store already holds a coverage artifact that is the same or + /// better. A deep walk from an earlier run gives such an artifact. async fn ingest_coverage_sidecar(&self, alignment_id: i64, sidecars: &SampleSidecars) -> Result { let read = |p: &Path| { let p = p.to_path_buf(); @@ -443,7 +513,8 @@ impl App { .map_err(|e| AppError::Import(format!("{}: {e}", p.display()))) } }; - // Per-contig stats + callable counts from samtools coverage (empty base if absent). + // The statistics of each contig, and the callable counts, from the samtools coverage + // output. The code starts from an empty value when that file is absent. let lite = match &sidecars.coverage { Some(cp) => { let cov = read(cp).await?; @@ -455,8 +526,9 @@ impl App { } None => CoverageResult::default(), }; - // Overlay Picard's genome-wide depth distribution onto the per-contig breakdown: start from - // the Picard result (median/sd/MAD, exclusion fractions, pct_Nx) and graft the contig stats. + // Write the genome-wide depth distribution of Picard onto the table of contigs. Start from + // the Picard result, which holds the median, the sd, the MAD, the fractions that the tool + // excluded, and the pct_Nx values. Then add the statistics of each contig. let result = match &sidecars.wgs_metrics { Some(wp) => match sidecar::parse_wgs_metrics(&read(wp).await?) { Some(mut w) => { @@ -475,9 +547,11 @@ impl App { }, None => lite, }; - // Still `partial`: no per-base depth histogram (only the deep walk produces that), so the - // deep pass still upgrades this. Stored under the standard coverage key. Never downgrade a - // full deep-walk coverage on reimport — keep it if one is already present. + // The result keeps the `partial` mark. It holds no depth histogram for each base, because + // only the deep walk makes one. So the deep pass still replaces this result. + // + // The store holds it under the standard coverage key. A second import must never replace a + // full deep-walk result with this one. Keep the stored result when it exists. let wrote = self .save_analysis_no_downgrade( alignment_id, @@ -491,14 +565,20 @@ impl App { Ok(wrote) } - /// Self-referential callable intervals (BED 0-based half-open) for `contig` from the - /// alignment's own reads. Parameters adapt to the sample: long reads (HiFi) earn - /// callability at lower depth, and the CALLABLE-run gate scales with molecule length - /// (`f`·fragment), so long molecules clear it over far more of chrY. Requires the BAM. + /// The callable intervals of `contig`, from the reads of the alignment itself. The intervals + /// are in the BED form, which is 0-based and half open. + /// + /// The parameters change with the sample. A long read from a HiFi test becomes callable at a + /// lower depth. The limit on a CALLABLE run also grows with the length of the molecule, at + /// `f` times the fragment length. So a long molecule passes that limit across much more of + /// chrY. + /// + /// The method needs the BAM file. pub async fn callable_chr_intervals(&self, alignment_id: i64, contig: &str) -> Result, AppError> { - // Resolve the reference via the gateway when the alignment has no stored path — a CRAM can't - // be decoded without one, and most imported alignments leave `reference_path` null (the build - // alone is recorded). Same resolution the de-novo caller uses. + // Find the reference through the gateway when the alignment holds no path. No reader can + // decode a CRAM file without a reference, and most imported alignments hold a NULL + // `reference_path` value. The row then records the build only. The de-novo caller finds the + // reference in the same way. let (bam, reference) = self.alignment_bam_reference(alignment_id).await?; let reference = Some(reference); let contig = contig.to_string(); @@ -515,10 +595,14 @@ impl App { .map_err(Into::into) } - /// The **private bucket**: de-novo SNP calls on chrY that the Y placement does not - /// explain (not on the assigned backbone), classified as off-path-known (a finer/ - /// sibling FTDNA branch) or novel (a new-branch candidate). With `callable_bed` (e.g. - /// the Poznik/1KG `b38_sites.bed`), calls outside reliable regions are dropped. + /// The **private bucket**. It holds the de-novo SNP calls on chrY that the Y placement does not + /// explain. Those calls are not on the backbone that the code assigned. + /// + /// The method puts each call in one of two groups. A known call off the path marks a finer FTDNA + /// branch, or a branch beside the assigned one. A new call is a candidate for a new branch. + /// + /// With `callable_bed`, such as the Poznik file `b38_sites.bed` from 1KG, the method removes + /// each call outside a reliable region. pub async fn private_y_variants( &self, alignment_id: i64, @@ -531,13 +615,18 @@ impl App { self.private_y_core(alignment_id, mask).await } - /// [`private_y_variants`] using the sample's **own** callable-Y BED as the mask - /// (self-referential — adapts to the sample's depth and read tech; no external file). + /// The work of [`private_y_variants`], with the callable-Y BED of the sample itself as the + /// mask. That mask changes with the depth and the read technology of the sample, and it needs no + /// other file. + /// + /// With a GVCF sidecar for the sample, the method **does not** apply that mask. The confidence + /// values in the GVCF file are the evidence that a site is callable. /// - /// With a per-sample GVCF sidecar the self-mask is **skipped**: the GVCF's own confidence - /// gating is the callable evidence (re-imposing Navigator's callable-loci depth threshold would - /// discard GATK calls the whole point is to trust), and skipping it avoids a CRAM walk — so the - /// GVCF fast path stays fast. Reliability then comes from the cohort callable mask + GVCF GQ. + /// A second depth limit from Navigator would remove GATK calls, and the purpose of this path is + /// to trust those calls. The method also avoids a CRAM walk, so the GVCF fast path stays fast. + /// + /// The reliability then comes from the callable mask of the cohort and the GQ value of the GVCF + /// file. pub async fn private_y_variants_self_masked(&self, alignment_id: i64) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let mask = if chr_y_gvcf_for_alignment(&aln).is_some() { @@ -547,11 +636,15 @@ impl App { Some(navigator_analysis::mask::RegionMask::from_intervals(intervals)) }; let bucket = self.private_y_core(alignment_id, mask).await?; - // Persist the self-masked bucket so it reloads instead of recomputing next session. Version - // "3": prefers a per-sample GVCF sidecar as the derived-call source (was pileup-only in v2), - // so v2 blobs must recompute rather than reload. - // Version 4: private variants are now classified against structural masks lifted to the - // alignment's own build. A v3 bucket on a GRCh38 alignment saw no mask at all. + // Write the masked bucket to the store, so the next session reads it and calculates + // nothing. + // + // Version 3 takes the GVCF sidecar of the sample as its source of derived calls. Version 2 + // used the pileup only. So the code must calculate a version 2 value again, and it must not + // read it. + // + // Version 4 classifies each private variant against the structural masks in the build of + // the alignment. A version 3 bucket on a GRCh38 alignment used no mask. self.save_analysis(alignment_id, "private_y", "4", &bucket).await?; Ok(bucket) } @@ -561,27 +654,36 @@ impl App { self.load_analysis(alignment_id, "private_y", "4").await } - /// Shared core: assign Y, de-novo chrY, subtract the backbone, optionally mask, classify. - /// The curated CHM13 chrY structural regions (palindrome/amplicon/AZF-DYZ), resolving + - /// caching the three BEDs on first use. Best-effort: any download/parse failure yields - /// `None` so the annotation never blocks the analysis. - /// Genome-region metadata (centromere/telomere/cytoband/PAR) for a build, via the gateway's - /// 2-layer cache (fetches the UCSC cytoBand table on a cold miss). For QC / display context. + /// The shared core. It runs five steps. It assigns the Y haplogroup and calls de-novo variants + /// on chrY. It then removes the backbone calls, applies a mask when the caller asks for one, and + /// classifies each remaining call. + /// + /// It also reads the curated structural regions of chrY on CHM13, which are the palindromes, the + /// amplicons, and the AZF-DYZ regions. It finds and caches the three BED files at the first use. + /// Each step is optional, and a failed download or a failed parse gives `None`. So this + /// annotation never stops the analysis. + /// + /// It also reads the genome-region metadata of a build, which is the centromere, the telomeres, + /// the cytobands, and the PAR regions. Those values come from the two-layer cache of the + /// gateway, and the gateway reads the UCSC cytoBand table when the cache holds nothing. The UI + /// uses them for quality checks and for context. pub async fn genome_regions(&self, build: &str) -> Result, AppError> { Ok(self.gateway.genome_regions(build, &mut |_, _| {}).await?) } - /// Region annotation for a 1-based `position` on `contig` in `build` (centromere/telomere/PAR - /// membership + cytoband name). Uses the cached regions only — `None` if not yet fetched. + /// The region of a 1-based `position` on `contig` in `build`. The value states whether the + /// position is in a centromere, a telomere, or a PAR region, and it gives the cytoband name. The + /// method reads the cache only, and it returns `None` when the cache holds nothing. pub fn region_annotation(&self, build: &str, contig: &str, position: i64) -> Option { self.gateway .cached_genome_regions(build) .map(|r| r.annotate(contig, position)) } - /// Memo for [`y_structural_regions_for`]: lifting parses the whole chain file, and a project - /// pass over thousands of subjects would otherwise repeat that per subject — the same trap the - /// tree fetch fell into. The masks are static within a process, so resolve each build once. + /// A memory for [`y_structural_regions_for`]. A liftover parses the full chain file. A project + /// pass across thousands of subjects would repeat that parse for each subject, and the tree + /// fetch had the same fault. The masks do not change inside one process, so the code resolves + /// each build one time. fn y_regions_memo() -> &'static std::sync::Mutex>> { static MEMO: std::sync::OnceLock>>> = std::sync::OnceLock::new(); @@ -590,25 +692,30 @@ impl App { /// The curated chrY structural regions **in `build`'s coordinates**. /// - /// The three BEDs are CHM13-native, so anything else is lifted. That matters more than it - /// sounds: without it, a GRCh38 or GRCh37 source has no structural mask at all, every - /// palindromic and amplicon call counts as unique sequence, and private-variant counts inflate - /// into the hundreds — the difference between a donor averaging 4 and one averaging 661. + /// The three BED files are native to CHM13, so the code moves them to any other build. /// - /// Best-effort throughout: any download / chain / parse failure yields `None` so the annotation - /// never blocks the analysis, exactly as before. + /// That step is important. Without it, a GRCh38 source or a GRCh37 source has no structural + /// mask. Each call in a palindrome and in an amplicon then counts as unique sequence, and the + /// count of private variants grows into the hundreds. One donor gave a mean of 4 with the mask + /// and a mean of 661 without it. + /// + /// Each step is optional. A failed download, a failed liftover, and a failed parse each give + /// `None`, so this annotation never stops the analysis. async fn y_structural_regions_for(&self, build: &str) -> Option { - // Keyed by the *canonical* build: `hs1`, `CHM13v2.0` and the masked variant share coordinates - // and must share one entry rather than lifting three times. + // The key is the *canonical* build. The builds `hs1`, `CHM13v2.0`, and the masked build + // use the same coordinates. So they must share one entry, and the code must not do three + // liftovers. let key = canonical_build(build)?.as_str().to_string(); if let Some(hit) = Self::y_regions_memo().lock().unwrap().get(&key) { return hit.clone(); } let built = self.build_y_structural_regions(build).await.map(std::sync::Arc::new); if built.is_none() { - // Cached so a batch does not retry a failing download per subject — but *said*, because - // "no structural mask" is the condition that inflated private-variant counts into the - // hundreds in the first place, and it must never be reached silently again. + // The code caches this state, so a batch does not try a failed download again for + // each subject. It also writes a message. + // + // The state "no structural mask" grew the count of private variants into the hundreds. + // The app must never reach that state again with no message. eprintln!( "no chrY structural mask available for {key} — private-variant counts will include \ palindromic and amplicon calls" @@ -642,8 +749,9 @@ impl App { if matches!(target, ReferenceBuild::Chm13v2 | ReferenceBuild::Chm13v2MaskedRcrs) { return Some(native); } - // PAR and heterochromatin are taken natively per build rather than lifted — a chain is least - // trustworthy in exactly those places (PAR is shared with chrX, Yq12 is satellite). + // The code reads the PAR regions and the heterochromatin natively for each build. It does + // not move them from CHM13. A chain file is least reliable in those places. Both chrX and + // chrY hold the PAR regions, and Yq12 is satellite sequence. let landmarks = navigator_analysis::mask::y_landmarks(build)?; self.gateway @@ -656,8 +764,9 @@ impl App { .lift_intervals(ReferenceBuild::Chm13v2.as_str(), target.as_str(), "chrY", m.intervals()) .ok()?; if iv.is_empty() { - // A mask that lifted to nothing is not a mask; better to annotate nothing than to - // report "no structural regions here" as though it had been checked. + // A mask that gives no interval after the liftover is not a mask. The code then + // writes no annotation. It must not report "no structural regions here" as a + // result of a real check. eprintln!("chrY {what} mask lifted CHM13→{} to nothing; skipping", target.as_str()); return None; } @@ -676,23 +785,28 @@ impl App { RegionMask::from_intervals(landmarks.par.to_vec()), lift(palindrome_m, "palindrome")?, lift(amplicon_m, "amplicon")?, - // The satellite arrays rarely survive a chain, so the build's heterochromatin bound is - // the load-bearing part here and the lifted AZF/DYZ intervals only refine it. + // A satellite array rarely survives a chain file. So the heterochromatin bound of the + // build carries this test, and the AZF and DYZ intervals from the liftover only make it + // more exact. lift(native.heterochromatin_mask(), "AZF/DYZ") .unwrap_or_else(|| RegionMask::from_intervals(vec![])) .union(&[landmarks.heterochromatin]), )) } - /// Derive chrY private-variant candidates from a per-sample GVCF, returning the same - /// [`VariantCall`] shape the pileup de-novo path produces so `private_y_core`'s downstream - /// classification is identical. GATK's reassembly recovers SNVs the pileup caller misses. + /// Find the private-variant candidates on chrY from the GVCF file of one sample. + /// + /// The method returns the same [`VariantCall`] shape that the pileup de-novo path returns. So + /// the classification in `private_y_core` is the same for both paths. + /// + /// The reassembly step of GATK finds SNVs that the pileup caller does not find. async fn run_denovo_from_gvcf(&self, gvcf: &Path) -> Result, AppError> { let gvcf = gvcf.to_path_buf(); let snvs = tokio::task::spawn_blocking(move || { - // min_dp 4 (over the reader's permissive default 2): a real private SNV is covered by ≥4 - // reads, whereas a misaligned-read cluster smears 2–3 reads across many nearby false SNVs - // — so the depth floor removes those artifact clusters without touching the (DP≥4) truth. + // Use min_dp 4, and not the default of 2 that the reader allows. A real private SNV + // has 4 reads or more. A group of misaligned reads gives 2 or 3 reads across many false + // SNVs that are near each other. So this depth limit removes each artefact group, and + // it keeps each true call with a DP of 4 or more. let params = navigator_analysis::gvcf::GvcfReadParams { min_dp: 4, min_gq: 20 }; navigator_analysis::gvcf::read_derived_snvs(&gvcf, "chrY", ¶ms) }) @@ -717,15 +831,21 @@ impl App { alignment_id: i64, mask: Option, ) -> Result { - // Classify novels against the **DecodingUs** tree — the app's placement authority, which - // folds in the cohort-derived branches (from the de-novo tree pipeline). A shared lineage - // variant is named there, so it reads as OffPathKnown, not a false "novel"; a variant absent - // from this tree yet shared across the cohort is genuinely suspect. FTDNA fallback keeps the - // report working when the AppView tree is unavailable or the build has no DecodingUs coords. + // Classify each new call against the **DecodingUs** tree. That tree has the authority for + // a placement in this app, and it holds the branches that the de-novo tree pipeline found + // in the cohort. + // + // A variant of a shared lineage has a name in that tree, so the code marks it OffPathKnown + // and not "novel". A variant that the tree does not hold, and that the cohort shares, is + // doubtful. + // + // The FTDNA tree is the second choice. It keeps the report correct when the code can not + // read the AppView tree, and when the build has no DecodingUs coordinates. let (tree, tree_calls) = match self.y_decodingus_tree_calls(alignment_id).await { Ok(tc) => tc, - // A gone alignment file is not a tree problem: the fallback reads the same absent file, - // so it can only fail again while logging a tree provider that was never at fault. + // An absent alignment file is not a fault of the tree. The second path reads the same + // absent file, so it also fails. It then writes a log entry that names a tree provider + // with no fault. Err(e) if e.is_missing_alignment_file() => return Err(e), Err(e) => { eprintln!("DecodingUs Y tree unavailable ({e}); private-Y classifying against FTDNA"); @@ -741,27 +861,42 @@ impl App { let path = navigator_analysis::haplo::path_positions(&tree, terminal.id); let known = navigator_analysis::haplo::tree_positions(&tree); - // The structural BEDs are in CHM13 chrY coordinates, so they only annotate a CHM13 alignment. - // The cohort masks apply per build: native for CHM13, CrossMap-lifted (hs1→hg38) for GRCh38. + // The structural BED files hold CHM13 chrY coordinates, so they annotate a CHM13 alignment + // only. The cohort masks apply to each build: the CHM13 files are native, and CrossMap + // moved the GRCh38 files from hs1 to hg38. let aln = self.alignment_or_err(alignment_id).await?; let regions = self.y_structural_regions_for(&aln.reference_build).await; - // L2: the cohort **callable mask** (Poznik-style, CALLABLE in ≥90% of a ~3k-male cohort) — - // only ~25% of non-PAR chrY is reliably callable cohort-wide. L3: a **cohort-shared-sites** - // blocklist — every position that varies with ≥2 carriers across the cohort (plus homoplasy - // hotspots). A real shared lineage variant belongs in the DecodingUs tree (and so classifies - // as off-path-known above); one that is cohort-shared yet *absent* from the tree is a suspect - // recurrent artifact, not a private SNP. A truly private variant has a single cohort carrier, - // so it survives this filter. This is the single-sample stand-in for the de-novo pipeline's - // cohort carrier filter. Bundled per build (CHM13 native, GRCh38 lifted); absent ⇒ skipped. + // Layer 2 is the **callable mask** of the cohort, in the Poznik form. A position is in that + // mask when it is CALLABLE in 90% or more of a cohort of about 3,000 men. Only about 25% of + // chrY outside the PAR regions is reliably callable across a cohort. + // + // Layer 3 is a blocklist of the **sites that the cohort shares**. It holds each position + // that varies with two carriers or more across the cohort. It also holds the homoplasy + // hotspots. + // + // A real variant of a shared lineage is in the DecodingUs tree, and the step above marks it + // off-path-known. A variant that the cohort shares and the tree does not hold is a + // recurrent artefact. It is not a private SNP. + // + // A true private variant has one carrier in the cohort, so it passes this filter. This + // layer takes the place of the cohort carrier filter of the de-novo pipeline, for one + // sample. + // + // The bundle holds a file for each build. The CHM13 file is native, and CrossMap moved the + // GRCh38 file. The code skips this layer when the file is absent. let mask_token = y_mask_build_token(&aln.reference_build); let cohort_mask = mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_CALLABLE_MASK", "chrY_callable_mask", t)); let cohort_shared = mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_COHORT_SHARED", "chrY_cohort_shared_sites", t)); - // Derived-call source. Prefer a per-sample chrY GVCF (GATK HaplotypeCaller's local haplotype - // reassembly resolves misaligned-ref ~50/50 sites the pileup caller drops — see the WGS229 - // recall gap); fall back to Navigator's de-novo pileup caller when no sidecar is present. + // The source of the derived calls. Take the chrY GVCF file of the sample first. + // + // GATK HaplotypeCaller builds the local haplotypes again. Take a site with about 50% + // reference reads, where the mapper placed the reference reads wrongly. That step gives an + // answer at such a site, and the pileup caller removes it. See the recall gap of WGS229. + // + // When the sample has no sidecar file, use the de-novo pileup caller of Navigator. let denovo = match chr_y_gvcf_for_alignment(&aln) { Some(gvcf) => { eprintln!("private-Y: sourcing chrY calls from GVCF sidecar {}", gvcf.display()); @@ -817,7 +952,7 @@ mod gvcf_discovery_tests { } } - /// Each case gets its own directory — these run in parallel. + /// Each case has its own directory, because these tests run at the same time. fn scratch(name: &str) -> PathBuf { let d = std::env::temp_dir().join(format!("nav-gvcf-{}-{name}", std::process::id())); let _ = std::fs::remove_dir_all(&d); @@ -836,9 +971,11 @@ mod gvcf_discovery_tests { #[test] fn finds_a_bare_named_gvcf_in_a_caller_subdirectory() { - // The D2C per-run layout: `/CP086569.2/gatk4/chrY.g.vcf.gz`, with no sample prefix and - // one directory down. Matching only `*.chry.g.vcf.gz` beside the CRAM found none of these, - // so every subject fell back to decoding the whole chromosome. + // The D2C layout for one run: `/CP086569.2/gatk4/chrY.g.vcf.gz`. That name holds no + // sample prefix, and the file is one directory below the alignment. + // + // An earlier version matched `*.chry.g.vcf.gz` beside the CRAM file only, and it found no + // file of this kind. So each subject decoded the full chromosome. let d = scratch("subdir"); std::fs::write(d.join("chrYM.cram"), "").unwrap(); std::fs::create_dir_all(d.join("gatk4")).unwrap(); @@ -885,72 +1022,100 @@ mod gvcf_discovery_tests { /// path's floor: below this, a call is far more likely a misaligned-read cluster than a real SNV. const VCF_PRIVATE_MIN_DP: u32 = 4; -/// Genotype-quality floor, matching the GVCF path's `min_gq`. Aligns the two sources' gates as far -/// as the evidence allows — though it does not make them comparable: see -/// [`App::private_y_from_variant_set`] on why a vendor caller's call set is a different instrument. +/// The lowest genotype quality that a call can have. The value equals the `min_gq` value of the GVCF +/// path. +/// +/// This value makes the gates of the two sources as similar as the evidence permits. It does not +/// make their results comparable. [`App::private_y_from_variant_set`] gives the reason: the call set +/// of a vendor comes from a different instrument. const VCF_PRIVATE_MIN_GQ: u32 = 20; -/// Derived-allele fraction a call must reach to count as **deterministic** on a haploid chromosome. -/// chrY carries one copy, so a genuine call is essentially all-alt; a middling fraction is an -/// ambiguous locus, and an ambiguous call can not support a private-variant claim. +/// The fraction of derived reads that a call needs to be **deterministic** on a haploid chromosome. +/// +/// A cell holds one copy of chrY. So a true call has almost no reference read. A fraction between +/// the two states marks a locus with two answers, and such a call can not support a claim about a +/// private variant. const VCF_PRIVATE_MIN_AF: f64 = 0.95; -/// Depth ceiling, as a multiple of the donor's own typical depth at good calls. +/// The maximum depth of a call, as a factor of the usual depth of that donor at a good call. /// -/// chrY carries one copy, so a locus drawing far more reads than the rest of the chromosome is -/// collecting them from somewhere else — a collapsed repeat. This was found by reviewing a candidate -/// branch whose two carriers sat at DP 413 and 504 against a median of 57, each holding a stubborn -/// ~5% reference allele: the shape of a paralogous pile-up, and it cleared every other gate. Three -/// times the median keeps ~91% of quality calls while removing that tail. +/// A cell holds one copy of chrY. A locus with many more reads than the rest of the chromosome takes +/// those reads from another place. That place is a collapsed repeat. +/// +/// A review of one candidate branch found this fault. Its two carriers had a DP of 413 and 504, +/// against a median of 57. Each one also held about 5% reference reads. That shape is a pile of +/// reads from a paralog, and the call passed each other gate. +/// +/// A limit of three times the median keeps about 91% of the good calls and removes that group. const VCF_PRIVATE_MAX_DEPTH_RATIO: u32 = 3; -/// Minimum quality-passing calls before a depth ratio is trustworthy. Below this the median is not a -/// description of the donor's coverage, and the rule abstains rather than judging against noise. +/// The count of good calls that the code needs before it can trust a depth ratio. Below this count, +/// the median does not describe the coverage of the donor. The rule then does nothing, and it makes +/// no comparison against noise. const VCF_PRIVATE_MIN_CALLS_FOR_RATIO: usize = 20; impl App { - /// The **private bucket for a variant set** — the VCF counterpart of [`Self::private_y_variants`]. + /// The **private bucket of a variant set**. It is the VCF form of + /// [`Self::private_y_variants`]. + /// + /// The key of the private-Y data was always an alignment. The code walks a BAM file or a CRAM + /// file, or its GVCF sidecar, and it caches the result under `alignment_id`. + /// + /// A subject whose Y data arrived as a VCF file from another tool has no alignment. So the app + /// never offered this option to that subject. On R1b-CTS4466Plus, about 1,600 of the 1,881 + /// members are in that group. For this reason, each cohort feature that needs private variants + /// had almost no data. + /// + /// The classification is the same as the classification of the alignment path, by design. The + /// code removes the backbone that it placed. It then removes each call outside the callable mask + /// of the cohort, and each call on the blocklist of the cohort. It then separates the known + /// off-path calls from the new ones. /// - /// Private-Y has always been keyed on an alignment: it walks a BAM/CRAM (or its GVCF sidecar) and - /// caches against `alignment_id`. A subject whose Y data arrived as an externally processed VCF - /// has no alignment, so the option was never offered — on R1b-CTS4466Plus that is ~1,600 of 1,881 - /// members, and it is why cohort features that depend on private variants had almost nothing to - /// work with. + /// Two things are different: the source of the evidence, and the test of the reliability of the + /// donor. /// - /// The classification is deliberately the same as the alignment path's: subtract the placed - /// backbone, drop anything outside the cohort callable mask or on the cohort-shared blocklist, - /// then split off-path-known from novel. What differs is where the evidence comes from and how - /// the donor's own reliability is judged: + /// - **The placement** uses [`Self::vset_base_calls`]. So the terminal comes from the genotypes + /// at each tree position, and those genotypes include the hom-ref calls. It does not come from + /// the few derived calls alone. + /// - **There is no self-callable mask**, because a VCF file holds no coverage track. The + /// evidence of each call takes its place. The code removes a call with a `FILTER` flag, a call + /// below [`VCF_PRIVATE_MIN_DP`], and each heterozygous call on chrY. A cell holds one copy of + /// chrY, so such a call comes from a paralog or from a read that the mapper placed wrongly. + /// Those calls are about two thirds of the chrY rows of a Big Y test. + /// - **The depth and the allele fraction come from the source**, so [`PublishGate`] judges these + /// calls on real read evidence. A set that the app imported before it stored the evidence has + /// a `call_schema` of 1. Such a set gives nothing that the app can publish, and that result is + /// correct. /// - /// - **Placement** uses [`Self::vset_base_calls`], so the terminal is derived from tree-position - /// genotypes (including hom-ref) rather than the handful of derived calls alone. - /// - **There is no self-callable mask** — a VCF carries no coverage track. Its place is taken by - /// the source's own per-call evidence: a `FILTER`-flagged call is dropped, as is one below - /// [`VCF_PRIVATE_MIN_DP`], and a chrY heterozygote is dropped outright — on a haploid - /// chromosome that is a paralog or mismapping artefact, and it is ~2/3 of a Big Y's chrY rows. - /// - **Depth and allele fraction are the source's**, so [`PublishGate`] judges these calls on real - /// read evidence. A set imported before evidence capture (`call_schema` 1) therefore yields - /// nothing publishable, which is the honest outcome rather than a fabricated one. + /// **These counts do not compare with the counts of the alignment path, and they are not yet + /// good enough for branch inference.** /// - /// **Not comparable to the alignment path's counts, and not yet fit for branch inference.** This - /// yields a median ~175 novel calls per donor against the GVCF path's 3–13. The gap is the - /// instrument, not a defect here: the alignment path reads GATK HaplotypeCaller at ploidy 1, - /// while a vendor export is a diploid caller emitting far more chrY calls, and only ~10% of the - /// difference is reachable by matching DP/GQ gates. Note also that `Novel` means "not - /// branch-defining in *this* tree" — the tree is FTDNA's supported branches plus splits solved - /// from the cohort, not a catalogue of known Y variation — so a real, well-known variant that - /// defines no branch classifies as novel here. Feeding these buckets to the block tree's - /// candidate detection took CTS4466 from 3 candidates to 20 (39 conflicts, 105 recurrent - /// positions dropped) on only 111 of ~1,600 sets, which is why - /// [`Self::private_y_for_biosamples`] does not union them yet. + /// This path gives a median of about 175 new calls for each donor. The GVCF path gives 3 to 13. + /// The difference is the instrument and not a fault here. The alignment path reads GATK + /// HaplotypeCaller at ploidy 1. A vendor export comes from a diploid caller, and that caller + /// writes many more chrY calls. A match of the DP gates and the GQ gates removes only about 10% + /// of the difference. + /// + /// Note also that `Novel` means "this variant defines no branch in *this* tree". The tree holds + /// the branches that FTDNA supports, and the splits that the cohort resolved. It is not a + /// catalogue of each known Y variant. So a real, well-known variant that defines no branch is + /// `Novel` here. + /// + /// The block tree read these buckets for its candidate detection. On CTS4466 the count of + /// candidates went from 3 to 20, with 39 conflicts and 105 recurrent positions removed. Only 111 + /// of about 1,600 sets took part. For this reason, + /// [`Self::private_y_for_biosamples`] does not yet join them. pub async fn private_y_from_variant_set(&self, set: &VariantSet) -> Result { use navigator_analysis::haplo; - // Without per-call evidence every quality gate below is a no-op, and the result is a list of - // whatever the vendor's caller emitted — on a real set that is 400-550 "novel" calls against - // ~70 for the same donor's evidence-bearing set. A call we can not judge is the most - // non-deterministic kind there is, so refuse rather than publish a number that looks like a - // finding. Re-importing the source populates `CallEvidence` (migration 0042). + // With no evidence for each call, every quality gate below does nothing. The result is + // then the list that the caller of the vendor wrote. On a real set that list holds 400 to + // 550 "new" calls. The set of the same donor with evidence holds about 70. + // + // A call that the app can not judge is the least deterministic call of all. So the method + // refuses, and it does not publish a number that looks like a result. + // + // A second import of the source writes the `CallEvidence` rows. See migration 0042. if !set.has_evidence() { return Err(AppError::Import(format!( "variant set {} carries no per-call evidence (call_schema {}); re-import it to enable private-Y", @@ -966,9 +1131,12 @@ impl App { .values() .flat_map(|n| n.loci.iter().map(|l| l.position)) .collect(); - // `pv2`: the chrY structural masks are now lifted to the set's own build, so a `pv1` - // bucket was classified with **no** structural mask on anything but CHM13 and its counts - // are inflated. The version is the invalidation — `--force` can not reach this cache. + // The `pv2` key. The code now moves the chrY structural masks to the build of the + // set. + // + // A `pv1` bucket used **no** structural mask on any build except CHM13, so its counts + // are too high. The version number removes the old value from the cache, and the + // `--force` option can not reach this cache. format!("pv2:{}", crate::haplogroup::genotype_cache_key("chrY", None, &targets)) }; if let Ok(Some(json)) = variant_set_private_y::get(self.store.pool(), set.id, &cache_key).await { @@ -992,12 +1160,14 @@ impl App { mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_CALLABLE_MASK", "chrY_callable_mask", t)); let cohort_shared = mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_COHORT_SHARED", "chrY_cohort_shared_sites", t)); - // The structural BEDs are CHM13-native and lifted to whatever this set is in — without that - // a GRCh38 set has no structural mask and its private counts inflate into the hundreds. + // The structural BED files are native to CHM13, and the code moves them to the build of + // this set. Without that step, a GRCh38 set has no structural mask, and its count of private + // variants grows into the hundreds. let regions = self.y_structural_regions_for(&build).await; - // Quality-passing calls first, so the depth ceiling below is measured against the donor's own - // good coverage rather than against a median dragged down by the junk we are about to drop. + // Take the calls that pass the quality gates first. The depth limit below then compares + // against the good coverage of the donor. Without this order, it compares against a median + // that the calls below the gates make smaller. let passing: Vec<&navigator_domain::variants::VariantCall> = set .calls .iter() @@ -1025,8 +1195,9 @@ impl App { position: c.position, reference, alternate, - // The source's own numbers; absent when it gave none, which the publish gate - // then (correctly) refuses rather than treating as evidence. + // The numbers of the source. The value is absent when the source gave none. + // The publish gate then refuses that call, and that decision is correct. It + // must not read an absent value as evidence. depth: c.evidence.dp.unwrap_or(0), alt_depth: c.evidence.ad_alt.unwrap_or(0), allele_fraction: c.evidence.allele_fraction().unwrap_or(0.0), @@ -1051,12 +1222,17 @@ impl App { } } -/// Whether a genotype is a single-allele (hemizygous / homozygous-alt) call. +/// Shows whether a genotype holds one allele. Such a call is hemizygous or homozygous for the +/// alternate allele. +/// +/// A cell holds one copy of chrY. So a heterozygous call there is not possible in biology. It marks +/// a paralog, or a locus where the mapper placed the reads wrongly. /// -/// chrY is haploid, so a heterozygous call there has no biological reading: it is a paralogous or -/// mismapped locus. In a real Big Y export those are ~2/3 of the chrY rows, and admitting them would -/// make the private set mostly artefact. An absent genotype is admitted — a source that reports no GT -/// is not asserting heterozygosity. +/// In a real Big Y export, those calls are about two thirds of the chrY rows. With them, most of the +/// private set is an artefact. +/// +/// The function accepts a call with no genotype. A source that writes no GT field makes no statement +/// about heterozygosity. fn is_hemizygous(gt: Option<&str>) -> bool { let Some(gt) = gt else { return true }; let alleles: Vec<&str> = gt.split(['/', '|']).filter(|a| *a != ".").collect(); @@ -1069,8 +1245,9 @@ mod vcf_private_y_tests { #[test] fn a_chr_y_heterozygote_is_rejected() { - // chrY is haploid: a het call is a paralog or a mismapping, and it is ~2/3 of a Big Y's - // chrY rows — admitting them would make the private set mostly artefact. + // A cell holds one copy of chrY. So a heterozygous call marks a paralog, or a read that + // the mapper placed wrongly. Those calls are about two thirds of the chrY rows of a Big Y + // test. With them, most of the private set is an artefact. assert!(!is_hemizygous(Some("0/1"))); assert!(!is_hemizygous(Some("1|2"))); assert!(!is_hemizygous(Some("1/2"))); @@ -1086,16 +1263,18 @@ mod vcf_private_y_tests { #[test] fn a_source_that_reports_no_genotype_is_not_treated_as_heterozygous() { - // Absence of a GT is not an assertion about ploidy; rejecting it would silently discard - // every sites-only or CSV-derived set. + // An absent GT field makes no statement about the ploidy. A rule that refused it would + // remove each set with sites only, and each set from a CSV file, with no message. assert!(is_hemizygous(None)); assert!(is_hemizygous(Some("1/.")), "a partial call still carries one allele"); } } -/// Median read depth across `calls`, or `None` when too few carry one to describe the donor's -/// coverage. Median rather than mean: the pile-ups this exists to find would drag a mean upward and -/// hide themselves behind it. +/// The median read depth across `calls`. The function returns `None` when too few calls hold a depth +/// to describe the coverage of the donor. +/// +/// The function takes the median and not the mean. The piles of reads that this code must find make +/// a mean larger, and they then hide behind that larger value. fn median_depth(calls: &[&navigator_domain::variants::VariantCall]) -> Option { let mut depths: Vec = calls.iter().filter_map(|c| c.evidence.dp).collect(); if depths.len() < VCF_PRIVATE_MIN_CALLS_FOR_RATIO { @@ -1127,7 +1306,7 @@ mod depth_ratio_tests { #[test] fn the_median_ignores_the_pile_ups_it_exists_to_find() { - // A mean would be dragged up by the outliers and hide them behind itself. + // The extreme values make a mean larger, and they then hide behind that larger value. let mut calls: Vec = (0..VCF_PRIVATE_MIN_CALLS_FOR_RATIO).map(|_| call(Some(50))).collect(); calls.push(call(Some(2584))); calls.push(call(Some(1191))); diff --git a/crates/navigator-app/src/import_unified.rs b/crates/navigator-app/src/import_unified.rs index ab1e13a3..0453962d 100644 --- a/crates/navigator-app/src/import_unified.rs +++ b/crates/navigator-app/src/import_unified.rs @@ -5,20 +5,31 @@ use super::*; impl App { // ---- unified import ---------------------------------------------------- - /// Detect a file's type and route it to the right subject importer (STR / variants / - /// chip / mtDNA), using sensible defaults. Returns the detected type. Alignment files - /// are rejected here — they attach to a sequencing test, not directly to a subject. - /// Probe a BAM/CRAM header for the build/aligner/platform/test-type (best-effort). + /// Find the type of a file and send it to the correct importer for a subject. The importers + /// cover STR data, variants, a chip export, and mtDNA data. The method uses a default value + /// where it needs one, and it returns the type that it found. + /// + /// This method refuses an alignment file. Such a file belongs to a sequence test, and it does + /// not attach to a subject directly. + /// + /// The method also reads the header of a BAM file or a CRAM file. From that header it finds + /// the build, the aligner, the platform, and the test type. That step is optional. pub async fn probe_alignment(&self, path: PathBuf) -> Result { tokio::task::spawn_blocking(move || navigator_analysis::probe::probe_alignment(&path)) .await? .map_err(AppError::from) } - /// Scan a bounded prefix of an alignment's reads to infer the instrument/library identity — - /// the `@RG SM/LB/PU` tags plus the most-frequent instrument/flowcell/platform from read names - /// (the crowd-source input for resolving the lab). Off-thread (blocking IO + CRAM decode); - /// `reference` is required for CRAM. Best-effort — callers tolerate an error. + /// Read a limited count of records from the start of an alignment, and find the identity of the + /// instrument and the library. + /// + /// That identity is the `@RG` tags `SM`, `LB`, and `PU`. It also holds the most frequent + /// instrument, flowcell, and platform in the read names. The AppView uses those values to find + /// the laboratory. + /// + /// The method runs on another thread, because it blocks on I/O and decodes a CRAM file. A CRAM + /// file also needs the `reference` value. The step is optional, and each caller continues after + /// an error. pub async fn library_stats( &self, path: PathBuf, @@ -35,20 +46,30 @@ impl App { .map_err(AppError::from) } - /// Auto-import an alignment file by probing its header: create the sequencing run (test type, - /// platform, instrument) and the alignment (reference build + aligner) with no questions - /// asked. The reference FASTA is **not** required — it is resolved from the build on demand; - /// if already cached it is stored so every analysis step has it immediately. + /// Import an alignment file with no question to the user. The method reads the header of that + /// file. + /// + /// It then makes the sequence run, with the test type, the platform, and the instrument. It also + /// makes the alignment, with the reference build and the aligner. + /// + /// The method does **not** need the reference FASTA file. It finds that file from the build when + /// a step needs it. When the cache already holds the file, the method stores its path, and each + /// analysis step then has it at once. async fn import_alignment_file( &self, biosample_guid: SampleGuid, path: &Path, test_type_override: Option<&str>, ) -> Result<(), AppError> { - // Idempotent per subject: skip only if *this* subject already has the alignment. Dedup used - // to be global (any subject), which silently skipped importing a file into a new subject when - // another subject already had it — leaving an empty subject and a misleading "imported" toast - // (e.g. re-importing a file after deleting its old subject, when a sibling subject also has it). + // A second import is safe for one subject. The code skips the file only when *this* + // subject already holds the alignment. + // + // An earlier version compared across each subject. So the code skipped a file for a new + // subject when another subject already held it, and it gave no message. The new subject + // stayed empty, and the app showed an "imported" message that was not true. + // + // One case is a second import of a file after the user deleted its earlier subject, when + // another subject also holds that file. let path_str = path.to_string_lossy().into_owned(); if alignment::list_for_biosample(self.store.pool(), biosample_guid) .await? @@ -57,25 +78,31 @@ impl App { { return Ok(()); } - // Best-effort: a probe failure falls back to filename/defaults rather than aborting. + // The step is optional. After a failed read of the header, the code uses the file name and + // its default values. It does not stop the import. let probe = self.probe_alignment(path.to_path_buf()).await.unwrap_or_default(); - // Resolve the reference first — the read-name scan needs it to decode a CRAM. + // Find the reference first. The scan of the read names needs it to decode a CRAM file. let reference_build = probe .reference_build .clone() .unwrap_or_else(|| reference_build_for(path)); - // Store the cached reference path if we have it; otherwise leave it unset (resolved on - // demand) — never block import on a download. + // Store the path of the reference when the cache holds that file. If not, leave the field + // empty, and the code finds the file when a step needs it. An import must never wait for a + // download. let reference_path = self .gateway .cached_reference(&reference_build) .map(|p| p.to_string_lossy().into_owned()); - // Read-name scan → instrument/library identity (the lab crowd-source input). Best-effort: - // it fills the platform/model the header `@RG` left blank, and the instrument/flowcell that - // never live in the header. Skipped silently if the file can't be read (e.g. CRAM with no - // resolved reference yet). + // The scan of the read names gives the identity of the instrument and the library, and the + // AppView uses those values to find the laboratory. + // + // The step is optional. It fills the platform and the model when the `@RG` header holds + // neither. It also fills the instrument and the flowcell, which no header holds. + // + // The code skips this step when it can not read the file. One case is a CRAM file with no + // reference yet. let stats = self .library_stats(path.to_path_buf(), reference_path.as_deref().map(PathBuf::from)) .await @@ -97,13 +124,19 @@ impl App { .clone() .or_else(|| stats.as_ref().and_then(|s| s.instrument_model.clone())); - // Test type: refine the header/platform guess with coverage *shape* from the BAI index — - // a targeted-Y pile-up (autosomes empty) → Big Y / Y Elite / YSEQ; an mtDNA pile-up → - // mtFull. Best-effort and cheap (O(contigs), no read scan); CRAM / unindexed BAMs have no - // profile and keep the platform-based guess. - // An explicit override (e.g. a Big_Y-700/500 directory the caller recognized) wins over - // inference — CRAMs ship no `.bai`, so the coverage-shape detector below can't see the - // targeted-Y pile-up and would otherwise fall back to the platform default (WGS). + // The test type. The code reads the *shape* of the coverage from the BAI index, and that + // shape corrects the value from the header and the platform. + // + // Many reads on chrY, with no read on an autosome, mark a Big Y test, a Y Elite test, or a + // YSEQ test. Many reads on chrM mark an mtFull test. + // + // The step is optional and fast. It costs O(contigs) and reads no record. A CRAM file and a + // BAM file with no index hold no such profile, and they keep the value from the platform. + // + // A value from the caller wins over each value above. One example is a Big_Y-700 directory + // or a Big_Y-500 directory that the caller recognized. A CRAM file has no `.bai` file, so + // the detector below can not see the reads on chrY. Without the value from the caller, the + // code would use the default of the platform, which is WGS. let test_type = match test_type_override { Some(t) => t.to_string(), None => { @@ -134,9 +167,11 @@ impl App { }) .await?; - // Persist the inferred lab/instrument identity block (the crowd-source key). The lab - // (`sequencing_facility`) stays unset — set manually, or resolved from `instrument_id` - // once the AppView lookup ships (roadmap D8). + // Write the identity of the laboratory and the instrument that the code found. The AppView + // uses those values as its key. + // + // The `sequencing_facility` field stays empty. The user sets it, or the AppView lookup + // gives it from `instrument_id` after that feature ships. See roadmap D8. if let Some(s) = &stats { let _ = sequence_run::set_library_stats( self.store.pool(), @@ -149,10 +184,15 @@ impl App { s.read_type.as_deref(), ) .await; - // Resolve the lab from the instrument id via the AppView (best-effort, cached). The - // FTDNA Big Y generation comes from the header `@RG LB` label (already in `test_type` - // above) or, on older headers that omit it, from the callable-chrY footprint after - // analysis ([`Self::refine_big_y_generation`]) — not guessed from the lab here. + // Find the laboratory from the instrument id, through the AppView. The step is + // optional, and the result goes into the cache. + // + // The generation of an FTDNA Big Y test comes from the `@RG LB` label of the header, + // and the step above already put it in `test_type`. An older header holds no such + // label. For such a file, the callable area of chrY gives the generation after the + // analysis, in [`Self::refine_big_y_generation`]. + // + // This code does not estimate the generation from the laboratory. if let Some(inst) = s.instrument_id.as_deref() { if let Some(lab) = self.lookup_lab_by_instrument(inst).await { let _ = sequence_run::set_facility(self.store.pool(), run.id, &lab).await; @@ -160,10 +200,13 @@ impl App { } } - // Defer the content hash (the file's identity, used to invalidate cached analyses): a - // whole-file SHA-256 of a multi-GB alignment would block this import for minutes with no - // feedback. Like the batch path, leave it `None` — `alignment_content_hash` computes and - // caches it lazily on the first analysis that needs it. + // Do not calculate the content hash here. That hash is the identity of the file, and the + // app uses it to find an old cache entry. + // + // A SHA-256 hash of a full alignment of many GB stops this import for some minutes, and the + // user sees nothing. The batch path also leaves the field `None`. The function + // `alignment_content_hash` calculates the hash at the first analysis that needs it, and it + // writes the value to the cache. self.record_alignment(NewAlignment { sequence_run_id: run.id, reference_build, @@ -172,8 +215,8 @@ impl App { bam_path: Some(path.to_string_lossy().into_owned()), reference_path, content_sha256: None, - // An imported alignment is an original — nothing derived it. Only realignment sets - // these, and it registers its own row. + // An imported alignment is an original alignment, and no other row made it. Only a + // realignment writes these fields, and it adds its own row. derived_from_alignment_id: None, derivation: None, }) @@ -185,9 +228,12 @@ impl App { self.add_data_with_test_type(biosample_guid, path, None).await } - /// Like [`add_data`], but forces the sequencing-run `test_type` for an alignment file instead - /// of inferring it (e.g. a bulk Big Y import where the directory layout names the test). The - /// override is ignored for non-alignment inputs (their type is intrinsic to the file). + /// The same work as [`add_data`], but the caller gives the `test_type` of the sequence run for + /// an alignment file. The code does not find that value itself. One case is a bulk Big Y import, + /// where the layout of the directories names the test. + /// + /// The method ignores that value for each other kind of file, because the file itself gives the + /// type. pub async fn add_data_with_test_type( &self, biosample_guid: SampleGuid, @@ -199,9 +245,12 @@ impl App { .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_default(); let lower = name.to_ascii_lowercase(); - // Binary/structured formats are detected by extension; only text needs a sniff. A VCF is - // sniffed too now — an all-sites (genotyped) VCF is a 1240K call set, a variant-only VCF is a - // plain variant set (see `filetype::looks_like_genotyped_callset_vcf`) — so `.vcf*` is NOT here. + // The extension of a file gives its type for each binary format and each structured + // format. Only a text file needs a look at its content. + // + // The code now also looks inside a VCF file. A VCF with each site is a 1240K call set. A VCF + // with the variants only is a plain variant set. See + // `filetype::looks_like_genotyped_callset_vcf`. So this list holds no `.vcf` pattern. let by_ext = lower.ends_with(".bam") || lower.ends_with(".cram") || lower.ends_with(".geno") @@ -255,12 +304,21 @@ impl App { Ok(detected) } - /// Batch [`add_data`]: expand any directories among `paths` into their recognized data files, - /// then auto-detect + import each into the subject, collecting a [`BatchImportSummary`]. A - /// failed/unrecognized file is recorded (not propagated) so one bad file does not abort the - /// batch. `progress(done, total)` ticks per file. The unified multi-file / folder importer - /// behind the GUI's Add Data button + drag-and-drop. (Distinct from [`import_project_dir`], - /// which builds a *new* multi-subject project from a NAS layout; this adds to *this* subject.) + /// The batch form of [`add_data`]. + /// + /// The method expands each directory in `paths` into the data files that it recognizes. It then + /// finds the type of each file and imports it into the subject. It collects the results in a + /// [`BatchImportSummary`] value. + /// + /// The summary records a file that failed, and a file that the code does not recognize. The + /// method does not return an error for such a file, so one bad file does not stop the batch. + /// + /// The method calls `progress(done, total)` after each file. + /// + /// The Add Data button of the GUI calls this method, and a drag-and-drop action also calls it. + /// + /// This method is not [`import_project_dir`]. That method makes a *new* project with many + /// subjects from a NAS layout. This method adds files to *this* subject. pub async fn add_data_batch( &self, biosample_guid: SampleGuid, @@ -269,9 +327,12 @@ impl App { ) -> Result { let mut files = Vec::new(); for p in &paths { - // Guard against a single picked folder that is really a *parent* of several per-sample - // folders (e.g. an FTDNA download root): recursing it would silently merge sibling - // samples into this one subject. Refuse with guidance rather than import the wrong data. + // Guard against one folder that the user picked and that is the *parent* of the + // folders of many samples. An FTDNA download root is one example. + // + // A read of each folder below it would add the samples of each folder to this one + // subject, with no message. The method refuses and gives the user a hint. It must not + // import the wrong data. if p.is_dir() { let mut these = Vec::new(); collect_data_files(p, &mut these, 0); @@ -308,18 +369,25 @@ impl App { Ok(summary) } - /// Ingest one staged **sample directory** onto an existing subject (the CLI `ingest` fast path - /// for the D2C bulk side-load). Scans `dir` into a single sample, records its alignment(s) onto - /// `biosample_guid` from the **header only** (no read decode / library scan), imports any variant - /// files, then — when `fast_path` and a haplogroup GVCF is present — runs [`Self::ingest_sidecars`] - /// to place Y + mt from the BGZF GVCFs and fill sex / read-metrics / lite-coverage from the text - /// sidecars, still **without decoding the CRAM**. Per-file [`Self::add_data`] can't do this: it - /// can't group `*.callable.bed` / `coverage.txt` / `stats.txt` to their alignment, and it would - /// route a `*.g.vcf.gz` through the plain-VCF importer instead of the GVCF haplogroup fast path. + /// Import one **sample directory** onto a subject that exists. The CLI `ingest` command uses + /// this fast path for the D2C bulk load. + /// + /// The method reads `dir` as one sample. It records each alignment onto `biosample_guid` from + /// the **header only**. It decodes no read and scans no library. It then imports each variant + /// file. + /// + /// When the caller sets `fast_path` and the directory holds a haplogroup GVCF file, the method + /// calls [`Self::ingest_sidecars`]. That method places the Y haplogroup and the mt haplogroup + /// from the BGZF GVCF files. It also fills the sex, the read metrics, and a small coverage + /// result from the text sidecar files. It decodes **no CRAM file**. /// - /// A directory that holds no alignment, variant, or haplogroup GVCF falls back to a best-effort - /// per-file [`Self::add_data`] of its contents — so a plain folder of chip/STR/mtDNA exports - /// still imports as before. + /// A call of [`Self::add_data`] for each file can not do this work. That method can not join a + /// `*.callable.bed` file, a `coverage.txt` file, or a `stats.txt` file to its alignment. It also + /// sends a `*.g.vcf.gz` file to the plain-VCF importer, and not to the GVCF fast path. + /// + /// A directory with no alignment, no variant file, and no haplogroup GVCF takes another path. + /// The method then calls [`Self::add_data`] for each file, and each call is optional. So a plain + /// folder of chip exports, STR exports, and mtDNA exports imports as it did before. pub async fn add_sample_dir( &self, biosample_guid: SampleGuid, @@ -330,8 +398,9 @@ impl App { let sample = tokio::task::spawn_blocking(move || navigator_analysis::scan::scan_sample(&scan_dir)).await?; let mut summary = SampleDirSummary::default(); - // No primary sequencing data (no alignment, no variant, no haplogroup GVCF): treat the - // directory as a loose bundle of subject files and import each as add_data would. + // The directory holds no primary sequence data: no alignment, no variant file, and no + // haplogroup GVCF file. So the code reads it as a set of separate subject files, and it + // imports each file as add_data does. let has_primary = !sample.alignment_files.is_empty() || !sample.variant_files.is_empty() || sample.sidecars.has_haplogroup_gvcf(); @@ -367,8 +436,8 @@ impl App { } }; - // Record each alignment from the header only — cheap, no read decode (the whole point of the - // fast path). Idempotent on the alignment's stored path. + // Record each alignment from the header only. That read is fast and decodes no record, + // which is the purpose of the fast path. A second call with the same stored path is safe. let existing = alignment::list_for_run(self.store.pool(), run.id).await?; for aln_path in &sample.alignment_files { let path_str = aln_path.to_string_lossy().into_owned(); @@ -401,13 +470,21 @@ impl App { summary.alignments_created += 1; } - // Import bundled variant files ONLY when there is no haplogroup GVCF. When a GVCF is present - // the fast path below is the authoritative Y/mt source, so a called `chrY.vcf.gz` sitting - // beside it (the GATK repo layout ships both) is redundant — importing it would fire a second - // Y placement and, because variant-set import is not content-idempotent, would duplicate the - // set on a resumable re-run. Non-GVCF tiers (e.g. the b38 aengine `variants.vcf.gz`) still - // import here: there the VCF *is* the Y source. GVCFs themselves are `.g.vcf.gz`, which `scan` - // also lists as variant files — the guard keeps them out of this loop too. + // Import the variant files of this directory ONLY when it holds no haplogroup GVCF file. + // + // With a GVCF file, the fast path below is the source of the Y value and the mt value. A + // called `chrY.vcf.gz` file beside it holds the same data, and the GATK layout ships both + // files. + // + // An import of that file starts a second Y placement. A second import of a variant set also + // adds a second copy, because that import does not compare the content. So a run that + // continues an earlier run would duplicate the set. + // + // A directory with no GVCF file still imports its variant files here. In the b38 aengine + // layout, for example, the `variants.vcf.gz` file *is* the Y source. + // + // A GVCF file has the name `*.g.vcf.gz`, and `scan` also lists it as a variant file. This + // guard keeps it out of this loop. if !sample.sidecars.has_haplogroup_gvcf() { for vcf in &sample.variant_files { let name = vcf @@ -429,9 +506,10 @@ impl App { } } - // Fast path: place Y + mt from the GVCFs and fill sex / read-metrics / lite-coverage from the - // text sidecars onto the build-matching alignment — no CRAM walk. Best-effort (mirrors the - // project-import chooser at import_project_sample). + // The fast path. It places the Y haplogroup and the mt haplogroup from the GVCF files. It + // fills the sex, the read metrics, and a small coverage result from the text sidecar files, + // onto the alignment with the same build. It walks no CRAM file. The step is optional, and + // the chooser in import_project_sample works in the same way. if fast_path && sample.sidecars.has_haplogroup_gvcf() { let alns = alignment::list_for_run(self.store.pool(), run.id).await?; let chosen = sample @@ -460,10 +538,14 @@ impl App { } } - // Progressive consensus (docs §7.17): fold whatever autosomal dosages are now available into - // the subject's consensus. Cheap — chips/WGS-VCFs resolve without a decode, and a freshly- - // imported WGS alignment (dosages not yet cached) is simply skipped until the panel batch mode - // genotypes it. Best-effort: a consensus hiccup must not fail the import. + // The progressive consensus, in docs §7.17. The code adds each autosomal dosage that the + // store now holds to the consensus of the subject. + // + // The step is fast. A chip and a WGS VCF resolve with no decode. A WGS alignment from a + // recent import has no dosage in the cache. The code skips such an alignment until the + // batch mode of the panel genotypes it. + // + // The step is optional. A fault in the consensus must not fail the import. if let Err(e) = self.refresh_autosomal_consensus(biosample_guid).await { summary.errors.push(format!("consensus refresh: {e}")); } @@ -471,14 +553,22 @@ impl App { Ok(summary) } - /// Batch-import a NAS project directory: scan `{dir}/{sample}/…` and create the Project - /// plus its Biosample → SequenceRun → Alignment rows. The reference is resolved per - /// alignment: pass `Some(fasta)` to use a specific FASTA (validated with its `.fai`) for - /// every alignment, or `None` to let the gateway resolve each file's inferred build from - /// the cache. If a needed build is not cached, returns [`AppError::ReferenceNeeded`] - /// **before any DB writes** so the UI can prompt + download, then retry. Idempotent: an - /// existing project (by name), biosample (by donor id), or alignment (by path) is reused. - /// Coverage is NOT computed here — run it per alignment or via the project report. + /// Import a NAS project directory as a batch. The method reads `{dir}/{sample}/…` and makes + /// the project with its Biosample, SequenceRun, and Alignment rows. + /// + /// The method finds the reference of each alignment. With `Some(fasta)`, it uses that one FASTA + /// file for each alignment, and it checks the `.fai` file. With `None`, the gateway finds the + /// build of each file in the cache. + /// + /// When the cache holds no file for a build that the import needs, the method returns + /// [`AppError::ReferenceNeeded`] **before it writes to the database**. The UI can then ask the + /// user, download the file, and call the method again. + /// + /// A second call is safe. The method uses a project with the same name, a biosample with the + /// same donor id, and an alignment with the same path. + /// + /// The method does NOT calculate the coverage. Run that step for one alignment, or from the + /// project report. pub async fn import_project_dir( &self, dir: &Path, @@ -490,12 +580,19 @@ impl App { .await } - /// Re-run the sidecar fast path for every alignment of a subject whose source directory still - /// carries the pipeline GVCFs — restoring external (GATK4) Y/mt calls that an older build's - /// internal walk had overwritten before provenance existed. Cheap: reads the small GVCFs, never - /// the CRAM. The external calls land on their own `:ext` keys (they can not clobber, and with the - /// "prefer external caller" policy they win the consensus). Returns `(y_placed, mt_placed)`. - /// This is the operational fix for a workspace imported before external-caller precedence. + /// Run the sidecar fast path again, for each alignment of a subject whose source directory + /// still holds the GVCF files of the pipeline. + /// + /// The method returns the external Y calls and mt calls from GATK4. An older build ran its + /// internal walk and replaced those calls, before the app recorded a provenance. + /// + /// The method is fast. It reads the small GVCF files and never the CRAM file. + /// + /// Each external call goes to its own `:ext` key. So it can replace no other call, and the + /// "prefer external caller" policy makes it win the consensus. + /// + /// The method returns `(y_placed, mt_placed)`. It is the correction for a workspace that a user + /// imported before the app had external-caller precedence. pub async fn reingest_external_for_biosample( &self, biosample_guid: SampleGuid, @@ -521,10 +618,14 @@ impl App { Ok((y_placed, mt_placed)) } - /// [`Self::import_project_dir`] with a per-sample progress callback `progress(done, total, - /// sample_id)`, invoked before each sample so a large NAS import (thousands of samples) can - /// stream a status bar instead of appearing frozen. `done` is the 0-based index about to - /// process; the first call fires only after the (potentially slow) header-probe pre-flight. + /// The work of [`Self::import_project_dir`], with a progress callback for each sample. The + /// method calls `progress(done, total, sample_id)` before each sample. + /// + /// So a large NAS import of some thousands of samples can move a status bar. Without it, the app + /// looks stopped. + /// + /// The `done` value is the 0-based index of the next sample. The first call comes after the + /// header probe of the preflight, and that step can be slow. pub async fn import_project_dir_with_progress( &self, dir: &Path, @@ -533,7 +634,8 @@ impl App { fast_path: bool, mut progress: impl FnMut(usize, usize, &str), ) -> Result { - // An explicit FASTA must exist and be indexed; it applies to every alignment. + // A FASTA file from the caller must exist and must have an index. It applies to each + // alignment. if let Some(path) = &reference { if !path.exists() { return Err(AppError::Import(format!( @@ -553,10 +655,12 @@ impl App { let scan_dir = dir.to_path_buf(); let discovered = tokio::task::spawn_blocking(move || navigator_analysis::scan::scan(&scan_dir)).await??; - // Detect each alignment's reference build from its **header** (only the header, so it is - // cheap and needs no reference FASTA). The filename is an unreliable signal — most NAS - // project layouts do not put the build in the name — so probe first, fall back to the - // filename, and record how each build was decided for the import diagnostics. + // Find the reference build of each alignment from its **header**. The code reads the + // header only, so this step is fast and needs no reference FASTA file. + // + // The file name is not a reliable source, because most NAS layouts do not put the build in + // that name. So the code reads the header first and uses the file name second. It also + // records the source of each build, for the import report. let all_paths: Vec = discovered .samples .iter() @@ -573,17 +677,22 @@ impl App { }) .await?; - // Resolve each *distinct* detected build to a reference path. A build the gateway can't - // canonicalize falls back to the CHM13v2.0 default rather than aborting the whole batch; - // a known build that is not cached is surfaced as a recoverable download need. `effective_of` - // maps a detected build to the one actually stored on the alignment (after any fallback). + // Find a reference path for each *distinct* build that the code detected. + // + // A build that the gateway does not recognize takes the CHM13v2.0 default, so the batch + // continues. A known build with no file in the cache becomes a download that the UI can + // start. + // + // The `effective_of` map takes a build that the code detected and gives the build that the + // alignment row holds, after each default above. let explicit = reference.as_ref().map(|p| p.to_string_lossy().into_owned()); let mut resolved: HashMap = HashMap::new(); // effective build -> FASTA path let mut effective_of: HashMap = HashMap::new(); // detected build -> effective build let mut needs: Vec = Vec::new(); let mut reference_notes: Vec = Vec::new(); - // Alignment count + a representative detection source, per distinct detected build. + // The count of alignments, and one example of the detection source, for each distinct + // build. let mut per_build: BTreeMap = BTreeMap::new(); for (build, source) in detected.values() { let e = per_build.entry(build.clone()).or_insert((0, *source)); @@ -592,9 +701,10 @@ impl App { for (detected_build, (count, source)) in &per_build { let count = *count; - // Effective build: keep the detected one when the gateway recognizes it (or an explicit - // FASTA overrides everything); otherwise fall back to the default so unlabeled files - // still import instead of killing the batch. + // The build that the row holds. Keep the build that the code detected when the gateway + // recognizes it. A FASTA file from the caller replaces each such value. If neither + // applies, use the default build. A file with no label then still imports, and the + // batch continues. let (effective, defaulted) = if explicit.is_some() || !matches!(self.gateway.reference_status(detected_build), RefStatus::Unknown) { (detected_build.clone(), false) @@ -603,9 +713,12 @@ impl App { }; effective_of.insert(detected_build.clone(), effective.clone()); - // Resolve the effective build to a FASTA once (explicit > already-resolved > cache > - // gateway status). A download need is collected; an unresolvable build is recorded - // without a FASTA (resolved on demand at analysis time) rather than aborting. + // Find one FASTA file for that build. The order is the file from the caller, a file + // that the code already found, the cache, and then the status of the gateway. + // + // The method collects each download that the import needs. It records a build with no + // file and no FASTA path, and the analysis finds that file later. It does not stop the + // import. let path: Option = if let Some(ref p) = explicit { Some(p.clone()) } else if let Some(p) = resolved.get(&effective) { @@ -680,9 +793,10 @@ impl App { fast_path: FastPathSummary::default(), }; - // Import each sample independently: a single sample's failure (unreadable file, DB hiccup) - // is logged + tallied into `sample_errors` and the batch continues with the rest, rather - // than one bad sample aborting the whole import. + // Import each sample on its own. A failure in one sample goes into the log and into the + // `sample_errors` count, and the batch continues with the other samples. The causes are a + // file that the code can not read, and a fault in the database. One bad sample must not + // stop the full import. let total = discovered.samples.len(); for (i, sample) in discovered.samples.iter().enumerate() { progress(i, total, &sample.sample_id); @@ -708,10 +822,13 @@ impl App { Ok(summary) } - /// Import one sample's subject, run, alignments, and fast-path sidecars. Extracted so a failure - /// here bubbles up as this sample's error (caught by [`Self::import_project_dir`]) instead of - /// aborting the whole batch. `detected`/`effective_of`/`resolved` are the pre-flight reference - /// maps from the caller; `summary` is updated in place with what this sample contributed. + /// Import the subject, the run, the alignments, and the fast-path sidecar files of one sample. + /// + /// This method is separate, so a failure here becomes the error of this sample. + /// [`Self::import_project_dir`] catches that error, and the batch continues. + /// + /// The `detected`, `effective_of`, and `resolved` maps come from the preflight of the caller. + /// The method writes what this sample added to `summary`. #[allow(clippy::too_many_arguments)] async fn import_project_sample( &self, @@ -723,10 +840,14 @@ impl App { resolved: &HashMap, summary: &mut ProjectImportSummary, ) -> Result<(), AppError> { - // Biosample: reuse an existing subject with this donor identifier **anywhere in the - // workspace** — a person is one subject across projects. Scoping the lookup to the target - // project duplicated everyone when the same folder was re-imported under a different - // project name (a person then existed once per project). Create only when truly new. + // The biosample. Use a subject with this donor identifier from **any place in the + // workspace**. One person is one subject in each project. + // + // An earlier version looked in the target project only. So a second import of the same + // folder, under another project name, made a second subject for each person. A person then + // had one subject in each project. + // + // Make a subject only when the workspace holds none. let biosample = match biosample::find_by_donor(self.store.pool(), &sample.sample_id).await? { Some(b) => b, None => { @@ -740,8 +861,9 @@ impl App { .await? } }; - // Ensure the subject is a member of this project (idempotent on the (guid, project) PK). - // A reused subject whose *home* project is another one still joins this project's roster. + // Make sure that the subject is a member of this project. A second call is safe, because + // the primary key is the pair (guid, project). A subject whose *home* project is another + // project also joins the list of this project. biosample_project::add( self.store.pool(), biosample.guid, @@ -792,8 +914,8 @@ impl App { variant_caller: None, bam_path: Some(path_str), reference_path, - // Batch import: hash lazily on first analysis (do not stall a bulk NAS import - // hashing every multi-GB file up front). + // This is a batch import. Calculate the hash at the first analysis. A bulk NAS + // import must not stop while it hashes each file of many GB. content_sha256: None, // An imported alignment is an original; see above. derived_from_alignment_id: None, @@ -803,9 +925,14 @@ impl App { summary.alignments_created += 1; } - // Fast path: ingest the pipeline sidecars onto the build-matching alignment — - // places Y + mt from the GVCFs and fills sex/metrics/lite-coverage from the text - // sidecars, no CRAM walk. Best-effort; a failure is tallied and import continues. + // The fast path. It reads the pipeline sidecar files onto the alignment with the same + // build. + // + // It places the Y haplogroup and the mt haplogroup from the GVCF files. It also fills the + // sex, the metrics, and a small coverage result from the text sidecar files. It walks no + // CRAM file. + // + // The step is optional. A failure goes into a count, and the import continues. if fast_path && sample.sidecars.has_haplogroup_gvcf() { let alns = alignment::list_for_run(self.store.pool(), run.id).await?; let chosen = sample @@ -840,8 +967,9 @@ impl App { self.gateway.reference_status(build) } - /// Resolve a reference build to a cached, indexed `.fa`, downloading on a miss. - /// `progress(received, total)` is invoked as bytes arrive. + /// Find the indexed `.fa` file of a reference build in the cache. The method downloads that + /// file when the cache holds none. It calls `progress(received, total)` as each part of the file + /// arrives. pub async fn resolve_reference( &self, build: &str, @@ -850,8 +978,9 @@ impl App { Ok(self.gateway.resolve_reference(build, progress).await?) } - /// Resolve (and cache) a liftover chain for a build pair, downloading on a miss. The - /// cached `.chain` is then available for the haplogroup/liftover path. + /// Find the liftover chain of a build pair, and write it to the cache. The method downloads + /// that file when the cache holds none. The haplogroup path and the liftover path then read the + /// `.chain` file from the cache. pub async fn resolve_chain( &self, from: &str, @@ -861,19 +990,26 @@ impl App { Ok(self.gateway.resolve_chain(from, to, progress).await?) } - /// Re-hash a cached reference against its integrity sidecar (gap §7) — detects on-disk - /// corruption of the cached `.fa`. Runs on a blocking thread (re-reads the whole FASTA), so it is - /// an explicit, user-triggered check (Settings), not the hot path. + /// Calculate the hash of a reference in the cache again, and compare it with the sidecar file + /// that holds the correct value. See gap §7. The method finds a `.fa` file that the disk + /// damaged. + /// + /// The method reads the full FASTA file, so it runs on its own thread. The user starts it from + /// the Settings screen, and no analysis calls it. pub async fn verify_reference(&self, build: &str) -> Result { let gw = self.gateway.clone(); let build = build.to_string(); Ok(tokio::task::spawn_blocking(move || gw.verify_reference(&build)).await??) } - /// Lift a whole VCF from `source` build to `target` build (gap §7 — the GATK `LiftoverVcf` - /// replacement). Ensures the source→target chain and the target reference are resolved - /// (downloading on a miss, with progress), then runs the line-level lift on a blocking thread. - /// Returns lift/drop counts. + /// Move a full VCF file from the `source` build to the `target` build. See gap §7. This method + /// takes the place of the GATK `LiftoverVcf` tool. + /// + /// The method first finds the chain from the source to the target, and the reference of the + /// target. It downloads each file that the cache does not hold, and it reports the progress. + /// + /// It then moves each line on its own thread. It returns the count of the lines that it moved + /// and the count of the lines that it removed. pub async fn lift_vcf( &self, source: &str, @@ -883,12 +1019,14 @@ impl App { opts: navigator_refgenome::VcfLiftOpts, progress: &mut (dyn FnMut(u64, Option) + Send), ) -> Result { - // Resolve the inputs (chain + target FASTA), downloading on a miss. + // Find the two input files, which are the chain and the FASTA file of the target. Download + // each file that the cache does not hold. self.gateway.resolve_chain(source, target, progress).await?; let target_fa = self.gateway.resolve_reference(target, progress).await?; let lo = self.gateway.load_liftover(source, target)?; - // Target chrY PAR intervals (only needed when filtering them out). + // The PAR intervals of chrY on the target build. The code needs them only when it removes + // those intervals. let target_par: Vec<(i64, i64)> = if opts.filter_par { let regions = self.gateway.genome_regions(target, progress).await?; regions @@ -918,31 +1056,37 @@ impl App { Ok(stats) } - /// See [`asset_action`] for the present/stale/absent decision this drives. + /// Make sure that the ancestry asset or IBD asset at `path` is present **and current**. + /// [`asset_action`] makes the decision that this method acts on. + /// + /// The method downloads the asset, and the manifest that it checks the asset against, from the + /// published GitHub release. A user receives each panel in this way, and no user runs the + /// offline `panelbuild` tool. /// - /// Ensure a prebuilt ancestry/IBD asset at `path` is present **and current**, downloading it — - /// and the asset manifest it is verified against — from the published GitHub release. End users - /// get the panels this way instead of running the offline `panelbuild` tool. + /// There are three cases, and the manifest decides each one. /// - /// Three cases, all manifest-driven: + /// * The asset is **absent**. Download it when the manifest lists it. An optional asset that + /// the team did not publish stays absent, and its feature gives less data. + /// * The **manifest does not list the asset**. Read the manifest again one time, then test + /// again. The code refreshes a cached manifest at no other time. So an installation from + /// before the publication of an asset would never learn that the asset exists. That fault + /// occurred with `ancestry_haps`. + /// * The asset is **present with the wrong size**. The team published a new version, so replace + /// the file. A test for the file alone can not see a new version. So without this test, an + /// installation with the asset keeps the old file for all time. /// - /// * **Absent** → download it, provided the manifest lists it (an unpublished optional asset - /// simply stays absent and its feature degrades). - /// * **Manifest does not list it** → re-fetch the manifest once, then re-check. The cached - /// manifest is otherwise never refreshed, so an install that predates an asset's publication - /// would never learn the asset exists — which is exactly what happened to `ancestry_haps`. - /// * **Present but the wrong size** → the published asset was revised; replace it. Without this - /// an install that already has an asset keeps the stale one forever, because a revision is - /// invisible to a plain existence check. The stale file is moved aside, not deleted, and put - /// back if the download fails — a stale asset beats no asset. + /// The code moves the old file to another name and does not delete it. It puts that file back + /// when the download fails, because an old asset is better than no asset. /// - /// An explicit `$NAVIGATOR_*` path override is never fetched over or repaired: that file is the - /// user's own. Best-effort throughout — network failures leave on-disk state alone. + /// The method never downloads over a path from a `$NAVIGATOR_*` variable, and it never repairs + /// such a file. That file belongs to the user. Each step is optional, and a network failure + /// changes nothing on the disk. pub(crate) async fn ensure_ancestry_asset(&self, build: ReferenceBuild, path: &Path) -> Result<(), AppError> { let Some(name) = path.file_name().and_then(|n| n.to_str()).map(str::to_string) else { return Ok(()); }; - // Only auto-fetch to the default cache location — an explicit override is the user's own file. + // Download only to the default place in the cache. A path from a variable names a file of + // the user. let default = refgenome_cache::base_dir().join("ancestry").join(&name); if path != default { return Ok(()); @@ -951,9 +1095,12 @@ impl App { let manifest_name = format!("ancestry_manifest_{}.json", build.as_str()); let manifest_path = default.with_file_name(&manifest_name); - // (1) The manifest: fetch when absent, and re-fetch when it does not list this asset (a - // manifest cached before the asset was published). Keep the old copy in memory so a - // failed refresh does not cost us the integrity data we already had. + // (1) The manifest. Download it when the cache holds none. Download it again when it does + // not list this asset, because the cache can hold a manifest from before the team + // published that asset. + // + // Keep the old copy in memory. A failed download then does not remove the check values + // that the app already has. let listed = |m: &Option| { m.as_ref().is_some_and(|m| m.assets.contains_key(&name)) }; @@ -984,8 +1131,8 @@ impl App { return Ok(()); }; - // (2) What to do with what is on disk. Content is verified at read time by - // `read_verified_asset`; hashing every asset here would cost seconds per paint. + // (2) The decision about the file on disk. `read_verified_asset` checks the content at + // each read. A hash of each asset here costs some seconds at each paint. let on_disk = std::fs::metadata(&default).ok().map(|m| m.len()); let action = asset_action(Some(&entry), on_disk); if action == AssetAction::Ready { @@ -1035,7 +1182,8 @@ impl App { Ok(()) } Err(e) => { - // Put the old asset back: analysing against a superseded panel beats not analysing. + // Put the old asset back. An analysis against an old panel is better than no + // analysis. if let Some(aside) = stale { let _ = std::fs::rename(&aside, &default); eprintln!("ancestry assets: {name} download failed ({e}) — kept the existing copy"); @@ -1046,9 +1194,12 @@ impl App { } } - /// Load the CHM13 IBD panel — downloading the prebuilt asset from the release on first use (no - /// `panelbuild` needed). The single entry point for the panel: replaces the five call sites that - /// each errored "build it with `panelbuild ibd-panel`" when the asset was absent. + /// Read the CHM13 IBD panel. At the first use, the method downloads the asset from the release, + /// and no user runs `panelbuild`. + /// + /// This method is the one entry point for that panel. It takes the place of five call sites. + /// Each of those sites gave the error "build it with `panelbuild ibd-panel`" when the asset was + /// absent. pub(crate) async fn load_ibd_panel(&self) -> Result { let build = ReferenceBuild::Chm13v2; let path = ibd_panel_path(build); @@ -1063,10 +1214,16 @@ impl App { Ok(navigator_analysis::ibd_panel::IbdPanel::from_bytes(&bytes)?) } - /// Resolve an imported chip's genotypes to canonical CHM13 **IBD-panel** dosages — the chip→IBD - /// path (no alignment, no runtime liftover: the multi-build panel pre-computes coordinates). The - /// output [`SiteGenotype`]s are over the same CHM13 sites a WGS caller would hit, so a chip and a - /// WGS sample compare uniformly. Errors if the IBD panel asset is not built yet. + /// Change the genotypes of an imported chip into dosages at the canonical CHM13 **IBD panel** + /// sites. This method is the path from a chip to the IBD data. + /// + /// It needs no alignment and does no liftover at run time, because the panel holds the + /// coordinates of each build. + /// + /// The [`SiteGenotype`] values that the method returns cover the same CHM13 sites that a WGS + /// caller reaches. So a chip sample and a WGS sample compare in the same way. + /// + /// The method fails when the IBD panel asset does not exist. pub async fn chip_ibd_dosages(&self, chip_profile_id: i64) -> Result, AppError> { let chip = chip_profile::get(self.store.pool(), chip_profile_id) .await? @@ -1085,14 +1242,24 @@ impl App { Ok(panel.resolve_chip(&from_build, &tuples)) } - /// Import a trusted external caller's autosomal **1240K EIGENSTRAT call set** - /// (`.geno`/`.snp`/`.ind`) for a subject — the autosomal counterpart to the Y/mt GVCF sidecar - /// fast path. Resolves the target individual's genotypes to canonical CHM13 panel dosages (no - /// CRAM decode; `resolve_chip` self-orients against the CHM13 alleles), persists them as an - /// `external` source, and refreshes the autosomal consensus so modern/fine/deep ancestry and IBD - /// pick them up. `path` may point at any member of the triplet (siblings resolved by basename). - /// The `.snp` build is GRCh37 (AADR 1240K) unless `NAVIGATOR_CALLSET_BUILD` overrides it. Returns - /// the number of resolved panel sites. + /// Import the autosomal **1240K EIGENSTRAT call set** of a trusted external caller for a + /// subject. The files are `.geno`, `.snp`, and `.ind`. This method is the autosomal form of the + /// Y and mt sidecar fast path. + /// + /// The method changes the genotypes of the target individual into dosages at the canonical CHM13 + /// panel sites. It decodes no CRAM file, and `resolve_chip` orients each call against the CHM13 + /// alleles itself. + /// + /// It stores those dosages as an `external` source, and it builds the autosomal consensus again. + /// The modern ancestry, the fine ancestry, the deep ancestry, and the IBD steps then read them. + /// + /// The `path` value can name any of the three files, because the code finds the other two from + /// the base name. + /// + /// The build of the `.snp` file is GRCh37, which the AADR 1240K set uses. The + /// `NAVIGATOR_CALLSET_BUILD` variable replaces that value. + /// + /// The method returns the count of the panel sites that it resolved. pub async fn import_callset_from_file(&self, biosample_guid: SampleGuid, path: &Path) -> Result { // Resolve the triplet from any member by shared basename. let stem = path @@ -1113,7 +1280,8 @@ impl App { } } - // AADR 1240K is GRCh37/hg19; allow a GRCh38-built call set via the env override. + // The AADR 1240K set uses GRCh37, which is also hg19. The variable lets a user import a + // call set on GRCh38. let build = std::env::var("NAVIGATOR_CALLSET_BUILD").unwrap_or_else(|_| "GRCh37".to_string()); let (g, s, i, b) = (geno.clone(), snp.clone(), ind.clone(), build.clone()); let callset = @@ -1138,13 +1306,21 @@ impl App { .await } - /// Import a trusted external caller's autosomal genotypes for a subject from a **diploid VCF/gVCF** - /// — the VCF path of the autosomal fast path (Phases 4/5). Handles a GATK4 gVCF (variant records + - /// hom-ref ref blocks) **and** a genotyped all-sites VCF (e.g. `bcftools mpileup`/`call` over the - /// 1240K sites, where every site carries an explicit `GT`). Genotypes the panel loci directly with - /// **no CRAM decode**, re-keys to canonical CHM13 (`resolve_chip`), stores the dosages as an - /// `external` source, and refreshes the autosomal consensus. Build is auto-detected from the VCF - /// header (`NAVIGATOR_CALLSET_BUILD` overrides). Returns the number of resolved panel sites. + /// Import the autosomal genotypes of a trusted external caller for a subject, from a **diploid + /// VCF file or gVCF file**. This method is the VCF path of the autosomal fast path, in phases 4 + /// and 5. + /// + /// The method reads a GATK4 gVCF file, which holds variant records and hom-ref blocks. It also + /// reads a VCF file with each site genotyped, such as the output of `bcftools mpileup` and + /// `bcftools call` across the 1240K sites. In that second file, each site holds a `GT` value. + /// + /// The method genotypes the panel loci directly and decodes **no CRAM file**. It then re-keys + /// each call to the canonical CHM13 sites with `resolve_chip`, stores the dosages as an + /// `external` source, and builds the autosomal consensus again. + /// + /// The code finds the build in the header of the VCF file, and the `NAVIGATOR_CALLSET_BUILD` + /// variable replaces that value. The method returns the count of the panel sites that it + /// resolved. pub async fn import_gvcf_callset_from_file( &self, biosample_guid: SampleGuid, @@ -1154,8 +1330,9 @@ impl App { let panel = self.load_ibd_panel().await?; - // Panel loci in the gVCF's build, grouped + sorted per contig; keep each site's reference - // allele so a hom-ref block resolves to (ref, ref). + // The panel loci in the build of the gVCF file. The code groups them by contig and sorts + // each group. It keeps the reference allele of each site, so a hom-ref block gives the pair + // (ref, ref). let mut targets_by_contig: std::collections::HashMap> = std::collections::HashMap::new(); let mut ref_allele: std::collections::HashMap<(String, i64), char> = std::collections::HashMap::new(); for site in &panel.sites { @@ -1238,7 +1415,8 @@ impl App { created_at: chrono::Utc::now().to_rfc3339(), }; navigator_store::external_panel_dosage::upsert(self.store.pool(), &row).await?; - // Fold into the autosomal consensus immediately — cheap, no decode (best-effort). + // Add the dosages to the autosomal consensus at once. The step is fast, it decodes + // nothing, and it is optional. let _ = self.refresh_autosomal_consensus(biosample_guid).await; Ok(site_count) } @@ -1259,11 +1437,16 @@ impl App { Ok(detect_ibd(&ga, &gb, ReferenceBuild::Chm13v2, config)) } - /// IBD comparison between two **subjects** from their autosomal consensuses — each subject's - /// pooled best genotype per site (across all its WGS + chip sources), no per-source genotyping. - /// This is the subject-level IBD path (consensus-driven); both subjects must have a built - /// autosomal consensus. A near-complete genome-wide match is the cross-subject identity (dedup) - /// signal — read it off the returned [`MatchSummary`]'s relationship estimate. + /// Compare two **subjects** for IBD segments, from their autosomal consensus values. + /// + /// The consensus of a subject holds the best genotype at each site, from each of its WGS sources + /// and chip sources. The method genotypes no source again. + /// + /// This path works at the level of a subject, and the consensus drives it. Both subjects must + /// have an autosomal consensus. + /// + /// A match across almost the full genome shows that the two subjects are the same person. Read + /// that result from the relationship estimate of the [`MatchSummary`] value. pub async fn compare_ibd_consensus( &self, a: SampleGuid, @@ -1283,9 +1466,11 @@ impl App { Ok(detect_ibd(&ga, &gb, ReferenceBuild::Chm13v2, config)) } - /// Dosages over the canonical CHM13 IBD-panel sites for a comparison source. A chip resolves - /// directly ([`Self::chip_ibd_dosages`]); an alignment genotypes the panel's CHM13 sites from - /// its BAM (cached per alignment, ploidy-2 autosomal). + /// The dosages at the canonical CHM13 IBD-panel sites, for one comparison source. + /// + /// A chip resolves directly, in [`Self::chip_ibd_dosages`]. An alignment genotypes the CHM13 + /// sites of the panel from its BAM file. The code caches that result for each alignment, and it + /// uses ploidy 2 on an autosome. pub async fn ibd_panel_dosages(&self, source: IbdSource) -> Result, AppError> { match source { IbdSource::Chip(id) => self.chip_ibd_dosages(id).await, @@ -1296,16 +1481,19 @@ impl App { self.variant_set_panel_dosages(&set).await } IbdSource::Alignment(id) => { - // Salt the cache key with the panel asset's manifest hash, so regenerating the panel - // (e.g. the probe superset) auto-invalidates stale per-alignment genotypes instead of - // silently serving genotypes taken over an older site set. + // Add the manifest hash of the panel asset to the cache key. A new panel, such as + // one with more probes, then makes each stored genotype of an alignment invalid. So + // the app does not return a genotype from an older set of sites with no message. let kind = ibd_panel_cache_kind(); if let Some(g) = self.load_analysis(id, &kind, caller::GENOTYPE_VERSION).await? { return Ok(g); } - // Resolve the reference for decode (see alignment_reference_for_decode): required for - // a CRAM, None for a BAM. Panel genotyping tallies SNP sites (ref/alt come from the - // panel), so a BAM consults no reference bases — do not force a download for it. + // Find the reference for the decoder. See alignment_reference_for_decode. A CRAM + // file needs it, and a BAM file uses None. + // + // The panel genotype step counts the reads at each SNP site, and the panel gives the + // reference allele and the alternate allele. So the code reads no reference base for + // a BAM file, and it must not start a download for one. let build = self.alignment_or_err(id).await?.reference_build; let (bam, reference) = self.alignment_reference_for_decode(id).await?; let panel = self.load_ibd_panel().await?; @@ -1315,8 +1503,9 @@ impl App { ); let genotypes = if is_chm13 { - // Native CHM13: genotype directly at the panel's canonical CHM13 loci — the dosage - // is already CHM13-oriented, no re-keying. + // A native CHM13 alignment. Genotype it directly at the canonical CHM13 loci + // of the panel. Each dosage is then already in the CHM13 space, and the code + // changes no key. let sites: Vec = panel .sites .iter() @@ -1341,10 +1530,15 @@ impl App { }) .await?? } else if panel.sites.iter().any(|s| s.locus(&build).is_some()) { - // GRCh37/GRCh38: the panel already carries this build's coordinates (offline - // allele-aware liftover). Genotype at the build's loci, then re-key the dosages to - // canonical CHM13 ([`IbdPanel::resolve_alignment`]) — no runtime liftover needed. - // Match the panel's per-build contig names to the file's naming (`chr1` vs `1`). + // A GRCh37 alignment or a GRCh38 alignment. The panel already holds the + // coordinates of that build, from an offline liftover that read the alleles. + // + // Genotype the sample at the loci of that build. Then re-key each dosage to the + // canonical CHM13 sites, in [`IbdPanel::resolve_alignment`]. The code does no + // liftover at run time. + // + // Match the contig names of the panel for that build to the names in the file. + // One file uses `chr1`, and another file uses `1`. let (bam_h, ref_h) = (bam.clone(), reference.clone()); let file_contigs = tokio::task::spawn_blocking(move || { navigator_analysis::reader::contig_names(&bam_h, ref_h.as_deref()) @@ -1384,8 +1578,9 @@ impl App { .await??; panel.resolve_alignment(&build, &raw) } else { - // A build the panel does not carry — nothing to genotype (degrade gracefully rather - // than probe the wrong loci). + // The panel holds no coordinates for this build, so there is nothing to + // genotype. The code gives a smaller result. It must not read the wrong + // loci. Vec::new() }; self.save_analysis(id, &kind, caller::GENOTYPE_VERSION, &genotypes) @@ -1395,10 +1590,13 @@ impl App { } } - /// Cached IBD-panel dosages for an alignment, **without genotyping** — `Ok(None)` when they - /// have not been computed yet (so callers can reduce over what is available progressively rather - /// than triggering a whole-genome decode). [`Self::ibd_panel_dosages`] is the compute-and-cache - /// path; this is the read-only companion used by the progressive-consensus refresh. + /// The IBD-panel dosages of an alignment from the cache. The method genotypes **nothing**. + /// + /// It returns `Ok(None)` when no earlier run calculated those dosages. So a caller can work with + /// the data that the store holds, and it does not start a decode of the full genome. + /// + /// [`Self::ibd_panel_dosages`] calculates the dosages and writes them to the cache. This method + /// only reads them, and the progressive-consensus refresh calls it. pub async fn cached_alignment_panel_dosages( &self, alignment_id: i64, @@ -1407,9 +1605,13 @@ impl App { .await } - /// Subject-level identity verification (gap §8) — "are these two subjects the same individual?" - /// (duplicate detection). Pooled autosomal consensus genotype concordance (no panel selection), - /// corroborated by Y-STR distance. Both subjects need a built autosomal consensus. + /// A test of identity at the level of a subject, in gap §8. It answers the question "are these + /// two subjects the same person?", and the app uses it to find a duplicate. + /// + /// The method compares the genotypes of the two pooled autosomal consensus values, and it + /// selects no panel. The distance between the Y-STR values supports that comparison. + /// + /// Both subjects need an autosomal consensus. pub async fn verify_identity_consensus( &self, a: SampleGuid, @@ -1441,22 +1643,30 @@ impl App { } } -/// What [`App::ensure_ancestry_asset`] must do for one asset, from the manifest entry (`None` when -/// the manifest does not list it) and the on-disk size (`None` when the file is absent). +/// The action that [`App::ensure_ancestry_asset`] must take for one asset. +/// +/// The decision reads two values. The first is the manifest entry, which is `None` when the manifest +/// does not list the asset. The second is the size of the file on disk, which is `None` when the +/// file is absent. +/// +/// The decision uses the size and not a hash. The team makes a new version of an asset when it +/// builds that asset again, and the new file has a different length. The read step already checks +/// the content, and that check has authority. /// -/// Size, not hash: a published asset is revised by rebuilding it, which changes its length, and the -/// authoritative content check already happens at read time. Hashing a 133 MB panel on every call -/// would cost seconds per paint to catch a case (same size, different bytes) that read-time -/// verification catches anyway. +/// A hash of a panel of 133 MB at each call costs some seconds at each paint. It finds only one more +/// case, where two files have the same size and different bytes, and the read step finds that case +/// also. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum AssetAction { - /// Present at the published size — use it. + /// The file is present, and its size equals the published size. Use it. Ready, - /// Absent — fetch it. + /// The file is absent. Download it. Download, - /// Present but superseded (or truncated) — move aside and fetch. + /// The file is present, but a newer version exists, or the file is not complete. Move it to + /// another name and download the new file. Replace, - /// Not published for this build — leave it absent and let the feature degrade. + /// The team published no file of this asset for this build. Leave it absent, and let its + /// feature give a smaller result. Skip, } diff --git a/crates/navigator-app/src/queries.rs b/crates/navigator-app/src/queries.rs index 43f51515..5a98a310 100644 --- a/crates/navigator-app/src/queries.rs +++ b/crates/navigator-app/src/queries.rs @@ -148,20 +148,26 @@ impl App { Ok(runs) } - /// Batch-populate the read-profile fields backing the standardized test label - /// ([`du_domain::testprofile`]) on runs imported before those fields existed — for the CLI - /// `backfill-profiles` command. Idempotent; only fills what is missing. + /// Fill the read-profile fields of many runs at once. Those fields support the standard test + /// label in [`du_domain::testprofile`]. The CLI command `backfill-profiles` calls this method + /// for a run that the app imported before the fields existed. A second call is safe, and the + /// method fills only an empty field. /// - /// - **`total_bases`** — recovered for free from a cached `read_metrics` artifact on any of the - /// run's alignments (`Σ read_length_histogram`), no file walk. - /// - **`read_type`** — inferred cheaply from `platform_name` / `test_type` (`SHORT`, `ONT_SIMPLEX`, - /// or `HIFI`/`CLR` when the code already says so), falling back to the cached mean read length - /// as evidence (a short mean ⇒ `SHORT`, which resolves sidecar-imported `UNKNOWN`-platform - /// runs). When `rescan` is set, runs still missing it (long reads — HiFi vs CLR needs the read - /// names) get a bounded [`library_stats`](Self::library_stats) scan of a primary alignment file. + /// - **`total_bases`** comes from a cached `read_metrics` artifact on any alignment of the run. + /// The value is the sum of `read_length_histogram`, and the method walks no file. + /// - **`read_type`** comes from `platform_name` and `test_type` at a low cost. The values are + /// `SHORT`, `ONT_SIMPLEX`, and `HIFI` or `CLR` when the code already holds one of them. /// - /// `project_id` restricts to a project's subjects (legacy home column, matching - /// `rebuild_signatures`). Returns per-field counts. + /// If those fields give nothing, the method reads the cached mean read length. A short mean + /// gives `SHORT`, and that rule resolves a run from a sidecar import with an `UNKNOWN` + /// platform. + /// + /// With `rescan`, a run that still has no value gets a limited + /// [`library_stats`](Self::library_stats) scan of one alignment file. A long-read run needs + /// that scan, because only the read names separate HiFi from CLR. + /// + /// `project_id` limits the work to the subjects of one project. The method reads the old home + /// column, as `rebuild_signatures` does. It returns a count for each field. pub async fn backfill_read_profiles( &self, project_id: Option, @@ -179,8 +185,8 @@ impl App { out.runs_examined += 1; let alns = alignment::list_for_run(self.store.pool(), run.id).await?; - // One representative cached read-metrics artifact for the run — reused for both the - // yield and the read-length evidence below (no re-read). + // One cached read-metrics artifact stands for the run. The code uses it for the + // yield and for the read-length value below, and it reads the artifact one time. let mut metrics = None; for a in &alns { if let Some(m) = self.cached_read_metrics(a.id).await? { @@ -199,9 +205,11 @@ impl App { } } - // read_type: cheap platform/test-type inference, then the cached mean read length as - // evidence (resolves sidecar-imported UNKNOWN-platform runs), then an optional file - // rescan for long reads that need the read names to tell HiFi from CLR. + // The `read_type` value. The code first reads the platform and the test type, + // which costs little. It then reads the cached mean read length, and that value + // resolves a run from a sidecar import with an UNKNOWN platform. It can then scan + // the file. A long-read run needs that scan, because only the read names separate + // HiFi from CLR. if run.read_type.is_none() { let inferred = infer_read_type_cheap(&run.platform_name, &run.test_type).or_else(|| { metrics @@ -228,16 +236,20 @@ impl App { Ok(out) } - /// Bounded library-stats scan of the first readable alignment file in `alns`, returning its - /// inferred `read_type` (HiFi vs CLR from the read names). `None` when no file is accessible or - /// nothing decodable was found. + /// A limited library-stats scan of the first alignment file in `alns` that the code can read. + /// The method returns the `read_type` value that it deduces, and the read names separate HiFi + /// from CLR. + /// + /// The method returns `None` when it can open no file, and when it finds nothing that it can + /// decode. async fn rescan_read_type(&self, alns: &[navigator_domain::workspace::Alignment]) -> Option { for a in alns { if a.bam_path.is_none() { continue; } - // Resolve the reference for decode (see alignment_reference_for_decode): required for a - // CRAM, None for a BAM. Best-effort — skip an alignment that can't be resolved. + // Find the reference for the decoder. See alignment_reference_for_decode. A CRAM file + // needs it, and a BAM file uses None. The step is optional, and the code skips an + // alignment with no reference. let Ok((path, reference)) = self.alignment_reference_for_decode(a.id).await else { continue; }; @@ -253,8 +265,9 @@ impl App { None } - /// Cached coverage for several alignments at once (Data Sources alignment rows). `None` for any - /// alignment without a persisted coverage artifact. No genotyping/walking — pure cache reads. + /// The cached coverage of many alignments at once, for the alignment rows of the Data Sources + /// tab. The value is `None` for an alignment with no stored coverage artifact. The method reads + /// the cache only. It calls no caller and walks no file. pub async fn cached_coverage_bulk( &self, alignment_ids: &[i64], @@ -266,14 +279,18 @@ impl App { Ok(out) } - /// Alignments for a sequence run. - /// The best alignment to drive a subject's analysis tabs (subject-centric default): the - /// highest mean-coverage alignment with a cached coverage result, else the first with a BAM, - /// else the first. Returns `(sequence_run_id, alignment_id)` so the UI can select the run then - /// the alignment without the user navigating Data Sources. + /// The alignments of a sequence run. /// - /// Where a realignment exists, its output is preferred over the source it was derived from — - /// as a tie-break only, after breadth and depth. See the ranking comment below. + /// The method returns the alignment that drives the analysis tabs of a subject. It takes the + /// alignment with the highest mean coverage that also has a cached coverage result. If there is + /// none, it takes the first alignment with a BAM file, and then the first alignment. + /// + /// The method returns `(sequence_run_id, alignment_id)`. The UI can then select the run and the + /// alignment, and the user does not open the Data Sources tab. + /// + /// When a realignment exists, the method takes its output before the source of that + /// realignment. This rule applies only when the breadth and the depth are equal. See the + /// comment on the order below. pub async fn default_alignment_for_subject( &self, biosample_guid: SampleGuid, @@ -282,12 +299,17 @@ impl App { if alignments.is_empty() { return Ok(None); } - // Rank by (breadth, then depth, then file-present): a whole-genome test (WGS/HiFi) is the - // subject's representative test over a targeted Y/mt test **even when the targeted test is - // deeper**. A Y-only test's mean depth is a chrY-only number (coverage is scoped to the - // target contigs), so ranking on depth alone lets a deep Y Elite outscore a genome-wide - // WGS — and surfacing it as "your test" contradicts the autosomal ancestry the brief shows - // beside it. Depth and file presence only break ties within a breadth class. + // The order is the breadth, then the depth, then the presence of a file. + // + // A whole-genome test, such as WGS or HiFi, represents the subject before a targeted Y test + // or mt test. This rule applies **even when the targeted test has more depth**. + // + // The mean depth of a Y-only test covers chrY alone, because the coverage covers the target + // contigs only. So an order on the depth alone puts a deep Y Elite test above a WGS test. + // The app then names the Y test as "your test", and that name disagrees with the autosomal + // ancestry that the brief shows beside it. + // + // The depth and the presence of a file apply only inside one breadth class. let mut best: Option<(u8, f64, bool, bool, &Alignment)> = None; for a in &alignments { let target = match navigator_store::sequence_run::get(self.store.pool(), a.sequence_run_id).await? { @@ -296,12 +318,16 @@ impl App { }; let breadth = test_breadth_rank(target); let depth = self.cached_coverage(a.id).await?.map_or(0.0, |c| c.mean_coverage); - // Last, and only as a tie-break: a realigned alignment beats the source it was made - // from. Both describe the same library at the same breadth and near-identical depth, - // so without this the winner is whichever the list happened to yield first — and a - // default that changes between runs is worse than either choice. The realigned one is - // preferred because the user asked for it and it is on the newer reference; this is a - // *default*, not a restriction, and the source stays selectable. + // The last rule, and it applies only when the values above are equal. A realigned + // alignment goes before the source that made it. + // + // Both rows describe the same library, at the same breadth and at almost the same + // depth. Without this rule the first row in the list wins, and that row can change + // between two runs. A default that changes is worse than either choice. + // + // The realigned row goes first because the user asked for it, and because it is on the + // newer reference. This rule sets a *default*. It is not a limit, and the user can + // still select the source. let derived = a.is_derived(); let key = (breadth, depth, a.bam_path.is_some(), derived); if best.as_ref().map_or(true, |(b, d, f, r, _)| key > (*b, *d, *f, *r)) { @@ -311,13 +337,17 @@ impl App { Ok(best.map(|(_, _, _, _, a)| (a.sequence_run_id, a.id))) } - /// Donor-level ancestry: the modern super-population **`ADMIXTURE`** estimate — the consensus one - /// ([`CONSENSUS_SOURCE_ID`], pools all sources) when present, else the best-quality per-alignment - /// one (most genotyped SNPs) for back-compat with results predating the consensus path. + /// The ancestry of a donor. The value is the modern super-population **`ADMIXTURE`** estimate. + /// + /// The method takes the consensus estimate, [`CONSENSUS_SOURCE_ID`], which pools each source. + /// If the store holds none, it takes the estimate of one alignment with the best quality, which + /// is the estimate with the most genotyped SNPs. That second path supports a result from before + /// the consensus feature. /// - /// Must filter to `ADMIXTURE` specifically: the consensus source now also carries `FINE_ADMIXTURE` - /// and `ANCIENT_ADMIXTURE` rows, and picking the *first* consensus row would surface the deep - /// (ancient) breakdown here — a separate report — instead of the modern super-population one. + /// The method must filter on `ADMIXTURE`. The consensus source now also holds a + /// `FINE_ADMIXTURE` row and an `ANCIENT_ADMIXTURE` row. A read of the *first* consensus row + /// gives the deep, or ancient, breakdown, which is a separate report. This method must give the + /// modern super-population breakdown. pub async fn donor_ancestry(&self, biosample_guid: SampleGuid) -> Result, AppError> { let all = ancestry_result::for_biosample(self.store.pool(), biosample_guid).await?; if let Some(c) = all @@ -332,9 +362,11 @@ impl App { .max_by_key(|(_, r)| r.snps_with_genotype)) } - /// A specific persisted consensus ancestry estimate (keyed on the consensus pseudo-source + - /// `method`) — e.g. `"FINE_ADMIXTURE"` (detailed modern populations) or `"PCA_PROJECTION_GMM"` - /// (ancient components). Filtered per-subject (alignment_id 0 is not biosample-unique on its own). + /// One stored consensus ancestry estimate. The key is the consensus source with the `method` + /// value. Two examples are `"FINE_ADMIXTURE"`, which gives the detailed modern populations, and + /// `"PCA_PROJECTION_GMM"`, which gives the ancient components. + /// + /// The query also filters on the subject. An `alignment_id` of 0 does not name one biosample. pub async fn consensus_ancestry( &self, biosample_guid: SampleGuid, @@ -347,9 +379,11 @@ impl App { .map(|(_, r)| r)) } - /// Donor-level private-Y: the **union** of cached (self-masked) private-Y calls across all of - /// the subject's alignments, deduped by position (keeping the deepest observation). The - /// terminal is taken from the deepest-covered source bucket. + /// The private-Y calls of a donor. The value is the **union** of the cached private-Y calls of + /// each alignment of the subject, and the code applied the self-mask to those calls. + /// + /// The method removes a duplicate position and keeps the observation with the most depth. The + /// terminal comes from the source bucket with the most coverage. pub async fn donor_private_y(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut by_pos: std::collections::HashMap = std::collections::HashMap::new(); @@ -383,16 +417,22 @@ impl App { })) } - /// [`donor_private_y`](Self::donor_private_y) for **many** subjects at once, in two queries - /// rather than two per subject: the alignments, then the artifact rows, then per-subject merging - /// in memory. Subjects with no cached private-Y are simply absent from the map — that is "never - /// computed", not "none found". + /// The work of [`donor_private_y`](Self::donor_private_y) for **many** subjects at once. /// - /// Deliberately **not** built on `AlignmentArtifacts`: that stats every alignment file up front - /// to check freshness, and private-Y is computed for only a small fraction of a typical cohort. - /// Statting the rest buys nothing and costs everything — on a collection living on an external - /// volume those stats dominated the whole block-tree build. Here only the alignments that - /// actually carry a `private_y` row are statted. + /// The method sends two queries in total, and not two for each subject. The first reads the + /// alignments, and the second reads the artifact rows. The method then joins the rows of each + /// subject in memory. + /// + /// The map holds no entry for a subject with no cached private-Y data. That state means "no + /// analysis ran". It does not mean "the analysis found nothing". + /// + /// This method does **not** use `AlignmentArtifacts`, by design. That type stats each alignment + /// file first, to check the age of the cache. The app calculates the private-Y data of only a + /// small part of a cohort. + /// + /// A stat call on each other file gives no result and costs much. On a collection that sits on + /// an external volume, those stat calls were most of the time of the block-tree build. This + /// method stats only the alignments with a `private_y` row. pub(crate) async fn private_y_for_biosamples( &self, guids: &[SampleGuid], @@ -412,9 +452,11 @@ impl App { return Ok(HashMap::new()); } - // VCF-derived buckets, for subjects whose Y data never came with an alignment — the large - // majority of a Y project. Read from the cache only, as the alignment path does: opening a - // tab must not start classifying thousands of call sets. + // The buckets from a VCF file. They cover a subject whose Y data arrived with no + // alignment, and such subjects are most of a Y project. + // + // The code reads the cache only, as the alignment path does. A user who opens a tab must + // not start a classification of thousands of call sets. let mut out: HashMap = HashMap::new(); for guid in guids { let rows = @@ -439,7 +481,7 @@ impl App { let mut any = false; for a in alignments { let Some(row) = stored.get(&a.id) else { continue }; - // Only now is a stat worth paying for. + // A stat call is worth its cost only at this point. let current = a.bam_path.as_deref().and_then(|p| file_signature(Path::new(p))); if !artifact_is_fresh(row.source_sig.as_deref(), current.as_deref()) { continue; @@ -495,8 +537,9 @@ impl App { /// Projects with their sample counts, for a dashboard/list view. pub async fn project_overview(&self) -> Result, AppError> { - // One grouped count for the whole workspace rather than a COUNT per project — this runs on - // every projects-list load and on every CLI `--project` name lookup. + // One grouped count covers the full workspace. The code does not send a COUNT query for + // each project. This code runs at each load of the projects list, and at each `--project` + // name lookup in the CLI. let counts: HashMap = biosample::member_counts(self.store.pool()).await?.into_iter().collect(); Ok(project::list(self.store.pool()) .await? @@ -509,15 +552,18 @@ impl App { .collect()) } - /// Per-sample report for a project: each biosample's alignment count, coverage roll-up - /// (the first alignment with cached coverage), and Y/mtDNA haplogroup consensus. - /// Composes existing per-subject queries (no new join) — coverage/haplogroup cells are - /// `None` until those analyses have run. + /// A report of each sample in a project. Each row holds the count of alignments of one + /// biosample, a coverage summary, and the Y and mtDNA haplogroup consensus. The coverage comes + /// from the first alignment with a cached coverage result. + /// + /// The method calls the queries for one subject that already exist, and it adds no join. A + /// coverage cell and a haplogroup cell hold `None` until those analyses run. pub async fn project_report(&self, project_id: i64) -> Result, AppError> { let members = biosample::list_members_for_project(self.store.pool(), project_id).await?; - // Everything this report needs, in four queries rather than a per-cell round-trip: the - // members, their alignments, every artifact of those alignments, and the haplogroup - // reconciliation. Each cell below is then a lookup, not a query. + // Four queries read each value that this report needs. The code does not send one query + // for each cell. The queries read the members, their alignments, each artifact of those + // alignments, and the haplogroup reconciliation. Each cell below is then a lookup in + // memory. let guids: Vec = members.iter().map(|b| b.guid).collect(); let mut by_subject: HashMap> = HashMap::new(); for (guid, aln) in alignment::list_for_biosamples(self.store.pool(), &guids).await? { @@ -525,7 +571,8 @@ impl App { } let all_alignments: Vec<&Alignment> = by_subject.values().flatten().collect(); let artifacts = AlignmentArtifacts::load(&self.store, &all_alignments).await?; - // Same precedence (per-run vote → placed label → manual override) as `haplogroup_consensus`. + // The order is the same as the order in `haplogroup_consensus`. It is the vote of each + // run, then the placed label, then a value that the user set. let terminals = self.haplogroup_terminals().await?; let mut out = Vec::new(); @@ -540,7 +587,8 @@ impl App { break; } } - // A lite (sidecar) coverage is flagged so the UI can badge it and offer a deep walk. + // The row marks a small coverage result from a sidecar. The UI can then show a badge + // and offer a full walk. let coverage_partial = match coverage_aln { Some(id) => matches!( artifacts.provenance(id, "coverage", coverage::COVERAGE_VERSION), @@ -548,7 +596,8 @@ impl App { ), None => false, }; - // Prefer the coverage-bearing alignment; else fall back to the first. + // Take the alignment with a coverage result. If there is none, take the first + // alignment. let primary_alignment_id = coverage_aln.or_else(|| alignments.first().map(|a| a.id)); let (y_haplogroup, mt_haplogroup) = terminals.get(&biosample.guid).cloned().unwrap_or_default(); // Sex + read-metrics from whichever alignment has them cached. @@ -589,9 +638,11 @@ impl App { median_insert_size: metrics.as_ref().map(|m| m.median_insert_size), sv_count, coverage_partial, - // Surface a persisted failure (corrupt/undecodable file) only when there is no - // coverage to show — a successful re-walk clears the marker anyway. Read without a - // freshness check, as `analysis_error` does: the marker stands until a success clears it. + // Show a stored failure only when the row has no coverage. Such a failure comes + // from a file that the decoder refuses. A good walk removes the mark. + // + // The code reads the mark and does not check its age, as `analysis_error` does. The + // mark stays until a good walk removes it. decode_error: match (coverage.is_none(), primary_alignment_id) { (true, Some(id)) => artifacts .raw(id, ERROR_KIND, ERROR_VERSION) @@ -605,14 +656,20 @@ impl App { Ok(out) } - /// Per-member Y-STR overview for a project (the FTDNA-style "Y-DNA Results Overview"): each - /// member that has at least one STR profile, with identity columns, terminal Y haplogroup, the - /// reached STR panel/tier, and the consensus marker values (uppercase marker → value). Members - /// with no STR data are omitted. Composes existing per-subject queries (no new join). + /// A Y-STR overview of each member of a project. The table has the shape of the FTDNA "Y-DNA + /// Results Overview". + /// + /// The result holds each member with one STR profile or more. A row holds the identity columns, + /// the terminal Y haplogroup, the STR panel that the test reached, and the consensus marker + /// values. Each marker name is in upper case. + /// + /// The result holds no member with no STR data. The method calls the queries for one subject + /// that already exist, and it adds no join. pub async fn project_str_overview(&self, project_id: i64) -> Result, AppError> { use navigator_domain::{strpanel, strprofile}; let mut out = Vec::new(); - // One bulk reconciliation for the whole workspace rather than two queries per member. + // One reconciliation covers the full workspace. The code does not send two queries for + // each member. let terminals = self.haplogroup_terminals().await?; for biosample in biosample::list_members_for_project(self.store.pool(), project_id).await? { let profiles = self.list_str_profiles(biosample.guid).await?; @@ -654,12 +711,20 @@ impl App { Ok(out) } - /// Build the precomputed FTDNA-style Y-STR overview for a project: members grouped by their - /// **assigned** (consensus) Y haplogroup, ordered by tree topology (basal → derived, children - /// nested under their ancestor subgroups), with per-subgroup MIN/MAX/MODE and per-cell deviation - /// from the modal value precomputed. Members without a SNP haplogroup fall into an "Unassigned" - /// bucket at the base. All heavy work happens here (off the UI thread); the renderer just - /// iterates [`ProjectStrChart::rows`]. + /// Build the Y-STR overview of a project, in the FTDNA form, with each value calculated in + /// advance. + /// + /// The method groups the members by their **assigned** Y haplogroup, which is the consensus + /// value. It then orders the groups by the shape of the tree, from the basal node to the + /// derived nodes. Each child group goes below its ancestor group. + /// + /// For each group, the method calculates the MIN, MAX, and MODE values. For each cell, it + /// calculates the difference from the modal value. + /// + /// A member with no SNP haplogroup goes into an "Unassigned" group at the base. + /// + /// Each large calculation happens here, away from the UI thread. The renderer only reads + /// [`ProjectStrChart::rows`]. pub async fn project_str_chart(&self, project_id: i64) -> Result { use navigator_domain::{strchart, strpanel}; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -828,11 +893,18 @@ impl App { }) } - /// Analyze every sample in a project: compute coverage and assign the Y haplogroup on each - /// sample's primary (first BAM-bearing) alignment, so the project report fills in. Coverage - /// already cached and Y already recorded are skipped (idempotent re-run). Best-effort: one - /// sample's failure is recorded and the rest continue. mtDNA is intentionally not assigned - /// here (provisional on CHM13 — see the reconciliation/liftover notes). + /// Analyze each sample in a project. The method calculates the coverage and assigns the Y + /// haplogroup on the primary alignment of each sample. That alignment is the first one with a + /// BAM file. The project report then holds a value in each cell. + /// + /// The method skips a coverage result that the cache holds, and a Y value that the store + /// already holds. So a second run is safe. + /// + /// A failure on one sample goes into the report, and the method continues with the other + /// samples. + /// + /// The method does not assign mtDNA, by design. That value is not final on CHM13. See the notes + /// on the reconciliation and the liftover. pub async fn analyze_project( &self, project_id: i64, @@ -848,8 +920,8 @@ impl App { errors: Vec::new(), }; for biosample in biosample::list_members_for_project(self.store.pool(), project_id).await? { - // Between samples as well as inside them: a cancel that lands while a sample's walk is - // finishing must not start the next one. + // The code checks for a stop between two samples, and also inside one sample. A stop + // that arrives at the end of the walk of one sample must not start the next sample. if cancel.is_cancelled() { break; } @@ -867,13 +939,21 @@ impl App { Ok(summary) } - /// Deep-analyze one biosample's primary (first BAM-bearing) alignment: coverage, Y - /// haplogroup, sex, and read metrics. **Not** structural variants — see the note at the end of - /// the body. Idempotent — a *full* coverage and a recorded Y/sex/metrics are skipped; a - /// `partial` (lite sidecar) coverage is upgraded by - /// the per-base walk, which overwrites it. Best-effort: a per-step failure is recorded in - /// `errors` (prefixed with the donor id) and the remaining steps still run. This is the - /// per-sample unit the project pass and the streaming deep-analyze job both drive. + /// Do the full analysis of the primary alignment of one biosample. That alignment is its first + /// alignment with a BAM file. The steps are the coverage, the Y haplogroup, the sex, and the + /// read metrics. + /// + /// The method does **not** call structural variants. See the note at the end of the body. + /// + /// A second run is safe. The method skips a *full* coverage result, and a Y value, a sex value, + /// and a metrics value that the store holds. It replaces a `partial` coverage result from a + /// sidecar, because the walk across each base gives a better result. + /// + /// A failure in one step goes into `errors`, with the donor id at the start of the message. The + /// other steps still run. + /// + /// This method is the unit of work for one sample. The project pass and the deep-analyze job + /// both call it. pub async fn analyze_biosample( &self, biosample: &Biosample, @@ -881,12 +961,16 @@ impl App { ) -> Result { let mut o = SampleAnalyzeOutcome::default(); let alignments = alignment::list_for_biosample(self.store.pool(), biosample.guid).await?; - // Prefer an alignment whose file is actually still there. Selecting merely by "a path was - // recorded" meant a subject with two alignments — one whose vendor download has since been - // cleaned out, one intact — could pick the gone one, fail preflight, and be skipped whole, - // when the other would have analyzed fine. Falling back to any recorded path keeps the - // no-file-present case reporting a real preflight diagnosis rather than silently reading as - // "this subject has no alignment at all". + // Take an alignment whose file is still on disk. + // + // An earlier rule took any alignment with a recorded path. Take a subject with two + // alignments, where a user removed the vendor download of one and kept the other. That rule + // could take the absent one. The preflight then failed, and the code skipped the full + // subject. The other alignment would have given a good result. + // + // When no file is on disk, the code takes any recorded path. The preflight then gives a + // real diagnosis. Without that step, the report reads as "this subject has no alignment", + // which is not true. let Some(aln) = alignments .iter() .find(|a| Self::alignment_file(a).is_ok()) @@ -897,20 +981,22 @@ impl App { o.had_alignment = true; let label = &biosample.donor_identifier; - // Preflight before spending any I/O on the steps below. A batch is the worst place to - // discover a file problem the slow way: without this, an unreadable alignment produces one - // near-identical `io error on ` per step — each after a walk that had to - // fail first — and none of them names the file actually at fault. + // Run the preflight before any I/O in the steps below. A batch is the worst place to find + // a file problem the slow way. + // + // Without the preflight, an alignment that the code can not read gives one + // `io error on ` message for each step. Each message arrives after a walk + // that had to fail first, and no message names the file at fault. // - // What it does *not* do is skip the sample on any failure. A broken index blocks only the - // region-query steps; the unified metrics walk falls back to a sequential pass and still - // produces coverage, read metrics and sex. Skipping on that would throw away results the - // user would otherwise get, so only a failure that blocks sequential reads short-circuits. + // The preflight does *not* skip the sample on each failure. A broken index stops only the + // steps that query a region. The unified metrics walk then reads the file from start to end + // and still gives the coverage, the read metrics, and the sex. A skip would remove results + // that the user can get. So only a failure that stops a sequential read ends the work. // - // Both sinks get the *first failure*, not the whole report: `o.errors` renders as one line - // per entry, and `record_analysis_error` truncates to 500 chars keeping the head — which - // for a full report is the path preamble and the checks that passed, so the diagnosis - // itself would be the part cut off. The full report stays available via `navigator doctor`. + // Both destinations receive the *first failure* and not the full report. `o.errors` shows + // one line for each entry. `record_analysis_error` keeps the first 500 characters. For a + // full report, those characters are the path and the checks that passed, so the tool would + // cut the diagnosis itself. The command `navigator doctor` still gives the full report. match self.diagnose_alignment(aln.id).await { Ok(report) if report.failed() => { let cause = report @@ -926,16 +1012,21 @@ impl App { return Ok(o); } } - // A preflight that itself failed to run is not evidence about the alignment — fall - // through and let the real steps report whatever they hit. + // A preflight that did not run says nothing about the alignment. Continue, and let + // the real steps report the fault that they find. Ok(_) | Err(_) => {} } - // Coverage + read-metrics + sex in ONE pass (the unified walker) instead of three separate - // reads of the BAM/CRAM — a 3x I/O cut per subject, which dominates the batch on a slow / - // network volume (the single-subject Full Analysis already does this; the batch path did not). - // Walk only when something's missing: a full, correctly-scoped coverage (a stale whole-genome - // result for a targeted-Y test is recomputed) plus cached read-metrics and sex = all done. + // The unified walker gives the coverage, the read metrics, and the sex in ONE pass. It + // does not read the BAM file or the CRAM file three times. This change divides the I/O of + // each subject by three. That I/O is most of the time of a batch on a slow volume or a + // network volume. The Full Analysis of one subject already used this walker, and the batch + // path did not. + // + // The code walks the file only when a value is absent. The work is complete when the store + // holds a full coverage result with the correct scope, the read metrics, and the sex. A + // whole-genome coverage result for a targeted Y test has the wrong scope, and the code + // calculates it again. let coverage_full = matches!( self.analysis_provenance(aln.id, "coverage", coverage::COVERAGE_VERSION).await?, Some((_, ref c)) if c == "full" @@ -959,13 +1050,18 @@ impl App { o.coverage_done = true; o.metrics_done = true; o.sex_done = true; - // A prior run may have left a failure marker (corrupt file since replaced); clear it. + // An earlier run can leave a failure mark, from a bad file that the user then + // replaced. Remove that mark. self.clear_analysis_error(aln.id).await; } - // A cancellation is the user's decision, not a property of the file: recording it - // would persist a "Failed" marker that survives the run and makes the sample look - // broken forever, and counting it as an error would inflate the batch summary. Stop - // the sample here instead — the remaining steps would only be cancelled too. + // A stop is the decision of the user. It says nothing about the file. + // + // A record of it writes a "Failed" mark that stays after the run, and the sample + // then looks broken for all time. A count of it as an error also makes the batch + // summary wrong. + // + // So the code stops the work on this sample here. The user would stop each + // remaining step also. Err(e) if e.is_cancellation() => return Ok(o), Err(e) => { // Persist the failure so the report can show "Failed" instead of a silent blank @@ -986,38 +1082,50 @@ impl App { } } - // Build the genome-consensus Y signature (deep placement + variant profile → descent report) - // here, so the Y-DNA descent report is populated by batch analysis rather than requiring an - // explicit "Build descent report" click (that button stays for an on-demand rebuild). Built - // once — skipped when a profile already exists — and best-effort. The chrY genotypes it needs - // were just cached by the Y assignment above, so this adds no extra read of the file. + // Build the genome-consensus Y signature here. That work is the deep placement and the + // variant profile, and it gives the descent report. + // + // So a batch analysis fills the Y-DNA descent report, and the user does not press "Build + // descent report". That button stays, and it rebuilds the report at any time. + // + // The code builds the report one time, and it skips a profile that already exists. The step + // is optional. The Y assignment above wrote the chrY genotypes to the cache a moment ago, + // so this step reads the file no more times. if o.y_done && self.cached_y_profile(biosample.guid).await?.is_none() { if let Err(e) = self.build_y_profile(biosample.guid).await { o.errors.push(format!("{label} Y signature: {e}")); } } - // SV deliberately does NOT run here. It is experimental, nothing else consumes its output, - // and it is the one step that walks every read in the file for its own sake: measured at - // 2–5 h per whole-genome sample on this workspace's CRAMs, against ~1 h for everything - // above put together. In a 148-sample project that is the difference between a batch that - // finishes overnight and one that takes weeks. Run it deliberately instead — the "Call SV" - // button, or `analyze --sv` — via `plan_full_analysis(.., include_sv = true)`. + // The SV step does NOT run here, by design. It is experimental, and no other step reads + // its output. It is also the one step that reads each record in the file for its own + // result. + // + // A measurement on the CRAM files of this workspace gave 2 to 5 hours for one whole-genome + // sample. Each step above needs about 1 hour in total. In a project of 148 samples, that + // difference is a batch that completes in one night against a batch that needs weeks. + // + // The user starts this step: the "Call SV" button, or `analyze --sv`. Both call + // `plan_full_analysis(.., include_sv = true)`. Ok(o) } } // ---- Y-tree topology helpers for the project STR chart ordering -------------------------------- -/// Normalize a haplogroup / node name for matching: uppercase, and (since our consensus labels and -/// the tree nodes both use the "R-CTS4466" convention) keep the full name. Tolerates a bare SNP by -/// also being comparable to the suffix after the last '-' (callers index both forms). +/// Change a haplogroup name or a node name into the form that the code compares. The function makes +/// each letter upper case and keeps the full name. Our consensus labels and the tree nodes both use +/// the "R-CTS4466" form. +/// +/// The result also compares with the part after the last `-`. So the code accepts a plain SNP name, +/// and a caller indexes both forms. fn norm_hg(name: &str) -> String { name.trim().to_ascii_uppercase() } -/// Map every tree node name (and its bare-SNP suffix) to a node id, for resolving haplogroup labels. -/// Full names win over suffix aliases on collision. +/// A map from each tree node name to a node id. The map also holds the plain SNP part of each name. +/// The code uses it to find the node of a haplogroup label. When two entries have the same key, the +/// full name wins. fn tree_name_index(tree: &navigator_analysis::haplo::HaploTree) -> std::collections::HashMap { let mut idx = std::collections::HashMap::new(); // Pass 1: suffix aliases (lower priority). @@ -1044,8 +1152,9 @@ fn tree_parent_map(tree: &navigator_analysis::haplo::HaploTree) -> std::collecti parent } -/// Pre-order DFS rank for every node (basal → derived; children follow their parent, siblings in -/// stored order) so groups can be ordered to mirror the tree. +/// The pre-order rank of each node, from a depth-first search. The order goes from the basal node +/// to the derived nodes. Each child follows its parent, and the code keeps the stored order of the +/// children. A caller then orders its groups in the same shape as the tree. fn tree_preorder(tree: &navigator_analysis::haplo::HaploTree) -> std::collections::HashMap { let mut rank = std::collections::HashMap::new(); let mut next = 0usize; @@ -1074,9 +1183,11 @@ fn tree_preorder(tree: &navigator_analysis::haplo::HaploTree) -> std::collection rank } -/// Infer a run's `read_type` without touching the alignment file — from the `test_type` code (which -/// may already name the chemistry) then the platform. Returns `None` for generic-WGS PacBio, where -/// HiFi vs CLR genuinely needs the read names (a file rescan). +/// Find the `read_type` of a run and read no alignment file. The function reads the `test_type` +/// value, which can already name the chemistry, and then the platform. +/// +/// The function returns `None` for a PacBio run with a plain WGS test type. Only the read names +/// separate HiFi from CLR, and a scan of the file gives them. fn infer_read_type_cheap(platform_name: &str, test_type: &str) -> Option<&'static str> { let tt = test_type.to_ascii_uppercase(); if tt.contains("HIFI") { @@ -1094,7 +1205,8 @@ fn infer_read_type_cheap(platform_name: &str, test_type: &str) -> Option<&'stati } else if p.contains("NANOPORE") || p == "ONT" { Some("ONT_SIMPLEX") } else { - // PacBio (or an unrecognized platform) — can't tell HiFi from CLR here. + // A PacBio platform, or a platform that the code does not know. The code can not separate + // HiFi from CLR here. None } } @@ -1107,12 +1219,17 @@ fn read_type_from_mean_len(mean: f64) -> Option<&'static str> { (mean > 0.0 && mean <= 1000.0).then_some("SHORT") } -/// How well a test represents a whole person, for picking a subject's default/representative -/// alignment (see [`App::default_alignment_for_subject`]). Higher is broader: a whole-genome test -/// carries paternal, maternal *and* autosomal ancestry; an autosomal/chip test carries the ancestry -/// composition the brief leads with; a Y/mt/X test is a single-lineage close-up. `None` is an -/// unrecognized test type — ranked above targeted (it may be a broad test whose label we did not -/// recognize) but below anything we know is genome-wide. +/// The rank of a test by the part of a person that it covers. The code uses this rank to select the +/// default alignment of a subject. See [`App::default_alignment_for_subject`]. +/// +/// A higher value covers more. A whole-genome test carries the paternal ancestry, the maternal +/// ancestry, *and* the autosomal ancestry. An autosomal test or a chip test carries the ancestry +/// composition, which is the first section of the brief. A Y test, an mt test, or an X test covers +/// one lineage only. +/// +/// A `None` value is a test type that the code does not know. Its rank is above a targeted test, +/// because the test can be a wide test with a label that the code does not hold. Its rank is below +/// each test that the code knows to be genome-wide. fn test_breadth_rank(target: Option) -> u8 { use navigator_domain::testtype::TargetType::*; match target { @@ -1131,8 +1248,9 @@ mod breadth_tests { #[test] fn whole_genome_outranks_targeted_regardless_of_depth() { - // The reported quirk: a deep Y Elite must not outrank a genome-wide WGS/HiFi as the - // representative test, because a Y-only test can't produce the ancestry shown beside it. + // The fault that a user reported. A deep Y Elite test must not rank above a genome-wide + // WGS test or HiFi test. A Y-only test can not give the ancestry that the app shows beside + // it. let wgs = test_breadth_rank(target_of("WGS")); let hifi = test_breadth_rank(target_of("WGS_HIFI")); let y_elite = test_breadth_rank(target_of("Y_ELITE")); @@ -1146,7 +1264,8 @@ mod breadth_tests { #[test] fn ancestry_bearing_and_unknown_tiers() { - // Chips / exomes carry the autosomal ancestry the brief leads with — above targeted, below WGS. + // A chip and an exome carry the autosomal ancestry, which is the first section of the + // brief. Their rank is above a targeted test and below a WGS test. assert!(test_breadth_rank(Some(TargetType::WholeGenome)) > test_breadth_rank(target_of("ARRAY_23ANDME_V5"))); assert!(test_breadth_rank(target_of("ARRAY_23ANDME_V5")) > test_breadth_rank(target_of("Y_ELITE"))); // An unrecognized test type ranks above targeted but below a known genome-wide test. @@ -1171,7 +1290,7 @@ mod read_profile_tests { assert_eq!(infer_read_type_cheap("MGI", "WGS"), Some("SHORT")); // Nanopore by platform alone. assert_eq!(infer_read_type_cheap("NANOPORE", "WGS"), Some("ONT_SIMPLEX")); - // Generic-WGS PacBio — unresolved without a rescan. + // A PacBio platform with a plain WGS test type. The code needs a scan of the file. assert_eq!(infer_read_type_cheap("PACBIO", "WGS"), None); } @@ -1181,16 +1300,19 @@ mod read_profile_tests { // Sidecar-imported short-read WGS (mean 62–150 bp) → SHORT. assert_eq!(read_type_from_mean_len(150.0), Some("SHORT")); assert_eq!(read_type_from_mean_len(62.5), Some("SHORT")); - // Long reads can't be split HiFi/CLR by length — unresolved. + // The length of a long read does not separate HiFi from CLR. The code needs a scan of the + // file. assert_eq!(read_type_from_mean_len(21_563.0), None); // No metrics. assert_eq!(read_type_from_mean_len(0.0), None); } } -/// Fold `extra` into `into`, deduping by position and keeping the deeper observation — the same rule -/// the per-subject alignment union uses. A donor with both a CRAM and a vendor VCF should end up with -/// one private set, not two competing ones. +/// Add `extra` to `into`. The function removes a duplicate position and keeps the observation with +/// the most depth. The union of the alignments of one subject uses the same rule. +/// +/// A donor with a CRAM file and a vendor VCF must have one private set. Two sets that disagree are +/// not correct. fn merge_bucket(into: &mut PrivateBucket, extra: PrivateBucket) { if into.terminal.is_empty() { into.terminal = extra.terminal; diff --git a/crates/navigator-app/src/realign_job.rs b/crates/navigator-app/src/realign_job.rs index 6834bb4e..a1cefc8a 100644 --- a/crates/navigator-app/src/realign_job.rs +++ b/crates/navigator-app/src/realign_job.rs @@ -1,9 +1,10 @@ -//! Running a realignment end to end — stages A through D as one cancellable job. +//! One realignment from start to end. This module runs stage A to stage D as one job, and the user +//! can stop that job. //! -//! The stages exist as independent, separately-tested pieces -//! ([`revert`](navigator_analysis::revert), [`navigator_align`], -//! [`postprocess`](navigator_analysis::postprocess), and `crate::realign`). This is the part that -//! knows the order, what to do between them, and how to stop. +//! Each stage is a separate piece of code with its own tests. They are +//! [`revert`](navigator_analysis::revert), [`navigator_align`], +//! [`postprocess`](navigator_analysis::postprocess), and `crate::realign`. This module knows their +//! order, the work between them, and the way to stop them. //! //! ## Shape of the job //! @@ -11,23 +12,25 @@ //! preflight ──► revert ──► index ──► map ──► sort ──► markdup ──► finalize ──► register //! ``` //! -//! It is hours of work on tens of GB of scratch, so three things matter more than they would in a -//! shorter job: +//! The job needs hours of work and tens of GB of scratch space. So three rules matter more here +//! than in a short job. //! -//! - **Nothing is destroyed.** The source alignment is read and never written. If any stage fails -//! or is cancelled, the workspace is exactly as it was — the new row is inserted last, after the -//! final artifact exists, so there is no window where a half-built alignment is registered. -//! - **Scratch is cleaned up.** Intermediates are the size of the input, several times over. -//! [`JobScratch`] removes them on the way out whether the job succeeded, failed, or was -//! cancelled. -//! - **It can be stopped.** Every stage takes the cancel token, and a cancelled job reports itself -//! as cancelled rather than as a failure — the user pressed the button. +//! - **The job destroys nothing.** It reads the source alignment and never writes to it. After a +//! failed stage, and after a stop, the workspace holds what it held before. The code adds the new +//! row last, after the final file exists. So the workspace never holds a row for an alignment +//! that the job did not complete. +//! - **The job removes its scratch files.** Those files are some times the size of the input. +//! [`JobScratch`] removes them at the end, after a success, after a failure, and after a stop. +//! - **The user can stop the job.** Each stage takes the cancel token. A job that the user stopped +//! reports a stop and not a failure, because the user pressed the button. //! //! ## Preflight //! -//! Checking disk and memory before starting is not politeness. Filling a disk partway through hour -//! three fails the job *and* leaves the machine unusable until someone finds the scratch directory; -//! and the index sizing needs the RAM figure anyway. See [`preflight`]. +//! The job checks the disk and the memory before it starts, and that check is necessary. +//! +//! A disk that fills in the third hour fails the job. It also leaves the machine unusable until +//! somebody finds the scratch directory. The size of the index also needs the RAM value. See +//! [`preflight`]. use std::path::{Path, PathBuf}; @@ -40,10 +43,10 @@ use navigator_domain::workspace::Alignment; use crate::error::AppError; use crate::App; -/// The stages, in order, for progress reporting. +/// The stages, in their order, for the progress report. /// -/// Named rather than numbered in the UI: "sorting" tells a user waiting two hours something that -/// "step 5 of 8" does not. +/// The UI shows a name and not a number. The word "sorting" tells a user who waits two hours more +/// than the text "step 5 of 8". #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RealignStage { Preflight, @@ -87,34 +90,38 @@ impl RealignStage { } } -/// Progress from a running job. +/// The progress of a job that is in operation. #[derive(Debug, Clone)] pub struct RealignProgress { pub stage: RealignStage, pub total_stages: usize, - /// Free-text detail — a record count, a part number. May be empty. + /// Text with more detail, such as a count of records or a part number. The value can be + /// empty. pub detail: String, } -/// What a finished job produced. +/// The result of a job that is complete. +/// +/// Each count is optional, because a job that continues an earlier job does not always run the stage +/// that makes that count. /// -/// The counts are optional because a resumed job did not necessarily run the stage that produces -/// them. A run that picks up from a previous attempt's sorted BAM never reverted anything, so it -/// can not report how many unmapped reads that revert saw; [`ScratchState`] carries the figure -/// across when the earlier attempt recorded it, and `None` says plainly that nobody measured it -/// rather than reporting a zero that reads like a finding. +/// A run that starts from the sorted BAM file of an earlier run does no revert step. So it can not +/// report the count of unmapped reads that the revert step saw. +/// +/// [`ScratchState`] holds that value when the earlier run wrote it. A value of `None` states that no +/// code measured the count. A zero value looks like a measurement, and it would be wrong. #[derive(Debug, Clone)] pub struct RealignOutcome { pub alignment: Alignment, - /// Reads that were unmapped in the source and therefore had a chance to be placed by the new - /// reference. This is the number the module exists to move, so it is surfaced rather than - /// buried in a log. + /// The count of reads with no place in the source alignment. The new reference can give each of + /// them a place. This module exists to increase that count, so the report holds it. A log entry + /// alone would hide it. pub source_unmapped_reads: Option, pub reads_written: Option, pub duplicates_marked: Option, } -/// Tuning a caller may override; the defaults are what the UI uses. +/// The values that a caller can change. The UI uses the default values. #[derive(Debug, Clone)] pub struct RealignParams { /// Target build, e.g. `chm13v2.0`. @@ -125,27 +132,29 @@ pub struct RealignParams { pub preset: Option, /// Where intermediates live. `None` puts them beside the output. pub scratch_root: Option, - /// Pick up from a previous attempt's intermediates instead of starting over. + /// Use the intermediate files of an earlier run, and do not start again from the source. /// - /// Off by default, because reusing files a *different* job left behind would be a correctness - /// bug, and the scratch path alone can not prove they came from this source and this target. - /// The caller opting in is what supplies that knowledge. See [`Resumed`]. + /// The default is off. A run that used the files of a *different* job gives a wrong result. + /// The scratch path alone does not prove that those files came from this source and this + /// target. The caller who sets this field supplies that knowledge. See [`Resumed`]. pub resume: bool, } -/// How far a previous attempt got, judged from what it left in the scratch directory. +/// The last stage that an earlier run completed. The code decides from the files in the scratch +/// directory. /// -/// Ordered by how much work it saves, and checked newest-artifact-first: a complete `marked.bam` -/// makes the sort behind it irrelevant, so there is no reason to ask about `sorted.bam` as well. +/// The order is the quantity of work that each value saves. The code tests the newest file first. A +/// complete `marked.bam` file makes the sort behind it unnecessary, so the code does not test +/// `sorted.bam` also. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] enum Resumed { /// Start from the source alignment. Nothing, - /// Reads were recovered and mapped; sorting is next. + /// The run recovered the reads and mapped them. The sort is next. Mapped, - /// The mapped BAM was sorted; duplicate marking is next. + /// The run sorted the mapped BAM file. The duplicate mark step is next. Sorted, - /// Duplicates were marked; only finalising and registration remain. + /// The run marked the duplicates. Only the finalize step and the register step remain. Marked, } @@ -161,23 +170,29 @@ impl Resumed { } } -/// Counts a resumed job can not re-derive, left beside the intermediates they describe. +/// The counts that a job which continues an earlier job can not calculate again. The file sits +/// beside the intermediate files that it describes. +/// +/// Each stage measures its part of [`RealignOutcome`] while that stage runs, and that value is gone +/// after the stage ends. A job that continues an earlier job skips stages, by design. So the code +/// writes each number to this file as a stage produces it. /// -/// Each stage's contribution to [`RealignOutcome`] is measured while that stage runs and is gone -/// once it has. A resumed job skips stages by design, so the numbers are written down as they are -/// produced. Best-effort throughout: failing to write this file must not fail a job that has -/// otherwise done hours of correct work, and a missing or unreadable file simply means the counts -/// come back `None`. +/// Each write is optional. A failed write of this file must not fail a job that did hours of correct +/// work. An absent file, and a file that the code can not read, give counts of `None`. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] struct ScratchState { unmapped_reads: Option, reads_written: Option, duplicates_marked: Option, - /// The furthest stage that *returned successfully*, as opposed to merely leaving a file behind. + /// The last stage that *returned a success*. A stage that only left a file behind does not + /// count. /// - /// Written after the stage returns, never before, so it says something the file itself can not: - /// see [`discard_partial`] for why a finished-looking BAM is not proof on its own. A scratch - /// directory predating this field has `None`, and then the marker is all there is to go on. + /// The code writes this value after the stage returns, and never before. So the value states + /// something that the file itself can not state. [`discard_partial`] gives the reason: a BAM + /// file that looks complete is not proof on its own. + /// + /// A scratch directory from before this field holds `None`. The code then has the marker + /// only. completed_through: Option, } @@ -198,20 +213,26 @@ impl ScratchState { } } -/// Remove a stage's output when the stage did not finish. +/// Remove the output of a stage that did not complete. +/// +/// The BGZF end-of-file marker alone does not prove that a file is complete. This function exists, +/// so that no user learns that fact from a broken result. +/// +/// The writer of noodles uses many threads, and it closes its stream from `Drop`. So a stage that +/// *unwinds* leaves a partial file with the marker of a complete file. A stop, a failure, and a +/// panic each unwind. +/// +/// One measurement showed this. A user stopped a merge at 13.2 GB of an expected 30 GB, and +/// `is_complete_bam` then accepted that output. A run from that file would mark the duplicates of a +/// short alignment and add the result to the workspace. No record anywhere would state that the +/// reads were absent. /// -/// The BGZF end-of-file marker can not carry the whole weight of "this file is complete", and -/// finding that out the hard way is what this function exists to prevent. noodles' multithreaded -/// writer finishes its stream from `Drop`, so a stage that *unwinds* — cancelled, failed, panicked -/// — leaves a partial file wearing a finished file's marker. Measured: a merge cancelled at 13.2 GB -/// of an expected ~30 GB, whose output `is_complete_bam` then accepted. Resuming from it would have -/// marked duplicates on a truncated alignment and registered the result, with nothing anywhere -/// saying the reads were missing. +/// A hard kill is the opposite case, and the resume feature exists for it. No `Drop` runs, the code +/// writes no marker, and it correctly refuses the file. /// -/// A hard kill is the opposite case, and the one resume was built for: no `Drop` runs, no marker is -/// written, and the file is correctly refused. So the rule that makes the marker trustworthy again -/// is this one — whenever the job is still alive to notice a stage did not finish, its output does -/// not survive to be mistaken for one that did. +/// So this rule makes the marker reliable again. While the job runs and can see that a stage did not +/// complete, the output of that stage must not survive. If it survives, a later run reads it as the +/// output of a stage that did complete. fn discard_partial(path: &Path) { let _ = std::fs::remove_file(path); } @@ -227,13 +248,16 @@ async fn stage(output: &Path, work: impl std::future::Future Resumed { let by_marker = resumable_by_marker(scratch); match state.completed_through { @@ -242,7 +266,8 @@ fn resumable(scratch: &Path, state: &ScratchState) -> Resumed { } } -/// The marker-only half of [`resumable`], kept separate so the two rules can be tested apart. +/// The half of [`resumable`] that reads the marker only. It is a separate function, so a test can +/// cover each of the two rules alone. fn resumable_by_marker(scratch: &Path) -> Resumed { if postprocess::is_complete_bam(&scratch.join("marked.bam")) { Resumed::Marked @@ -255,21 +280,25 @@ fn resumable_by_marker(scratch: &Path) -> Resumed { } } -/// Clear a stage's working directory before that stage runs. +/// Empty the directory of a stage before that stage runs. +/// +/// A job that continues an earlier job runs the first stage that it can not skip. The files of that +/// stage from the earlier run give nothing. /// -/// A resumed job re-runs the first stage it can not skip, and that stage's leftovers from the -/// killed attempt are pure cost: the sort ignores run files it did not write itself, so stale ones -/// are not a correctness problem, but at WGS scale they are tens of GB held against a disk that -/// the same job is about to need. Best-effort — a directory that will not clear is not a reason to -/// refuse to start. +/// They are not a correctness problem, because the sort ignores a run file that it did not write. +/// But for a WGS sample they hold tens of GB on a disk that this same job needs. +/// +/// The step is optional. A directory that the code can not empty is not a reason to refuse the +/// job. fn clear_stage_dir(dir: &Path) { let _ = std::fs::remove_dir_all(dir); } impl App { - /// Realign `source_id` onto another reference, end to end. + /// Realign `source_id` onto another reference, from start to end. /// - /// Long-running and cancellable. `progress` is called as each stage begins. + /// The job runs for a long time, and the user can stop it. The code calls `progress` at the + /// start of each stage. pub async fn realign_alignment( &self, source_id: i64, @@ -291,8 +320,8 @@ impl App { .scratch_root .clone() .unwrap_or_else(|| output.with_extension("scratch")); - // Removes every intermediate on the way out, however the job ends — unless - // [`keep_scratch_on_failure`] is set and the job dies. + // This value removes each intermediate file at the end of the job, after each result. The + // one exception is a failed job with [`keep_scratch_on_failure`] set. let mut scratch = JobScratch::new(scratch)?; let report = |stage: RealignStage, detail: &str| { @@ -307,7 +336,7 @@ impl App { let sorted = scratch.path().join("sorted.bam"); let marked = scratch.path().join("marked.bam"); - // What a previous attempt left behind, and what it measured while it was running. + // The files that an earlier run left, and the values that it measured. let previous = ScratchState::load(scratch.path()); let resumed = if params.resume { resumable(scratch.path(), &previous) @@ -320,12 +349,12 @@ impl App { previous }; - // Watch the machine for as long as the job runs. It reports; it never intervenes — see - // `navigator_resource`. Started here so that it covers every stage, including the ones a - // resumed job skips over quickly. + // Watch the machine while the job runs. This watch reports, and it never stops the job. + // See `navigator_resource`. It starts here, so it covers each stage. That set holds the + // stages that a job which continues an earlier job passes quickly. let _watch = navigator_resource::ResourceWatch::start(navigator_resource::DEFAULT_INTERVAL, |sample| { - // Anything short of trouble is a log line; the bands exist so that trouble is - // greppable afterwards rather than buried in six hours of normal readings. + // A normal reading is one line in the log. The bands exist so that a user can find a + // problem in that log later. Without them, six hours of normal readings hide it. if sample.pressure == navigator_resource::Pressure::Normal { eprintln!("realign: {}", sample.summary()); } else { @@ -337,9 +366,11 @@ impl App { report(RealignStage::Preflight, resumed.detail()); cancel.check()?; let source_size = std::fs::metadata(&source_bam).map(|m| m.len()).unwrap_or(0); - // A resumed job is sized from the intermediate it is resuming from, not from the source. - // The source's expansion — CRAM to FASTQ and back to BAM — has already happened and is - // sitting on the disk; charging for it a second time would refuse a job that fits. + // A job that continues an earlier job takes its size from the intermediate file that it + // starts from, and not from the source. + // + // The growth of the source, from CRAM to FASTQ and then back to BAM, already happened, and + // those files are on the disk. A second count of that space refuses a job that fits. let plan = if resumed == Resumed::Nothing { preflight(scratch.path(), Path::new(&source_bam), source_size)? } else { @@ -444,14 +475,17 @@ impl App { state.completed_through = Some(Resumed::Mapped); state.store(scratch.path()); } else { - // Still reported, even though there is nothing to do: a progress display that skips - // from stage 3 to stage 5 reads as a missing step rather than a saved one. + // Report the stage, although it has no work. A progress display that goes from stage + // 3 to stage 5 looks like a step that the code lost. It does not look like a step that + // the code saved. report(RealignStage::Map, resumed.detail()); } - // Every stage's input is dead once the next stage has read it, and at WGS scale each is - // tens of GB. Holding them all until the job ends — which is what this did first — roughly - // doubles the peak and is the difference between fitting on a normal disk and not. + // The input of a stage has no more use after the next stage reads it. For a WGS sample, + // each input is tens of GB. + // + // An earlier version kept each input until the end of the job. That version needed about + // two times the peak space, and a normal disk was then too small. let discard = discard_partial; if let Some(reverted) = &reverted { discard(&reverted.read1); @@ -464,8 +498,9 @@ impl App { if resumed < Resumed::Sorted { let (input, out) = (mapped.clone(), sorted.clone()); let dir = scratch.path().join("sort"); - // A resumed job re-sorts from the start, so the killed attempt's spilled runs are dead - // weight — and at WGS scale they are tens of GB the sort is about to want back. + // A job that continues an earlier job sorts from the start. So the spilled runs of the + // earlier run have no use. For a WGS sample they are tens of GB, and the sort needs + // that space. clear_stage_dir(&dir); let token = cancel.clone(); let params = SortParams::default(); @@ -480,17 +515,20 @@ impl App { }) .await?; - // The runs are consumed by the merge; the sorted BAM is what the next stage reads. + // The merge reads each run file. The next stage reads the sorted BAM file. clear_stage_dir(&scratch.path().join("sort")); state.completed_through = Some(Resumed::Sorted); state.store(scratch.path()); } - // Only what *this* run produced is disposable. A resumed run that skipped the sort has - // re-derived nothing, and deleting the mapped BAM it resumed past would throw away the most - // expensive artifact in the pipeline on the strength of a belief about a file it did not - // write. That is not hypothetical: combined with the marker bug in `discard_partial`, it is - // exactly how a 59 GB `mapped.bam` — 3 h 58 m of revert and mapping — was destroyed. + // The code can remove only the files that *this* run made. + // + // A run that continues an earlier run and skips the sort made nothing again. A delete of the + // mapped BAM file that it started from removes the file that costs the most in this + // pipeline. The code would remove a file that it did not write, on a belief about that file. + // + // That fault occurred. With the marker fault in `discard_partial`, it destroyed a 59 GB + // `mapped.bam` file, which was 3 hours and 58 minutes of revert work and mapping work. if resumed < Resumed::Sorted { discard(&mapped); } @@ -499,8 +537,8 @@ impl App { if resumed < Resumed::Marked { let (input, out) = (sorted.clone(), marked.clone()); let token = cancel.clone(); - // Long-read libraries are typically PCR-free and long reads rarely share endpoints, so - // marking them would discard real coverage. + // A long-read library usually needs no PCR step, and two long reads rarely have the + // same end points. So a mark on those reads removes real coverage. let md_params = MarkDupParams { enabled: preset.is_paired(), ..Default::default() @@ -533,16 +571,18 @@ impl App { .map_err(|e| AppError::Join(e.to_string()))?? }; - // The output exists, so the intermediates behind it are disposable even if registration - // then fails — re-registering is seconds of work, not hours. `marked` is not discarded - // here because finalising *moved* it into place rather than copying it. + // The output file exists, so the code can remove each intermediate file behind it. A failed + // registration then costs seconds of work and not hours. + // + // The code does not remove `marked` here. The finalize stage *moved* that file into its + // place, and it made no copy. scratch.completed(); // ---- stage D: register ---- // - // Last, deliberately. The row is only inserted once the artifact it points at exists, so a - // job that fails or is cancelled leaves no alignment referring to a file that was never - // finished. + // This stage is last, by design. The code adds the row only after the file that it names + // exists. So a failed job, and a job that the user stopped, leave no alignment that names a + // file with no content. report(RealignStage::Register, ""); cancel.check()?; let alignment = self @@ -570,8 +610,9 @@ impl App { .await? .ok_or(AppError::MissingPaths(source.id))?; Preset::infer(Some(&run.test_type), Some(&run.platform_name)).map_err(|e| { - // A refusal here is the right outcome — mapping long reads under a short-read preset - // produces plausible, wrong alignments rather than failing — so it is surfaced as-is. + // A refusal here is the correct result. A map of long reads under a short-read preset + // does not fail. It gives alignments that look correct and are wrong. So the code + // reports the refusal to the user. AppError::Import(format!( "cannot choose a mapper preset for alignment #{}: {e}", source.id @@ -585,43 +626,51 @@ impl App { pub struct RealignPlan { /// Index batch size, chosen from the machine's memory. pub batch: BatchSize, - /// Bytes of scratch the job is expected to need. + /// The count of scratch bytes that the job needs. pub scratch_needed: u64, /// Bytes free where the scratch will live. pub scratch_free: u64, } -/// Scratch multiple of the source's **uncompressed** volume. +/// The factor between the scratch space and the **uncompressed** volume of the source. /// -/// Calibrated against a measured run, not reasoned from stage sizes — the reasoned figure was 25% -/// low. Realigning WGS229 (17.3 GB CRAM) peaked at **276 GB of scratch**, i.e. 16x the source -/// file, which this reproduces as `4` here times the CRAM expansion factor below. +/// This value comes from a measured run. An estimate from the size of each stage gave a value that +/// was 25% too small. /// -/// The peak is inside the revert, not where the stage list suggests: its spill runs are the -/// bespoke uncompressed encoding while its FASTQ output is gzipped, so the two coexist at very -/// different densities and the sum beats anything later in the pipeline. +/// A realignment of WGS229, from a CRAM file of 17.3 GB, reached a peak of **276 GB of scratch**. +/// That peak is 16 times the size of the source file. The value `4` here, times the CRAM expansion +/// factor below, gives that result. +/// +/// The peak is inside the revert stage, and not at the place that the stage list suggests. The +/// spill runs of that stage use an uncompressed format of this project, and its FASTQ output uses +/// gzip. So the two files exist together at very different densities, and their sum is larger than +/// any later stage. const SCRATCH_MULTIPLE: u64 = 4; -/// How much larger the data is than the file holding it. +/// The factor between the size of the data and the size of the file that holds it. +/// +/// This correction is important, and a wrong value here is not a small error. /// -/// This is the correction that matters, and getting it wrong is not a rounding error. A CRAM is -/// reference-compressed: 17 GB of CRAM is ~70 GB of BAM-equivalent data and, reverted to FASTQ, -/// larger still — FASTQ spends an ASCII byte per base on quality where BAM packs. Estimating -/// scratch from the *file* size would have told a user that a job needing ~200 GB needed 69, and -/// the disk would have filled somewhere in the third hour. +/// A CRAM file uses the reference to compress its data. A CRAM file of 17 GB holds about 70 GB of +/// data in the BAM form. The revert step writes FASTQ, and that file is larger again. A FASTQ file +/// uses one ASCII byte for the quality of each base, and a BAM file packs those values. +/// +/// An estimate from the size of the *file* tells a user that a job needs 69 GB, when it needs about +/// 200 GB. The disk then fills in the third hour. fn expansion_factor(source: &Path) -> u64 { match source.extension().and_then(|e| e.to_str()) { Some(e) if e.eq_ignore_ascii_case("cram") => 4, - // BAM is bgzf-compressed at roughly 4x, but the reverted FASTQ that comes out of it is - // gzipped too, so the ratio between them is far closer to 1. + // A BAM file uses bgzf compression, at a factor of about 4. But the revert step writes a + // FASTQ file with gzip compression. So the ratio between the two files is near 1. _ => 2, } } -/// Check that the job can finish before starting it. +/// Check that the job can complete, before it starts. /// -/// Running out of disk in hour three fails the job *and* leaves the machine wedged until someone -/// finds the scratch directory. The RAM figure is needed regardless, to size the index. +/// A disk that fills in the third hour fails the job. It also leaves the machine unusable until +/// somebody finds the scratch directory. The code also needs the RAM value in each case, to size the +/// index. pub fn preflight(scratch: &Path, source: &Path, source_size: u64) -> Result { let needed = source_size .saturating_mul(expansion_factor(source)) @@ -629,28 +678,33 @@ pub fn preflight(scratch: &Path, source: &Path, source_size: u64) -> Result Result { let size = |path: &Path| std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); let largest = size(mapped).max(size(sorted)).max(size(marked)); plan_for(scratch, largest.saturating_mul(3), "resume the realignment") } -/// Measure the disk, refuse a job that can not finish on it, and describe what was decided. +/// Measure the disk, refuse a job that it can not hold, and report the decision. +/// +/// The two preflight functions differ only in the way that they calculate `needed`. Each later step +/// was the same code two times, and a developer had to change both. Those steps are the read of the +/// free space, the refusal, the text, and the plan. /// -/// The two preflights differ only in how they size `needed`; everything after that — probing free -/// space, the refusal, the wording, the plan — was written out twice and had to be kept in step by -/// hand. `what` is the verb in the refusal, so the two messages stay exactly as they were. +/// The `what` value is the verb of the refusal, so each message stays as it was. fn plan_for(scratch: &Path, needed: u64, what: &str) -> Result { let free = free_space(scratch); @@ -677,29 +731,32 @@ fn gb(bytes: u64) -> u64 { bytes / 1_000_000_000 } -/// Record the spill budget a stage is about to run with. +/// Record the spill budget that a stage will use. /// -/// Both budgets now come from the machine rather than a constant, so the run count they produce is -/// no longer inferable from the version number. Without this line, "it spilled 400 runs" in a bug -/// report is a fact with nothing to attach it to. +/// The machine now gives both budgets, and no constant holds them. So the version number no longer +/// gives the count of runs that a stage produces. Without this log line, the statement "it spilled +/// 400 runs" in a bug report has no context. fn log_buffer(stage: &str, bytes: usize) { eprintln!("realign: {stage} with a {} MB buffer", bytes / (1024 * 1024)); } -/// Whether a job needing `needed` bytes may start given `free` bytes available. +/// Shows whether a job that needs `needed` bytes can start with `free` bytes available. /// -/// Split from the syscall so the *decision* is testable on any machine; the probe itself is not. -/// `free == 0` means the platform would not say, and is treated as permission to proceed — -/// refusing a multi-hour job because a free-space call failed would be a worse outcome than -/// letting it run and fail honestly on a real write. +/// This function is separate from the system call, so a test can cover the *decision* on any +/// machine. No test can cover the system call itself. +/// +/// A `free` value of 0 means that the platform gave no answer, and the function then permits the +/// job. A job of many hours must not stop because a call for the free space failed. It is better to +/// run that job and let it fail on a real write. fn has_room(needed: u64, free: u64) -> bool { free == 0 || free >= needed } -/// Free bytes on the filesystem holding `path`, or 0 when it can not be determined. +/// The count of free bytes on the file system of `path`. The function returns 0 when it can not +/// find that value. /// -/// Zero means "unknown", and preflight treats it as "do not block" — refusing a job because the -/// free-space call failed would be worse than letting it run and fail honestly on a real write. +/// A zero means "unknown", and the preflight then permits the job. A refusal, because a call for the +/// free space failed, is worse than a job that runs and then fails on a real write. fn free_space(path: &Path) -> u64 { // Walk up to the nearest existing ancestor: the scratch directory itself may not exist yet. let mut probe = path; @@ -732,25 +789,27 @@ fn fs_free_space(path: &Path) -> u64 { } } -/// Free bytes on the volume holding `path`. +/// The count of free bytes on the volume of `path`. +/// +/// The code reads `lpFreeBytesAvailableToCaller`, and not the total free space of the volume. That +/// choice is the important part of this call. On a volume with a disk quota, the two values differ, +/// and the preflight needs the quantity that *this user* can write. /// -/// `lpFreeBytesAvailableToCaller` rather than the volume's total free space, which is the subtle -/// half of this call: on a volume with disk quotas the two differ, and the number preflight needs is -/// what *this user* may actually write. It is the same choice the Unix arm makes in taking -/// `f_bavail` (blocks available to an unprivileged process) over `f_bfree`. +/// The Unix code makes the same choice. It reads `f_bavail`, which is the count of blocks for a +/// process with no special rights, and not `f_bfree`. #[cfg(windows)] fn fs_free_space(path: &Path) -> u64 { use std::os::windows::ffi::OsStrExt; use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW; - // `encode_wide` does not NUL-terminate, and the API requires it. + // `encode_wide` writes no NUL character at the end, and the API needs one. let mut wide: Vec = path.as_os_str().encode_wide().collect(); wide.push(0); let mut available: u64 = 0; - // SAFETY: `wide` is a valid NUL-terminated UTF-16 buffer owned here and outliving the call, and - // `available` is a `u64` we own. The two totals this does not need are passed as null, which the - // API documents as permitted. + // SAFETY: `wide` is a correct UTF-16 buffer with a NUL character at the end. This function owns + // it, and it lives longer than the call. This function also owns `available`, which is a `u64`. + // The two other totals are null, and the API permits that value. let ok = unsafe { GetDiskFreeSpaceExW( wide.as_ptr(), @@ -760,8 +819,8 @@ fn fs_free_space(path: &Path) -> u64 { ) }; - // A failure is reported as "unknown" rather than as zero free space, so that a probe that could - // not answer does not masquerade as a full disk and refuse the job. See `has_room`. + // A failure gives "unknown" and not a free space of zero. So a call that gave no answer does + // not look like a full disk, and it does not refuse the job. See `has_room`. if ok == 0 { return 0; } @@ -770,16 +829,17 @@ fn fs_free_space(path: &Path) -> u64 { #[cfg(not(any(unix, windows)))] fn fs_free_space(_path: &Path) -> u64 { - // No probe for this platform. Preflight reports "unknown" and declines to block — see + // This platform has no such call. The preflight reports "unknown" and permits the job. See // `free_space`. 0 } /// A scratch directory that removes itself. /// -/// Intermediates are several times the size of the input, so leaving them behind after a failed or -/// cancelled job would quietly consume a disk. Removal is best-effort: it must never mask the error -/// that is already unwinding. +/// The intermediate files are some times the size of the input. After a failed job, and after a job +/// that the user stopped, those files fill a disk and the user sees no message. +/// +/// The removal is optional. It must never hide the error that already unwinds the code. struct JobScratch { path: PathBuf, /// Set once the job has produced its output, so [`Drop`] can tell "finished" from "died". @@ -788,22 +848,25 @@ struct JobScratch { keep_on_failure: bool, } -/// Opt in to keeping a failed job's intermediates (`NAVIGATOR_REALIGN_KEEP_SCRATCH=1`). +/// Keep the intermediate files of a failed job. Set `NAVIGATOR_REALIGN_KEEP_SCRATCH=1`. +/// +/// The default is off, by design. For a WGS sample this directory holds hundreds of GB. A desktop +/// application must not leave that data after a failure. The user then loses that disk space for a +/// fault that they did not cause. /// -/// Off by default, and that default is deliberate: at WGS scale this directory is hundreds of GB, -/// and a desktop application that leaves that behind after a failure has taken the user's disk -/// hostage over a bug they did not cause. +/// The setting exists because the other default has its own fault. A realignment that fails in +/// stage 7 of 8 removes the work of the seven stages that succeeded. On a 30x WGS sample, one +/// measurement gave 10.7 hours of such work. A developer who works on the pipeline needs the +/// opposite behaviour, so this setting gives it. /// -/// It exists because the opposite default has a matching failure of its own. A realignment that -/// dies in stage 7 of 8 discards the seven stages that worked — a measured 10.7 hours on a 30x WGS. -/// For anyone iterating on the pipeline, that trade is the wrong way round, so they can invert it. +/// This setting also makes [`RealignParams::resume`] useful. A new run reads the intermediate files +/// of an earlier run, and by default a failed run leaves none. /// -/// This is now also the switch that makes [`RealignParams::resume`] worth anything: resume reads -/// the intermediates a previous attempt left, and by default a failed attempt does not leave any. -/// The two are meant to be used together when a run is expected to be interrupted — which, at six -/// hours a run on a machine someone is still using, is a reasonable thing to expect. A job killed -/// outright (a session teardown, a power loss) never runs `Drop` at all, so its scratch survives -/// regardless of this setting; that is the case resume was built for. +/// Use the two together when something can stop a run. A run of six hours, on a machine that +/// somebody also uses, meets that condition often. +/// +/// A job that stops at once runs no `Drop` call, so its scratch files stay in each case. The causes +/// are the end of a login session and a power loss. The resume feature exists for that case. fn keep_scratch_on_failure() -> bool { std::env::var("NAVIGATOR_REALIGN_KEEP_SCRATCH") .map(|v| v != "0" && !v.is_empty()) @@ -856,12 +919,12 @@ mod tests { } } - /// Preflight must refuse a job that can not finish, and say how much is needed rather than - /// leaving the user to guess. + /// The preflight must refuse a job that it can not complete. It must also state the quantity of + /// space that the job needs, so the user makes no estimate. /// - /// Runs wherever [`fs_free_space`] has a real implementation, which is now both desktop - /// families. Platforms without one report "unknown" and can not refuse anything — pinned - /// separately by `preflight_cannot_refuse_where_free_space_is_unknown`. + /// This test runs on each platform where [`fs_free_space`] has real code, and that set now + /// holds both desktop families. A platform with no such code reports "unknown" and can refuse + /// nothing. The test `preflight_cannot_refuse_where_free_space_is_unknown` covers that case. #[cfg(any(unix, windows))] #[test] fn preflight_refuses_when_the_disk_is_too_small() { @@ -880,8 +943,9 @@ mod tests { assert!(plan.batch.bases() > 0); } - /// The correction that matters: a CRAM holds several times its own size in read data, so - /// estimating scratch from the file size understates a WGS job by well over 100 GB. + /// The important correction. A CRAM file holds some times its own size in read data. So an + /// estimate of the scratch space from the size of that file is more than 100 GB too small for a + /// WGS job. #[test] fn a_cram_is_estimated_larger_than_a_bam_of_the_same_size() { let dir = std::env::temp_dir(); @@ -896,9 +960,9 @@ mod tests { assert_eq!(cram.scratch_needed, 1_000_000 * 4 * SCRATCH_MULTIPLE); } - /// Free space is a check, not a gate on the check working: if the platform will not say, the - /// job runs and fails honestly on a real write rather than being refused for an unrelated - /// reason. + /// The free space is a check. The check itself must not depend on it. When the platform gives + /// no answer, the job runs and fails on a real write. It must not stop for a reason that has no + /// connection to the data. #[test] fn an_unknown_free_space_does_not_block_the_job() { assert!(has_room(u64::MAX, 0), "0 means unknown, not full"); @@ -906,12 +970,12 @@ mod tests { assert!(!has_room(101, 100), "one byte short is short"); } - /// The scratch directory does not exist when preflight runs — it is created by the job — so the - /// probe has to answer for the filesystem that will hold it, which means walking up to the - /// nearest existing ancestor rather than giving up. + /// The scratch directory does not exist while the preflight runs, because the job makes it. So + /// the call must answer for the file system that will hold that directory. The code moves up the + /// path to the nearest directory that exists, and it does not stop. /// - /// Both desktop families, for the same reason as above: the ancestor walk is platform- - /// independent, but it is only meaningful where the call it ends at can answer. + /// The test covers both desktop families, for the reason above. The move up the path does not + /// depend on the platform, but it has a result only where the call at the end can answer. #[cfg(any(unix, windows))] #[test] fn free_space_resolves_through_a_directory_that_does_not_exist_yet() { @@ -923,13 +987,15 @@ mod tests { ); } - /// What the platforms without a free-space probe actually do, stated as a test rather than left - /// as the absence of one. + /// The behaviour of a platform with no call for the free space. This test states that + /// behaviour, and the absence of a test does not. + /// + /// `preflight` can refuse no job that it did not measure. A refusal on an unknown value stops + /// each realignment on that platform. So the code starts a job that clearly does not fit, and + /// that job fails on a real write. /// - /// `preflight` can not refuse a job it has no measurement for, and refusing on an unknown would - /// block every realignment on that platform. So a job that would obviously not fit is allowed - /// to start and fail honestly on a real write. Windows used to be in this bucket; it now has - /// `GetDiskFreeSpaceExW` and is tested by the two above. + /// Windows was in this group. It now has `GetDiskFreeSpaceExW`, and the two tests above cover + /// it. #[cfg(not(any(unix, windows)))] #[test] fn preflight_cannot_refuse_where_free_space_is_unknown() { @@ -941,7 +1007,8 @@ mod tests { assert!(plan.scratch_needed > 0, "the estimate is still made and reported"); } - /// Scratch is several times the size of the input; a cancelled job must not leave it behind. + /// The scratch files are some times the size of the input. A job that the user stopped must + /// remove them. #[test] fn scratch_is_removed_when_the_job_ends() { let dir = std::env::temp_dir().join(format!("dun-jobscratch-{}", std::process::id())); @@ -958,7 +1025,7 @@ mod tests { mod resume_tests { use super::*; - /// A scratch directory of this test's own, holding whatever files it names. + /// A scratch directory for this test alone. It holds the files that the test names. fn scratch(tag: &str, files: &[(&str, bool)]) -> PathBuf { let dir = std::env::temp_dir().join(format!("navigator-resume-{tag}")); let _ = std::fs::remove_dir_all(&dir); @@ -966,8 +1033,8 @@ mod resume_tests { for (name, complete) in files { let path = dir.join(name); - // A "complete" BAM is one carrying the BGZF end-of-file marker; an incomplete one is - // the same bytes with the marker cut short, which is what a killed writer leaves. + // A "complete" BAM file holds the BGZF end-of-file marker. An incomplete file holds + // the same bytes with a short marker, and a writer that stopped leaves such a file. let mut bytes = vec![0u8; 64]; let eof: [u8; 28] = [ 0x1f, 0x8b, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x06, 0x00, 0x42, 0x43, 0x02, 0x00, 0x1b, @@ -989,13 +1056,19 @@ mod resume_tests { assert_eq!(resumable(&dir, &ScratchState::default()), Resumed::Nothing); } - /// The bug this pair of rules exists for, reproduced. + /// The fault that these two rules exist for. This test reproduces it. + /// + /// A user stopped a merge. It left a `sorted.bam` file of 13.2 GB, and the full file is about + /// 30 GB. + /// + /// That partial file held a correct BGZF end-of-file marker. The writer of noodles uses many + /// threads, and it closes its stream from `Drop` while the code unwinds. + /// + /// The marker test alone accepted that file. The next run then marked the duplicates of a short + /// alignment. /// - /// A cancelled merge left 13.2 GB of an expected ~30 GB `sorted.bam` carrying a valid BGZF - /// end-of-file marker, because noodles' multithreaded writer finishes its stream from `Drop` - /// while the job unwinds. The marker check alone accepted it and the next run went straight to - /// marking duplicates on a truncated alignment. The stage record is what refuses it: the sort - /// never returned, so nothing ever claimed it completed. + /// The stage record refuses the file. The sort never returned, so no code stated that it + /// completed. #[test] fn a_marker_written_by_an_unwinding_drop_is_not_enough() { let dir = scratch("unwound", &[("mapped.bam", true), ("sorted.bam", true)]); @@ -1017,8 +1090,9 @@ mod resume_tests { ); } - /// The record can not promise more than the files deliver either — a scratch directory whose - /// `marked.bam` was removed must not be resumed from just because a stale record mentions it. + /// The record can also promise no more than the files hold. A scratch directory with no + /// `marked.bam` file must not start a new run, and an old record that names that file changes + /// nothing. #[test] fn the_record_cannot_outrun_the_files() { let dir = scratch("stale-record", &[("mapped.bam", true)]); @@ -1030,18 +1104,18 @@ mod resume_tests { assert_eq!(resumable(&dir, &state), Resumed::Mapped); } - /// A scratch directory written before the stage record existed still has to work: the Aug-13 - /// `mapped.bam` was left by a process killed outright, so no `Drop` ran and the marker means - /// exactly what it says. + /// A scratch directory from before the stage record must still work. A process that stopped at + /// once left the `mapped.bam` file of 13 August. No `Drop` call ran, so the marker states the + /// truth. #[test] fn a_scratch_without_a_record_falls_back_to_the_marker() { let dir = scratch("legacy", &[("mapped.bam", true)]); assert_eq!(resumable(&dir, &ScratchState::default()), Resumed::Mapped); } - /// The 2026-08-13 case exactly: mapping finished, the sort was killed partway through writing - /// its output. The mapped BAM is worth four hours and must be picked up; the half-written - /// sorted BAM must not be. + /// The exact case of 2026-08-13. The mapping stage completed, and the sort stopped while it + /// wrote its output. The mapped BAM file holds four hours of work, and a new run must use it. + /// That run must not use the partial sorted BAM file. #[test] fn a_complete_map_and_a_truncated_sort_resumes_from_the_map() { let dir = scratch("mid-sort", &[("mapped.bam", true), ("sorted.bam", false)]); @@ -1060,8 +1134,9 @@ mod resume_tests { assert_eq!(resumable(&dir, &ScratchState::default()), Resumed::Sorted); } - /// The stage guards are written as `resumed < Resumed::Sorted`, so the ordering is load-bearing - /// rather than cosmetic: getting it backwards would skip work that has not been done. + /// Each stage guard is a test of the form `resumed < Resumed::Sorted`. So the order of these + /// values controls the code, and it is not only for the reader. A wrong order skips a stage that + /// did not run. #[test] fn the_stages_are_ordered_by_how_much_they_skip() { assert!(Resumed::Nothing < Resumed::Mapped); @@ -1090,8 +1165,8 @@ mod resume_tests { assert_eq!(loaded.completed_through, Some(Resumed::Marked)); } - /// A resumed job is sized from what is already on disk. Sizing it from the source again would - /// charge twice for an expansion that has already happened and refuse a job that fits. + /// A job that continues an earlier job takes its size from the files on the disk. A size from + /// the source counts the growth of that source two times, and it refuses a job that fits. #[test] fn resume_preflight_sizes_from_the_surviving_intermediate() { let dir = scratch("preflight", &[("mapped.bam", true)]); diff --git a/crates/navigator-app/tests/app.rs b/crates/navigator-app/tests/app.rs index 6372f640..bb880786 100644 --- a/crates/navigator-app/tests/app.rs +++ b/crates/navigator-app/tests/app.rs @@ -13,9 +13,9 @@ async fn app() -> App { App::new(Store::open_in_memory().await.unwrap()) } -/// `App::new` reloads the active account, so every `app()` above would read the *production* -/// keychain service if the OS backend were ever switched on in a test process. Only the shipped -/// binary's `main` may switch it on; assert this test binary never did. +/// `App::new` reads the active account again. So each `app()` call above reads the *production* +/// keychain service if a test process turns the OS backend on. Only the `main` function of the +/// shipped binary can turn it on. This test asserts that the test binary never did. #[test] fn tests_never_touch_the_os_keychain() { assert!( @@ -24,9 +24,12 @@ fn tests_never_touch_the_os_keychain() { ); } -/// Serializes tests that mutate the process-global `NAVIGATOR_TREE_DIR`: one test's `remove_var` -/// would otherwise yank the seeded tree dir out from under another running concurrently. Held for -/// the whole test body; ignores poisoning so a panicking test does not wedge the rest. +/// A lock for each test that changes `NAVIGATOR_TREE_DIR`, which belongs to the full process. +/// +/// Without the lock, a `remove_var` call in one test removes the tree directory of another test at +/// the same time. Each test holds the lock for its full body. +/// +/// The lock ignores a poisoned state, so a test with a panic does not stop the other tests. static TREE_DIR_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Reuse the analysis crate's committed fixtures (workspace-relative). @@ -34,13 +37,16 @@ fn fixtures() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../navigator-analysis/tests/fixtures") } -/// Serializes the `NAVIGATOR_REFGENOME_DIR` env write (read once in `App::new`) so -/// parallel tests pointing the gateway cache at different temp dirs do not race. +/// A lock for the write to `NAVIGATOR_REFGENOME_DIR`. `App::new` reads that variable one time. The +/// lock stops a race between two tests that give the gateway cache two different temporary +/// directories. static REF_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); -/// An `App` whose reference-gateway cache is `cache`. The store is opened first (async), then -/// the env write + `App::new` happen synchronously under the lock so the gateway captures the -/// right base dir without racing other tests. +/// An `App` value whose reference-gateway cache is `cache`. +/// +/// The function opens the store first, and that call is asynchronous. It then writes the variable +/// and calls `App::new` on one thread, under the lock. The gateway then reads the correct base +/// directory, and no other test can change it at the same time. async fn app_with_ref_cache(cache: &std::path::Path) -> App { let store = Store::open_in_memory().await.unwrap(); let _g = REF_ENV_LOCK.lock().unwrap(); @@ -84,7 +90,8 @@ async fn import_variants_from_csv_keeps_only_snps() { let subject = app.add_biosample(None, "HG002", None, None).await.unwrap(); let path = std::env::temp_dir().join(format!("variants-{}.csv", subject.guid.0)); - // header layout; one indel row that must be dropped (SNP-only) + // The header layout, and one indel row. The importer must remove that row, because it keeps + // only SNPs. std::fs::write( &path, "contig,position,ref,alt,rsid,genotype\nchr1,1000,A,G,rs1,0/1\nchr1,2000,A,AT,rs2,0/1\nchrM,73,G,A,.,1/1\n", @@ -134,8 +141,9 @@ async fn import_vendor_big_y_vcf_is_tagged() { let app = app().await; let subject = app.add_biosample(None, "HG002", None, None).await.unwrap(); - // A Big Y bundle: a generically-named variants.vcf with the FTDNA aengine signature + a - // sibling readme, in a per-sample directory (the parent dir disambiguates the label). + // A Big Y set of files. It holds a variants.vcf file with a general name, and that file + // carries the FTDNA aengine signature. A readme file is in the same directory, and that + // directory holds one sample. The name of the parent directory gives the label. let dir = std::env::temp_dir().join(format!("bigy-{}", subject.guid.0)); std::fs::create_dir_all(&dir).unwrap(); let vcf = dir.join("variants.vcf"); @@ -255,7 +263,7 @@ async fn import_mtdna_fasta_round_trips() { assert_eq!(listed.len(), 1); assert_eq!(listed[0].sequence.len(), 16_569); - // a too-short sequence is rejected + // The importer refuses a sequence that is too short. let bad = std::env::temp_dir().join(format!("mtdna-bad-{}.fasta", subject.guid.0)); std::fs::write(&bad, ">x\nACGT\n").unwrap(); assert!(matches!( @@ -314,9 +322,11 @@ async fn derive_mtdna_variants_detects_a_deletion() { let subject = app.add_biosample(None, "HG002", None, None).await.unwrap(); let dir = std::env::temp_dir(); - // rCRS: A-runs around a 9-base landmark at positions 301-309; the sample lacks it. The - // landmark is A-free so the flanking A-runs can't absorb any of it — the 9-base - // deletion is unambiguous. + // In rCRS, a run of A bases is on each side of a 9-base landmark at positions 301 to 309. The + // sample does not hold that landmark. + // + // The landmark holds no A base. So the two runs of A bases can take no part of it, and the + // 9-base deletion has one position only. let landmark = "CCCCCCCCC"; let reference = format!("{}{}{}", "A".repeat(300), landmark, "A".repeat(16_569 - 309)); let sample = "A".repeat(16_560); @@ -371,10 +381,16 @@ async fn assign_mtdna_haplogroup_ranks_best() { let _ = std::fs::remove_file(&samp_path); } -/// Real-data validation: assign mt + Y haplogroups from a GRCh38-aligned HG002 BAM -/// (chrM = rCRS, chrY = GRCh38 — matching the FTDNA trees). Needs network (live FTDNA -/// fetch). Run: `HG002_B38_BAM=/path/HG002.b38.bam cargo test -p navigator-app --test app \ -/// validate_hg002 -- --ignored --nocapture`. +/// A check against real data. The test assigns the mt haplogroup and the Y haplogroup from an +/// HG002 BAM file on GRCh38. The chrM contig is rCRS, and the chrY contig is GRCh38. Those contigs +/// match the FTDNA trees. +/// +/// The test needs the network, because it reads the FTDNA trees. To run it: +/// +/// ```bash +/// HG002_B38_BAM=/path/HG002.b38.bam \ +/// cargo test -p navigator-app --test app validate_hg002 -- --ignored --nocapture +/// ``` #[tokio::test] #[ignore = "requires HG002_B38_BAM (GRCh38) + network"] async fn validate_hg002_haplogroups() { @@ -433,7 +449,8 @@ async fn validate_hg002_haplogroups() { for r in y.ranked.iter().skip(1).take(3) { eprintln!(" alt: {} ({:.3})", r.name, r.score); } - // Why descent stopped: child branches and their defining-SNP states. + // The reason that the descent stopped: the child branches, and the state of each SNP that + // defines them. use navigator_app::CallState; for b in &y.branches { eprintln!(" child {} — {}/{} derived:", b.name, b.derived, b.snps.len()); @@ -448,7 +465,8 @@ async fn validate_hg002_haplogroups() { } assert!(top.depth > 0 && top.matched > 0, "Y should resolve below root"); - // Private bucket (de-novo chrY off the backbone) — gated separately (slow + needs ref). + // The private bucket, which holds the de-novo chrY calls that are not on the backbone. This + // step has its own gate, because it is slow and it needs the reference. if std::env::var("PRIVATE_Y").is_ok() { use navigator_app::PrivateClass; // Y_MASK_BED=path -> external mask; SELF_MASK set -> self-referential; else none. @@ -495,13 +513,22 @@ async fn validate_hg002_haplogroups() { } } -/// Real-data validation that liftover gives the right answer: a CHM13-aligned HiFi BAM of a -/// donor whose GRCh38 terminals are known (Y: R-FGC29071, mt: U5a1b1g). Y is assigned by -/// lifting the GRCh38 tree positions onto CHM13 via the cached chain (auto-downloaded); -/// mtDNA is a direct chrM query (this BAM's chrM is 16,569 bp = rCRS). The calls should match -/// the GRCh38 result. Needs network (FTDNA tree + the GRCh38→CHM13 chain). Run: -/// GFX_CHM13_BAM=/Users/jkane/Genomics/GFX0457637/GFX0457637.pbmm2.chm13v2.bam \ +/// A check against real data that the liftover gives the correct answer. +/// +/// The input is a HiFi BAM file on CHM13, from a donor with known GRCh38 terminals. The Y terminal +/// is R-FGC29071, and the mt terminal is U5a1b1g. +/// +/// The test assigns the Y haplogroup with the cached chain, which the app downloads. That chain +/// moves the GRCh38 tree positions onto CHM13. The mtDNA step queries chrM directly, because the +/// chrM contig of this BAM file is 16,569 bp and equals rCRS. +/// +/// The two calls must equal the GRCh38 result. The test needs the network for the FTDNA tree and +/// for the GRCh38 to CHM13 chain. To run it: +/// +/// ```bash +/// GFX_CHM13_BAM=/Users/jkane/Genomics/GFX0457637/GFX0457637.pbmm2.chm13v2.bam \ /// cargo test -p navigator-app --test app validate_gfx_chm13 -- --ignored --nocapture +/// ``` #[tokio::test] #[ignore = "requires GFX_CHM13_BAM (CHM13) + network (FTDNA tree + liftover chain)"] async fn validate_gfx_chm13_haplogroups() { @@ -518,8 +545,9 @@ async fn validate_gfx_chm13_haplogroups() { .record_sequence_run(NewSequenceRun::new(b.guid, "PACBIO", "WGS")) .await .unwrap(); - // mt now needs the CHM13 reference (to self-generate the rCRS↔chrM map): resolve it - // (downloads ~1 GB on first run, then cached), or take GFX_CHM13_REF if provided. + // The mt step now needs the CHM13 reference, because the code makes the map between rCRS and + // chrM itself. Find that reference, which downloads about 1 GB at the first run and then stays + // in the cache. Use GFX_CHM13_REF when the caller gives it. let reference = match std::env::var("GFX_CHM13_REF") { Ok(p) => p, Err(_) => app @@ -545,8 +573,8 @@ async fn validate_gfx_chm13_haplogroups() { .unwrap() .id; - // mtDNA: now lifted via the self-generated rCRS↔CHM13-chrM map — expect the U5a1b1g lineage, - // matching the GRCh38 result. + // The mtDNA step now uses the map between rCRS and the CHM13 chrM contig, which the code makes + // itself. The result must be the U5a1b1g lineage, which equals the GRCh38 result. let mt = app .assign_mtdna_haplogroup_from_alignment(aln) .await @@ -565,7 +593,8 @@ async fn validate_gfx_chm13_haplogroups() { top.lineage.join(" › ") ); - // Y: lifted GRCh38 tree → CHM13 chrY — expect the R-FGC29071 clade. + // The Y step moves the GRCh38 tree onto the CHM13 chrY contig. The result must be the + // R-FGC29071 clade. let y = app.assign_y_haplogroup(aln).await.expect("Y assign"); let top = &y.ranked[0]; eprintln!( @@ -603,16 +632,24 @@ async fn validate_gfx_chm13_haplogroups() { } } -/// End-to-end DecodingUs Y-tree provider against a locally-running AppView, using the CHM13 -/// alignment's **native `hs1` coordinates** (no liftover). Verifies the integration places the -/// GFX sample deep onto the decoding-us backbone (the K2b clade, en route to its known -/// R-FGC29071 terminal). Reaching the R tips requires the AppView to enrich `hs1` coords for the -/// FTDNA-grafted variants (today `hs1` covers the backbone only); until then deep CHM13 placement -/// stops at the backbone. Gated on a reachable AppView. Run (AppView up on :9000, default URL): -/// GFX_CHM13_BAM=/Users/jkane/Genomics/GFX0457637/GFX0457637.pbmm2.chm13v2.bam \ +/// An end-to-end test of the DecodingUs Y-tree provider against an AppView on this machine. The +/// test uses the **native `hs1` coordinates** of the CHM13 alignment and does no liftover. +/// +/// The test checks that the code places the GFX sample deep on the decoding-us backbone. That place +/// is the K2b clade, on the path to its known R-FGC29071 terminal. +/// +/// A placement at the R tips needs `hs1` coordinates for the variants that FTDNA added to the tree. +/// The AppView must supply them, and today its `hs1` data covers the backbone only. Until then, a +/// deep CHM13 placement stops at the backbone. +/// +/// The test runs only when it can reach an AppView. To run it, with an AppView on port 9000: +/// +/// ```bash +/// GFX_CHM13_BAM=/Users/jkane/Genomics/GFX0457637/GFX0457637.pbmm2.chm13v2.bam \ /// GFX_CHM13_REF=/Users/jkane/Genomics/chm13v2.0/chm13v2.0.fa \ /// DECODINGUS_APPVIEW_URL=http://localhost:9000 \ /// cargo test -p navigator-app --test app validate_gfx_decodingus_y -- --ignored --nocapture +/// ``` #[tokio::test] #[ignore = "requires GFX_CHM13_BAM + a running DecodingUs AppView (DECODINGUS_APPVIEW_URL)"] async fn validate_gfx_decodingus_y() { @@ -655,9 +692,11 @@ async fn validate_gfx_decodingus_y() { top.name, top.matched, top.expected, top.score ); eprintln!(" lineage: {}", top.lineage.join(" › ")); - // Native hs1 coords place GFX deep on the decoding-us backbone (K2b, toward R-FGC29071). - // Substantial match count + reaching the K backbone confirms the end-to-end provider works; - // the R tips need AppView hs1 enrichment (see fn docs). + // The native hs1 coordinates place GFX deep on the decoding-us backbone, at K2b, on the path + // to R-FGC29071. + // A large count of matches, together with a placement on the K backbone, shows that the full + // provider path works. The R tips need `hs1` data from the AppView. See the doc comment of this + // function. assert!( top.matched >= 50, "expected a substantial native-hs1 match count, got {}", @@ -718,8 +757,9 @@ async fn gvcf_y_placement_smoke() { top.name, top.matched, top.expected, top.score ); eprintln!(" lineage: {}", top.lineage.join(" › ")); - // HG00096 is a 1000G GBR sample → deep R1b. Confirm the GVCF places it deep on the R - // backbone (not a shallow veto), with a substantial match count. + // HG00096 is a GBR sample from 1000G, and it belongs to a deep R1b clade. This test confirms + // that the GVCF path places it deep on the R backbone, with a large count of matches. The path + // must not stop near the root. assert!( top.matched >= 100, "expected a deep match count from the GVCF, got {}", @@ -732,11 +772,17 @@ async fn gvcf_y_placement_smoke() { ); } -/// **The fast-path correctness gate.** Placing a sample's Y (and mt) from the precomputed -/// pipeline GVCF must reach the same terminal as walking the CRAM — otherwise the fast path -/// is silently wrong. Set the env to a sample dir that has BOTH the CRAM and the GVCFs -/// (the ytree layout), with a running DecodingUs AppView (or a warm tree cache). -/// GVCF_PARITY_CRAM, GVCF_PARITY_REF, GVCF_PARITY_Y_GVCF[, GVCF_PARITY_M_GVCF] +/// **The correctness gate of the fast path.** +/// +/// The fast path reads the GVCF file of the pipeline. It must place the Y haplogroup and the mt +/// haplogroup at the same terminal as a walk of the CRAM file. If the two differ, the fast path +/// gives a wrong answer and reports nothing. +/// +/// Point the variables at a sample directory that holds BOTH the CRAM file and the GVCF files, in +/// the ytree layout. The test also needs a DecodingUs AppView, or a tree in the cache. +/// +/// The variables are `GVCF_PARITY_CRAM`, `GVCF_PARITY_REF`, `GVCF_PARITY_Y_GVCF`, and the optional +/// `GVCF_PARITY_M_GVCF`. #[tokio::test] #[ignore = "requires a ytree sample dir (CRAM + GVCFs) + DecodingUs tree (AppView/cache)"] async fn gvcf_fast_path_matches_cram_walk() { @@ -780,10 +826,13 @@ async fn gvcf_fast_path_matches_cram_walk() { "Y GVCF: {} ({}/{}) CRAM: {} ({}/{})", ft.name, ft.matched, ft.expected, st.name, st.matched, st.expected ); - // Same-lineage consistency, not exact-terminal equality: the GVCF path uses robust - // (proportional-top) selection and the CRAM path the strict guard, so they can stop at - // different depths on the *same* path. The gate is that neither places on a different - // branch — one lineage must contain the other's terminal. + // The test compares the lineage, and it does not compare the two terminals for equality. + // + // The GVCF path selects a node by a proportional rule, and the CRAM path uses the strict guard. + // So the two can stop at different depths on the *same* path. + // + // The test fails only when one path places the sample on a different branch. One lineage must + // hold the terminal of the other. assert!( ft.lineage.contains(&st.name) || st.lineage.contains(&ft.name), "GVCF and CRAM placed on different Y branches: {} vs {}", @@ -956,7 +1005,7 @@ async fn haplogroup_consensus_combines_recorded_calls() { assert_eq!(c.run_count, 2); assert_eq!(c.warnings.len(), 1); // flags the deeper HiFi placement - // re-recording the same source key replaces (no duplicate) + // A second write of the same source key replaces the first row and adds no duplicate. let calls = app.haplogroup_calls(subject.guid, DnaType::Y).await.unwrap(); assert_eq!(calls.len(), 2); // mt has nothing recorded @@ -966,7 +1015,8 @@ async fn haplogroup_consensus_combines_recorded_calls() { .unwrap() .is_none()); - // manual override replaces the computed consensus and is flagged + audited. + // A value from the user replaces the consensus that the code calculated. The row carries a + // mark, and the audit log holds an entry. app.set_manual_override(subject.guid, DnaType::Y, "R-FGC29071", Some("Sanger-confirmed")) .await .unwrap(); @@ -1067,8 +1117,9 @@ async fn add_data_detects_and_routes() { ); assert_eq!(app.list_chip_profiles(subject.guid).await.unwrap().len(), 1); - // A BAM/CRAM auto-imports: it creates a sequencing run + alignment (header probed - // best-effort; here the bytes are not a real BAM so detection falls back to defaults). + // A BAM file or a CRAM file imports without a question. The code makes a sequence run and an + // alignment. It reads the header when it can. Here the bytes are not a real BAM file, so the + // code uses its default values. let bam = dir.join(format!("data-{}.bam", subject.guid.0)); std::fs::write(&bam, b"\x1f\x8b").unwrap(); assert_eq!(app.add_data(subject.guid, &bam).await.unwrap(), DetectedData::Alignment); @@ -1076,10 +1127,10 @@ async fn add_data_detects_and_routes() { assert_eq!(runs.len(), 1); let alns = app.list_alignments(runs[0].id).await.unwrap(); assert_eq!(alns.len(), 1); - // The content hash is deferred (not computed at import) so a multi-GB alignment imports - // instantly; it is filled in lazily on the first analysis that needs it. + // The code does not calculate the content hash at the import. So an alignment of many GB + // imports at once. The first analysis that needs the hash calculates it. assert_eq!(alns[0].content_sha256, None, "content hash is deferred at import"); - // Idempotent: re-adding the same path does not duplicate the run/alignment. + // A second import of the same path is safe. It adds no second run and no second alignment. assert_eq!(app.add_data(subject.guid, &bam).await.unwrap(), DetectedData::Alignment); assert_eq!(app.list_sequence_runs(subject.guid).await.unwrap().len(), 1); @@ -1265,8 +1316,9 @@ async fn run_coverage_persists_and_reads_back_from_cache() { assert_eq!(result.genome_territory, 50); // chrM fixture assert_eq!(result.callable_bases, 10); - // now cached for this version (integer fields exact; floats survive round-trip to - // ~1 ULP, so compare those about rather than with fragile float ==) + // The cache now holds the result for this version. An integer field is exact. A float field + // changes by about 1 ULP in the round trip, so the test compares a float with a tolerance and + // not with `==`. let cached = app.cached_coverage(aln).await.unwrap().unwrap(); assert_eq!(cached.genome_territory, result.genome_territory); assert_eq!(cached.callable_bases, result.callable_bases); @@ -1274,7 +1326,8 @@ async fn run_coverage_persists_and_reads_back_from_cache() { assert_eq!(cached.coverage_histogram, result.coverage_histogram); assert!((cached.mean_coverage - result.mean_coverage).abs() < 1e-9); - // re-running is idempotent (upsert in place; store-layer test covers row count) + // A second run is safe. The code replaces the row, and a test in the store layer counts the + // rows. let rerun = app .run_coverage( aln, @@ -1339,7 +1392,8 @@ async fn publish_coverage_summary_requires_cached_coverage() { let app = app().await; let aln = diploid_alignment(&app).await; // has a BAM but no coverage run - // Bearer client is never reached — the missing-coverage check fails first. + // The code never reaches the Bearer client. The check for an absent coverage result fails + // first. let client = navigator_app::PdsClient::bearer(reqwest::Client::new(), "http://127.0.0.1:1", "did:plc:x", "tok"); let err = app.publish_coverage_summary(&client, aln).await; assert!( @@ -1351,8 +1405,9 @@ async fn publish_coverage_summary_requires_cached_coverage() { ); } -/// Full path: run coverage on the fixture → publish the summary (a real CoverageResult, -/// floats encoded as strings) to a live PDS via a throwaway Bearer account. +/// The full path. The test calculates the coverage of the fixture file and publishes the summary to +/// a live PDS. The summary is a real `CoverageResult`, and each float is a string. The test signs in +/// with a temporary Bearer account. #[tokio::test] #[ignore = "requires PDS_TEST_URL (local atproto PDS container)"] async fn publish_coverage_summary_to_live_pds() { @@ -1484,8 +1539,9 @@ async fn import_project_dir_creates_rows_is_idempotent_and_coverage_runs_on_cram let app = app().await; let fx = fixtures(); - // Build a temp project tree: /HG00096/HG00096.chm13.cram(+.crai), reusing the - // committed CRAM fixture (the .crai is index-by-offset, so the rename is fine). + // Build a temporary project tree at /HG00096/HG00096.chm13.cram, with its .crai file. + // The test copies the CRAM fixture of this repository. The .crai file holds offsets, so a new + // name for the CRAM file is correct. let root = std::env::temp_dir().join(format!("dun-import-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); let sample = root.join("HG00096"); @@ -1519,7 +1575,8 @@ async fn import_project_dir_creates_rows_is_idempotent_and_coverage_runs_on_cram assert_eq!(again.alignments_skipped, 1); assert_eq!(app.project_overview().await.unwrap().len(), 1); - // Coverage recompute works on the imported CRAM (reference_path was stamped). + // A second coverage calculation works on the imported CRAM file, because the import wrote + // `reference_path`. let aln = app.list_all_alignments().await.unwrap(); assert_eq!(aln.len(), 1); assert_eq!(aln[0].reference_build, "chm13v2.0"); @@ -1531,8 +1588,9 @@ async fn import_project_dir_creates_rows_is_idempotent_and_coverage_runs_on_cram #[tokio::test] async fn reimport_under_different_project_name_reuses_subject() { - // A person is one subject across projects: re-importing the same sample folder under a - // different project name must reuse the subject (join it to the new project), not duplicate it. + // One person is one subject in each project. A second import of the same sample directory, + // under another project name, must use that subject again. It adds the subject to the new + // project, and it makes no second subject. let app = app().await; let fx = fixtures(); let reference = fx.join("ref.fa"); @@ -1577,8 +1635,9 @@ async fn reimport_under_different_project_name_reuses_subject() { #[tokio::test] async fn delete_project_detaches_members_and_keeps_subjects() { - // A project is a grouping — deleting a non-empty one must succeed by detaching its members, - // not refuse ("N subjects still belong to it"). The subjects themselves survive. + // A project is a group. A delete of a project with members must succeed. The code removes each + // membership. It must not refuse with the message "N subjects still belong to it". Each subject + // stays in the workspace. let app = app().await; let p = app .create_project(NewProject { @@ -1638,7 +1697,8 @@ async fn project_report_rolls_up_coverage_and_csv_round_trips() { let aln = app.list_all_alignments().await.unwrap(); - // A lite (sidecar) coverage is flagged `partial` in the report so the UI can badge it. + // A small coverage result from a sidecar carries the `partial` mark in the report. The UI can + // then show a badge. let lite = app.run_coverage_for_alignment(aln[0].id).await.unwrap(); app.save_analysis_with_provenance( aln[0].id, @@ -1686,7 +1746,8 @@ async fn import_without_reference_resolves_from_cache_else_reports_needed() { std::fs::copy(fx.join("coverage.cram"), sample.join("HG00096.chm13.cram")).unwrap(); std::fs::copy(fx.join("coverage.cram.crai"), sample.join("HG00096.chm13.cram.crai")).unwrap(); - // Empty cache → import (no explicit reference) reports the chm13v2.0 build is needed, no writes. + // With an empty cache and no reference from the caller, the import reports that it needs the + // chm13v2.0 build. It writes nothing. match app.import_project_dir(&root, None, "tester".into(), false).await { Err(AppError::ReferenceNeeded(needs)) => { assert_eq!(needs.len(), 1); @@ -1786,14 +1847,18 @@ async fn assign_y_haplogroup_lifts_grch38_tree_onto_chm13_alignment() { } #[tokio::test] -// Holds TREE_DIR_ENV_LOCK across awaits on purpose — it serializes tests that mutate the -// process-global NAVIGATOR_TREE_DIR env var; the guard must outlive the async body. +// This code holds TREE_DIR_ENV_LOCK across each await, by design. The lock orders the tests that +// change the NAVIGATOR_TREE_DIR variable, which belongs to the full process. The guard must live +// longer than the async body. #[allow(clippy::await_holding_lock)] async fn analyze_project_runs_coverage_and_attempts_y_per_sample() { let _env = TREE_DIR_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - // Seed the Y-tree cache so assign_y is offline; a root-only tree (no loci) means no query - // targets and no chain — exercises the orchestration without network. Force the FTDNA provider - // so the seeded tree is used (the default DecodingUs provider would reach out to the AppView). + // Write a tree to the Y-tree cache, so `assign_y` needs no network. The tree holds the root + // only and no locus. So there is no query target and no chain, and the test covers the control + // flow with no network. + // + // The test also selects the FTDNA provider, so the code reads the tree in the cache. The + // default DecodingUs provider sends a request to the AppView. let trees = std::env::temp_dir().join(format!("dun-trees-{}", std::process::id())); let _ = std::fs::remove_dir_all(&trees); std::fs::create_dir_all(&trees).unwrap(); @@ -1834,7 +1899,8 @@ async fn analyze_project_runs_coverage_and_attempts_y_per_sample() { .unwrap(); assert_eq!(s.samples, 1); assert_eq!(s.coverage_done, 1, "coverage computed on the CRAM"); - // Y was attempted: recorded, or (here) errored on the chrM-only fixture lacking chrY. + // The code tried the Y step. It wrote a result, or it gave an error. Here it gives an error, + // because the fixture holds chrM only and has no chrY contig. assert_eq!(s.y_done + s.errors.iter().filter(|e| e.contains("Y:")).count(), 1); // The report now shows coverage filled for the sample. @@ -1846,10 +1912,14 @@ async fn analyze_project_runs_coverage_and_attempts_y_per_sample() { let _ = std::fs::remove_dir_all(&trees); } -/// The AppView instrument→lab lookup (D8): a seeded `sequencer-lab-instruments.json` cache stands -/// in for the live endpoint (a fresh cache short-circuits the network). The returned lab name is -/// normalized to the local labs catalog's canonical display name when it matches; unknown labs -/// pass through; an unassociated instrument resolves to `None`. +/// The lookup from an instrument to a laboratory on the AppView (D8). +/// +/// A `sequencer-lab-instruments.json` file in the cache takes the place of the live endpoint, +/// because a new cache entry stops the network call. +/// +/// The code changes the laboratory name to the display name of the local labs catalog, when the two +/// match. A name that the catalog does not hold passes through with no change. An instrument with +/// no laboratory gives `None`. #[tokio::test] #[allow(clippy::await_holding_lock)] // see analyze_project_* — env-var serialization guard held across awaits async fn lookup_lab_by_instrument_resolves_and_normalizes_from_cache() { @@ -1885,20 +1955,27 @@ async fn lookup_lab_by_instrument_resolves_and_normalizes_from_cache() { let _ = std::fs::remove_dir_all(&trees); } -/// A 23andMe import stores the haploid Y/MT genotype rows as a `Chip` variant set and places -/// BOTH a Y and an mtDNA haplogroup on import (best-effort), offline against seeded FTDNA trees. -/// The file declares build 38 so Y placement uses the FTDNA fallback (no DecodingUs AppView). +/// A 23andMe import writes the haploid Y rows and MT rows as a `Chip` variant set. It also places +/// BOTH a Y haplogroup and an mtDNA haplogroup at the import, when it can. +/// +/// The test runs offline against FTDNA trees in the cache. The file names build 38, so the Y +/// placement uses the FTDNA path and needs no DecodingUs AppView. #[tokio::test] #[allow(clippy::await_holding_lock)] // see analyze_project_* — env-var serialization guard held across awaits async fn import_23andme_stores_calls_and_places_y_and_mt() { use navigator_app::DnaType; let _env = TREE_DIR_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - // Seed real (subset) FTDNA Y + mt trees so both placements run with NO network/integration — - // small connected subtrees of the FTDNA trees, committed under tests/fixtures: Y is - // R-L761 → R-L389 → R-P297 → R-M269 (GRCh38 coords); mt is H2a → H2a2 → H2a2a → H2a2a1 (rCRS). - // Force the FTDNA provider so Y placement uses the seeded tree, not the (mutable) DecodingUs - // instance — a hard assert on a terminal label can't depend on live curated data. + // Write real FTDNA Y and mt trees to the cache, so both placements run with NO network. Each + // tree is a small connected part of the full FTDNA tree, and this repository holds both under + // tests/fixtures. + // + // The Y tree is R-L761, R-L389, R-P297, R-M269, in GRCh38 coordinates. The mt tree is H2a, + // H2a2, H2a2a, H2a2a1, in rCRS coordinates. + // + // The test also selects the FTDNA provider, so the Y placement reads the tree in the cache. The + // DecodingUs instance changes over time, and a strict assert on a terminal label must not + // depend on curated data that changes. let trees = std::env::temp_dir().join(format!("dun-chip-trees-{}", std::process::id())); let _ = std::fs::remove_dir_all(&trees); std::fs::create_dir_all(&trees).unwrap(); @@ -1915,10 +1992,15 @@ async fn import_23andme_stores_calls_and_places_y_and_mt() { std::env::set_var("NAVIGATOR_TREE_DIR", &trees); std::env::set_var("NAVIGATOR_Y_TREE_PROVIDER", "ftdna"); - // A synthetic 23andMe export (GRCh38): one autosomal row (ignored), the four informative Y rows - // derived along the R-M269 lineage (L761 G, L389 G, P297 C, M269 C), the four informative MT - // rows derived along the H2a2a1 lineage (4769/750/8860/263 → A), plus filler MT rows so the MT - // marker count clears the "real mt panel" threshold (≥20). + // A 23andMe export that this test makes, on GRCh38. It holds one autosomal row, and the code + // ignores that row. + // + // It holds four Y rows with a derived state on the R-M269 lineage: L761 G, L389 G, P297 C, and + // M269 C. It holds four MT rows with a derived state on the H2a2a1 lineage: positions 4769, + // 750, 8860, and 263, each with the base A. + // + // It also holds more MT rows, so the count of MT markers goes above the limit of 20 that marks + // a real mt panel. let mut file = String::from( "# This data file generated by 23andMe at human assembly build 38\n\ rsid\tchromosome\tposition\tgenotype\n\ @@ -1949,7 +2031,8 @@ async fn import_23andme_stores_calls_and_places_y_and_mt() { .unwrap(); assert_eq!(profile.provider, "23andMe"); - // The haploid Y/MT rows are stored as a Chip variant set on the vendor build (GRCh38 here). + // The code stores the haploid Y rows and MT rows as a Chip variant set, on the build of the + // vendor. That build is GRCh38 here. let sets = app.list_variant_sets(b.guid).await.unwrap(); assert_eq!(sets.len(), 1); assert_eq!(sets[0].source_type, navigator_app::SourceType::Chip); @@ -1960,7 +2043,7 @@ async fn import_23andme_stores_calls_and_places_y_and_mt() { "4 Y + 20 MT haploid calls (autosomal row dropped)" ); - // Both haplogroups are placed on import, against the seeded FTDNA subset trees. + // The code places both haplogroups at the import, against the FTDNA trees in the cache. let y = app .haplogroup_consensus(b.guid, DnaType::Y) .await @@ -1980,13 +2063,22 @@ async fn import_23andme_stores_calls_and_places_y_and_mt() { let _ = std::fs::remove_dir_all(&dir); } -/// Exact GRCh38-vs-CHM13 mtDNA comparison on the SAME donor (GFX0457637): the GRCh38 BAM -/// queries chrM directly (rCRS), the CHM13 BAM lifts via the self-generated rCRS↔chrM map. -/// Prints both terminals and the per-SNP lineage states, and diffs them position-by-position -/// so any difference is attributable (NoCall = coverage/unmapped vs Ancestral = different base). -/// GFX_B38_BAM=/Users/jkane/Genomics/GFX0457637/GFX0457637.b38.bam \ +/// An exact mtDNA comparison between GRCh38 and CHM13, on the SAME donor, GFX0457637. +/// +/// The GRCh38 BAM file gives a direct query of chrM, in rCRS coordinates. The CHM13 BAM file uses +/// the map between rCRS and chrM that the code makes itself. +/// +/// The test prints both terminals and the lineage state of each SNP. It then compares the two lists +/// position by position, so a reader can name the cause of each difference. A `NoCall` state comes +/// from absent coverage or an unmapped read. An `Ancestral` state comes from a different base. +/// +/// To run it: +/// +/// ```bash +/// GFX_B38_BAM=/Users/jkane/Genomics/GFX0457637/GFX0457637.b38.bam \ /// GFX_CHM13_BAM=/Users/jkane/Genomics/GFX0457637/GFX0457637.pbmm2.chm13v2.bam \ /// cargo test -p navigator-app --test app compare_mt_grch38_vs_chm13 -- --ignored --nocapture +/// ``` #[tokio::test] #[ignore = "requires GFX_B38_BAM + GFX_CHM13_BAM (+ network for the mt tree / CHM13 reference)"] async fn compare_mt_grch38_vs_chm13() { @@ -2040,9 +2132,12 @@ async fn compare_mt_grch38_vs_chm13() { c.ranked[0].name, c.ranked[0].matched, c.ranked[0].expected ); - // Compare the lineage element-wise (same tree + terminal → same ordered path). A position - // can RECUR with opposite polarity (e.g. C182T then a T182C reversal), so a position-keyed - // map would falsely diff the two occurrences against each other — match 1:1 by order. + // Compare the two lineages one element at a time. The same tree and the same terminal give the + // same ordered path. + // + // A position can occur again with the opposite polarity. One example is C182T, and then a + // reversal T182C. A map with the position as its key compares those two entries with each + // other, and that comparison is wrong. So the test matches the two lists by their order. let _ = (&g_calls, &c_calls); assert_eq!( g_lin.len(), @@ -2138,8 +2233,9 @@ async fn gfx_sex_is_male() { assert_eq!(s.inferred_sex, navigator_app::InferredSex::Male); } -/// Analysis-cache staleness: a cached artifact is reused while the source file is unchanged, and -/// invalidated (recomputed) once the file's signature changes (BAM-mtime invalidation, §6). +/// The age rule of the analysis cache. The app uses a cached artifact while the source file does +/// not change. It calculates the result again after the signature of that file changes. The mtime of +/// the BAM file gives that signature. See §6. #[tokio::test] async fn cached_artifact_invalidated_when_source_file_changes() { let app = app().await; @@ -2164,7 +2260,7 @@ async fn cached_artifact_invalidated_when_source_file_changes() { .unwrap() .id; - // Save → load round-trips while the source is unchanged. + // A save and then a load give the same value, while the source file does not change. app.save_analysis(aln, "testkind", "v1", &vec![1u32, 2, 3]) .await .unwrap(); @@ -2176,7 +2272,7 @@ async fn cached_artifact_invalidated_when_source_file_changes() { let stale: Option> = app.load_analysis(aln, "testkind", "v1").await.unwrap(); assert_eq!(stale, None, "changed source invalidates the cached artifact"); - // Recomputing re-stamps the new signature → served again. + // A second calculation writes the new signature, and the cache then gives the result again. app.save_analysis(aln, "testkind", "v1", &vec![9u32]).await.unwrap(); let fresh: Option> = app.load_analysis(aln, "testkind", "v1").await.unwrap(); assert_eq!(fresh, Some(vec![9]), "recomputed cache is fresh again"); @@ -2184,8 +2280,11 @@ async fn cached_artifact_invalidated_when_source_file_changes() { let _ = std::fs::remove_file(&bam); } -/// FTDNA project import — the B5163↔GFX merge scenario plus a new subject and an orphan -/// (ancestry without a roster row). Exercises plan → resolve fuzzy → commit end to end. +/// The FTDNA project import. The test covers the merge of kit B5163 with subject GFX. It also +/// covers a new subject, and an orphan row, which is ancestry data with no roster row. +/// +/// The test runs the full sequence: the plan, then the decision on each candidate that the engine is +/// not sure about, then the commit. #[tokio::test] async fn ftdna_project_import_plans_and_commits_merge_new_and_orphan() { use navigator_app::{DnaType, FtdnaImportOptions, FtdnaResolution, MatchKind}; @@ -2203,7 +2302,8 @@ async fn ftdna_project_import_plans_and_commits_merge_new_and_orphan() { .await .unwrap(); - // The existing WGS Subject: GFX, already placed at R-FGC29071 — the same person as kit B5163. + // The WGS subject that already exists. It is GFX, at R-FGC29071, and it is the same person as + // kit B5163. let gfx = app .add_biosample(Some(project.id), "GFX0457637", None, None) .await @@ -2288,7 +2388,8 @@ async fn ftdna_project_import_plans_and_commits_merge_new_and_orphan() { "no Y-STR profile attached on merge" ); - // GFX gained the FTDNA kit identity, member labels, and MDKA — without a duplicate Subject. + // GFX now holds the FTDNA kit identity, the member labels, and the MDKA rows. The workspace + // holds no second subject. let ids = app.external_ids(gfx.guid).await.unwrap(); assert_eq!(ids.len(), 1); assert_eq!(ids[0].external_id, "B5163"); @@ -2315,7 +2416,8 @@ async fn ftdna_project_import_plans_and_commits_merge_new_and_orphan() { .unwrap() .contains(&project.id)); - // The exact-kit path now auto-merges on a re-plan (the kit# is attached). + // The exact-kit path now merges with no question at the next plan, because the subject holds + // the kit number. let replan = app .plan_ftdna_import( Some(project.id), @@ -2334,8 +2436,9 @@ async fn ftdna_project_import_plans_and_commits_merge_new_and_orphan() { } } -/// FTDNA import with no pre-selected project creates one (named from the caller) at commit — the -/// fix for the "dead Import button". A cancelled dry-run (no commit) creates nothing. +/// An FTDNA import with no project makes one at the commit, with the name that the caller gives. +/// This behaviour corrects the fault that users called the "dead Import button". A plan with no +/// commit makes nothing. #[tokio::test] async fn ftdna_import_into_new_project_creates_it_at_commit() { use navigator_app::{FtdnaImportOptions, MatchKind}; @@ -2344,7 +2447,8 @@ async fn ftdna_import_into_new_project_creates_it_at_commit() { let ftdna = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ftdna"); let app = app().await; - // Plan into a NEW project (no project_id) — read-only, so nothing is created yet. + // Plan the import into a NEW project, with no `project_id`. This step only reads, so the code + // makes nothing. let plan = app .plan_ftdna_import( None, @@ -2376,9 +2480,12 @@ async fn ftdna_import_into_new_project_creates_it_at_commit() { assert_eq!(overview[0].project.id, summary.project_id); } -/// FTDNA matching via Y-STR genetic distance: an existing subject whose Y haplogroup is an ISOGG -/// long-form label (no SNP terminal to compare) but which carries the same Y-STR profile still -/// surfaces as a fuzzy candidate (the real KANE-0001 = GFX case). +/// A match on the genetic distance of the Y-STR values. +/// +/// A subject in the workspace can hold a long ISOGG label as its Y haplogroup. Such a label has no +/// SNP terminal for the engine to compare. When that subject carries the same Y-STR profile, the +/// engine must still offer it as a candidate. The real case is KANE-0001, which is the same person +/// as GFX. #[tokio::test] async fn ftdna_matches_existing_subject_by_ystr_distance() { use navigator_app::{DnaType, FtdnaImportOptions, MatchKind}; @@ -2401,7 +2508,8 @@ async fn ftdna_matches_existing_subject_by_ystr_distance() { .await .unwrap(); - // Give KANE-0001 B5163's Y-STR markers (from the overview fixture) via a tall CSV. + // Give the Y-STR markers of kit B5163 to KANE-0001. The values come from the overview fixture, + // in a tall CSV file. let ydna = std::fs::read_to_string(ftdna.join("YDNA_Results_Overview.csv")).unwrap(); let per_kit = navigator_domain::ftdna::parse_ydna_overview(&ydna).unwrap(); let (_, markers) = per_kit.iter().find(|(k, _)| k == "B5163").unwrap(); @@ -2455,9 +2563,12 @@ async fn ftdna_matches_existing_subject_by_ystr_distance() { other => panic!("expected B5163 NeedsConfirm via Y-STR, got {other:?}"), } - // Commit the merge into the (new) project. KANE-0001 has NO home project (`project_id` is NULL) — - // the merge adds an M:N membership row only. The project report must still surface it (regression - // for "matched samples do not appear in the Project report" — it reads membership ∪ home column). + // Commit the merge into the new project. KANE-0001 has NO home project, and its `project_id` + // column is NULL. So the merge adds one M:N membership row and nothing more. + // + // The project report must still show that subject. This test covers the fault "matched samples + // do not appear in the Project report". The report reads the union of the membership table and + // the home column. let mut res = std::collections::BTreeMap::new(); res.insert("B5163".to_string(), navigator_app::FtdnaResolution::Merge(kane.guid)); let summary = app.commit_ftdna_import(&plan, &res).await.unwrap(); @@ -2480,8 +2591,8 @@ async fn ftdna_matches_existing_subject_by_ystr_distance() { let _ = std::fs::remove_file(&tmp); } -/// Deleting a sequencing run purges the haplogroup calls + consensus placement derived from its -/// alignments, so a wrong haplogroup does not linger after the run is removed. +/// A delete of a sequence run also removes the haplogroup calls and the consensus placement that +/// came from its alignments. So a wrong haplogroup does not stay after the user removes the run. #[tokio::test] async fn deleting_run_purges_derived_haplogroup_and_consensus() { use navigator_app::DnaType; @@ -2524,11 +2635,16 @@ async fn deleting_run_purges_derived_haplogroup_and_consensus() { assert!(app.haplogroup_calls(b.guid, DnaType::Y).await.unwrap().is_empty()); } -/// End-to-end `branch_report` over the mtDNA path (finding: nothing drove the query before this). -/// Seeds a tiny FTDNA-schema mt tree offline and genotypes the committed `coverage.bam` fixture -/// (all-`A` reads, callable chrM 1-10, ref `ACGT…`). Loci are placed so the report exercises all -/// three states at once: pos 2 (ref C, reads A → derived), pos 1 (ref A, reads A → ancestral off -/// this branch), pos 40 (no coverage → no-call). +/// A full test of `branch_report` on the mtDNA path. Before this test, no code called that query. +/// +/// The test writes a small mt tree in the FTDNA schema to the cache, and it runs offline. It then +/// genotypes the `coverage.bam` fixture of this repository. Each read of that file holds the base +/// `A`, the callable region of chrM is positions 1 to 10, and the reference is `ACGT…`. +/// +/// The loci give each of the three states in one report. Position 2 has the reference base C and +/// reads with A, which gives a derived state. Position 1 has the reference base A and reads with A, +/// which gives an ancestral state below this branch. Position 40 has no coverage, which gives a +/// no-call state. #[tokio::test] // Holds TREE_DIR_ENV_LOCK across awaits: it serializes the process-global NAVIGATOR_TREE_DIR write. #[allow(clippy::await_holding_lock)] @@ -2603,9 +2719,12 @@ async fn branch_report_genotypes_the_mt_subtree_end_to_end() { assert_eq!(nc.state, CallState::NoCall); assert!(nc.note.contains("no call"), "no-call note, got {:?}", nc.note); - // An insertion (empty ancestral allele) at a no-coverage site is BOTH an indel and a no-call. - // The note must carry both tags — picking only the first would let "indel/MNV" mask the - // no-call, and the asymmetric SNV test would have mislabeled the empty allele as a clean SNV. + // An insertion has an empty ancestral allele. At a site with no coverage, such a variant is + // BOTH an indel and a no-call. + // + // The note must hold both tags. With the first tag only, the "indel/MNV" tag hides the no-call + // state. The asymmetric SNV test then reads the empty allele as a clean SNV, and that reading + // is wrong. let ins = row("41.1A"); assert_eq!(ins.state, CallState::NoCall); assert!( @@ -2694,10 +2813,16 @@ async fn mt_alignment_pick_skips_a_y_only_run() { #[tokio::test] async fn add_sample_dir_records_alignment_from_header_no_decode_and_is_idempotent() { - // The CLI `ingest` fast path for one staged sample directory: the CRAM alignment is recorded - // from the header/filename (no read decode) and text sidecars sit alongside. No haplogroup GVCF - // here, so the sidecar fast-path block is skipped (that path is covered by the fastpath tests); - // this pins the alignment recording, build detection, and idempotency of `add_sample_dir`. + // The fast path of the CLI `ingest` command, for one sample directory. + // + // The code writes the CRAM alignment from the header and the file name, and it decodes no read. + // The text sidecar files are in the same directory. + // + // This directory holds no haplogroup GVCF file, so the code skips the sidecar block. The + // fastpath tests cover that block. + // + // This test covers three things: the record of the alignment, the build that the code finds, + // and a second call of `add_sample_dir`. let app = app().await; let fx = fixtures(); @@ -2723,7 +2848,7 @@ async fn add_sample_dir_records_alignment_from_header_no_decode_and_is_idempoten assert_eq!(aln.len(), 1); assert_eq!(aln[0].reference_build, "chm13v2.0"); - // Re-ingest the same directory: the alignment is reused, nothing new is created. + // A second import of the same directory uses the alignment again and makes nothing new. let again = app.add_sample_dir(subject.guid, &dir, false).await.unwrap(); assert_eq!(again.alignments_created, 0); assert_eq!(again.alignments_skipped, 1); @@ -2734,8 +2859,9 @@ async fn add_sample_dir_records_alignment_from_header_no_decode_and_is_idempoten #[tokio::test] async fn add_sample_dir_falls_back_to_per_file_for_a_loose_bundle() { - // A directory with no alignment/variant/GVCF is a loose bundle of subject files: each is - // imported as `add_data` would (here a 23andMe chip export → chip profile). + // A directory with no alignment, no variant file, and no GVCF file holds separate files of one + // subject. The code imports each file as `add_data` does. Here a 23andMe chip export gives a + // chip profile. let app = app().await; let dir = std::env::temp_dir().join(format!("dun-loosebundle-{}", std::process::id())); @@ -2762,10 +2888,15 @@ async fn add_sample_dir_falls_back_to_per_file_for_a_loose_bundle() { #[tokio::test] async fn add_sample_dir_skips_called_vcf_when_gvcf_present() { - // GATK repo layout: a bare `chrY.g.vcf.gz` (the Y source → fast path) sits beside the called - // `chrY.vcf.gz`. With the GVCF present the called VCF must NOT be imported — it is redundant and - // (variant-set import not being content-idempotent) would duplicate on a resumable re-run. The - // GVCF is a stub here, so the placement itself is a best-effort no-op; this pins the routing. + // The GATK layout. A `chrY.g.vcf.gz` file, which is the Y source for the fast path, is beside + // the called `chrY.vcf.gz` file. + // + // With the GVCF file present, the code must NOT import the called VCF. That file holds the same + // data. A second import of a variant set adds a second copy, because that import does not + // compare the content. So a run that continues an earlier run would duplicate the set. + // + // The GVCF file here holds no real data, so the placement does nothing. This test covers the + // choice between the two files. let app = app().await; let fx = fixtures(); let dir = std::env::temp_dir().join(format!("dun-gvcf-skip-{}", std::process::id())); @@ -2793,9 +2924,13 @@ async fn add_sample_dir_skips_called_vcf_when_gvcf_present() { let _ = std::fs::remove_dir_all(&dir); } -/// The full-analysis pipeline is planned in one place so the GUI worker and the `analyze` CLI can -/// never drift again — they had, and the CLI's copy re-genotyped Y unconditionally, overwriting a -/// trusted external call. These assert the plan's shape and its skip conditions. +/// One function plans the full-analysis pipeline. So the GUI worker and the `analyze` command of +/// the CLI can never become different again. +/// +/// They did become different. The copy in the CLI genotyped the Y chromosome at each run, and it +/// replaced a trusted external call. +/// +/// These tests assert the shape of the plan and each condition that skips a step. mod full_analysis_plan { use super::*; use navigator_app::AnalysisStep; @@ -2819,8 +2954,9 @@ mod full_analysis_plan { .id } - /// With no coverage yet the mitochondrial steps are assumed present (an unknown genome is not - /// silently narrowed), quality metrics always leads, and ancestry is opt-in. + /// With no coverage result, the plan holds the mitochondrial steps. The code must not remove a + /// step from an unknown genome. The quality metrics step is always first, and the user must + /// select the ancestry step. #[tokio::test] async fn plans_the_full_pipeline_for_an_unanalyzed_alignment() { let app = app().await; @@ -2835,7 +2971,8 @@ mod full_analysis_plan { ] { assert!(steps.contains(&expected), "{expected:?} missing from {steps:?}"); } - // Subject-level: the Y signature is planned so the descent report needs no extra click. + // A step for the subject. The plan holds the Y signature, so the descent report needs no + // second action from the user. assert!( steps.iter().any(|s| matches!(s, AnalysisStep::YSignature { .. })), "Y signature missing from {steps:?}" @@ -2848,7 +2985,8 @@ mod full_analysis_plan { "ancestry must be opt-in: {steps:?}" ); - // Opting in appends both ancestry steps, profile before estimate (the estimate reads it). + // The option adds both ancestry steps. The profile step comes before the estimate step, + // because the estimate reads the profile. let with = app.plan_full_analysis(id, true, false, None).await.unwrap(); let profile_at = with .iter() @@ -2862,15 +3000,16 @@ mod full_analysis_plan { assert_eq!(ancestry_at, with.len() - 1, "ancestry runs last (heaviest)"); } - /// Coverage showing no chrM reads (a Big Y) drops both mitochondrial steps — scoring zero chrM - /// data would only record a meaningless RSRS root. + /// A coverage result with no chrM read comes from a Big Y test. The plan then holds neither + /// mitochondrial step. A placement with no chrM data writes the RSRS root, and that result has + /// no value. #[tokio::test] async fn drops_the_mitochondrial_steps_without_chrm_reads() { use navigator_analysis::coverage::{ContigCoverageStats, CoverageResult}; let app = app().await; let id = alignment(&app).await; - // Coverage carrying a single chrM entry, with `num_reads` the only field under test. + // A coverage result with one chrM entry. This test reads the `num_reads` field only. let chrm_with = |num_reads: u64| CoverageResult { contig_coverage_stats: vec![ContigCoverageStats { contig: "chrM".into(), @@ -2901,15 +3040,20 @@ mod full_analysis_plan { !steps.iter().any(|s| matches!(s, AnalysisStep::MitoDenovo { .. })), "{steps:?}" ); - // The rest of the pipeline is untouched. + // The plan does not change the other steps. assert!(steps.contains(&AnalysisStep::YHaplogroup)); } - /// SV is opt-in, and this is the guard on that. It is experimental, nothing else consumes its - /// output, and it is the only step that walks every read in the file for its own sake — 2–5 h - /// per whole-genome sample, measured, against ~1 h for the whole rest of the pipeline. Folded - /// into a 148-sample batch that is the difference between overnight and weeks, which is exactly - /// what it cost before. It runs when a caller asks for it and not otherwise. + /// The user must select the SV step, and this test is the guard on that rule. + /// + /// The step is experimental, and no other step reads its output. It is also the one step that + /// reads each record in the file for its own result. + /// + /// A measurement gave 2 to 5 hours for one whole-genome sample, against about 1 hour for each + /// other step together. In a batch of 148 samples, that difference is one night against some + /// weeks, and the batch did cost that before. + /// + /// The step runs when a caller asks for it, and at no other time. #[tokio::test] async fn structural_variants_is_planned_only_when_asked_for() { let app = app().await; @@ -2926,21 +3070,24 @@ mod full_analysis_plan { with.contains(&AnalysisStep::StructuralVariants), "SV must be planned when requested: {with:?}" ); - // Opting in adds SV and changes nothing else. + // The option adds the SV step and changes no other step. let added: Vec<_> = with.iter().filter(|s| !without.contains(s)).collect(); assert_eq!(added, vec![&AnalysisStep::StructuralVariants], "{with:?}"); } } -/// The workspace-chore survey is what the Dashboard's maintenance panel renders, and every number -/// on it has to mean something specific. An empty workspace must report *nothing due* rather than -/// nothing found — the distinction is the whole point of showing `due` against `total`. +/// The maintenance panel of the Dashboard draws the survey of the workspace chores. Each number on +/// that panel must be clear to the reader. +/// +/// An empty workspace must report that *no work is due*. It must not report that the survey found +/// nothing. That difference is the reason for the pair `due` and `total`. #[tokio::test] async fn maintenance_survey_reports_every_chore() { let app = app().await; let survey = app.maintenance_survey().await.expect("survey"); - // Every chore is present, in a fixed order, so the panel can not silently lose one. + // The result holds each chore, in a fixed order. So the panel can not lose one with no + // message. let chores: Vec<_> = survey.iter().map(|s| s.chore).collect(); assert_eq!(chores, navigator_app::Chore::ALL.to_vec()); @@ -2948,8 +3095,8 @@ async fn maintenance_survey_reports_every_chore() { assert!(s.due <= s.total || s.total == 0, "{:?}: due exceeds total", s.chore); } - // Nothing to publish, and no account to publish with — the chore reports *why* it can not run - // rather than offering a button that would fail. + // There is no record to publish, and there is no account for a publish. The chore reports the + // *reason* that it can not run. It does not offer a button that fails. let publish = survey .iter() .find(|s| s.chore == navigator_app::Chore::PublishOrigins) @@ -2969,8 +3116,8 @@ async fn maintenance_survey_reports_every_chore() { ); } -/// A subject with no alignment and no variant set is not a failure — it is a subject with nothing -/// to compute, and counting it as an error would bury the real ones. +/// A subject with no alignment and no variant set is not a failure. It is a subject with no work +/// to do. A count of it as an error hides the real errors. #[tokio::test] async fn refresh_private_y_on_a_bare_subject_is_not_a_failure() { let app = app().await; @@ -2986,7 +3133,7 @@ async fn refresh_private_y_on_a_bare_subject_is_not_a_failure() { // ---- realignment provenance (stage D) -------------------------------------- -/// A biosample with one sequence run — the minimum context an alignment needs. +/// A biosample with one sequence run. This is the smallest set of rows that an alignment needs. async fn subject_with_run( app: &App, ) -> ( @@ -3004,8 +3151,9 @@ async fn subject_with_run( (b, run) } -/// A realigned alignment is a new row under the *same* library, pointing back at what it came -/// from. Nothing about the source may change — that is what makes realignment safe to offer. +/// A realigned alignment is a new row under the *same* library, and it names the alignment that it +/// came from. The code must change nothing in the source row. That rule makes a realignment safe to +/// offer. #[tokio::test] async fn registering_a_realignment_is_additive_and_records_its_source() { let app = app().await; @@ -3050,15 +3198,15 @@ async fn registering_a_realignment_is_additive_and_records_its_source() { "the file was just written, so hashing it now is nearly free" ); - // The source is untouched. + // The code did not change the source row. let source_now = app.alignment(source.id).await.unwrap().unwrap(); assert_eq!(source_now, source, "realignment must not modify its source"); let _ = std::fs::remove_dir_all(&dir); } -/// Realigning to the build a sample is already on costs hours and produces a duplicate. The rule -/// lives in the app rather than the UI so the CLI is covered by it too. +/// A realignment to the build that a sample already uses costs hours and gives a duplicate. The +/// rule is in the app layer and not in the UI, so it also covers the CLI. #[tokio::test] async fn realigning_to_the_same_build_is_refused() { let app = app().await; @@ -3086,7 +3234,8 @@ async fn realigning_to_the_same_build_is_refused() { assert!(format!("{err}").contains("already on"), "unhelpful message: {err}"); } -/// The UI asks this before offering "Realign", so a sample is not silently given a second copy. +/// The UI calls this method before it offers the "Realign" action. So the app does not make a +/// second copy of a sample with no message. #[tokio::test] async fn derived_alignments_are_discoverable_from_their_source() { let app = app().await; @@ -3129,8 +3278,8 @@ async fn derived_alignments_are_discoverable_from_their_source() { ); } -/// Every row that predates the migration is an original, and must read back that way rather than -/// as something with unknown provenance. +/// Each row from before the migration is an original alignment. A read of such a row must give that +/// answer. It must not give an unknown provenance. #[tokio::test] async fn existing_alignments_read_back_as_originals() { let app = app().await; @@ -3150,10 +3299,11 @@ async fn existing_alignments_read_back_as_originals() { assert!(!read_back.is_derived()); } -/// When a realignment exists, the subject's default alignment is its output rather than the source -/// it came from. Both describe the same library at the same breadth, so without a rule the winner -/// is whichever the list happened to yield first — and a default that changes between runs is -/// worse than either choice. +/// When a realignment exists, the default alignment of the subject is its output. It is not the +/// source of that realignment. +/// +/// Both rows describe the same library at the same breadth. Without this rule, the first row in the +/// list wins. A default that changes between two runs is worse than either choice. #[tokio::test] async fn a_realigned_alignment_becomes_the_subjects_default() { let app = app().await; @@ -3167,7 +3317,7 @@ async fn a_realigned_alignment_becomes_the_subjects_default() { .await .unwrap(); - // Before realigning, the source is the default. + // Before the realignment, the source is the default alignment. assert_eq!( app.default_alignment_for_subject(b.guid).await.unwrap(), Some((run.id, source.id)) @@ -3197,8 +3347,8 @@ async fn a_realigned_alignment_becomes_the_subjects_default() { ); } -/// A batch counts only what it would really act on, so the number shown before a job measured in -/// days is the honest one rather than an upper bound. +/// A batch counts only the alignments that it acts on. So the number before a job of some days is +/// the true number and not a maximum. #[tokio::test] async fn a_project_batch_skips_what_it_would_refuse() { let app = app().await; @@ -3239,7 +3389,7 @@ async fn a_project_batch_skips_what_it_would_refuse() { let queue = app.realignable_in_project(project.id, "chm13v2.0").await.unwrap(); assert_eq!(queue, vec![eligible.id]); - // Once it has been realigned, a second batch has nothing left to do. + // After the realignment, a second batch has no work. app.record_alignment(NewAlignment { bam_path: Some("/tmp/c.cram".into()), derived_from_alignment_id: Some(eligible.id), From b407ed16cab072a3c248f74ef70525789e483d5f Mon Sep 17 00:00:00 2001 From: James Kane Date: Wed, 19 Aug 2026 07:57:32 -0500 Subject: [PATCH 08/33] docs(ste): analysis.rs to zero, and lib.rs started MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit analysis.rs joins the converted set; 31 of 33 files in the crate are at zero. lib.rs goes 678 to 524 and is the first file left partly done — the type documentation at the top is converted, the `impl App` body below it is not. Two findings in analysis.rs were worth the care they got: - the localized-copy leak. Copies were cached in a directory that one caller cleared while three created them, so every other path copied ~400 MB per alignment and cleaned up nothing — 687 files and 145 GB, which filled the volume mid-run. Removal now hangs off `Drop`, and the comment explains why a refcount keeps the original benefit. - why two concurrent copiers must never name the same scratch file: sharing one lets them open a single inode, where one truncates what the other is writing and keeps writing into it after the other has renamed it into place and started reading. `cargo check -p navigator-app --all-targets` passes on this tree. Co-Authored-By: Claude Opus 5 (1M context) --- crates/navigator-app/src/analysis.rs | 399 ++++++++++++++++--------- crates/navigator-app/src/lib.rs | 431 +++++++++++++++++---------- 2 files changed, 522 insertions(+), 308 deletions(-) diff --git a/crates/navigator-app/src/analysis.rs b/crates/navigator-app/src/analysis.rs index f618706b..c72df2a2 100644 --- a/crates/navigator-app/src/analysis.rs +++ b/crates/navigator-app/src/analysis.rs @@ -879,9 +879,8 @@ impl App { )) } - /// The alignments of the subject on the **most frequent reference build**. That build is the one - /// that the most alignments use, and the code compares the canonical build, so `chm13v2` and - /// `hs1` are the same build here. + /// The alignments of the subject on the **most frequent reference build**. The code compares the + /// canonical build, so `chm13v2` and `hs1` count as one build here. /// /// The consensus diploid genotype pools the alignments of one build only. The position of a /// de-novo variant does not compare across two builds, and a join by position needs a liftover @@ -909,13 +908,23 @@ impl App { .collect()) } - /// **Subject-level consensus** diploid genotype across the subject's same-build WGS alignments — - /// the joint genotype (opportunity #3). Per [`reconcile_site_genotypes`]: call each alignment's - /// variants (cached [`run_diploid_calls`]), union the SNV sites, force-genotype **every** - /// alignment at the union (so a site absent from one run is its real hom-ref / no-call), and vote - /// a depth-weighted 0/1/2 dosage per site. Returns the variant (het/hom-alt) consensus sites. - /// `contigs` limits the scan (None = all primary chromosomes). Heavy (a call pass + a force-call - /// pass per alignment) — an explicit export action; nothing is persisted. + /// The **consensus diploid genotype of a subject**, across its WGS alignments on one build. This + /// value is the joint genotype, which is opportunity #3. + /// + /// [`reconcile_site_genotypes`] does the work in four steps. + /// + /// It calls the variants of each alignment, and [`run_diploid_calls`] gives those calls from the + /// cache. It joins the SNV sites of each alignment into one set. It then genotypes **each** + /// alignment at each site of that set. So a site that one run does not hold gets its real + /// hom-ref call or no-call. It then votes a dosage of 0, 1, or 2 at each site, and a deeper run + /// has more weight. + /// + /// The method returns the consensus sites with a variant, which are the heterozygous sites and + /// the homozygous alternate sites. The `contigs` value limits the scan, and `None` reads each + /// primary chromosome. + /// + /// The method costs much: one call pass and one forced-call pass for each alignment. The user + /// starts it from the export screen, and the method stores nothing. pub async fn consensus_diploid_calls( &self, biosample_guid: SampleGuid, @@ -927,7 +936,8 @@ impl App { return Ok(Vec::new()); } - // (bam, reference) per same-build alignment, resolved once. + // The pair (bam, reference) of each alignment on this build. The code finds each pair one + // time. let mut paths = Vec::new(); for id in &aln_ids { paths.push((*id, self.alignment_bam_reference(*id).await?)); @@ -949,8 +959,8 @@ impl App { } }; for contig in clist { - // Tolerate a contig absent from this alignment's header (heterogeneous inputs) — - // skip it for this source rather than aborting the whole consensus. + // The header of this alignment can hold no such contig, because the inputs differ. + // Skip that contig for this source, and do not stop the full consensus. let Ok(variants) = self.run_diploid_calls(*id, contig, cancel.clone()).await else { continue; }; @@ -987,14 +997,15 @@ impl App { per_aln.push(g); } - // 4. Vote per site → consensus. min_depth = 2: a run abstains only when essentially - // uncovered; depth-weighting lets deep runs dominate the rest. + // 4. Vote at each site to get the consensus. The value min_depth = 2 means that a run + // gives no vote only when it has almost no coverage there. A deeper run has more weight than + // a shallow one. Ok(caller::reconcile_site_genotypes(&per_aln, 2)) } - /// A **consensus** diploid VCF (VCFv4.2) for the subject — the joint genotype across same-build - /// alignments (see [`consensus_diploid_calls`]), sample column `consensus`. Heavy; the export - /// path runs it off the UI thread. + /// A **consensus** diploid VCF file for the subject, in VCFv4.2. It holds the joint genotype + /// across the alignments on one build. See [`consensus_diploid_calls`]. The sample column is + /// `consensus`. The method costs much, and the export path runs it away from the UI thread. pub async fn consensus_diploid_vcf(&self, biosample_guid: SampleGuid) -> Result { let calls = self .consensus_diploid_calls(biosample_guid, None, CancelToken::none()) @@ -1002,11 +1013,14 @@ impl App { Ok(navigator_analysis::vcf::write_diploid_vcf("consensus", &calls)) } - /// Run de-novo calling on `contig` using the alignment's own stored paths. - /// The alignment's BAM + a usable reference FASTA: the stored path, else resolved from the - /// alignment's build via the gateway (cached, else downloaded). Errors only if no BAM is - /// recorded. Use this in steps that *require* the reference, so the user never has to supply - /// one (it follows from the header-detected build). + /// Call the de-novo variants on `contig` with the stored paths of the alignment. + /// + /// The method returns the BAM path of the alignment and a reference FASTA path that the code can + /// use. That reference is the stored path. When the alignment holds none, the gateway finds one + /// from the build of the alignment, from the cache or by a download. + /// + /// The method fails only when the alignment holds no BAM path. Use it in a step that *needs* the + /// reference. The user then supplies no reference, because the build in the header gives it. pub(crate) async fn alignment_bam_reference(&self, alignment_id: i64) -> Result<(PathBuf, PathBuf), AppError> { let aln = self.alignment_or_err(alignment_id).await?; let bam = Self::alignment_file(&aln)?; @@ -1021,12 +1035,20 @@ impl App { Ok((bam, reference)) } - /// The alignment's path and a reference suitable for **decoding** it: a CRAM can't be read - /// without the reference, so resolve it (stored path, else from the build via the gateway, - /// cache-first); a BAM decodes without one, so return the stored path as-is (usually `None`) and - /// never force a reference download. Use this for record/pileup reads and SNP-site genotyping - /// that do not consult reference bases; use [`alignment_bam_reference`](Self::alignment_bam_reference) - /// for calling paths (de-novo SNV/indel) that need the reference even on a BAM. + /// The path of the alignment, and a reference that the code can use to **decode** it. + /// + /// No reader can open a CRAM file without its reference. So for a CRAM file the method takes + /// the stored path first, and then the build through the gateway. It reads the cache before it + /// starts a download. + /// + /// A reader can open a BAM file with no reference. So for a BAM file the method returns the + /// stored path with no change, and that value is usually `None`. It never starts a download. + /// + /// Use this method to read records, to read a pileup, and to genotype a SNP site. None of those + /// steps reads a reference base. + /// + /// Use [`alignment_bam_reference`](Self::alignment_bam_reference) for a caller path, such as a + /// de-novo SNV call or indel call. Those paths need the reference for a BAM file also. pub(crate) async fn alignment_reference_for_decode( &self, alignment_id: i64, @@ -1052,9 +1074,12 @@ impl App { self.gateway.cached_reference(build).is_some() } - /// The distinct reference builds across a subject's alignments — the builds whose FASTA an - /// analysis of this subject may need. Used to pre-resolve references (with a progress bar) after - /// import and before a subject-level analysis, so on-demand downloads are not silent. + /// Each distinct reference build across the alignments of a subject. An analysis of that subject + /// can need the FASTA file of any of them. + /// + /// The code reads this list after an import, and before an analysis of the subject. It then + /// downloads each file with a progress bar. So a download during the analysis never surprises + /// the user. pub async fn reference_builds_for_subject(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alns = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut builds: Vec = alns.into_iter().map(|a| a.reference_build).collect(); @@ -1063,16 +1088,18 @@ impl App { Ok(builds) } - /// The reference build of a single alignment (`None` if it no longer exists) — for pre-resolving - /// that alignment's reference before a per-alignment analysis. + /// The reference build of one alignment. The method returns `None` when the store holds no such + /// alignment. The code reads this value to find the reference before it analyzes that + /// alignment. pub async fn reference_build_of_alignment(&self, alignment_id: i64) -> Result, AppError> { Ok(alignment::get(self.store.pool(), alignment_id) .await? .map(|a| a.reference_build)) } - /// The alignment IDs (BAM/CRAM only) across a subject's alignments — for pre-building each one's - /// coordinate index (with a progress bar) after import and before a subject-level analysis. + /// The id of each alignment of a subject that has a BAM file or a CRAM file. The code reads this + /// list to make the coordinate index of each one, with a progress bar. It does that work after an + /// import, and before an analysis of the subject. pub async fn alignment_ids_for_subject(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alns = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; Ok(alns @@ -1082,12 +1109,21 @@ impl App { .collect()) } - /// Ensure the alignment's coordinate index (`.bai`/`.crai`) exists, **building it if missing** so - /// the query-driven analyses (the per-contig walker, callable intervals, the de-novo / STR - /// callers) can seek by region instead of erroring or degrading to a whole-file linear scan. - /// Returns the index path if one was built, `None` if it was already present. `progress(done, - /// total)` reports a byte fraction for a BAM and indeterminate progress (`total = None`) for a - /// CRAM. The build is a single sequential pass, run on a decode-safe blocking thread. + /// Make sure that the coordinate index of the alignment exists. That index is a `.bai` file or a + /// `.crai` file, and the method **makes it** when the disk holds none. + /// + /// Each analysis that queries a region needs that index. Those analyses are the walker that + /// works on one contig, the step that finds the callable intervals, the de-novo caller, and the + /// STR caller. Without an index, such a step fails, or it reads the full file from start to end. + /// + /// The method returns the path of the index when it made one. It returns `None` when the index + /// already existed. + /// + /// The method calls `progress(done, total)`. For a BAM file, that call gives a fraction of the + /// bytes. For a CRAM file, the `total` value is `None`, and the progress has no end value. + /// + /// The method reads the file one time, from start to end, on a thread that can decode + /// safely. pub async fn ensure_alignment_index( &self, alignment_id: i64, @@ -1103,15 +1139,18 @@ impl App { Ok(built) } - /// Diagnose why an alignment can't be read, naming the **exact file** at fault rather than the - /// one the failing call happened to be handed. See [`navigator_analysis::preflight`] for why - /// that distinction is the whole point: an unreadable `.crai` and an unreadable CRAM produce - /// the same `io error on …cram` message today, and on macOS a privacy (TCC) denial and a Unix - /// permission denial are told apart only by the raw errno. + /// Report the reason that the app can not read an alignment. The report names the **exact + /// file** at fault, and not the file that the failed call received. + /// + /// [`navigator_analysis::preflight`] gives the reason for that rule. A `.crai` file that the app + /// can not read, and a CRAM file that it can not read, give the same `io error on …cram` message + /// today. On macOS, only the raw errno separates a privacy denial from TCC and a Unix permission + /// denial. /// - /// Deliberately **cache-only** for the reference: a diagnostic has to describe the machine as - /// it is, so resolving (and silently downloading) a missing FASTA here would paper over exactly - /// the state we were asked to report. A CRAM with no cached reference is a finding, not a task. + /// For the reference, the method reads the **cache only**, by design. A diagnostic must describe + /// the machine as it is. A download of an absent FASTA file here hides the exact state that the + /// user asked about. A CRAM file with no reference in the cache is a result of this check. It + /// is not a task for this method. pub async fn diagnose_alignment( &self, alignment_id: i64, @@ -1141,9 +1180,11 @@ impl App { .await } - /// The [`PublishGate`] for an alignment, adapted to its mean read length (HiFi relaxes the - /// supporting-read floor — see [`PublishGate::for_read_len`]). Samples the BAM head; any error - /// falls back to the short-read default. + /// The [`PublishGate`] of an alignment, for its mean read length. A HiFi read needs fewer reads + /// at a site than a short read. See [`PublishGate::for_read_len`]. + /// + /// The method reads the first records of the BAM file. After any error, it returns the default + /// gate for a short read. pub async fn publish_gate_for_alignment(&self, alignment_id: i64) -> Result { let (bam, reference) = self.alignment_bam_reference(alignment_id).await?; let read_len = tokio::task::spawn_blocking(move || { @@ -1156,15 +1197,20 @@ impl App { } } -/// True for a path on a removable/network mount (macOS `/Volumes/…`), where per-record random access -/// is slow but a bulk sequential copy is fast — the case [`App::localize`] copies to local disk. +/// True for a path on a removable mount or a network mount. On macOS such a mount is below +/// `/Volumes/…`. +/// +/// A read of one record at a random position is slow on that mount, and a bulk copy from start to +/// end is fast. [`App::localize`] copies such a file to the local disk. fn is_removable_volume(p: &Path) -> bool { p.starts_with("/Volumes/") } -/// A collision-free local filename for a remote alignment. Every kit's file is named `chrYM.cram`, -/// so the basename alone collides; hash the full remote path and keep the extension so the reader -/// still finds the sibling index at `.crai` / `.bai`. +/// A local file name for a remote alignment. Two such names are never the same. +/// +/// The file of each kit has the name `chrYM.cram`, so the base name alone gives the same local name +/// for two kits. The function hashes the full remote path and keeps the extension. So the reader +/// still finds the index beside the file, at `.crai` or `.bai`. fn local_cache_name(remote: &Path) -> String { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -1173,11 +1219,16 @@ fn local_cache_name(remote: &Path) -> String { format!("{:016x}.{ext}", h.finish()) } -/// A scratch name for one caller's in-progress copy of `local`. **Unique per call**: a temp path -/// derived from the destination alone (`.partial`) is shared by every concurrent copier of -/// that alignment, which lets two of them open the same inode — one truncating what the other is -/// writing, and continuing to write into it after the other has renamed it into place and started -/// reading. Uniqueness makes that unrepresentable rather than merely unlikely. +/// A scratch name for the copy of `local` that one caller is writing. Each call gives a **different +/// name**. +/// +/// A name from the destination alone, such as `.partial`, is the same name for each caller +/// that copies that alignment at the same time. Two callers then open the same inode. +/// +/// One caller empties the file that the other one writes. It also continues to write into that file +/// after the other caller renames it into place and starts to read it. +/// +/// A different name for each call makes that state impossible. It does not only make it rare. fn partial_path(local: &Path) -> PathBuf { static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -1185,18 +1236,22 @@ fn partial_path(local: &Path) -> PathBuf { local.with_file_name(format!("{stem}.partial.{}.{n}", std::process::id())) } -/// Copy `remote` → `local` plus its index sibling. The index is copied **first** and the main file -/// last (via a temp + rename), so a present `local` always implies its index is present too — the -/// cache check in [`App::localize`] can't see a half-copied pair. +/// Copy `remote` to `local`, and copy the index beside it. +/// +/// The function copies the index **first** and the main file last. For the main file it writes a +/// temporary file and then renames it. So a `local` file that exists always has its index. The cache +/// test in [`App::localize`] can then never see one file of the pair. +/// +/// `expect_len` is the size of the remote file. When the caller gives that value, the function +/// refuses a copy with a different size and publishes nothing. /// -/// `expect_len` is the remote's size; when known, a copy that does not match it is rejected rather -/// than published. A short copy is otherwise indistinguishable from corrupt data: it surfaces as a -/// decode error ("unexpected end of file", a bad container checksum) tens of gigabytes into a walk, -/// naming the cache path, and the copy is deleted on drop before anyone can look at it. +/// A short copy looks the same as damaged data. It gives a decode error tens of GB into a walk, such +/// as "unexpected end of file" or a bad container checksum. That error names the cache path, and the +/// code deletes the copy at the drop, before a user can look at it. /// -/// Nothing partial survives a failure — neither the temp nor the index copied ahead of it. A -/// leftover `.partial` used to sit in the cache indefinitely, occupying the disk while satisfying -/// no one. +/// No partial file survives a failure. That rule covers the temporary file and the index that the +/// function copied first. An old `.partial` file stayed in the cache for all time, and it filled the +/// disk with data that no code used. fn copy_with_index(remote: &Path, local: &Path, expect_len: Option) -> std::io::Result<()> { if let Some(parent) = local.parent() { std::fs::create_dir_all(parent)?; @@ -1242,19 +1297,22 @@ fn copy_with_index(remote: &Path, local: &Path, expect_len: Option) -> std: result } -/// One step of a full analysis of a single alignment, in the order [`App::plan_full_analysis`] -/// returns them. The variants carry whatever the step needs, so a caller's dispatch is total and it -/// can not silently run a step the plan excluded. +/// One step of a full analysis of one alignment, in the order that [`App::plan_full_analysis`] +/// gives. Each variant carries the values that its step needs. So a caller must handle each variant, +/// and it can not run a step that the plan left out. #[derive(Debug, Clone, PartialEq)] pub enum AnalysisStep { /// Coverage + callable, read-level QC, and sex inference in one pass over the alignment. QualityMetrics, - /// CNV + discordant pairs. Needs ≥10× — the step itself reports when the depth is too low. + /// The CNV calls and the discordant pairs. The step needs a depth of 10x or more, and it + /// reports a depth below that value. /// - /// **Opt-in only.** SV is experimental and, alone among the steps, walks every read in the file - /// for a result nothing else consumes — hours per whole-genome sample. It is planned only when - /// a caller asks for it (`include_sv`); the GUI's "Call SV" button and `analyze --sv` are the - /// ways in. Nothing runs it unattended. + /// **The user must select this step.** It is experimental. It is also the one step that reads + /// each record in the file for a result that no other step uses. It needs hours for one + /// whole-genome sample. + /// + /// The plan holds this step only when a caller sets `include_sv`. The "Call SV" button of the + /// GUI and the `analyze --sv` command do that. No code runs it without a user. StructuralVariants, /// De-novo calling on the mitochondrial contig (small and fully callable, unlike whole chrY). MitoDenovo { contig: String }, @@ -1266,7 +1324,8 @@ pub enum AnalysisStep { YSignature { biosample_guid: SampleGuid }, /// Genotype the ancestry markers into the autosomal consensus profile. AutosomalProfile { biosample_guid: SampleGuid }, - /// Estimate admixture/PCA *from* the autosomal profile — must follow [`Self::AutosomalProfile`]. + /// Estimate the admixture and the PCA values *from* the autosomal profile. This step must come + /// after [`Self::AutosomalProfile`]. Ancestry { biosample_guid: SampleGuid }, } @@ -1301,20 +1360,26 @@ impl AnalysisStep { } impl App { - /// The ordered steps a full analysis of `alignment_id` should run. + /// The steps of a full analysis of `alignment_id`, in their order. + /// + /// This function is the **one** definition of that pipeline. The GUI sends progress events, and + /// the CLI writes a log. So the two report a step in different ways. + /// + /// But the set of steps, and each condition that skips one, must be the same. The two did become + /// different. The copy in the CLI genotyped the Y chromosome at each run, and it replaced a + /// trusted external call. On ancient DNA it replaced that call with a worse one. /// - /// This is the **one** definition of that pipeline. The GUI streams progress events and the CLI - /// prints a log, so how a step is reported differs — but which steps run, and the conditions - /// under which one is skipped, must not: the two had drifted, and the CLI's copy re-genotyped Y - /// unconditionally, overwriting a trusted external call (on ancient DNA, with a worse one). + /// The `coverage` value is the result that [`AnalysisStep::QualityMetrics`] calculated, when + /// that step already ran. The mitochondrial decision then has the correct data. Before that + /// step, pass `None`. The function then reads the cached coverage. With no cached value, it + /// assumes that the file holds chrM reads. /// - /// `coverage` is the just-computed result when [`AnalysisStep::QualityMetrics`] has already run, - /// which makes the mitochondrial decision authoritative; pass `None` before that and the cached - /// coverage (or, with none, the assumption that chrM is present) is used instead. Callers that - /// show a step count should re-plan after the metrics step, as the count can drop. + /// A caller that shows a count of steps must call this function again after the metrics step, + /// because that count can become smaller. /// - /// `include_sv` adds the experimental [`AnalysisStep::StructuralVariants`]; see that variant for - /// why it is off by default. `include_ancestry` likewise gates the two heaviest ancestry steps. + /// `include_sv` adds the experimental step [`AnalysisStep::StructuralVariants`]. That variant + /// gives the reason for its default. `include_ancestry` adds the two ancestry steps that cost + /// the most. pub async fn plan_full_analysis( &self, alignment_id: i64, @@ -1322,9 +1387,12 @@ impl App { include_sv: bool, coverage: Option<&CoverageResult>, ) -> Result, AppError> { - // Skip the mitochondrial steps when the alignment has no chrM reads (e.g. an FTDNA Big Y): - // scoring zero chrM data just records a meaningless RSRS root. Unknown coverage (never run) - // keeps them rather than silently skipping. + // Leave out the mitochondrial steps when the alignment holds no chrM read. An FTDNA Big Y + // file is one example. A placement with no chrM data writes the RSRS root, and that result + // has no value. + // + // With no coverage result, the plan keeps those steps. The code must not remove a step with + // no message. let has_mtdna = match coverage { Some(c) => chrm_has_reads(c), None => self @@ -1335,7 +1403,8 @@ impl App { .map(|c| chrm_has_reads(&c)) .unwrap_or(true), }; - // Subject-level steps need the owning subject; an unattached alignment simply skips them. + // A step at the level of a subject needs that subject. The plan leaves out each such step + // for an alignment with no subject. let guid = self.biosample_of_alignment(alignment_id).await.ok(); let mut steps = vec![AnalysisStep::QualityMetrics]; @@ -1345,10 +1414,14 @@ impl App { if has_mtdna { steps.push(AnalysisStep::MitoDenovo { contig: "chrM".into() }); } - // Skip the internal Y/mt genotyping when a trusted external caller (GATK4 GVCF, sidecar fast - // path) already placed this alignment and the user prefers it — re-walking would only produce - // a secondary call that loses the vote, and on ancient DNA a wrong one. The assign_* commands - // guard this too; planning around it also avoids the wasted decode. See external-caller-precedence. + // Leave out the internal Y step and the internal mt step under two conditions. A trusted + // external caller already placed this alignment, and the user prefers that caller. Such a + // caller is a GATK4 GVCF file through the sidecar fast path. + // + // A second walk gives a call that loses the vote. On ancient DNA it also gives a wrong call. + // + // Each `assign_*` command has the same guard. A plan without the step also saves the decode. + // See external-caller-precedence. if !self .has_preferred_external_call(alignment_id, DnaType::Y) .await @@ -1364,9 +1437,12 @@ impl App { { steps.push(AnalysisStep::MtHaplogroup); } - // The Y signature makes the descent report ready without an explicit click (the button stays - // for an on-demand rebuild). Built once — an existing profile is left alone — and it needs no - // extra read of the file, since the Y assignment above just cached the chrY genotypes. + // The Y signature makes the descent report ready, and the user presses no button. That + // button stays, and it rebuilds the report at any time. + // + // The code builds the signature one time, and it does not change a profile that exists. It + // reads the file no more times, because the Y step above wrote the chrY genotypes to the + // cache. if let Some(guid) = guid { if self.cached_y_profile(guid).await?.is_none() { steps.push(AnalysisStep::YSignature { biosample_guid: guid }); @@ -1383,27 +1459,32 @@ impl App { } } -/// Whether a coverage result shows any reads on the mitochondrial contig, under either naming. +/// Shows whether a coverage result holds a read on the mitochondrial contig. The function accepts +/// both names of that contig. fn chrm_has_reads(c: &CoverageResult) -> bool { c.contig_coverage_stats .iter() .any(|s| contig::is_chr_m(&s.contig) && s.num_reads > 0) } -/// Live localized copies: local path → how many [`LocalAlignment`]s hold it. A copy is removed when -/// the count reaches zero, so its lifetime belongs to the copy rather than to a caller remembering -/// to clean up. +/// The local copies that exist now. The map takes a local path and gives the count of +/// [`LocalAlignment`] values that hold it. +/// +/// The code removes a copy when its count reaches zero. So the copy itself controls its life, and no +/// caller must remember to remove it. fn localized_registry() -> &'static std::sync::Mutex> { static REG: std::sync::OnceLock>> = std::sync::OnceLock::new(); REG.get_or_init(|| std::sync::Mutex::new(HashMap::new())) } -/// The lock serializing cache copies for one destination path — see [`App::localize`], which holds -/// it across the copy so a second caller waits for the first rather than duplicating it. +/// The lock that orders the cache copies for one destination path. [`App::localize`] holds it across +/// the copy, so a second caller waits for the first one and makes no second copy. /// -/// Async, because it is held across the copy's `await`. Entries are never removed: one small entry -/// per distinct alignment localized in this process, bounded by the workspace's alignment count, -/// which is cheaper than the bookkeeping needed to retire them safely. +/// The lock is async, because the code holds it across the `await` of the copy. +/// +/// The map never removes an entry. It holds one small entry for each alignment that this process +/// copies, and the count of alignments in the workspace limits that number. The code to remove an +/// entry safely costs more than those entries. fn copy_gate(local: &Path) -> std::sync::Arc> { #[allow(clippy::type_complexity)] static GATES: std::sync::OnceLock>>>> = @@ -1413,17 +1494,22 @@ fn copy_gate(local: &Path) -> std::sync::Arc> { std::sync::Arc::clone(gates.entry(local.to_path_buf()).or_default()) } -/// An alignment path to read from, owning any local copy made for it. +/// A path to read an alignment from. This value owns the local copy of that alignment, when one +/// exists. +/// +/// The earlier design kept each copy in a directory that one caller emptied, and that caller was +/// `analyze_biosample`. But three call sites made a copy. /// -/// The previous design cached copies in a directory cleared by one caller — `analyze_biosample` — -/// while three call sites created them. Every other path (notably the Y genotyping a batch drives) -/// copied ~400 MB per alignment and never cleaned up; that reached 687 files and 145 GB, and filled -/// the volume mid-run. Tying removal to `Drop` makes that leak unrepresentable, and the refcount -/// keeps the original benefit: a subject's several passes still share one copy instead of re-copying -/// per pass. +/// Each other path copied about 400 MB for one alignment and removed nothing. The Y genotype step of +/// a batch is the main example. That fault reached 687 files and 145 GB, and it filled the volume +/// during a run. +/// +/// A removal at the `Drop` call makes that fault impossible. The count of holders keeps the first +/// advantage. The passes of one subject still share one copy, and the code copies that file one +/// time. pub(crate) struct LocalAlignment { path: PathBuf, - /// False when `path` is the original (no copy was made, so nothing to remove). + /// False when `path` is the original file. The code made no copy, so it removes nothing. owned: bool, } @@ -1439,17 +1525,22 @@ impl LocalAlignment { Self { path, owned: true } } - /// Register interest in an existing copy; `true` when one was present and is now retained. + /// Take a share of a copy that exists. The method returns `true` when such a copy was there and + /// now has one more holder. + /// + /// A file with no entry in the registry is a **file from an earlier process**. No `Drop` call + /// ran there, so something stopped that run. + /// + /// The method compares such a file with `expect_len`, which is the size of the remote file. It + /// removes the file when the two differ. /// - /// A file with no registry entry is a **leftover from an earlier process** — `Drop` never ran, - /// so the run was killed — and is validated against `expect_len` (the remote's size) before - /// being trusted, then discarded if it does not match. Adopting a leftover on its existence - /// alone is how a truncated copy gets read as though it were the alignment: the failure then - /// appears as a decode error deep into a walk, pointing at a cache path whose file is deleted - /// moments later. A wrong-sized copy is worth exactly one re-copy to be rid of. + /// A method that took such a file on its existence alone would read a short copy as the + /// alignment. The fault then appears as a decode error deep in a walk. It names a cache path, + /// and the code deletes that file a moment later. One more copy is a small price to remove a + /// file of the wrong size. /// - /// An entry that *is* registered belongs to a live holder in this process and was validated - /// when it was made, so it is shared without re-statting. + /// A file *with* an entry belongs to a holder in this process, and the code checked it when it + /// made that copy. So the method shares it and reads no metadata. fn retain(local: &Path, expect_len: Option) -> bool { let mut reg = localized_registry().lock().unwrap(); if let Some(n) = reg.get_mut(local) { @@ -1477,7 +1568,8 @@ impl LocalAlignment { true } - /// The path to read from — the local copy when one was made, else the original. + /// The path to read from. The value is the local copy when the code made one, and the original + /// path when it made none. pub(crate) fn path(&self) -> &Path { &self.path } @@ -1500,8 +1592,8 @@ impl Drop for LocalAlignment { return; } reg.remove(&self.path); - // Best-effort: a copy left behind is a disk-space problem, not a correctness one, and a - // panic here would mask whatever the caller was actually doing. + // The step is optional. A copy that stays on the disk costs space, and it gives no wrong + // result. A panic here also hides the work of the caller. let _ = std::fs::remove_file(&self.path); let p = self.path.to_string_lossy().into_owned(); for suffix in [".crai", ".bai"] { @@ -1537,8 +1629,9 @@ mod local_alignment_tests { #[test] fn a_shared_copy_survives_until_the_last_holder_drops() { - // The reason for the refcount: a subject's passes each localize the same alignment, and - // removing it when the first finishes would force the rest to re-copy ~400 MB. + // The reason for the count of holders. Each pass of one subject localizes the same + // alignment. A removal after the first pass makes each later pass copy about 400 MB + // again. let d = scratch("shared"); let cram = d.join("b.cram"); std::fs::write(&cram, "x").unwrap(); @@ -1554,9 +1647,11 @@ mod local_alignment_tests { assert!(!cram.is_file(), "removed once nobody holds it"); } - /// A leftover from a killed run must be checked, not trusted. Adopting a short copy is how a - /// truncated cache entry gets read as though it were the alignment — surfacing as a decode - /// failure tens of gigabytes into a walk, blamed on a cache file that is deleted moments later. + /// The code must check a file from a run that stopped. It must not trust that file. + /// + /// A method that took a short copy would read that copy as the alignment. The fault then appears + /// as a decode failure tens of GB into a walk. The message names a cache file, and the code + /// deletes that file a moment later. #[test] fn a_wrong_sized_leftover_is_discarded_rather_than_adopted() { let d = scratch("stale"); @@ -1580,9 +1675,11 @@ mod local_alignment_tests { drop(LocalAlignment::owned(cram.clone())); } - /// Two concurrent copiers must never be able to name the same scratch file: sharing one let - /// them open a single inode, where one truncates what the other is writing — and keeps writing - /// into it after the other renames it into place and starts reading. + /// Two callers that copy at the same time must never use the same scratch file name. + /// + /// With one name, the two open the same inode. One caller then empties the file that the other + /// one writes. It also continues to write into that file after the other caller renames it into + /// place and starts to read it. #[test] fn each_copy_gets_its_own_partial_path() { let local = PathBuf::from("/tmp/nav-cache/abc.cram"); @@ -1595,8 +1692,9 @@ mod local_alignment_tests { } } - /// A copy that arrives short is rejected instead of published, and leaves nothing behind — not - /// the scratch file, and not the index copied ahead of it. Both used to accumulate in the cache. + /// The code refuses a copy that is too short, and it publishes nothing. It also leaves no file + /// behind. That rule covers the scratch file and the index that it copied first. The cache used + /// to collect both of them. #[test] fn a_short_copy_is_rejected_and_leaves_no_debris() { let d = scratch("short"); @@ -1604,7 +1702,8 @@ mod local_alignment_tests { std::fs::write(&remote, "0123456789").unwrap(); std::fs::write(d.join("src.cram.crai"), "idx").unwrap(); - // Claim the remote is larger than it is — the same shape as a copy cut short. + // Give a remote size that is larger than the real size. The result has the same shape as a + // copy that stopped early. let err = copy_with_index(&remote, &local, Some(64)).expect_err("a short copy must fail"); assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof, "{err}"); @@ -1618,9 +1717,10 @@ mod local_alignment_tests { assert!(debris.is_empty(), "scratch files left behind: {debris:?}"); } - /// The gate is what stops two overlapping callers from both copying the same alignment. Every - /// worker command is `tokio::spawn`ed, so that overlap is real, and the duplicate was a second - /// full pull of a 40 GB CRAM over the network whose only outcome was a failed rename. + /// The lock stops two callers that overlap, so only one of them copies the alignment. Each + /// worker command + /// runs under `tokio::spawn`, so that overlap does occur. The second copy read a 40 GB CRAM file + /// over the network again, and its only result was a failed rename. #[tokio::test] async fn one_destination_copies_at_a_time_and_others_are_not_blocked() { use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1672,8 +1772,9 @@ mod local_alignment_tests { #[test] fn the_original_is_never_removed() { - // A path we did not copy (local disk, or a failed copy falling back to the remote) must be - // left alone — deleting the user's own alignment would be catastrophic. + // The code must not change a path that it did not copy. Such a path is a file on the local + // disk, or the remote file after a failed copy. A delete of the alignment of the user is a + // very bad fault. let d = scratch("borrowed"); let original = d.join("c.cram"); std::fs::write(&original, "x").unwrap(); diff --git a/crates/navigator-app/src/lib.rs b/crates/navigator-app/src/lib.rs index 57ae4dec..acb770e9 100644 --- a/crates/navigator-app/src/lib.rs +++ b/crates/navigator-app/src/lib.rs @@ -1,8 +1,12 @@ -//! Navigator application/command layer — the single API the UI dispatches to, and the -//! antidote to the `WorkbenchViewModel` god object. Orchestrates `navigator-store` (and -//! later analysis/sync) behind commands and queries; holds policy the old dialogs -//! embedded (identity assignment, existence checks, result (de)serialization). The UI -//! holds only view-state and dispatch — no DB calls or domain decisions in widgets. +//! The application layer of Navigator, and its command layer. This crate is the one API that the UI +//! calls. It takes the place of the `WorkbenchViewModel` type, which held too much. +//! +//! The crate controls `navigator-store`, and later the analysis code and the sync code, behind its +//! commands and queries. It also holds each policy that an old dialog held. Those policies assign an +//! identity, test that a record exists, and read and write a result. +//! +//! The UI holds the state of its views and sends commands. No widget calls the database, and no +//! widget makes a decision about the domain. use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -33,9 +37,12 @@ pub use navigator_analysis::preflight::{ }; pub use navigator_analysis::CancelToken; -/// Diagnose a BAM/CRAM **path** with no workspace record behind it — the case that matters when a -/// user is reporting a file the app refuses to read and we need the answer before deciding whether -/// importing it is even possible. Blocking; call it off the async runtime. +/// Report the state of a BAM **path** or CRAM **path** that has no record in the workspace. +/// +/// This case matters when a user reports a file that the app refuses to read. The team needs the +/// answer before it decides whether an import of that file is possible. +/// +/// The function blocks, so call it away from the async runtime. pub fn diagnose_alignment_file(alignment: &std::path::Path, reference: Option<&std::path::Path>) -> PreflightReport { navigator_analysis::preflight::diagnose(alignment, reference) } @@ -56,107 +63,148 @@ pub use navigator_domain::ancestry::{ side_label_default, AncestryResult, AncestrySegment, ConfidenceInterval, PaintingResult, PopulationComponent, SuperPopulationSummary, }; -// The ancestry panel format, re-exported so panel tooling/tests depend only on navigator-app. +// The format of the ancestry panel. This crate exports it again, so a panel tool and a test depend +// on navigator-app alone. pub use navigator_analysis::ancestry::{AncestryPanel, PanelSite as AncestryPanelSite}; -/// A haplogroup assignment: the ranked candidates plus, for the reported terminal, the -/// child branches with per-SNP evidence (why descent stopped — unsupported splits show -/// ancestral SNPs, unresolved ones show no-calls). +/// One haplogroup assignment. It holds the candidates in their order. For the terminal node that it +/// reports, it also holds each child branch with the evidence of each SNP. +/// +/// That evidence shows the reason that the descent stopped. A split with no support shows an +/// ancestral SNP. A split with no answer shows a no-call. #[derive(Debug, Clone)] pub struct HaploAssignment { pub ranked: Vec, pub branches: Vec, - /// Per-SNP evidence along the placed lineage (root→terminal): every defining mutation the - /// sample carries (or does not), Derived/Ancestral/NoCall. This is the set the multi-source - /// variant/mutation **profile** reconciles — distinct from `branches`, which is the *untaken* - /// child branches (explaining why descent stopped, hence largely ancestral/no-call). + /// The evidence of each SNP along the lineage, from the root to the terminal node. The list + /// holds each mutation that defines a node, and the state of the sample at that mutation. The + /// three states are Derived, Ancestral, and NoCall. + /// + /// The variant **profile**, which pools many sources, reconciles this set. + /// + /// This list is not `branches`. That field holds the child branches that the descent did not + /// take. It shows the reason that the descent stopped, so most of its states are Ancestral or + /// NoCall. pub lineage: Vec, } -/// A YFull-YReport-style descent report for one lineage (Y or mtDNA): the root→terminal path, each -/// node carrying its defining SNPs with the subject's per-SNP call state. Generic over [`DnaType`] -/// so the Y-DNA and mtDNA tabs share one model + renderer. Built by [`App::descent_report`]. +/// A descent report for one lineage, which is the Y lineage or the mtDNA lineage. The report has the +/// shape of a YFull YReport. +/// +/// It holds the path from the root to the terminal node. Each node holds the SNPs that define it, +/// and the call state of the subject at each SNP. +/// +/// The type takes a [`DnaType`] parameter, so the Y-DNA tab and the mtDNA tab share one model and +/// one renderer. [`App::descent_report`] builds it. #[derive(Debug, Clone)] pub struct DescentReport { pub dna: DnaType, /// The reported terminal haplogroup name (e.g. "R-FGC29071", "U5a1b1g"). pub terminal: String, - /// Nodes root→terminal, each with its defining SNPs + the sample's state (`NodeEvidence`). + /// The nodes from the root to the terminal node. Each node holds the SNPs that define it and + /// the state of the sample, in a `NodeEvidence` value. pub nodes: Vec, } -/// A cohort **block tree** for one project: the induced subtree of the haplotree spanning the -/// members' terminal haplogroups, each node a *block* of phylogenetically equivalent SNPs, with the -/// members hanging off their own terminal. The group-project counterpart to [`DescentReport`] — -/// where that draws one subject's root→terminal path, this draws where a whole cohort sits relative -/// to each other. Built by [`App::project_block_tree`]; see -/// `documents/design/project-block-tree.md`. +/// The **block tree** of the cohort of one project. +/// +/// The tree is the part of the haplotree that covers the terminal haplogroups of the members. Each +/// node is a *block* of SNPs that the tree treats as equivalent. Each member appears below its own +/// terminal node. +/// +/// This type is the group-project form of [`DescentReport`]. That report draws the path of one +/// subject from the root to its terminal node. This tree draws the place of each member of a cohort +/// against the other members. /// -/// This view **reads** placements and never re-places, so it can not introduce a placement error. +/// [`App::project_block_tree`] builds it. See `documents/design/project-block-tree.md`. +/// +/// This view **reads** a placement and never makes one. So it can add no placement error. #[derive(Debug, Clone)] pub struct ProjectBlockTree { pub dna: DnaType, /// Induced-subtree blocks in pre-order (a parent always precedes its children). pub blocks: Vec, - /// Members with no placement, or whose terminal is absent from this tree. Reported rather than - /// dropped: on a multi-lab cohort provider/build skew is expected, and hiding it would - /// misrepresent how much of the project the tree actually accounts for. + /// The members with no placement, and the members whose terminal node this tree does not hold. + /// + /// The report names them and does not remove them. In a cohort from many laboratories, a + /// difference between providers and builds is normal. Without those members, the reader can not + /// see how much of the project the tree covers. pub unplaced: Vec, - /// The tree the view was drawn on (`"decodingus"` / `"ftdna"`) — `Block::loci` belong to it. + /// The tree of this view, which is `"decodingus"` or `"ftdna"`. The `Block::loci` values belong + /// to that tree. pub provider: String, - /// The coordinate space `Block::loci` positions are in. Node names and topology are - /// build-independent; only the positions are, so the view is labelled with the one build key it - /// was parsed under (the cohort's modal build). + /// The coordinate space of each position in `Block::loci`. + /// + /// The node names and the shape of the tree do not depend on the build. Only the positions + /// depend on it. So the view carries the one build key that the code parsed it under, which is + /// the most frequent build of the cohort. pub build_key: String, - /// Shared-private groupings that were **dropped** because they conflicted: their member sets - /// overlapped an accepted group without nesting inside it, so keeping both would not be a tree. - /// Surfaced rather than silently discarded — a non-zero count means recurrent calls or genuine - /// phylogenetic conflict in the cohort, which is worth knowing about. + /// The count of groups of shared private variants that the code **removed** for a conflict. + /// + /// The member set of such a group shares some members with an accepted group, and the accepted + /// group does not hold it. Two such sets can not both be a branch of one tree. + /// + /// The report gives this count and does not hide it. A count above zero shows recurrent calls, + /// or a real conflict in the phylogeny of the cohort. The reader needs that fact. pub candidate_conflicts: usize, - /// Positions rejected as **recurrent** — each would have defined a candidate branch under more - /// than one parent block, so it arose more than once and can not mark a new branch. Counted - /// rather than hidden: a high number says the cohort's private calls carry systematic noise. + /// The count of positions that the code refused as **recurrent**. Each one would define a + /// candidate branch below more than one parent block. So it occurred more than one time, and it + /// can not mark a new branch. + /// + /// The report gives this count and does not hide it. A high value shows that the private calls + /// of the cohort hold noise from the same cause. pub candidate_recurrent: usize, } -/// One block of a [`ProjectBlockTree`]: a branch plus the run of defining SNPs that are -/// phylogenetically equivalent on it — every member below carries all of them, and nothing observed -/// in this cohort separates them. +/// One block of a [`ProjectBlockTree`]. It holds a branch and the run of SNPs that define that +/// branch. The tree treats those SNPs as equivalent. Each member below the branch carries each of +/// them, and no observation in this cohort separates them. #[derive(Debug, Clone)] pub struct Block { pub node_id: i64, pub name: String, /// Parent within the induced subtree (`None` at a root). pub parent: Option, - /// Depth within the induced subtree, root = 0 — the layout's x coordinate. + /// The depth of this block in the subtree. The root has the value 0. The layout uses this value + /// as its x coordinate. pub depth: usize, - /// The equivalent SNPs defining this block. After a collapse this is the concatenation of the - /// absorbed branches' loci, root-most first: within *this cohort* they are one undivided block. + /// The equivalent SNPs of this block. After a collapse, the list holds the loci of each absorbed + /// branch, with the loci nearest the root first. Inside *this cohort*, those branches are one + /// block that nothing divides. pub loci: Vec, /// Members whose terminal *is* this block. pub members: Vec, - /// Members at or below this block — the count to badge on a collapsed branch. + /// The count of members at this block and below it. A collapsed branch shows that count. pub subtree_members: usize, /// Names of the member-less branches this block absorbed when collapsed (root-most first). /// Empty for an ordinary block. Kept so the UI can still name what it folded away. pub collapsed: Vec, - /// True when this is a **candidate branch** — not a node in the published tree, but a grouping - /// inferred from private (unnamed) variants that two or more members share. `node_id` is - /// synthetic and negative for these; `name` is empty, because the label is the view's to - /// localize. This is the thing a published tree can not tell you and we can: a branch that is - /// real in the data but has not been named yet. + /// True when this block is a **candidate branch**. The published tree holds no such node. The + /// code makes the group from the private variants that two members or more share, and those + /// variants have no name. + /// + /// For a candidate, `node_id` is a value that the code made, and it is negative. The `name` + /// field is empty, because the view supplies the label in the language of the user. + /// + /// A published tree can not give this answer, and the app can. The branch is real in the data, + /// and nobody has named it yet. pub candidate: bool, - /// For a candidate branch: every carrier's evidence at each shared position, so it can be - /// reviewed. Empty on a named block, whose SNPs are the tree's assertion rather than ours. + /// For a candidate branch, the evidence of each carrier at each shared position. A reader can + /// then judge the branch. The list is empty on a named block, because the tree states those SNPs + /// and this app does not. pub evidence: Vec, } -/// One carrier's evidence at one of a candidate branch's shared positions. +/// The evidence of one carrier at one shared position of a candidate branch. +/// +/// A candidate is a deduction, and the statement "three men share 1 SNP" is not enough to judge it. +/// +/// The read evidence behind the call of each carrier decides between a branch and a mapping +/// artefact. That evidence is the depth, and the share of the reads that hold the derived allele. A +/// cell holds one copy of that chromosome, so a true call has almost no other allele. /// -/// A candidate is an inference, and "1 SNP shared by three men" is not enough to judge it. What -/// decides whether it is a branch or a mapping artefact is the read evidence behind each carrier's -/// call — depth, and how cleanly the derived allele dominates on a chromosome that carries one copy. -/// Carried on the aggregate so the branch can be reviewed rather than taken on trust. +/// The aggregate carries this evidence, so a reader can judge the branch and does not trust it +/// without data. #[derive(Debug, Clone)] pub struct CandidateEvidence { pub guid: SampleGuid, @@ -167,9 +215,10 @@ pub struct CandidateEvidence { pub alternate: char, /// Read depth at the site; `0` when the source reported none. pub depth: u32, - /// Reads supporting the derived allele. + /// The count of reads that hold the derived allele. pub alt_depth: u32, - /// Derived-allele fraction — on haploid chrY a real call is essentially 1.0. + /// The share of the reads that hold the derived allele. A cell holds one copy of chrY, so a + /// true call gives a value near 1.0. pub allele_fraction: f64, /// Whether this call clears the federation publish gate. pub publishable: bool, @@ -179,16 +228,25 @@ pub struct CandidateEvidence { #[derive(Debug, Clone)] pub struct BlockMember { pub guid: SampleGuid, - /// Display name (donor identifier, else the guid) — what the tree leaf is labelled with. + /// The name for the display. The value is the donor identifier, and then the guid. The leaf of + /// the tree shows it. pub name: String, - /// Unnamed (private) variants below this member's terminal. `None` until private-Y has been - /// computed for the subject, which is distinct from `Some(0)` ("computed, none found"). - /// Populated in phase 3 (`documents/design/project-block-tree.md` §9); `None` before that. + /// The count of private variants below the terminal node of this member. Those variants have no + /// name. + /// + /// The value is `None` until the app calculates the private-Y data of the subject. That state is + /// not the same as `Some(0)`, which means that the app calculated the data and found no variant. + /// + /// Phase 3 writes this value. See `documents/design/project-block-tree.md` §9. Before that + /// phase, the value is `None`. pub private_novel: Option, - /// The **publishable** subset of the above: novel, unique-sequence, near-homozygous, with enough - /// supporting reads ([`PublishGate`]). This is the count we would stake a branch claim on, and - /// the one the block tree averages — the permissive `private_novel` is a working figure, not a - /// finding. + /// The part of the count above that the app can **publish**. Each such variant is new, is in + /// unique sequence, has almost no other allele, and has enough reads. [`PublishGate`] holds + /// those rules. + /// + /// The app makes a claim about a branch only from this count, and the block tree takes the mean + /// of it. The `private_novel` count above uses weaker rules. It is a value for work in progress, + /// and it is not a result. pub private_publishable: Option, pub private_total: Option, } @@ -202,24 +260,32 @@ pub struct UnplacedMember { pub terminal: Option, } -/// A per-marker branch report: the sample's genotype at every defining marker of a chosen tree -/// node's **descendant subtree** (Y or mtDNA), for spot-checking placement accuracy and exchanging -/// observations with other researchers. Built by [`App::branch_report`]; exported by -/// [`crate::export::branch_report_tsv`]. Unlike [`DescentReport`] (which walks the placement's -/// root→terminal ancestors from the persisted profile) this genotypes the subtree fresh, so -/// off-path branches the sample is *ancestral* for are reported too. +/// A branch report with one row for each marker. It holds the genotype of the sample at each marker +/// that defines a node in the **subtree below** a tree node that the user chose. The lineage is the Y +/// lineage or the mtDNA lineage. +/// +/// A researcher uses this report to check a placement, and to send observations to another +/// researcher. +/// +/// [`App::branch_report`] builds it, and [`crate::export::branch_report_tsv`] writes it to a file. +/// +/// This report is not a [`DescentReport`]. That report reads the ancestors of the placement from the +/// stored profile, from the root to the terminal node. This report genotypes the subtree again. So +/// it also holds each branch off the path where the sample is *ancestral*. #[derive(Debug, Clone)] pub struct BranchReport { pub dna: DnaType, /// The queried root node's haplogroup name (e.g. `R-FGC29071`). pub root: String, pub contig: String, - /// True when the observed bases + evidence came from a per-sample GVCF sidecar (else pileup). + /// True when the bases and the evidence came from the GVCF sidecar file of the sample. False + /// when they came from the pileup. pub gvcf_backed: bool, pub rows: Vec, } -/// One defining marker of a branch in a [`BranchReport`], with the sample's call + evidence. +/// One marker that defines a branch in a [`BranchReport`]. It holds the call of the sample and the +/// evidence for that call. #[derive(Debug, Clone)] pub struct BranchRow { pub node: String, @@ -230,7 +296,8 @@ pub struct BranchRow { pub derived: String, pub observed_base: Option, pub state: CallState, - /// `(ref, alt)` allele depths — `None` on ref blocks / the pileup path. + /// The depth of the reference allele and the depth of the alternate allele, as a pair. The value + /// is `None` on a reference block, and on the pileup path. pub ad: Option<(u32, u32)>, pub dp: Option, pub gq: Option, @@ -256,7 +323,8 @@ impl BranchReport { } impl DescentReport { - /// Total defining SNPs across the path the sample carries (derived). + /// The count of SNPs on the path that define a node and that the sample carries. Each one has a + /// derived state. pub fn derived(&self) -> usize { self.nodes .iter() @@ -265,7 +333,7 @@ impl DescentReport { .count() } - /// Total defining SNPs across the path (all states). + /// The count of SNPs on the path that define a node, in each state. pub fn total(&self) -> usize { self.nodes.iter().map(|n| n.snps.len()).sum() } @@ -274,9 +342,10 @@ impl DescentReport { /// How a private (off-backbone) variant relates to the tree. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum PrivateClass { - /// A known tree SNP off the assigned path — supports a finer/sibling branch. + /// A known SNP of the tree that is not on the path that the code assigned. It supports a finer + /// branch, or a branch beside the assigned one. OffPathKnown(String), - /// Not in the tree at all — a candidate for proposing a new branch. + /// The tree does not hold this SNP. It is a candidate for a new branch. Novel, } @@ -287,16 +356,23 @@ pub struct PrivateVariant { pub reference: char, pub alternate: char, pub depth: u32, - /// Reads supporting the derived (alternate) allele. `alt_depth / depth` ≈ `allele_fraction`; - /// carried explicitly so the publish gate can require a minimum supporting-read count. Old - /// cached buckets predate this field → `serde(default)` reads them as 0 (they recompute). + /// The count of reads that hold the derived allele, which is the alternate allele. The value + /// `alt_depth / depth` is about equal to `allele_fraction`. + /// + /// The field is explicit, so the publish gate can set a minimum count of such reads. An older + /// cached bucket holds no such field, and `serde(default)` reads it as 0. The app then + /// calculates that bucket again. #[serde(default)] pub alt_depth: u32, pub allele_fraction: f64, pub class: PrivateClass, - /// Curated CHM13 chrY structural class at this position (palindrome / amplicon / AZF-DYZ), - /// if any — a paralog-prone zone where short-read mapping is unreliable, so the call is - /// suspect (annotation only; not dropped). `None` = unique sequence, or a non-CHM13 build. + /// The structural class of this position on chrY in CHM13, from the curated list. The classes + /// are a palindrome, an amplicon, and an AZF-DYZ region. + /// + /// Such a region holds paralogs, and a short read maps there without reliability. So the call is + /// doubtful. This value is an annotation only, and the code removes no call for it. + /// + /// A value of `None` means unique sequence, or a build that is not CHM13. #[serde(default)] pub region: Option, } @@ -319,13 +395,15 @@ impl PrivateBucket { .filter(|v| matches!(v.class, PrivateClass::OffPathKnown(_))) .count() } - /// Calls that fall in a curated chrY structural (paralog-prone) region — suspect, to be - /// down-weighted in reports rather than treated as confident new variants. + /// The calls in a curated structural region of chrY, which holds paralogs. Such a call is + /// doubtful. A report must give it less weight, and it must not show it as a confident new + /// variant. pub fn in_structural_region(&self) -> usize { self.variants.iter().filter(|v| v.region.is_some()).count() } - /// Novel calls in *unique* sequence (no structural-region flag) — the high-confidence - /// new-branch candidates, separated from the paralog-zone noise. + /// The new calls in *unique* sequence, with no structural-region mark. These calls are the + /// candidates for a new branch with high confidence, and they are separate from the noise of a + /// paralog region. pub fn novel_in_unique_sequence(&self) -> usize { self.variants .iter() @@ -333,10 +411,14 @@ impl PrivateBucket { .count() } - /// The **publishable** subset: novel, unique-sequence calls that also clear the strict - /// novel-marker `gate` (near-homozygous allele fraction + a supporting-read floor). This is the - /// set we federate to the AppView as unverified singleton candidates — far stricter than the - /// caller's placement gates, so a paralog/contamination/low-evidence call never becomes a claim. + /// The part that the app can **publish**. Each such call is new, is in unique sequence, and also + /// passes the strict `gate` for a new marker. That gate needs a high share of derived reads and + /// a minimum count of reads. + /// + /// The app sends this set to the AppView as a set of single candidates that nobody verified. + /// + /// The rules are much stricter than the placement rules of the caller. So a call from a paralog, + /// from contamination, or with little evidence never becomes a claim. pub fn publishable(&self, gate: PublishGate) -> Vec<&PrivateVariant> { self.variants.iter().filter(|v| gate.admits(v)).collect() } @@ -353,20 +435,25 @@ impl PrivateBucket { } } -/// Thresholds gating which private variants are confident enough to **publish** to the AppView as -/// novel-branch candidates. Haploid chrY should be effectively homozygous, so a mixed/paralog -/// allele fraction (0.5–0.9, which the placement caller still accepts) is rejected here, as is a -/// call with too few supporting reads to trust as a real singleton. +/// The limits that decide which private variants the app can **publish** to the AppView as +/// candidates for a new branch. +/// +/// A cell holds one copy of chrY, so a call there has almost no second allele. The code refuses an +/// allele fraction between 0.5 and 0.9, which marks a mixture or a paralog. The placement caller +/// still accepts such a fraction. +/// +/// The code also refuses a call with too few reads to trust as a real single variant. #[derive(Debug, Clone, Copy, PartialEq)] pub struct PublishGate { /// Minimum derived-allele fraction (haploid → expect ≈1.0). pub min_allele_fraction: f64, - /// Minimum reads supporting the derived allele. + /// The minimum count of reads that hold the derived allele. pub min_alt_depth: u32, } impl Default for PublishGate { - /// Short-read WGS defaults: near-homozygous and ≥10 supporting reads. + /// The default values for a short-read WGS sample. The call needs almost no second allele, and + /// it needs 10 reads or more. fn default() -> Self { Self { min_allele_fraction: 0.9, @@ -376,9 +463,11 @@ impl Default for PublishGate { } impl PublishGate { - /// Gate adapted to the sample's mean read length: HiFi/long reads make a confident haploid - /// observation from far fewer reads (same rationale as [`adaptive_min_depth`]), so the - /// supporting-read floor drops to 3. The allele-fraction requirement is unchanged. + /// The gate for the mean read length of the sample. + /// + /// A HiFi read, and each other long read, gives a confident haploid observation from many fewer + /// reads. [`adaptive_min_depth`] uses the same reason. So the minimum count of reads becomes 3. + /// The rule for the allele fraction does not change. pub fn for_read_len(read_len: f64) -> Self { let mut g = Self::default(); if read_len > 1000.0 { @@ -387,8 +476,8 @@ impl PublishGate { g } - /// Whether a variant clears the gate: it must be an unnamed novel in unique sequence, with a - /// near-homozygous derived fraction and enough supporting reads. + /// Shows whether a variant passes the gate. Such a variant must be new and have no name, must + /// be in unique sequence, must have almost no second allele, and must have enough reads. pub fn admits(&self, v: &PrivateVariant) -> bool { v.class == PrivateClass::Novel && v.region.is_none() @@ -425,9 +514,10 @@ mod publish_gate_tests { assert!(!g.admits(&var(PrivateClass::OffPathKnown("M269".into()), None, 30, 1.0))); // Paralog-prone structural region → rejected even when deep/homozygous. assert!(!g.admits(&var(PrivateClass::Novel, Some(YRegionClass::Palindrome), 30, 1.0))); - // Mixed allele fraction (the placement caller accepts 0.5, publishing must not). + // The alleles are mixed. The placement caller accepts a fraction of 0.5, and a publish + // must not. assert!(!g.admits(&var(PrivateClass::Novel, None, 30, 0.6))); - // Too few supporting reads for short-read. + // A short-read sample needs more reads than this call holds. assert!(!g.admits(&var(PrivateClass::Novel, None, 4, 1.0))); } @@ -509,15 +599,21 @@ const KEYCHAIN_SERVICE: &str = "decodingus-navigator"; pub struct IbdComparison { pub summary: MatchSummary, pub segments: Vec, - /// Sites called in **both** samples — the effective comparison size. Sparse overlap (a - /// chip↔chip pair, or chip↔WGS limited to the chip's sites) weakens short-segment calls, so - /// it is surfaced rather than hidden. + /// The count of sites with a call in **both** samples. That count is the true size of the + /// comparison. + /// + /// A small overlap makes a call on a short segment weak. Two chips give such an overlap, and a + /// chip against a WGS sample also gives one, because the chip sites limit it. The report gives + /// this count and does not hide it. pub overlapping_sites: usize, } -/// A sample for an IBD comparison — either a WGS/CRAM **alignment** (genotyped at the IBD-panel -/// sites) or an imported **chip** profile (resolved to the same CHM13 sites). Both yield dosages -/// over the canonical IBD panel, so the comparison is data-type-agnostic. +/// One sample of an IBD comparison. It is a WGS **alignment** in a CRAM file, which the code +/// genotypes at the IBD-panel sites. It can also be a **chip** profile that a user imported, which +/// the code re-keys to the same CHM13 sites. +/// +/// Both forms give a dosage at each site of the canonical IBD panel. So the comparison does not +/// depend on the kind of data. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IbdSource { Alignment(i64), @@ -549,15 +645,20 @@ pub struct AssetStatus { pub verified: bool, } -/// A pseudonymous federated-IBD candidate from the AppView's match engine. The -/// `suggested_sample_guid` is the AppView's opaque handle for the counterpart (not a DID, -/// not PII) — used to request an introduction. `signals` names the sources that contributed -/// (e.g. `POPULATION_OVERLAP`, `HAPLOGROUP`, `SHARED_MATCH`) behind the composite `score`. +/// A federated-IBD candidate from the match engine of the AppView. The candidate has no name. +/// +/// The `suggested_sample_guid` value is the opaque handle of the AppView for the other person. It is +/// not a DID, and it holds no personal data. The app uses it to ask for an introduction. +/// +/// The `signals` field names each source behind the `score` value, such as `POPULATION_OVERLAP`, +/// `HAPLOGROUP`, and `SHARED_MATCH`. /// -/// `target_sample_guid` is the AppView's handle for **our own** sample the candidate was ranked -/// against. We already own it, so it discloses nothing — but a self-publishing client has no other -/// way to learn its server-side sample handle, and [`App::ibd_attest`] can not report a completed -/// comparison without it. `None` when talking to an AppView that predates that field. +/// The `target_sample_guid` value is the handle of the AppView for **our own** sample, and the +/// engine ranked the candidate against that sample. We already own it, so it gives away nothing. +/// +/// But a client that publishes its own records has no other way to learn its handle on the server. +/// [`App::ibd_attest`] can not report a complete comparison without it. The value is `None` on an +/// AppView from before that field. #[derive(Debug, Clone, PartialEq)] pub struct IbdSuggestion { pub target_sample_guid: Option, @@ -567,30 +668,34 @@ pub struct IbdSuggestion { pub signals: Vec, } -/// How much a federated-IBD candidate's composite score is worth believing, as the Simple-mode -/// "Genetic relatives" card frames it. +/// The level of trust in the score of a federated-IBD candidate. The "Genetic relatives" card of +/// Simple mode uses these words. +/// +/// This value reads the evidence, and it does not only draw it. The line between "strong" and +/// "possible" decides which of three statements the app makes about the relationship between a +/// stranger and the user. /// -/// This is a reading of the evidence, not a rendering of it: where the line falls between "strong" -/// and "merely possible" decides which of three claims the app makes about a stranger's relatedness -/// to the user. It lives beside [`IbdSuggestion`] rather than in the card that draws it so the rule -/// has one home — and so tuning it later is a change to the interpretation, not to a widget. +/// This code is beside [`IbdSuggestion`] and not in the card that draws it. So the rule has one +/// home, and a later change to it is a change to the reading of the evidence and not to a +/// widget. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MatchStrength { /// The signals agree strongly; presented as a likely relative. Strong, - /// Enough agreement to be worth pursuing. + /// The two samples agree enough for the user to act. Likely, /// Weak or single-signal evidence; presented as a possibility only. Possible, } impl IbdSuggestion { - /// Classify this candidate's composite `score` into the tier the UI names. + /// Change the `score` of this candidate into the level that the UI names. + /// + /// The AppView gives a score from 0 to 1, and that score joins each of the `signals` values. So + /// the limits here are careful. The method names a candidate strong only when the evidence is + /// far above the middle of that range. /// - /// The AppView's score is a 0–1 composite over the contributing `signals`, so the cutoffs are - /// deliberately conservative: a candidate is only called strong when the evidence is well clear - /// of the middle of the range, because overstating a match invites someone to contact a stranger - /// on the strength of it. + /// A statement that is too strong makes a user write to a stranger on weak evidence. pub fn strength(&self) -> MatchStrength { if self.score >= 0.8 { MatchStrength::Strong @@ -602,12 +707,15 @@ impl IbdSuggestion { } } -/// Result of requesting an introduction to a candidate: the AppView's request URI and its -/// status (initially `PENDING`, awaiting the consent round-trip). +/// The result of a request for an introduction to a candidate. It holds the request URI of the +/// AppView and the status of that request. The first status is `PENDING`, and the two parties then +/// exchange their consent. /// -/// `purpose` is chosen server-side from the suggestion's dominant signal (`IBD_AUTOSOMAL` / `IBD_Y` -/// / `IBD_MT`) — it decides which genomic region a later attestation is filed under, so it is worth -/// recording at introduction rather than waiting for the session to reveal it. +/// The server chooses the `purpose` value from the strongest signal of the suggestion. The values +/// are `IBD_AUTOSOMAL`, `IBD_Y`, and `IBD_MT`. +/// +/// That value decides the genomic region of a later attestation. So the app records it at the +/// introduction, and it does not wait for the session to give it. #[derive(Debug, Clone, PartialEq)] pub struct IbdIntroResult { pub request_uri: String, @@ -615,8 +723,9 @@ pub struct IbdIntroResult { pub purpose: String, } -/// An inbound, **symmetric-blind** exchange request awaiting this account's consent (the initiator -/// is hidden until both parties consent). From `GET /api/v1/exchange/incoming`. +/// An exchange request that arrived and that needs the consent of this account. The view is +/// **symmetric-blind**: the app does not see the sender until both parties agree. The value comes +/// from `GET /api/v1/exchange/incoming`. #[derive(Debug, Clone, PartialEq)] pub struct IncomingRequest { pub request_uri: String, @@ -635,8 +744,9 @@ pub struct ExchangeSessionInfo { pub partner_key_uri: Option, } -/// Outcome of `POST /api/v1/exchange/consent`: `CONSENTED` (with the opened `session_id`), -/// `DECLINED`, or `PENDING` (recorded, awaiting the counterpart). +/// The result of `POST /api/v1/exchange/consent`. The value is `CONSENTED`, with the `session_id` +/// of the new session. It can also be `DECLINED`. It can also be `PENDING`, which means that the +/// server recorded our answer and waits for the other party. #[derive(Debug, Clone, PartialEq)] pub struct ConsentOutcome { pub status: String, @@ -646,9 +756,9 @@ pub struct ConsentOutcome { /// Who opened a matching conversation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MatchingDirection { - /// We asked to be introduced. + /// This account asked for the introduction. Outbound, - /// Someone asked to be introduced to us. + /// Another person asked for an introduction to this account. Inbound, } @@ -668,23 +778,24 @@ impl MatchingDirection { } } -/// Where a matching conversation stands. Deliberately records only what this edge can *know*: -/// the broker is symmetric-blind, so a partner declining is indistinguishable from a partner who -/// has not answered yet — both stay [`MatchingStatus::Requested`], and [`MatchingStatus::Declined`] -/// means **we** declined. +/// The state of a matching conversation. The value records only what this device can *know*. +/// +/// The broker is symmetric-blind. So a partner who declined looks the same as a partner who did not +/// answer, and both stay at [`MatchingStatus::Requested`]. The value +/// [`MatchingStatus::Declined`] means that **this account** declined. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MatchingStatus { /// We asked; the counterpart has not consented (or has not answered). Requested, - /// Inbound and awaiting our decision. + /// The request arrived, and this account must decide. AwaitingConsent, /// We declined. Declined, - /// Both consented — a session is open and the encrypted exchange can run. + /// Both parties agreed. A session is open, and the encrypted exchange can run. Ready, - /// The exchange ran and a result is stored. + /// The exchange ran, and the store holds a result. Exchanged, - /// The exchange was attempted and failed; `last_error` says why. + /// The exchange ran and failed. The `last_error` field gives the reason. Failed, } @@ -710,8 +821,8 @@ impl MatchingStatus { _ => MatchingStatus::Requested, } } - /// True once the conversation has nothing further to do — it either produced a result or we - /// turned it down. The UI files these away from the actionable list. + /// True when the conversation has no more work. It gave a result, or this account declined it. + /// The UI keeps such a conversation out of the list of actions. pub fn is_terminal(self) -> bool { matches!(self, MatchingStatus::Exchanged | MatchingStatus::Declined) } @@ -725,12 +836,14 @@ pub struct MatchingEntry { pub direction: MatchingDirection, pub purpose: String, pub status: MatchingStatus, - /// Revealed only after mutual consent — `None` while the request is still blind. + /// The server gives this value only after both parties agree. It is `None` while the request + /// stays blind. pub partner_did: Option, pub session_id: Option, /// The local subject whose dosages this conversation exchanges. pub biosample_guid: Option, - /// AppView sample handles (ours / theirs) — what an attestation is filed under. + /// The two sample handles of the AppView, ours and theirs. An attestation uses them as its + /// key. pub my_sample_ref: Option, pub partner_sample_ref: Option, /// Our own consent decision; `None` until we make one. From 8e97248cf4b8c42b27a09208e17321ca9e530050 Mon Sep 17 00:00:00 2001 From: James Kane Date: Wed, 19 Aug 2026 09:27:47 -0500 Subject: [PATCH 09/33] docs(ste): the conversion recipe, and the tool it depends on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whoever picks this up next should not have to rediscover the method. The dictionary now carries it, and `scripts/ste-blocks.py` — the block dumper the workflow is built around — is committed rather than living in a scratch directory. `ste-check.py` gives the score; `ste-blocks.py` gives the text to rewrite, with the rules that fired on each block. The loop is: dump the blocks, rewrite ten to fifteen of them in one batch of exact `(old, new)` string pairs, re-score, repeat. Three things the recipe records because they cost time to learn: - **Expect about three passes per file, not one.** A first pass on 50 violations leaves 15 to 25, every time. Replacing one 40-word sentence with three explanatory ones reliably produces a new 26-word sentence, so rule 6 fires on a line that did not exist before. It converges, and each pass is smaller. - **Batch with exact string pairs and a MISS guard, never a regex over prose.** The one regex pass that touched prose corrupted 14 lines ("kit is own CSV", "Inenough", "--find out-sites") and had to be reverted wholesale. - **A violation that makes no sense is probably the checker.** Three real domain terms and one suffix-matching bug in the passive-voice rule were found that way. Read the checker before rewriting prose that is already correct. Also states what a rewrite must keep — every measured number, date, sample id and failure that motivated the code — and what it is expected to lose. The compression and the voice go. That is the trade, and it is deliberate. Ends with where the work stands, so the next session starts from a fact and not an assumption. Co-Authored-By: Claude Opus 5 (1M context) --- documents/STE-dictionary.md | 85 +++++++++++++++++++++++++++++++++++++ scripts/ste-blocks.py | 60 ++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100755 scripts/ste-blocks.py diff --git a/documents/STE-dictionary.md b/documents/STE-dictionary.md index 16021cc8..473fdf93 100644 --- a/documents/STE-dictionary.md +++ b/documents/STE-dictionary.md @@ -124,3 +124,88 @@ Do not use a contraction. Write `do not`, not `don't`. documentation. They are outside the scope of this standard. - **The rationale itself.** STE controls *how* you write a reason. It does not tell you to delete the reason. A comment must still say why the code is as it is. + +## How to convert a file + +This is the method that converted `navigator-resource` and 31 of the 33 files of `navigator-app`. +Follow it, and a file needs about three passes. + +### The two tools + +```bash +python3 scripts/ste-check.py crates/navigator-app/src/lib.rs # the count, by rule +python3 scripts/ste-blocks.py crates/navigator-app/src/lib.rs # each block, whole +``` + +`ste-check.py` gives the score. `ste-blocks.py` gives the text to rewrite, with the rules that fired +on each block. Both accept a file, a directory, or no argument, which reads each Rust file. A single +file returns in about 0.03 seconds, so run the check after each edit. + +### The loop + +1. Run `ste-blocks.py` on the file and read the first 10 to 15 blocks. +2. Rewrite them in one batch. Use a Python script with a list of `(old, new)` pairs and + `str.replace`. Do not edit by hand, and do not use a regular expression on prose. An exact string + pair either matches or reports a miss, and it can not damage a line that you did not read. +3. Run `ste-check.py` on the file. +4. Repeat until the count is 0. + +Write a small guard into the batch script, so a pair that no longer matches is loud: + +```python +for a, b in subs: + if a not in s: + print("MISS", a[:55].replace("\n", " ")) + continue + s = s.replace(a, b) +``` + +### Expect three passes, not one + +The count does not go to zero in one pass, and that is normal. A first pass on a file with 50 +violations usually leaves 15 to 25. + +The reason is mechanical: you replace one 40-word sentence with three explanatory sentences, and one +of those three is 26 words. Rule 6 then fires on a line that did not exist before. The process +converges, and each pass is smaller than the last. + +### What a rewrite must keep + +Keep every fact and every reason. The measured numbers, the dates, the sample ids, the failure that +caused the code — each is the value of the comment. `blocktree.rs` in `navigator-app` is the worked +example: almost every constant there is a limit that a measurement defends, and each defence +survived the conversion. + +Lose the compression and the voice. That is the trade this standard makes, and it is intended. + +### Common false positives + +The checker is careful but not perfect. Three cases came up often: + +- **A domain term reads as an error.** `handle`, `panel`, `build`, and `genotyping array` are + Technical Names. Add the term to this file, and the checker reads it from here. Do not rewrite + the sentence around a term that is correct. +- **An `-ing` Technical Name.** `mapping`, `sequencing`, `reasoning model`. Add it to the `-ing` + section above and to `ING_OK` in the checker. +- **A word inside a longer word.** The passive-voice rule once matched `present` as `pre` + `sent`. + If a violation makes no sense, read the checker before you rewrite the prose. + +### Verify + +`cargo check -p --all-targets` after each batch of files. A doc comment can break the build: +a `[link]` that no longer resolves, or a line that starts with `-` or `+`, which +`clippy::doc_lazy_continuation` reads as a Markdown list. + +`cargo fmt --all` before each commit. The pre-commit hook enforces it. + +### Where the work stands + +Converted to zero: `navigator-resource`, `navigator-store/src/sig_cache.rs`, and 31 of the 33 files +of `navigator-app`. + +Remaining in `navigator-app`: `src/lib.rs` (the type documentation at the top is converted, the +`impl App` body is not) and `src/haplogroup.rs`. + +Not started: `navigator-analysis`, `navigator-ui`, `navigator-domain`, `navigator-align`, +`navigator-panelbuild`, `navigator-store` beyond `sig_cache`, `navigator-refgenome`, +`navigator-sync`. Run `python3 scripts/ste-check.py` for the current count. diff --git a/scripts/ste-blocks.py b/scripts/ste-blocks.py new file mode 100755 index 00000000..8d7b6efd --- /dev/null +++ b/scripts/ste-blocks.py @@ -0,0 +1,60 @@ +"""Print each comment block that has an STE violation, whole and ready to rewrite. + +`ste-check.py` counts violations and names the rule. This prints the *block* around each one, so a +rewrite has the full context and does not have to be reassembled from line numbers. + + python3 scripts/ste-blocks.py crates/navigator-app/src/lib.rs + +The header of each block gives its line range and the rules that fired inside it. See +documents/STE-dictionary.md for the workflow this belongs to. +""" + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +CHECK = os.path.join(HERE, "ste-check.py") + +# ste-check.py runs its own main() at import, so exec the source with that call removed. +src = open(CHECK, encoding="utf-8").read().replace("\nmain()\n", "\n") +ns = {} +exec(compile(src, CHECK, "exec"), ns) + +if len(sys.argv) < 2: + sys.exit("usage: ste-blocks.py ") +path = sys.argv[1] +lines = open(path, encoding="utf-8").read().splitlines() +violations = ns["analyse"](ns["extract_rust"](path), "rust") + +# line -> the rules that fired on it, with a short excerpt where the rule carries one. +by_line = {} +for rule, hits in violations.items(): + for line_no, detail in hits: + tag = rule.split()[0] + (f":{detail[:22]}" if detail else "") + by_line.setdefault(line_no, set()).add(tag) + + +def is_comment(i): + return 0 <= i < len(lines) and lines[i].strip().startswith(("///", "//!", "//")) + + +printed = set() +for line_no in sorted(by_line): + first = last = line_no - 1 + while is_comment(first - 1): + first -= 1 + while is_comment(last + 1): + last += 1 + if (first, last) in printed: + continue + printed.add((first, last)) + tags = set() + for i in range(first + 1, last + 2): + tags |= by_line.get(i, set()) + print(f"===== {path}:{first + 1}-{last + 1} [{', '.join(sorted(tags))}]") + for i in range(first, last + 1): + print(lines[i]) + print() + +total = sum(len(h) for h in violations.values()) +print(f"# {total} violations in {len(printed)} blocks") From 6ef791040984525a7295e70e98304c2d90ca2499 Mon Sep 17 00:00:00 2001 From: James Kane Date: Wed, 19 Aug 2026 11:32:00 -0500 Subject: [PATCH 10/33] docs(ste): lib.rs to zero, and haplogroup.rs halfway `navigator-app/src/lib.rs` goes 524 -> 0, which leaves 32 of the crate's 33 files converted. `haplogroup.rs` goes 758 -> 384. Two tool changes came out of the work, both of the kind the recipe already warns about. The sentence splitter did not break on `.**`. A sentence that ends inside bold emphasis merged with the one after it, and rule 6 then reported a length that no sentence had. `SENT_SPLIT` now steps over a trailing `*` or `_`. That is the third time a violation that made no sense turned out to be the checker. `copying` joins the -ing Technical Names. The chromosome painter's published method is copying-LAI, so the word is a name here, not a verb. The recipe gains a failure mode that cost real time. Splitting a long doc comment into paragraphs with a blank line, rather than a `///` line, orphans the text above it: that is E0585, a hard build error, not a lint. It is easy to hit, because a few files hold two doc comments that ran together into one block. The dictionary now carries the scan that finds it. Verified: `cargo check -p navigator-app --all-targets` and `cargo fmt --all --check` both clean. Co-Authored-By: Claude Opus 5 (1M context) --- crates/navigator-app/src/haplogroup.rs | 1084 ++++++++++-------- crates/navigator-app/src/lib.rs | 1410 +++++++++++++----------- documents/STE-dictionary.md | 16 +- scripts/ste-check.py | 7 +- 4 files changed, 1421 insertions(+), 1096 deletions(-) diff --git a/crates/navigator-app/src/haplogroup.rs b/crates/navigator-app/src/haplogroup.rs index 1d67354b..644b1e6d 100644 --- a/crates/navigator-app/src/haplogroup.rs +++ b/crates/navigator-app/src/haplogroup.rs @@ -3,13 +3,13 @@ use super::*; use crate::fastpath::{chr_m_gvcf_for_alignment, chr_y_gvcf_for_alignment}; -/// Analysis-artifact `kind` for cached per-alignment tree-genotype base calls (see -/// [`App::base_calls`]). The `algorithm_version` carries the site-set hash, so distinct trees / -/// contigs / lift paths get distinct cache rows. +/// Analysis-artifact `kind` for the cached tree-genotype base calls of one alignment (see +/// [`App::base_calls`]). The `algorithm_version` carries the site-set hash, so a different tree, +/// contig, or lift path gets a different cache row. const GENOTYPE_KIND: &str = "tree-genotype"; -/// Parse a stored painting JSON into a [`PaintingResult`], tolerating the legacy form (a bare -/// `Vec` with no side labels) by wrapping it with the neutral Side A/B defaults. +/// Parse a stored painting JSON into a [`PaintingResult`]. It accepts the legacy form, which is a +/// bare `Vec` with no side labels, and gives it the neutral Side A/B defaults. fn parse_painting_json(s: &str) -> Result { match serde_json::from_str::(s) { Ok(r) => Ok(r), @@ -23,11 +23,15 @@ fn parse_painting_json(s: &str) -> Result { } } -/// Which phased side (0/1) carries the parent's transmitted alleles, by **transmission consistency**: -/// at sites where the child is heterozygous and the parent homozygous, the parent transmitted a known -/// allele — the side carrying it is the parent's. (Parent-child IBD is genome-wide IBD1, so IBD -/// overlap alone can't distinguish the sides; the phased alleles can.) `None` when there are too few -/// informative sites or the signal is ambiguous (guards against a mis-called relative). +/// Which phased side (0/1) carries the parent's transmitted alleles, by **transmission +/// consistency**. +/// +/// Take the sites where the child is heterozygous and the parent homozygous. There the parent +/// transmitted a known allele, and the side that holds it is the parent's. Parent-child IBD is +/// genome-wide IBD1, so IBD overlap alone can not separate the sides. The phased alleles can. +/// +/// `None` when there are too few informative sites, or when the signal is ambiguous. That guards +/// against a mis-called relative. fn anchor_side_to_parent(phased: &navigator_analysis::phasing::PhasedGenotypes, parent: &[SiteGenotype]) -> Option { let pd: std::collections::HashMap<(&str, i64), i32> = parent .iter() @@ -66,8 +70,8 @@ fn anchor_side_to_parent(phased: &navigator_analysis::phasing::PhasedGenotypes, } } -/// `(this-side, other-side)` labels from a parent's recorded sex — once one parent is anchored the -/// other side is definitionally the other parent. `None` if the sex is not a clear male/female. +/// `(this-side, other-side)` labels from a parent's recorded sex. Once one parent has an anchor, +/// the other side must be the other parent. `None` if the sex is not a clear male or female. fn parent_labels_for_sex(sex: Option<&str>) -> Option<(&'static str, &'static str)> { match sex.map(|s| s.trim().to_ascii_lowercase()).as_deref() { Some("female") | Some("f") => Some(("Mother", "Father")), @@ -76,8 +80,9 @@ fn parent_labels_for_sex(sex: Option<&str>) -> Option<(&'static str, &'static st } } -/// The two side labels for a painting: Mother/Father (or "Parent: "/"Other parent") when the -/// painting is phased and a parent side was anchored; otherwise the neutral Side A/Side B. +/// The two side labels for a painting. A phased painting with an anchor on a parent side gives +/// Mother and Father, or "Parent: " and "Other parent". Any other painting gives the neutral +/// Side A and Side B. fn build_side_labels(phased: bool, anchor: Option, parent: Option<&(Option, String)>) -> [String; 2] { match (phased, anchor, parent) { (true, Some(side), Some((sex, name))) => { @@ -94,23 +99,25 @@ fn build_side_labels(phased: bool, anchor: Option, parent: Option<&(Option>> = std::sync::OnceLock::new(); fn tree_memo() -> &'static std::sync::Mutex> { TREE_MEMO.get_or_init(|| std::sync::Mutex::new(HashMap::new())) } -/// Per-source weighted genotype calls (position → base) for one build's pooling group in -/// [`App::place_y_consensus`]: one `(source type, position→base)` entry per contributing source. +/// Weighted genotype calls (position → base) from each source, for one build's pool group in +/// [`App::place_y_consensus`]. Each source that contributes gives one `(source type, +/// position→base)` entry. type YSourceCalls = Vec<(SourceType, HashMap)>; -/// A stable cache key (used as the artifact `algorithm_version`) for a tree-genotype base call: -/// the queried `contig`, the lift source build, and an FNV-1a hash of the **sorted target -/// positions** + their count. A changed tree (added/removed/moved positions) changes the hash → -/// cache miss → fresh walk; the BAM `source_sig` handles a changed alignment file separately. +/// A stable cache key for a tree-genotype base call, which the artifact uses as its +/// `algorithm_version`. It holds the queried `contig`, the lift source build, and an FNV-1a hash of +/// the **sorted target positions** with their count. A tree that adds, removes, or moves a position +/// changes the hash, which misses the cache and forces a fresh walk. The BAM `source_sig` handles a +/// changed alignment file separately. pub(crate) fn genotype_cache_key(contig: &str, source_build: Option<&str>, targets: &HashSet) -> String { let mut sorted: Vec = targets.iter().copied().collect(); sorted.sort_unstable(); @@ -128,15 +135,16 @@ pub(crate) fn genotype_cache_key(contig: &str, source_build: Option<&str>, targe for p in &sorted { feed(&p.to_le_bytes()); } - // `g3`: chrY native genotyping now also resolves indel loci (additive derived sentinels), so the - // cached result differs from the SNP-only `g1` payload — bump on any genotyping-logic change so a - // stale payload is not reused (the site-set hash alone does not capture logic changes). + // `g3`: chrY native genotyping now also resolves indel loci, as additive derived sentinels. So + // the cached result is different from the SNP-only `g1` payload. Raise this on any change to + // the genotyping logic, so that no code reuses a stale payload. The site-set hash alone does + // not catch a change of logic. format!("g3:{contig}:{}:{h:016x}", sorted.len()) } -/// Callable chrY bases from a coverage result — the FTDNA Big Y generation discriminator (see -/// [`App::refine_big_y_generation`]). A Big Y has reads only on chrY, so this is its whole callable -/// footprint. Build-agnostic (`chrY`/`Y`). +/// Callable chrY bases from a coverage result. This is how the code tells the FTDNA Big Y +/// generations apart (see [`App::refine_big_y_generation`]). A Big Y has reads only on chrY, so +/// this is its whole callable footprint. It accepts either build name (`chrY` or `Y`). fn callable_chr_y_bases(cov: &Coverage) -> u64 { cov.contig_callable .iter() @@ -148,9 +156,10 @@ fn callable_chr_y_bases(cov: &Coverage) -> u64 { impl App { // ---- result exports (gap §6) ------------------------------------------- - /// Format a cached result as a shareable file body (TSV / HTML / BED). The UI writes the - /// returned string to the user-chosen path. Errors when the source result has not been computed - /// yet (`NotFound`). [`ExportRequest::CallableBed`] re-walks the BAM (no cached intervals). + /// Format a cached result as a file body to share (TSV / HTML / BED). The UI writes the + /// returned string to the path that the user chose. It gives an error (`NotFound`) when the app + /// has not computed the source result yet. [`ExportRequest::CallableBed`] walks the BAM again, + /// because the cache holds no intervals. pub async fn export_content(&self, req: &ExportRequest) -> Result { match req { ExportRequest::CoverageTsv(id) => Ok(export::coverage_tsv(&self.require_coverage(*id).await?)), @@ -206,9 +215,10 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("ancestry for alignment {alignment_id}")))) } - /// Walk each analyzed contig for its CALLABLE intervals (BED export). Re-reads the BAM — the - /// coverage artifact stores only per-contig callable *counts*, not the intervals. Uses the - /// contig list from the cached coverage result, so coverage must have been run first. + /// Walk each analyzed contig for its CALLABLE intervals (BED export). It reads the BAM again, + /// because the coverage artifact stores only a callable *count* for each contig, and not the + /// intervals. It takes the contig list from the cached coverage result, so coverage must run + /// first. async fn callable_intervals_all(&self, alignment_id: i64) -> Result)>, AppError> { let cov = self.require_coverage(alignment_id).await?; let contigs: Vec = cov.contig_coverage_stats.iter().map(|s| s.contig.clone()).collect(); @@ -250,7 +260,7 @@ impl App { alternate: v.alternate.to_string(), rs_id: None, genotype: None, - // Derived from an rCRS diff, not a source VCF — there is no evidence to carry. + // This comes from an rCRS diff, and not from a source VCF, so it has no evidence. evidence: Default::default(), }) .collect(); @@ -266,9 +276,9 @@ impl App { Ok(variant_set::create(self.store.pool(), &new).await?) } - /// Assign an mtDNA haplogroup to a stored sequence: fetch (and cache) the FTDNA mt-DNA - /// haplotree and rank haplogroups by the Kulczynski measure over the sample's base - /// calls. RSRS-anchored and reference-free (no rCRS needed). Best first. + /// Assign an mtDNA haplogroup to a stored sequence. Fetch the FTDNA mt-DNA haplotree, cache + /// it, and rank the haplogroups by the Kulczynski measure over the sample's base calls. The + /// method is RSRS-anchored and reference-free, and needs no rCRS. The best comes first. pub async fn assign_mtdna_haplogroup(&self, mtdna_id: i64) -> Result { let tree_json = self.fetch_ftdna_mt_tree().await?; let assignment = self.assign_mtdna_haplogroup_with_tree(mtdna_id, &tree_json).await?; @@ -295,7 +305,7 @@ impl App { } /// Record (upsert) a source's haplogroup call for donor-level reconciliation. Defaults to the - /// internal `NavigatorWalk` provenance tier (the external fast path records via `record_call_fp`). + /// internal `NavigatorWalk` provenance tier. The external fast path uses `record_call_fp`. pub async fn record_haplogroup_call( &self, biosample_guid: SampleGuid, @@ -315,7 +325,7 @@ impl App { } /// Like [`record_haplogroup_call`](Self::record_haplogroup_call) but stamps the input - /// fingerprint (file + tree content hashes) so a later run can skip re-scoring. + /// fingerprint (file and tree content hashes), so a later run does not score it again. async fn record_haplogroup_call_fp( &self, biosample_guid: SampleGuid, @@ -345,7 +355,8 @@ impl App { Ok(()) } - /// Record an assignment's top candidate as a per-source call (no-op if no match). + /// Record an assignment's top candidate as a call for that source. It does nothing if there + /// is no match. async fn record_call( &self, biosample_guid: SampleGuid, @@ -395,11 +406,13 @@ impl App { Ok(()) } - /// The preferred external (sidecar fast-path) call for an alignment's DNA type — present only - /// when the "prefer external caller" policy is on **and** such a call exists. When present, - /// Navigator's internal caller must not re-walk the CRAM: returning this call instead is what - /// protects an external GATK4/1240K placement from being diluted or overwritten (the - /// PRJEB37976 ancient-DNA fix). See `documents/design/external-caller-precedence.md`. + /// The preferred external (sidecar fast-path) call for an alignment's DNA type. It is present + /// only when the "prefer external caller" policy is on **and** such a call exists. + /// + /// When it is present, Navigator's internal caller must not walk the CRAM again. To give this + /// call back instead is what protects an external GATK4/1240K placement. Without it, a walk + /// could dilute or overwrite that placement, which was the PRJEB37976 ancient-DNA fix. See + /// `documents/design/external-caller-precedence.md`. pub(crate) async fn preferred_external_call( &self, biosample_guid: SampleGuid, @@ -416,8 +429,9 @@ impl App { Ok(haplogroup_call::get_one(self.store.pool(), biosample_guid, dna_type, &key).await?) } - /// Whether an alignment already carries a preferred external call for a DNA type — the gate the - /// UI worker uses to skip enqueuing the internal Y/mt genotyping in "Full Analysis". + /// Whether an alignment already carries a preferred external call for a DNA type. This is the + /// gate that the UI worker uses. With it, "Full Analysis" does not put the internal Y/mt + /// genotyping in the queue. pub async fn has_preferred_external_call(&self, alignment_id: i64, dna_type: DnaType) -> Result { let Ok(bio) = self.biosample_of_alignment(alignment_id).await else { return Ok(false); @@ -428,12 +442,16 @@ impl App { .is_some()) } - /// "Compare callers": the trusted external caller vs Navigator's internal caller for one - /// alignment. **Forces** the internal walk regardless of the prefer-external policy — it records - /// its own `aln:{id}` / `aln:{id}:mt` (`NavigatorWalk`) rows and never touches the external `:ext` - /// row, so the comparison is non-destructive to the external call. Returns Y (for Y-bearing - /// subjects) and mtDNA, each with both terminals; a divergence is the ancient-DNA-damage signal - /// the "skip the internal walk" default is protecting against. See external-caller-precedence §6. + /// "Compare callers": the trusted external caller against Navigator's internal caller, for one + /// alignment. + /// + /// It **forces** the internal walk, whatever the prefer-external policy says. It records its + /// own `aln:{id}` and `aln:{id}:mt` (`NavigatorWalk`) rows, and never touches the external + /// `:ext` row. So the comparison does no damage to the external call. + /// + /// It returns Y, for a subject that has a Y chromosome, and mtDNA. Each one carries both + /// terminals. A difference between them is the ancient-DNA-damage signal that the "skip the + /// internal walk" default protects against. See external-caller-precedence §6. pub async fn compare_callers(&self, alignment_id: i64) -> Result, AppError> { let bio = self.biosample_of_alignment(alignment_id).await.ok(); let mut out = Vec::new(); @@ -488,7 +506,7 @@ impl App { /// The reconciled donor-level haplogroup consensus across all recorded sources. A user /// manual override, when set, replaces the computed terminal (flagged `overridden`). /// - /// See [`names_a_branch`] for why the placed-label rule below is not simply + /// See [`names_a_branch`] for why the placed-label rule below is more than /// `prefer_external && has_external`. pub async fn haplogroup_consensus( &self, @@ -498,25 +516,32 @@ impl App { let calls = haplogroup_call::list_for_with_provenance(self.store.pool(), biosample_guid, dna_type).await?; let prefer_external = prefer_external_calls(); let has_external = calls.iter().any(|(p, _)| *p == CallProvenance::External); - // Per-run label reconciliation supplies the lineage / compatibility / divergence warnings — - // honoring provenance: when the user prefers the external caller and one placed this subject, - // it wins the vote (a damaged ancient-DNA CRAM walk can not out-score it). + // The label reconcile of each run supplies the lineage, compatibility, and divergence + // warnings, and it obeys the provenance. When the user prefers the external caller, and one + // placed this subject, that call wins the vote. A damaged ancient-DNA CRAM walk can not + // out-score it. let mut consensus = reconciliation::reconcile_with_provenance(&calls, prefer_external); - // …the genome-level PLACED call (consensus_profile.consensus_label, from build_{y,mt}_profile) - // is normally authoritative. Phase 2 makes that placement GVCF-sourced on preferred-external - // subjects (place_{y,mt}_consensus → consensus_base_calls, no CRAM walk), so a freshly built - // label already agrees with the external call. We still skip it here so a *stale* label left - // by a pre-Phase-2 (CRAM-pooled) build can not resurface before the profile is rebuilt — the - // external reconcile is the safe authority for these subjects. + // …the genome-level PLACED call is normally the authority. It is + // consensus_profile.consensus_label, from build_{y,mt}_profile. Phase 2 takes that + // placement from the GVCF on a preferred-external subject (place_{y,mt}_consensus → + // consensus_base_calls, with no CRAM walk). So a label that the app has just built already + // agrees with the external call. + // + // The code still skips it here. A *stale* label from a pre-Phase-2 build, which pooled the + // CRAM, must not come back before a rebuild of the profile. For these subjects the external + // reconcile is the safe authority. // // …*unless* the reconcile has no branch name to offer. A call whose stored haplogroup is a - // variant string rather than a branch (see [`names_a_branch`]) is not an authority worth - // protecting: `altai363p` held one external call reading `chrY:5216846A>C [Node721]` while a - // freshly re-placed profile said `R-YP1507`, and this guard suppressed the good label in - // favour of the raw one — so a re-place appeared to do nothing to the assigned branch name - // even though it had rewritten it correctly. Skipping the placed label is only ever right + // variant string, and not a branch (see [`names_a_branch`]), is not an authority worth + // protection. + // + // `altai363p` held one external call that read `chrY:5216846A>C [Node721]`, while a freshly + // re-placed profile gave `R-YP1507`. This guard suppressed the good label in favour of the + // raw one. A re-place then looked as though it did nothing to the assigned branch name, + // even though it had rewritten the name correctly. To skip the placed label is right only // when what replaces it is better. + // // Read it as: skip the placed label only when a preferred external call offers a real // branch name to skip it *for*. let reconciled_names_a_branch = consensus.as_ref().is_some_and(|c| names_a_branch(&c.haplogroup)); @@ -571,8 +596,8 @@ impl App { /// Donor-level Y and mtDNA terminal haplogroups for **every** subject, for the subjects /// list. Reconciles each subject's recorded calls (and applies any manual override) in - /// memory from two bulk queries. `(guid → (Y terminal, mt terminal))`; either is `None` - /// when nothing is recorded. + /// memory from two bulk queries. The map is `(guid → (Y terminal, mt terminal))`. A terminal + /// is `None` when the store holds nothing for it. pub async fn haplogroup_terminals( &self, ) -> Result, Option)>, AppError> { @@ -595,9 +620,10 @@ impl App { } } } - // The genome-level placed terminal (build_{y,mt}_profile) wins over the per-run label vote, - // so the subjects table matches the detail tab — except on a preferred-external subject, where - // the CRAM-pooled placement is skipped in favor of the external call (as in haplogroup_consensus). + // The genome-level placed terminal (build_{y,mt}_profile) wins over the label vote of each + // run, so the subjects table matches the detail tab. A preferred-external subject is the + // exception: there the code skips the CRAM-pooled placement, and takes the external call, + // as haplogroup_consensus does. for (guid_s, dna_type_s, label) in navigator_store::consensus_profile::list_labels(self.store.pool()).await? { let Ok(uuid) = guid_s.parse::() else { continue; @@ -678,13 +704,15 @@ impl App { Ok(()) } - /// Load the persisted **observations** for a subject + DNA type — the raw genotype snapshot with - /// no interpretation. Cheap (no genotyping). The shared loader behind [`cached_y_profile`] / - /// [`cached_mt_profile`]; those interpret it against the current tree. `None` until a build runs. + /// Load the stored **observations** for a subject and DNA type. This is the raw genotype + /// snapshot, with no interpretation. It is low-cost, with no genotyping. It is the shared + /// loader behind [`cached_y_profile`] and [`cached_mt_profile`], which interpret it against the + /// current tree. `None` until a build runs. /// - /// Backward-compat: a payload written before the observation-first switch is a baked - /// [`ConsensusProfile`] (no `schema_version`); it is normalized to an [`ObservedProfile`] using - /// its stored per-source bases (a source with no base becomes a no-call until the next rebuild). + /// For compatibility: a payload from before the observation-first change is a complete + /// [`ConsensusProfile`], with no `schema_version`. The loader normalizes it to an + /// [`ObservedProfile`] from the bases that it stored for each source. A source with no base + /// becomes a no-call until the next rebuild. async fn load_observed_profile( &self, biosample_guid: SampleGuid, @@ -705,9 +733,10 @@ impl App { } } - /// Persist a reconciled consensus snapshot — the low-level row writer shared by every DNA type - /// (Y / mt key on [`DnaType`], autosomal keys on `"Auto"`; the payload is whatever profile shape - /// that type uses). The scalar columns mirror the summary header for quick listing. + /// Store a reconciled consensus snapshot. This is the low-level row writer that every DNA type + /// shares. Y and mt key on [`DnaType`], and the autosomal type keys on `"Auto"`. The payload is + /// the profile shape that the type uses. The scalar columns mirror the summary header, for a + /// fast listing. #[allow(clippy::too_many_arguments)] async fn persist_consensus_row( &self, @@ -738,9 +767,10 @@ impl App { Ok(()) } - /// Persist a Y/mt **observation** snapshot (the payload) plus the interpreted `summary` header for - /// quick listing. Only observations are stored; state/status are re-derived on load by - /// [`interpret_y_profile`](Self::interpret_y_profile) / [`interpret_mt_profile`]. + /// Store a Y/mt **observation** snapshot, which is the payload, plus the interpreted `summary` + /// header for a fast listing. The store keeps only the observations. + /// [`interpret_y_profile`](Self::interpret_y_profile) and [`interpret_mt_profile`] derive the + /// state and the status again at load time. async fn persist_observed_profile( &self, biosample_guid: SampleGuid, @@ -761,10 +791,11 @@ impl App { .await } - /// The Y/mt polarity map (SNP name → ancestral/derived) from the **current** tree for the - /// configured provider — the input to [`navigator_domain::consensus::interpret`]. DecodingUs uses - /// the tree's true phylogenetic polarity; FTDNA the parsed FTDNA tree's polarity. Empty when the - /// tree is unavailable (interpret then falls back to each variant's stored ref/alt). + /// The Y/mt polarity map (SNP name → ancestral/derived) from the **current** tree of the + /// configured provider. It is the input to [`navigator_domain::consensus::interpret`]. + /// DecodingUs uses the tree's true phylogenetic polarity. FTDNA uses the polarity of the parsed + /// FTDNA tree. The map is empty when the tree is not available, and interpret then falls back + /// to each variant's stored ref/alt. async fn current_y_polarity(&self) -> std::collections::BTreeMap { match y_tree_provider() { YTreeProvider::DecodingUs => self @@ -803,13 +834,14 @@ impl App { interpret_observed(observed, &pol) } - /// The Y-profile for a subject, if one has been built — cheap (no genotyping). `None` until - /// [`build_y_profile`](Self::build_y_profile) runs. + /// The Y-profile for a subject, if the app has built one. It is low-cost, with no genotyping. + /// `None` until [`build_y_profile`](Self::build_y_profile) runs. /// - /// Loads the stored **observations** and interprets them against the **current** Y tree polarity - /// on every read — so a corrected/updated tree (or a provider switch) flips the derived/ancestral - /// states with no rebuild and no BAM re-read. Legacy profiles are normalized to observations on - /// load; those persisted before bases were stored show no-calls until one rebuild. + /// It loads the stored **observations** and interprets them against the **current** Y tree + /// polarity on every read. So a corrected tree, or a change of provider, flips the derived and + /// ancestral states with no rebuild and no second BAM read. The load normalizes a legacy + /// profile to observations. A profile that the app stored before it kept bases shows no-calls + /// until one rebuild. pub async fn cached_y_profile(&self, biosample_guid: SampleGuid) -> Result, AppError> { match self.load_observed_profile(biosample_guid, DnaType::Y).await? { Some(observed) => Ok(Some(self.interpret_y_profile(observed).await)), @@ -817,15 +849,19 @@ impl App { } } - /// Build (and persist) the multi-source Y-variant profile: reconcile each Y-bearing source's - /// per-SNP calls — every alignment's haplogroup placement, the combined chip/BISDNA placement, - /// and the private-Y bucket — into one concordance view (confirmed / novel / conflict / - /// single-source per SNP, with per-source provenance + per-observation quality weighting). - /// Expensive (re-genotypes each alignment), so it is an explicit action; the result is persisted - /// so [`cached_y_profile`](Self::cached_y_profile) reloads it instantly. Sources without Y data - /// are skipped. + /// Build the multi-source Y-variant profile, and store it. + /// + /// It reconciles the SNP calls of each source that has Y data into one concordance view. Those + /// sources are every alignment's haplogroup placement, the combined chip/BISDNA placement, and + /// the private-Y bucket. Each SNP gets a status of confirmed, novel, conflict, or + /// single-source, with the provenance of each source and a quality weight for each + /// observation. + /// + /// It costs a lot, because it genotypes each alignment again, so it is an explicit action. The + /// store keeps the result, and [`cached_y_profile`](Self::cached_y_profile) reloads it + /// immediately. It skips a source with no Y data. pub async fn build_y_profile(&self, biosample_guid: SampleGuid) -> Result { - // Females have no Y chromosome — do not build or persist a Y variant profile for them. + // A female has no Y chromosome. Do not build or store a Y variant profile for her. if !self.subject_has_y_dna(biosample_guid).await? { return Ok(YProfile { variants: Vec::new(), @@ -837,18 +873,20 @@ impl App { let mut sources: Vec<(String, SourceType, Vec)> = Vec::new(); - // WGS / Y-NGS evidence: the **genome-consensus** deep placement — every alignment's chrY - // calls pooled on ONE tree+coordinate space and placed once ([`place_y_consensus`]). Its - // root→terminal lineage is the SNP set the sample carries all the way down to the deep - // terminal, so the descent report renders a populated backbone. + // WGS / Y-NGS evidence: the **genome-consensus** deep placement. Every alignment's chrY + // calls pool on ONE tree and coordinate space, and place once ([`place_y_consensus`]). The + // root→terminal lineage is the SNP set that the sample carries all the way down to the + // deep terminal. So the descent report draws a full backbone. // - // Previously this looped each alignment through `y_assignment_full`, whose *per-alignment* - // placement is shallow on lifted CHM13 Big Y data (it stops a few clades down): the profile - // then carried only the root→shallow-terminal SNPs while the profile's terminal came from - // the deeper pooled consensus — so the descent walked terminal→root over SNPs the profile - // never recorded, rendering every node below the shallow terminal as no-call (the "all - // no-call below F / reversed SNPs" bug). Pooling first keeps the variants and the terminal - // on the same deep placement. Genotypes are cached, so this reuses the Y walk already paid. + // The old code looped each alignment through `y_assignment_full`. That placement, for one + // alignment, is shallow on lifted CHM13 Big Y data, and stops a few clades down. The + // profile then carried only the root→shallow-terminal SNPs, while its terminal came from + // the deeper pooled consensus. So the descent walked terminal→root over SNPs that the + // profile never recorded, and drew every node below the shallow terminal as a no-call. + // That was the "all no-call below F / reversed SNPs" bug. + // + // To pool first keeps the variants and the terminal on the same deep placement. The cache + // holds the genotypes, so this reuses the Y walk that the app already paid for. let consensus_assignment = self.place_y_consensus(biosample_guid).await?; if let Some(asg) = &consensus_assignment { let obs = snp_obs_from_assignment(asg, true); @@ -857,9 +895,10 @@ impl App { } } - // One source *per chip/BISDNA panel* (a distinct VariantSet per import — 23andMe, - // AncestryDNA, BISDNA chromo2, …), so the profile shows which test confirmed each SNP and a - // single mistyped panel surfaces as a conflict rather than being averaged into "consumer tests". + // One source for *each chip or BISDNA panel*. Each import gives a distinct VariantSet: + // 23andMe, AncestryDNA, BISDNA chromo2, and so on. So the profile shows which test + // confirmed each SNP, and one mistyped panel shows as a conflict. An average over + // "consumer tests" would hide it. let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; let chip_sets: Vec<&VariantSet> = vsets.iter().filter(|s| s.source_type == SourceType::Chip).collect(); if !chip_sets.is_empty() { @@ -889,10 +928,11 @@ impl App { } } - // One source *per vendor Y-NGS VCF* (FTDNA Big Y / YSEQ / Full Genomes / Nebula / Dante — - // every non-chip VariantSet with chrY calls). Placed against each set's stored build so a - // GRCh38 Big Y reconciles alongside any WGS alignment; tagged with the set's real source - // type (TargetedNgs / WgsShortRead / …) so it carries the right concordance weight. + // One source for *each vendor Y-NGS VCF*: FTDNA Big Y, YSEQ, Full Genomes, Nebula, Dante, + // and every other non-chip VariantSet with chrY calls. Each one places against its own + // stored build, so a GRCh38 Big Y reconciles beside any WGS alignment. Each carries the + // set's real source type (TargetedNgs, WgsShortRead, …), which gives it the correct + // concordance weight. let ngs_sets: Vec<&VariantSet> = vsets.iter().filter(|s| s.source_type != SourceType::Chip).collect(); if !ngs_sets.is_empty() { let mut tree_cache: HashMap = HashMap::new(); @@ -929,8 +969,9 @@ impl App { PrivateClass::OffPathKnown(n) => n.clone(), PrivateClass::Novel => String::new(), // keyed by position }; - // Carry the observed base (= the called alt) so interpret re-derives Derived from - // the call's own ref/alt — no baked state, consistent with every other source. + // Carry the observed base, which is the called alt, so that interpret derives + // Derived again from the call's own ref/alt. The code stores no state here, + // and this matches every other source. let mut o = YObsInput::observed( name, v.position, @@ -939,8 +980,9 @@ impl App { Some(v.alternate), false, ); - // De-novo calls carry read depth; a structural-region (palindrome/amplicon) call - // is paralog-suspect → down-weight via the region modifier. + // A de-novo call carries a read depth. A call in a structural region, a + // palindrome or an amplicon, can be a paralog, so the region modifier + // down-weights it. o.depth = Some(v.depth); // Down-weight by the structural-region quality modifier (palindrome 0.4, // ampliconic 0.3, heterochromatin/centromere 0.1…); unique sequence = 1.0. @@ -953,13 +995,14 @@ impl App { } } - // Group the sources into an observation-only snapshot — no baked state. State/status are - // interpreted against the current tree on read (`interpret_y_profile`), so a corrected tree - // (incl. an FTDNA tree whose reference-as-ancestral polarity is inverted at some sites) flips - // the display without a rebuild. + // Group the sources into a snapshot of observations only, and store no state. The read + // path interprets the state and the status against the current tree + // (`interpret_y_profile`). So a corrected tree flips the display with no rebuild. This + // also covers an FTDNA tree that inverts its reference-as-ancestral polarity at some sites. let mut observed = yprofile::to_observed(&sources); - // Genome-level placement: the pooled call set placed once (computed above — not a vote among - // the per-run terminal labels). Falls back to the label reconciliation when nothing places. + // Genome-level placement: the pooled call set, placed once. The code above computes it, + // and it is not a vote among the terminal labels of each run. It falls back to the label + // reconcile when nothing places. observed.terminal_hint = match &consensus_assignment { Some(a) => a.ranked.first().map(|r| r.name.clone()), None => self @@ -979,9 +1022,10 @@ impl App { Ok(profile) } - /// The mtDNA consensus profile for a subject, if built — cheap (no genotyping). Loads stored - /// observations and interprets them against the current rCRS tree polarity on every read (the - /// mtDNA half of the observation-first fix — previously mt states could not re-interpret at all). + /// The mtDNA consensus profile for a subject, if the app has built one. It is low-cost, with no + /// genotyping. It loads the stored observations and interprets them against the current rCRS + /// tree polarity on every read. This is the mtDNA half of the observation-first fix. Before it, + /// an mt state could not take a new interpretation at all. pub async fn cached_mt_profile(&self, biosample_guid: SampleGuid) -> Result, AppError> { match self.load_observed_profile(biosample_guid, DnaType::Mt).await? { Some(observed) => Ok(Some(self.interpret_mt_profile(observed).await)), @@ -989,24 +1033,31 @@ impl App { } } - /// Build (and persist) the multi-source mtDNA consensus profile — the mtDNA adapter over the - /// generic [`navigator_domain::consensus`] engine. Reconciles each mt-bearing source's - /// defining-mutation calls (every alignment's chrM placement, each imported mtDNA FASTA - /// sequence's placement, and the combined chip mtDNA placement) into one concordance view, - /// keyed by phylotree **mutation name** (rCRS-coordinate, build-independent). Persisted with - /// `dna_type='Mt'` so [`cached_mt_profile`](Self::cached_mt_profile) reloads it instantly. - /// Expensive (re-places each alignment's chrM), so it is an explicit action; mt-less sources skip. + /// Build the multi-source mtDNA consensus profile, and store it. This is the mtDNA adapter + /// over the generic [`navigator_domain::consensus`] engine. + /// + /// It reconciles the mutation calls of each source that has mt data into one concordance view. + /// Those sources are every alignment's chrM placement, the placement of each imported mtDNA + /// FASTA sequence, and the combined chip mtDNA placement. The key is the phylotree **mutation + /// name**, which is an rCRS coordinate and is independent of build. + /// + /// The store keeps it with `dna_type='Mt'`, so + /// [`cached_mt_profile`](Self::cached_mt_profile) reloads it immediately. It costs a lot, + /// because it places each alignment's chrM again, so it is an explicit action. It skips a + /// source with no mt data. pub async fn build_mt_profile(&self, biosample_guid: SampleGuid) -> Result { - // One mt tree in rCRS coordinates (DecodingUs remapped from hs1, FTDNA fallback), shared by - // the per-source placements below and the pooled terminal — so the variants and the terminal - // sit on the same tree + coordinate space (the Y-profile fix, applied to mtDNA). + // One mt tree in rCRS coordinates. DecodingUs remaps it from hs1, with FTDNA as the + // fallback. The placement of each source below shares it with the pooled terminal. So the + // variants and the terminal sit on the same tree and coordinate space. This is the + // Y-profile fix, applied to mtDNA. let (tree, provider) = self.mt_tree_rcrs().await?; let source_calls = self.mt_source_calls(biosample_guid, &tree).await?; - // One source per contributing test — each alignment's chrM, each imported FASTA, the chip mt - // panel — placed individually on the shared tree so the profile shows which test confirmed - // each mutation (name-keyed reconcile across sources). Sparse sources (chip) use the robust - // assembler; dense ones (WGS/FASTA) the exact one. + // One source for each test that contributes: each alignment's chrM, each imported FASTA, + // and the chip mt panel. Each places on its own on the shared tree, so the profile shows + // which test confirmed each mutation. The reconcile across sources uses the name as its + // key. A sparse source, such as a chip, uses the robust assembler. A dense one, such as WGS + // or FASTA, uses the exact assembler. let mut sources: Vec<(String, SourceType, Vec)> = Vec::new(); for (label, st, calls) in &source_calls { let assignment = if *st == SourceType::Chip { @@ -1024,12 +1075,13 @@ impl App { let mut observed = yprofile::to_observed(&sources); // Interpret once (against the current mt polarity) for the return value + summary header. let mut profile = self.interpret_mt_profile(observed.clone()).await; - // Genome-level placement of the pooled chrM call set on the same tree. A subject with no - // derived mutations carries no real placement — an alignment with a handful of off-target - // chrM reads (a Big Y) genotypes to nothing below the root. Report no terminal rather than a - // root label, and never resurrect a stale persisted root call (the old "very few mt reads → - // RSRS" artifact); the profile is meaningful only once some mutation is derived (checked on - // the *interpreted* variants). + // Genome-level placement of the pooled chrM call set on the same tree. + // + // A subject with no derived mutations has no real placement. An alignment with a few + // off-target chrM reads, such as a Big Y, genotypes to nothing below the root. Report no + // terminal, and do not report a root label. Never bring back a stale root call from the + // store, which was the old "very few mt reads → RSRS" artifact. The profile says nothing + // until some mutation goes derived, and the check runs on the *interpreted* variants. let terminal = if profile .variants .iter() @@ -1051,7 +1103,7 @@ impl App { observed.terminal_hint = terminal.clone(); profile.terminal = terminal; - // Persist observations (keyed dna_type='Mt') with the tree provider actually used. + // Store the observations, keyed dna_type='Mt', with the tree provider that the code used. self.persist_observed_profile( biosample_guid, DnaType::Mt, @@ -1063,31 +1115,37 @@ impl App { Ok(profile) } - /// **Genome-level Y placement**: pool every source's tree-locus genotype (each alignment's - /// native-build placement calls + each chip/BISDNA panel's chrY calls) into one call set by a - /// weighted [`pool_bases`] vote — keyed by SNP **name** so sources on different builds merge — - /// then place that pooled set on one canonical tree **once** via [`assemble_assignment`]. This - /// replaces voting among the per-run terminal *labels*: a sparse run no longer drags the call - /// shallow, and a branch confirmed by any source informs the placement. `Ok(None)` when the - /// subject has no Y-bearing source. Re-genotypes each source (like [`build_y_profile`]), so it is - /// only run as part of that explicit action. + /// **Genome-level Y placement**. + /// + /// Pool the tree-locus genotype of every source into one call set, by a weighted + /// [`pool_bases`] vote. Those genotypes are each alignment's native-build placement calls and + /// each chip or BISDNA panel's chrY calls. The key is the SNP **name**, so sources on different + /// builds merge. Then place that pooled set on one canonical tree **once**, through + /// [`assemble_assignment`]. + /// + /// This replaces a vote among the terminal *labels* of each run. A sparse run no longer pulls + /// the call shallow, and a branch that any source confirms informs the placement. + /// + /// `Ok(None)` when the subject has no source with Y data. It genotypes each source again, as + /// [`build_y_profile`] does, so it runs only as part of that explicit action. pub async fn place_y_consensus(&self, biosample_guid: SampleGuid) -> Result, AppError> { - // Females have no Y chromosome — no genome consensus to place. + // A female has no Y chromosome, so there is no genome consensus to place. if !self.subject_has_y_dna(biosample_guid).await? { return Ok(None); } - // The consensus follows the user's configured tree provider (Preferences / - // NAVIGATOR_Y_TREE_PROVIDER), same as the per-alignment placement. + // The consensus follows the user's configured tree provider (Preferences or + // NAVIGATOR_Y_TREE_PROVIDER), the same as the placement of one alignment. match y_tree_provider() { YTreeProvider::DecodingUs => self.place_y_consensus_decodingus(biosample_guid).await, YTreeProvider::Ftdna => self.place_y_consensus_ftdna(biosample_guid).await, } } - /// FTDNA-provider genome consensus: pool every WGS alignment + GRCh38 vendor Y-VCF on the FTDNA - /// GRCh38 tree (`base_calls` lifts CHM13/GRCh37 sources into GRCh38) and place once. One tree + - /// one coordinate space keeps polarity/coverage consistent; chips (sparse, various builds) stay in - /// the variant profile's name-keyed reconcile. + /// FTDNA-provider genome consensus. Pool every WGS alignment and GRCh38 vendor Y-VCF on the + /// FTDNA GRCh38 tree, and place once. `base_calls` lifts a CHM13 or GRCh37 source into GRCh38. + /// One tree and one coordinate space keep the polarity and the coverage consistent. A chip is + /// sparse and can be on any build, so it stays in the name-keyed reconcile of the variant + /// profile. async fn place_y_consensus_ftdna(&self, biosample_guid: SampleGuid) -> Result, AppError> { let tree_json = self.fetch_ftdna_y_tree().await?; let tree = navigator_analysis::haplo::parse_ftdna_json(&tree_json).map_err(AppError::Import)?; @@ -1095,9 +1153,10 @@ impl App { let mut sources: Vec<(SourceType, HashMap)> = Vec::new(); let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; for a in &alignments { - // Lifted GRCh38-coordinate calls; sources lacking chrY / a reference are skipped. A - // preferred-external alignment is genotyped from its chrY GVCF (lifted native→GRCh38 by - // `gvcf_base_calls`) instead of walking the CRAM. + // Lifted GRCh38-coordinate calls. The code skips a source with no chrY or no + // reference. It genotypes a preferred-external alignment from its chrY GVCF, which + // `gvcf_base_calls` lifts from the native build to GRCh38, and it does not walk the + // CRAM. let calls = match (prefer_external_calls(), chr_y_gvcf_for_alignment(a)) { (true, Some(gvcf)) => self .gvcf_base_calls(a.id, "chrY", &gvcf, &tree, tree_build_for_contig("chrY")) @@ -1132,27 +1191,32 @@ impl App { Ok(Some(assemble_assignment(&tree, &pooled))) } - /// DecodingUs-provider genome consensus (the default): genotype every WGS alignment against the - /// DecodingUs Y tree in each source's *native* build, group by build, pool by position, and place - /// on the build carrying the most evidence. + /// DecodingUs-provider genome consensus, which is the default. Genotype every WGS alignment + /// against the DecodingUs Y tree in each source's *native* build. Group by build, pool by + /// position, and place on the build with the most evidence. async fn place_y_consensus_decodingus( &self, biosample_guid: SampleGuid, ) -> Result, AppError> { - // Genotype every WGS alignment against the **DecodingUs** Y tree — the workspace's configured - // provider, served from the local cache — in each source's *native* build (`hs1` for CHM13, - // `GRCh38`, `GRCh37`). No liftover and no FTDNA dependency: the per-alignment genotype is - // exactly the one the Y assignment already cached, so this reuses that walk rather than paying - // a second, FTDNA-coordinate one. Sources are grouped by build and pooled by **position** - // within one coordinate space (a single build per subject is the norm); the build carrying the - // most evidence is placed once. Pooling across builds by position would mix coordinate systems - // — cross-build merging lives in the variant profile's name-keyed reconcile, not here. + // Genotype every WGS alignment against the **DecodingUs** Y tree, which is the workspace's + // configured provider, served from the local cache. Use each source's *native* build: + // `hs1` for CHM13, `GRCh38`, or `GRCh37`. There is no liftover and no FTDNA dependency, + // because the genotype of one alignment is exactly the one that the Y assignment already + // cached. So this reuses that walk, and does not pay for a second walk in FTDNA + // coordinates. + // + // The code groups the sources by build, and pools them by **position** inside one + // coordinate space. One build for each subject is the norm. It then places the build with + // the most evidence, once. A pool across builds by position would mix coordinate systems. + // The merge across builds lives in the name-keyed reconcile of the variant profile, and + // not here. let tree_json = self.fetch_decodingus_y_tree().await?; let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; - // Parse the DecodingUs tree once per distinct build the sources use (cheap — the JSON is - // memoized). Built up front so the async genotyping loop holds only shared borrows of `trees`. + // Parse the DecodingUs tree once for each distinct build that the sources use. This is + // low-cost, because a memo holds the JSON. The code builds them first, so the async + // genotyping loop holds only shared borrows of `trees`. let mut builds: std::collections::HashSet<&'static str> = alignments .iter() .filter_map(|a| decodingus_build_key(&a.reference_build)) @@ -1182,8 +1246,9 @@ impl App { continue; }; let Some(tree) = trees.get(bk) else { continue }; - // Native build → no liftover; the cache-key matches the Y assignment's, so a CRAM walk is - // a hit — but a preferred-external alignment is genotyped from its GVCF instead (no decode). + // The build is native, so there is no liftover. The cache key matches the one of the Y + // assignment, so a CRAM walk is a hit. But the code genotypes a preferred-external + // alignment from its GVCF instead, with no decode. let Ok(calls) = self.consensus_base_calls(a, "chrY", tree, None).await else { continue; }; @@ -1192,9 +1257,9 @@ impl App { } } - // Vendor Y-NGS VCFs (FTDNA Big Y / YSEQ / Full Genomes / Nebula) are dense direct Y-SNP calls; - // fold each into its own build's group (strand-reconciled to that build's tree). Chips stay in - // the variant profile's name-keyed reconcile. + // A vendor Y-NGS VCF (FTDNA Big Y, YSEQ, Full Genomes, Nebula) holds dense, direct Y-SNP + // calls. Fold each one into the group of its own build, strand-reconciled to that build's + // tree. A chip stays in the name-keyed reconcile of the variant profile. for set in &vsets { if set.source_type == SourceType::Chip { continue; @@ -1216,7 +1281,8 @@ impl App { } } - // Place on the build carrying the most evidence (the subject's primary coordinate space). + // Place on the build with the most evidence, which is the subject's primary coordinate + // space. let Some(bk) = by_build .iter() .max_by_key(|(_, s)| s.iter().map(|(_, c)| c.len()).sum::()) @@ -1228,11 +1294,15 @@ impl App { Ok(Some(assemble_assignment(&trees[bk], &pooled))) } - /// Diagnostic: dump the Y **descent** for one subject SNP-by-SNP — the reported state + observed - /// base against the **incoming tree's** polarity in every DecodingUs build (hs1 / GRCh38 / GRCh37). - /// This is the "compare the tree vs the calls" log: a backbone SNP the sample must carry that reads - /// ancestral shows here as `state=Ancestral base=` with a build whose polarity is - /// flipped (`hs1: A>G GRCh38: G>A`), pinpointing a tree-polarity problem vs a genotyping one. + /// Diagnostic: dump the Y **descent** for one subject, SNP by SNP. It shows the reported state + /// and the observed base against the polarity of the **new tree** in every DecodingUs build + /// (hs1, GRCh38, GRCh37). + /// + /// This is the "compare the tree against the calls" log. Take a backbone SNP that the sample + /// must carry, but that reads ancestral. Its line shows `state=Ancestral base=`. + /// Beside that line, one build shows a flipped polarity (`hs1: A>G GRCh38: G>A`). That + /// separates a problem in the tree polarity from a problem in the genotyping. + /// /// Read-only. TSV: `node snp pos state base hs1 GRCh38 GRCh37`. pub async fn debug_y_descent(&self, biosample_guid: SampleGuid) -> Result { use navigator_analysis::haplo; @@ -1284,13 +1354,16 @@ impl App { Ok(out) } - /// Diagnostic: genotype a **single alignment** against the DecodingUs Y tree in its native build - /// and dump, per SNP down the placed lineage, the raw read pileup **behind** each call — the - /// reference base, the A/C/G/T passing-read tally, the consensus base, the tree's ancestral/derived - /// alleles, and the resulting state. This is the "calls generated" log: it shows whether a backbone - /// SNP that reads ancestral is (a) genuinely ancestral in the reads, (b) a coordinate/position - /// mismatch (reads a different base than the tree allele), or (c) a low-depth artifact. Read-only. - /// TSV: `node snp pos tree(anc>der) ref A C G T depth called state`. + /// Diagnostic: genotype a **single alignment** against the DecodingUs Y tree in its native + /// build. For each SNP down the placed lineage, dump the raw read pileup **behind** the call. + /// The dump holds the reference base, the A/C/G/T tally of the reads that pass, the consensus + /// base, the two tree alleles, and the call state. + /// + /// This is the "calls the code made" log. Take a backbone SNP that reads ancestral. This says + /// whether it is truly ancestral in the reads, or a coordinate mismatch, or an artifact of low + /// depth. A coordinate mismatch gives a base other than the tree allele. + /// + /// Read-only. TSV: `node snp pos tree(anc>der) ref A C G T depth called state`. pub async fn debug_y_calls(&self, alignment_id: i64) -> Result { use navigator_analysis::{caller, haplo, reader}; let aln = self.alignment_or_err(alignment_id).await?; @@ -1302,16 +1375,17 @@ impl App { }; let json = self.fetch_decodingus_y_tree().await?; let tree = haplo::parse_decodingus_json(&json, bk).map_err(AppError::Import)?; - // Native-build genotyping (no liftover) — the same walk place_y_consensus uses; the cache hit - // means `base_calls` returns the identical winning bases we are auditing here. + // Genotype in the native build, with no liftover. This is the same walk that + // place_y_consensus uses. The cache hits, so `base_calls` gives back exactly the bases that + // won, which are the bases under audit here. let calls = self.base_calls(alignment_id, "chrY", &tree, None).await?; let assignment = assemble_assignment(&tree, &calls); if assignment.lineage.is_empty() { return Ok(format!("alignment {alignment_id} ({bk}): no Y placement\n")); } - // Localize the BAM/CRAM and resolve its reference exactly as `base_calls` does, then tally the - // raw reads at the lineage positions and read the reference base there. + // Localize the BAM/CRAM and resolve its reference exactly as `base_calls` does. Then tally + // the raw reads at the lineage positions, and read the reference base there. let bam = self .localize(Path::new( &aln.bam_path.clone().ok_or(AppError::MissingPaths(alignment_id))?, @@ -1339,7 +1413,8 @@ impl App { caller::tally_at(&bam2, &contig2, &targets2, ¶ms, ref2.as_deref()) }) .await??; - // Reference base per lineage position (0-based index into the contig), best-effort. + // The reference base at each lineage position (0-based index into the contig), + // best-effort. let refseq: Option> = match reference.as_deref() { Some(r) => { let (r, c) = (r.to_path_buf(), resolved.clone()); @@ -1390,9 +1465,10 @@ impl App { Ok(out) } - /// Pick a single alignment to target for [`Self::debug_y_calls`] when only a subject is given: - /// prefer a CHM13/HiFi alignment (native tree, no liftover — the cleanest to audit), else the - /// first. Returns `None` when the subject has no alignments. + /// Pick a single alignment for [`Self::debug_y_calls`] when the caller gives only a subject. + /// Prefer a CHM13/HiFi alignment, which uses the native tree with no liftover and is the + /// easiest to audit. If there is none, take the first. Returns `None` when the subject has no + /// alignments. pub async fn pick_y_debug_alignment(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let pick = alignments @@ -1410,10 +1486,10 @@ impl App { Ok(pick.map(|a| a.id)) } - /// Pick a single alignment to genotype **mtDNA** against: skip Y-only runs (an FTDNA Big-Y - /// carries no `chrM` reads, so it would yield an all-no-call report while a usable WGS - /// alignment sat unselected), then prefer CHM13, else the first survivor. If *every* run is - /// Y-only, fall back to the first alignment rather than reporting "no alignment". + /// Pick a single alignment to genotype **mtDNA** against. Skip a Y-only run: an FTDNA Big-Y + /// carries no `chrM` reads, so it would give an all-no-call report while a usable WGS alignment + /// sat unpicked. Then prefer CHM13, else take the first one left. If *every* run is Y-only, + /// fall back to the first alignment, and do not report "no alignment". pub async fn pick_mt_alignment(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut mt_capable: Vec<&Alignment> = Vec::new(); @@ -1447,10 +1523,12 @@ impl App { } } - /// Diagnostic: trace the DecodingUs genome-consensus Y placement for one subject — the pooled - /// build, the Kulczynski top candidates + admissibility, the assembled terminal, a strict - /// root→tip descent, and the per-node derived/ancestral tally down the assembled lineage (so an - /// over-deepening tunnel through ancestral branches is visible). Read-only. + /// Diagnostic: trace the DecodingUs genome-consensus Y placement for one subject. + /// + /// It shows the pooled build, the top Kulczynski candidates with their admissibility, and the + /// assembled terminal. It then shows a strict root→tip descent, and the derived and ancestral + /// tally at each node down the assembled lineage. That last part makes a tunnel through + /// ancestral branches visible. Read-only. pub async fn debug_y_placement(&self, biosample_guid: SampleGuid) -> Result { use navigator_analysis::haplo; let tree_json = self.fetch_decodingus_y_tree().await?; @@ -1559,10 +1637,15 @@ impl App { Ok(out) } - /// Assemble a subject's lightweight [`YMatchProfile`] from **cached** data only (no re-genotyping): - /// the persisted consensus Y profile (derived/novel SNP-name sets + terminal), the terminal's - /// root→tip lineage from `tree`, and the first imported Y-STR panel's markers. `Ok(None)` when the - /// subject has neither a placed Y profile nor an STR panel — nothing to match on. + /// Assemble a subject's lightweight [`YMatchProfile`] from **cached** data only, with no + /// second genotype pass. + /// + /// It takes three things. The first is the stored consensus Y profile, which holds the derived + /// and novel SNP-name sets and the terminal. The second is the terminal's root→tip lineage from + /// `tree`. The third is the markers of the first imported Y-STR panel. + /// + /// `Ok(None)` when the subject has no placed Y profile and no STR panel, because then there is + /// nothing to match on. async fn y_match_profile( &self, b: &Biosample, @@ -1620,14 +1703,17 @@ impl App { })) } - /// Rank every other workspace subject against `query_guid` by Y relatedness (gap §2) — shared - /// derived/novel SNPs, divergence haplogroup, Y-STR genetic distance, and rough SNP/STR TMRCA. - /// One-vs-all over the workspace (or one project when `project_id` is set); local-only. Consumes - /// **cached** profiles so it is cheap over hundreds of subjects (no re-genotyping). `Ok(vec![])` - /// when the query subject has no matchable Y data. + /// Rank every other workspace subject against `query_guid` by Y relatedness (gap §2). The + /// signals are the shared derived and novel SNPs, the divergence haplogroup, the Y-STR genetic + /// distance, and a rough SNP and STR TMRCA. + /// + /// It compares one subject against all, over the workspace, or over one project when + /// `project_id` has a value. It is local only. It reads **cached** profiles, with no second + /// genotype pass, so it is low-cost over hundreds of subjects. `Ok(vec![])` when the query + /// subject has no Y data to match on. pub async fn y_matches(&self, query_guid: SampleGuid, project_id: Option) -> Result, AppError> { - // The tree only supplies the divergence haplogroup; shared-SNP and STR matching work without - // it, so a fetch failure degrades gracefully rather than failing the whole search. + // The tree supplies the divergence haplogroup alone. The shared-SNP and STR match work + // without it, so a failed fetch loses only that one field, and the search continues. let tree = match self.fetch_ftdna_y_tree().await { Ok(json) => navigator_analysis::haplo::parse_ftdna_json(&json).ok(), Err(_) => None, @@ -1638,7 +1724,7 @@ impl App { None => self.list_all_biosamples().await?, }; - // The query subject may sit outside the chosen project — load it directly if so. + // The query subject can sit outside the chosen project. Load it directly if it does. let query_bio = match candidates.iter().find(|b| b.guid == query_guid).cloned() { Some(b) => b, None => biosample::get(self.store.pool(), query_guid) @@ -1663,12 +1749,14 @@ impl App { Ok(ranked) } - /// Per-source rCRS-coordinate mtDNA calls for a subject — `(label, type, calls)` keyed by rCRS - /// position — shared by [`place_mt_consensus`] (pooled placement) and [`build_mt_profile`] - /// (per-source concordance). `tree` must be in rCRS coordinates: each alignment's `chrM` is - /// genotyped against it (cached; `base_calls` maps a CHM13 `chrM` back to rCRS, GRCh38/rCRS - /// direct), each imported FASTA is sampled at every rCRS position, and the chip mt panel is - /// strand-reconciled to it. + /// The rCRS-coordinate mtDNA calls of each source for a subject, as `(label, type, calls)` + /// keyed by rCRS position. [`place_mt_consensus`] uses it for the pooled placement, and + /// [`build_mt_profile`] for the concordance across sources. + /// + /// `tree` must be in rCRS coordinates. Each alignment's `chrM` genotypes against it, from the + /// cache, and `base_calls` maps a CHM13 `chrM` back to rCRS, while GRCh38 and rCRS are direct. + /// Each imported FASTA samples at every rCRS position. The chip mt panel strand-reconciles to + /// it. async fn mt_source_calls( &self, biosample_guid: SampleGuid, @@ -1676,8 +1764,8 @@ impl App { ) -> Result)>, AppError> { let mut sources: Vec<(String, SourceType, HashMap)> = Vec::new(); - // Each alignment's chrM genotype. `None` source-build → rCRS-direct / CHM13-chrM lift. A - // preferred-external alignment is genotyped from its chrM GVCF instead of the CRAM. + // Each alignment's chrM genotype. A `None` source-build means rCRS-direct or a CHM13-chrM + // lift. The code takes a preferred-external alignment from its chrM GVCF, not the CRAM. let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; for a in &alignments { let Ok(calls) = self.consensus_base_calls(a, "chrM", tree, None).await else { @@ -1692,7 +1780,7 @@ impl App { } } - // Each imported mtDNA FASTA — the full sequence sampled at every rCRS position. + // Each imported mtDNA FASTA: the full sequence, sampled at every rCRS position. for s in &self.list_mtdna_sequences(biosample_guid).await? { let Some(seq) = mtdna_store::get(self.store.pool(), s.id).await? else { continue; @@ -1714,10 +1802,11 @@ impl App { let sets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; - // Each imported **non-chip** variant set carrying chrM SNPs — a whole-genome VCF or a - // CompleteGenomics masterVar. These report forward-strand ref/alt on rCRS coordinates - // (GRCh37/GRCh38 chrM = rCRS), so the alt base is used raw like an alignment's chrM call - // (no TOP-strand reconciliation, unlike the chip panel below). + // Each imported **non-chip** variant set that holds chrM SNPs: a whole-genome VCF, or a + // CompleteGenomics masterVar. Such a set gives forward-strand ref/alt on rCRS coordinates, + // because GRCh37 and GRCh38 chrM are rCRS. So the code takes the alt base raw, as it does + // for an alignment's chrM call. There is no TOP-strand reconcile, unlike the chip panel + // below. for set in sets.iter().filter(|s| s.source_type != SourceType::Chip) { let calls: HashMap = set .calls @@ -1759,12 +1848,15 @@ impl App { Ok(sources) } - /// **Genome-level mtDNA placement**: the mt counterpart to [`place_y_consensus`]. Pools every - /// source's rCRS-coordinate genotype ([`mt_source_calls`]) by [`pool_votes`] vote keyed by - /// **position** (rCRS is the only mt coordinate system → no name indirection), then places the - /// pooled set on the mt tree once. The tree is the **DecodingUs** mt tree (the configured - /// provider) remapped onto rCRS, with the FTDNA mt tree as fallback ([`mt_tree_rcrs`]). `Ok(None)` - /// when the subject has no mt-bearing source. + /// **Genome-level mtDNA placement**: the mt counterpart to [`place_y_consensus`]. + /// + /// It pools the rCRS-coordinate genotype of every source ([`mt_source_calls`]) by a + /// [`pool_votes`] vote, keyed by **position**. There is no indirection through a name, because + /// rCRS is the only mt coordinate system. It then places the pooled set on the mt tree once. + /// + /// The tree is the **DecodingUs** mt tree, from the configured provider, remapped onto rCRS. + /// The FTDNA mt tree is the fallback ([`mt_tree_rcrs`]). `Ok(None)` when the subject has no + /// source with mt data. pub async fn place_mt_consensus(&self, biosample_guid: SampleGuid) -> Result, AppError> { let (tree, _provider) = self.mt_tree_rcrs().await?; let sources = self.mt_source_calls(biosample_guid, &tree).await?; @@ -1784,12 +1876,17 @@ impl App { Ok(Some(assignment)) } - /// Build a YFull-style [`DescentReport`] for a subject's Y or mtDNA lineage from the **already - /// persisted** variant profile — no re-genotyping. Reads the cached profile for its terminal + - /// per-SNP states (keyed by build-independent SNP name), then walks the FTDNA tree from the - /// terminal to the root, attaching each node's defining SNPs with the sample's call (`NoCall` for - /// an untested equivalent). `Ok(None)` when the profile is not built yet or has no terminal — the - /// UI then offers to build it (one expensive, persisted step that also powers the variant tabs). + /// Build a YFull-style [`DescentReport`] for a subject's Y or mtDNA lineage from the variant + /// profile that the store **already holds**. There is no second genotype pass. + /// + /// It reads the cached profile for its terminal and the state of each SNP, keyed by the SNP + /// name, which is independent of build. It then walks the FTDNA tree from the terminal to the + /// root. At each node it attaches the SNPs that define the node, with the sample's call, and + /// gives `NoCall` for an equivalent that no test covered. + /// + /// `Ok(None)` when no build has run yet, or when the profile has no terminal. The UI then + /// offers to build it. That is one costly step, and the store keeps the result, which also + /// feeds the variant tabs. pub async fn descent_report( &self, biosample_guid: SampleGuid, @@ -1797,8 +1894,8 @@ impl App { ) -> Result, AppError> { use navigator_domain::consensus::ConsensusState; - // Cheap first: the persisted profile. No profile / no terminal → nothing to draw, and we - // skip the (multi-MB) tree fetch + parse entirely. + // The low-cost step first: the stored profile. With no profile, or no terminal, there is + // nothing to draw, and the code skips the multi-MB tree fetch and parse. let profile = match dna { DnaType::Y => self.cached_y_profile(biosample_guid).await?, DnaType::Mt => self.cached_mt_profile(biosample_guid).await?, @@ -1808,32 +1905,37 @@ impl App { return Ok(None); }; - // Render on the configured provider's tree so the node names + defining SNPs line up with the - // profile's placement (which followed the same provider). Y: DecodingUs in the subject's - // native build, or the FTDNA GRCh38 tree; mtDNA: DecodingUs remapped hs1→rCRS, or FTDNA — via - // `mt_tree_rcrs`, which already honors the provider. + // Draw on the tree of the configured provider. The node names and their SNPs then line up + // with the profile's placement, which followed the same provider. For Y that is DecodingUs + // in the subject's native build, or the FTDNA GRCh38 tree. For mtDNA it is DecodingUs + // remapped from hs1 to rCRS, or FTDNA, through `mt_tree_rcrs`, which already obeys the + // provider. let tree = match dna { DnaType::Y => match y_tree_provider() { YTreeProvider::DecodingUs => { let json = self.fetch_decodingus_y_tree().await?; // Parse in the tree's **native** hs1 space, not the subject's alignment build. // - // This report joins the profile to the tree by SNP *name* (`state_by_name` - // below); the loci positions it carries are for display and export only. So - // parsing under a narrower build buys nothing and costs loci: a variant with no - // coordinate in the parse build is silently dropped - // (`flatten_du_node`'s `coordinates.get(build_key)?`), and a node whose every - // defining variant is dropped survives as a real node with no SNPs — which the - // renderer then correctly hides as an empty block. + // This report joins the profile to the tree by SNP *name*, through + // `state_by_name` below. The loci positions that it carries are for display and + // export only. + // + // So a parse under a narrower build gains nothing and loses loci. A variant + // with no coordinate in the parse build drops out with no message, at + // `flatten_du_node`'s `coordinates.get(build_key)?`. A node that loses every + // one of its variants stays as a real node with no SNPs. The renderer then + // correctly hides it as an empty block. // - // That is not hypothetical. hs1 covers 99.8% of the tree's ~204k variants, - // GRCh38 only 86.5%: most DecodingUs-discovered (`DU`-named) SNPs exist in CHM13 - // coordinates alone, since only a few hundred were ever mapped back to the older - // references. `1087` is placed at `R-DU17762`, whose sole defining variant - // `DU17762` has an hs1 coordinate and nothing else — so parsing that subject - // under GRCh38 (which it was, its first alignment being GRCh38) emptied the - // terminal block and the descent visibly stopped one branch short, at - // `R-BY57568`, while the terminal name itself was right. + // That is not a theory. hs1 covers 99.8% of the tree's ~204k variants, and + // GRCh38 only 86.5%. Most DecodingUs-discovered (`DU`-named) SNPs exist in + // CHM13 coordinates alone, because only a few hundred ever mapped back to the + // older references. + // + // Subject `1087` places at `R-DU17762`. The one variant that defines that node, + // `DU17762`, has an hs1 coordinate and nothing else. A parse of that subject + // under GRCh38, which is what it got, because its first alignment is GRCh38, + // emptied the terminal block. The descent then stopped one branch short, at + // `R-BY57568`, although the terminal name itself was right. navigator_analysis::haplo::parse_decodingus_json(&json, DECODINGUS_NATIVE_BUILD) .map_err(AppError::Import)? } @@ -1860,7 +1962,8 @@ impl App { (v.name.clone(), state) }) .collect(); - // The actual consensus nucleotide per SNP, so the descent shows/exports the observed allele. + // The consensus nucleotide at each SNP, so the descent shows and exports the observed + // allele. let base_by_name: std::collections::HashMap = profile .variants .iter() @@ -1883,14 +1986,19 @@ impl App { Ok(Some(DescentReport { dna, terminal, nodes })) } - /// Build a [`BranchReport`]: the sample's genotype at every defining marker of `node_query`'s - /// descendant subtree (Y or mtDNA), with per-marker evidence — for spot-checking placement and - /// exchanging observations. `node_query` matches a haplogroup name (`R-FGC29071`) or a defining - /// marker (`FGC29071`); `max_depth` bounds descent (`None` = the whole subtree). + /// Build a [`BranchReport`]. It gives the sample's genotype at every marker that defines a + /// node in the descendant subtree of `node_query`, for Y or mtDNA. Each marker carries its + /// evidence. The report lets you spot-check a placement, and exchange observations. + /// + /// `node_query` matches a haplogroup name (`R-FGC29071`) or a marker name (`FGC29071`). + /// `max_depth` bounds the descent, and `None` means the whole subtree. /// - /// Genotypes the subtree **fresh** over the tree's loci (not the placement profile), so branches - /// the sample is *ancestral* for are reported too. Observed bases + evidence come from a per- - /// sample chrY GVCF sidecar when present (rich DP/AD/GQ, ref blocks), else the pileup caller. + /// It genotypes the subtree **fresh** over the tree's loci, and not from the placement profile. + /// So the report also covers a branch that the sample is *ancestral* for. + /// + /// The observed bases and the evidence come from the sample's own chrY GVCF sidecar when there + /// is one. That sidecar holds DP/AD/GQ and ref blocks. If there is none, they come from the + /// pileup caller. pub async fn branch_report( &self, alignment_id: i64, @@ -1905,9 +2013,10 @@ impl App { // Tree + observed base calls over ALL tree loci (covers off-path descendant branches). let (tree, calls, contig, gvcf) = match dna { DnaType::Y => { - // Probe the sidecar *before* genotyping: with one present the tree comes straight - // from JSON and the calls from the GVCF, so the per-locus pileup walk is skipped - // entirely (the point of the fast path — cf. `assign_y_from_gvcf`). + // Probe the sidecar *before* the genotype step. When one is present, the tree + // comes straight from JSON and the calls come from the GVCF. The code then skips + // the pileup walk at each locus, which is the point of the fast path. Compare + // `assign_y_from_gvcf`. match crate::fastpath::chr_y_gvcf_for_alignment(&aln) { Some(gvcf) => { let build_key = decodingus_build_key(&aln.reference_build).ok_or_else(|| { @@ -1931,8 +2040,8 @@ impl App { DnaType::Mt => { // `mt_tree_rcrs` hands back the *provider* (decodingus/ftdna), not a reference // build. `tree_source_build` must stay `None`: a non-build string there makes - // `lifted_targets` return early, skipping the rCRS↔chrM map a CHM13 alignment - // needs (its chrM is a circular permutation of rCRS). + // `lifted_targets` return early. It would then skip the rCRS↔chrM map that a + // CHM13 alignment needs, because its chrM is a circular permutation of rCRS. let (tree, _provider) = self.mt_tree_rcrs().await?; let calls = self.base_calls(alignment_id, "chrM", &tree, None).await?; (tree, calls, "chrM", None) @@ -1950,7 +2059,7 @@ impl App { let root = tree.nodes.get(&root_id).map(|n| n.name.clone()).unwrap_or_default(); let subtree = navigator_analysis::haplo::subtree_report(&tree, &calls, root_id, max_depth); - // GVCF per-marker evidence (Y with a sidecar): ungated DP/AD/GQ, off-thread. + // GVCF evidence at each marker (Y with a sidecar): DP/AD/GQ with no gate, off-thread. let evidence = match gvcf { Some(gvcf) => { let positions: HashSet = subtree.iter().map(|r| r.snp.position).collect(); @@ -1973,8 +2082,9 @@ impl App { None if gvcf_backed => ("gvcf", None, None, None), None => ("pileup", None, None, None), }; - // The conditions are orthogonal — an uncalled indel is both — so compose the tags - // rather than report only the first. Picking one let "indel/MNV" hide a no-call. + // The conditions are orthogonal, because an indel with no call is both. So build + // up the tags, and do not report the first alone. One tag let "indel/MNV" hide a + // no-call. let mut tags: Vec<&str> = Vec::new(); if is_indel { tags.push("indel/MNV"); @@ -2030,8 +2140,9 @@ impl App { )) } - /// The persisted autosomal consensus-profile snapshot for a subject, if built — cheap (no - /// genotyping). `None` until [`build_autosomal_profile`](Self::build_autosomal_profile) runs. + /// The stored autosomal consensus-profile snapshot for a subject, if the app has built one. It + /// is low-cost, with no genotyping. `None` until + /// [`build_autosomal_profile`](Self::build_autosomal_profile) runs. pub async fn cached_autosomal_profile( &self, biosample_guid: SampleGuid, @@ -2042,24 +2153,33 @@ impl App { } } - /// Build (and persist) the multi-source **autosomal** consensus profile — the diploid (0/1/2) - /// adapter over the generic [`navigator_domain::consensus`] engine. Genotypes every WGS alignment - /// and imported chip over the canonical CHM13 **IBD panel** ([`ibd_panel_dosages`](Self::ibd_panel_dosages)) - /// and reconciles the per-site dosages into a voted genotype (confirmed where sources agree, - /// conflict where they do not), keyed by rsID. Persisted with `dna_type='Auto'`. Requires the IBD - /// panel asset (built with `panelbuild ibd-panel`); errors if it is missing. + /// Build the multi-source **autosomal** consensus profile, and store it. This is the diploid + /// (0/1/2) adapter over the generic [`navigator_domain::consensus`] engine. + /// + /// It genotypes every WGS alignment and imported chip over the canonical CHM13 **IBD panel** + /// ([`ibd_panel_dosages`](Self::ibd_panel_dosages)). It then reconciles the dosage at each site + /// into a voted genotype, keyed by rsID. A site where the sources agree counts as confirmed, + /// and a site where they disagree counts as a conflict. + /// + /// The store keeps it with `dna_type='Auto'`. It needs the IBD panel asset, which + /// `panelbuild ibd-panel` builds, and it gives an error when that asset is absent. pub async fn build_autosomal_profile(&self, biosample_guid: SampleGuid) -> Result { // Full build: genotype any alignment whose panel dosages are not cached yet. self.build_autosomal_profile_inner(biosample_guid, false).await } - /// **Progressive refresh** of the autosomal consensus (progressive-consensus, docs §7.17): reduce - /// over the per-source dosages that are **already available** — every chip / WGS-VCF (which resolve - /// cheaply with no decode) plus any alignment whose panel dosages are *cached* - /// ([`Self::cached_alignment_panel_dosages`]) — **without** decoding an uncached alignment. Cheap - /// and safe to call after every import; alignments get their dosages populated separately by the - /// panel batch-process mode, and the next refresh folds them in. Returns the refreshed profile, or - /// `Ok(None)` when the subject has no available autosomal source yet. + /// **Progressive refresh** of the autosomal consensus (progressive-consensus, docs §7.17). + /// + /// It reduces over the dosages of each source that are **already available**. Those are every + /// chip and WGS-VCF, which resolve at low cost with no decode, plus any alignment whose panel + /// dosages are in the cache ([`Self::cached_alignment_panel_dosages`]). It **never** decodes an + /// alignment that the cache does not hold. + /// + /// It is low-cost and safe to call after every import. The panel batch-process mode fills in + /// the dosages of an alignment separately, and the next refresh folds them in. + /// + /// It returns the refreshed profile, or `Ok(None)` when the subject has no autosomal source + /// available yet. pub async fn refresh_autosomal_consensus( &self, biosample_guid: SampleGuid, @@ -2068,28 +2188,35 @@ impl App { .await .map(Some) .or_else(|e| match e { - // "no source" is not an error for a refresh — the subject just has nothing cached yet. + // "no source" is not an error for a refresh. The cache holds nothing for this + // subject yet. AppError::Import(_) => Ok(None), other => Err(other), }) } - /// **Panel batch-process mode** (progressive-consensus, docs §7.17): genotype one alignment at - /// the full-1240k IBD panel and **cache** the dosages ([`Self::ibd_panel_dosages`]) — the - /// expensive per-source step (a whole-genome decode) that populates the consensus progressively. - /// Returns the number of panel sites genotyped. **Does not** refresh the consensus — the caller - /// refreshes **once** after a batch (reconciling millions of observations per source is wasted - /// work if repeated per alignment); use [`Self::refresh_autosomal_consensus`] at the batch - /// boundary. If the dosages are already cached this is a cheap read. + /// **Panel batch-process mode** (progressive-consensus, docs §7.17). Genotype one alignment at + /// the full-1240k IBD panel, and **cache** the dosages ([`Self::ibd_panel_dosages`]). That is + /// the costly step for each source, a whole-genome decode, and it fills the consensus one + /// source at a time. It returns the count of panel sites that it genotyped. + /// + /// It **does not** refresh the consensus. The caller refreshes **once** after a batch, with + /// [`Self::refresh_autosomal_consensus`] at the batch boundary. One reconcile handles millions + /// of observations for each source, so to repeat it for each alignment wastes work. + /// + /// If the cache already holds the dosages, this is a low-cost read. pub async fn genotype_panel_for_alignment(&self, alignment_id: i64) -> Result { Ok(self.ibd_panel_dosages(IbdSource::Alignment(alignment_id)).await?.len()) } - /// The subject's best alignment for panel genotyping, by **callable quality**: - /// `genome_territory × pct_10x × (1 − pct_exc_mapq)` — well-mapped, diploid-callable bases. Build- - /// agnostic (the IBD panel re-keys GRCh37/38 as well as CHM13), so it picks the cleanest - /// whole-genome WGS over a deep-but-targeted or too-shallow test. Requires a recorded BAM/CRAM and - /// a cached coverage artifact; `None` when the subject has neither. + /// The subject's best alignment for panel genotyping, by **callable quality**, which is + /// `genome_territory × pct_10x × (1 − pct_exc_mapq)`. That measures the bases that are + /// well-mapped and callable as a diploid. + /// + /// It works on any build, because the IBD panel re-keys GRCh37, GRCh38, and CHM13. So it picks + /// the cleanest whole-genome WGS over a test that is deep but targeted, or one that is too + /// shallow. It needs a recorded BAM/CRAM and a cached coverage artifact, and gives `None` when + /// the subject has neither. async fn best_callable_alignment(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut best: Option<(f64, i64)> = None; @@ -2108,13 +2235,17 @@ impl App { Ok(best.map(|(_, id)| id)) } - /// **Panel batch-process mode, subject-level** (progressive-consensus, docs §7.17): genotype the - /// subject's single **best-callable** alignment ([`Self::best_callable_alignment`]) at the 1240k - /// panel and refresh the autosomal consensus **once**. Chips and WGS-VCFs need no genotyping — - /// they resolve into the consensus during the refresh — so this pays at most one whole-genome - /// decode per subject (vs one per redundant same-person alignment). Returns - /// `(alignment_id, sites)`, or `None` when the subject has no callable alignment (its chips/VCFs - /// still get folded into the consensus). + /// **Panel batch-process mode, at subject level** (progressive-consensus, docs §7.17). + /// Genotype the subject's single **best-callable** alignment + /// ([`Self::best_callable_alignment`]) at the 1240k panel, and refresh the autosomal consensus + /// **once**. + /// + /// A chip or a WGS-VCF needs no genotyping, because it resolves into the consensus during the + /// refresh. So this pays for one whole-genome decode for each subject at most, and not one for + /// each redundant alignment of the same person. + /// + /// It returns `(alignment_id, sites)`, or `None` when the subject has no callable alignment. + /// The refresh still folds that subject's chips and VCFs into the consensus. pub async fn genotype_panel_for_subject( &self, biosample_guid: SampleGuid, @@ -2125,7 +2256,8 @@ impl App { } else { None }; - // Reconcile once — folds the freshly-cached alignment (if any) plus every chip / WGS-VCF. + // Reconcile once. This folds in the alignment that the code has just cached, if there is + // one, plus every chip and WGS-VCF. let _ = self.refresh_autosomal_consensus(biosample_guid).await?; Ok(picked) } @@ -2152,19 +2284,23 @@ impl App { }; let mut sources: Vec<(String, SourceType, Vec)> = Vec::new(); - // Remember the last source error: if *every* source fails (e.g. the panel asset is missing), - // surface it rather than silently returning an empty profile; a one-off per-source failure - // (a chip with no stored raw file, an alignment lacking a BAM) is just skipped. + // Remember the last source error. If *every* source fails, because the panel asset is + // absent for example, show that error. Do not return an empty profile with no message. + // + // The code skips a failure in one source on its own. That covers a chip with no stored raw + // file, or an alignment with no BAM. let mut last_err: Option = None; - // One source per WGS alignment (panel-genotyped, cached per alignment). The IBD panel carries - // every build's coordinates, so `ibd_panel_dosages` genotypes a CHM13 alignment at its native - // loci and a GRCh37/GRCh38 alignment at that build's loci, re-keying the result to canonical - // CHM13. A build the panel does not cover yields no genotypes and is skipped downstream. + // One source for each WGS alignment, genotyped at the panel, with a cache entry for each + // alignment. The IBD panel carries the coordinates of every build. So `ibd_panel_dosages` + // genotypes a CHM13 alignment at its native loci, and a GRCh37 or GRCh38 alignment at that + // build's loci. It then re-keys the result to canonical CHM13. A build that the panel does + // not cover gives no genotypes, and the code downstream skips it. let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; for a in &alignments { - // Progressive refresh reduces over cached dosages only — an uncached alignment is skipped - // (its dosages get populated by the panel batch mode), never decoded inline here. + // The progressive refresh reduces over cached dosages only. It skips an alignment that + // the cache does not hold, and never decodes one here. The panel batch mode fills in + // the dosages of such an alignment. let dosages = if cached_alignments_only { self.cached_alignment_panel_dosages(a.id).await? } else { @@ -2184,7 +2320,8 @@ impl App { } } - // One source per imported chip (resolved to canonical panel dosages, no alignment needed). + // One source for each imported chip, resolved to canonical panel dosages, with no + // alignment. let chips = self.list_chip_profiles(biosample_guid).await?; for c in &chips { match self.ibd_panel_dosages(IbdSource::Chip(c.id)).await { @@ -2198,11 +2335,13 @@ impl App { } } - // One source per **genome-wide** imported variant set — a WGS VCF or a CompleteGenomics - // masterVar (no alignment needed; resolved to panel dosages with unlisted sites taken as - // hom-reference). Only `WgsShortRead`/`WgsLongRead`: that hom-ref default is valid solely - // for a source that genotyped the whole genome — a targeted Big Y (`TargetedNgs`) or Sanger - // panel lists only a handful of sites and must NOT imply hom-ref everywhere else. + // One source for each **genome-wide** imported variant set: a WGS VCF, or a + // CompleteGenomics masterVar. Neither needs an alignment. Each resolves to panel dosages, + // and a site that it does not list counts as hom-reference. + // + // Accept only `WgsShortRead` and `WgsLongRead`. That hom-ref default is correct only for a + // source that genotyped the whole genome. A targeted Big Y (`TargetedNgs`), or a Sanger + // panel, lists a few sites, and must NOT mean hom-ref everywhere else. let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; for set in &vsets { if !matches!(set.source_type, SourceType::WgsShortRead | SourceType::WgsLongRead) { @@ -2219,9 +2358,10 @@ impl App { } } - // One source per imported **external autosomal call set** (a trusted 1240K EIGENSTRAT set — - // GATK4 / pileupCaller). Resolved to CHM13 panel dosages at import and stored, so it pools in - // with no CRAM decode (available to both the full build and the progressive refresh). + // One source for each imported **external autosomal call set**, which is a trusted 1240K + // EIGENSTRAT set from GATK4 or pileupCaller. The import resolves it to CHM13 panel dosages + // and stores them. So it pools in with no CRAM decode, and both the full build and the + // progressive refresh can use it. for row in navigator_store::external_panel_dosage::list_for_biosample(self.store.pool(), biosample_guid).await? { match serde_json::from_str::>(&row.dosages) { @@ -2276,10 +2416,10 @@ impl App { Ok(profile) } - /// Build the `com.decodingus.atmosphere.haplogroupReconciliation` record JSON for a - /// subject + DNA type from the stored consensus, per-run calls, manual override, and - /// audit log. mtDNA heteroplasmy observations and an optional identity-verification - /// result are passed in (the caller computes them from the relevant alignments). + /// Build the `com.decodingus.atmosphere.haplogroupReconciliation` record JSON for a subject + /// and DNA type. It reads the stored consensus, the calls of each run, the manual override, and + /// the audit log. The caller supplies the mtDNA heteroplasmy observations and an optional + /// identity-verification result, and computes both from the relevant alignments. async fn reconciliation_record( &self, biosample_guid: SampleGuid, @@ -2405,7 +2545,7 @@ impl App { } /// Publish a subject's haplogroup reconciliation record to the signed-in account's PDS - /// (with refresh-on-expiry and retry/backoff via [`AsyncSync`]). + /// [`AsyncSync`] refreshes the token when it expires, and retries with a backoff. pub async fn publish_reconciliation( &self, biosample_guid: SampleGuid, @@ -2428,7 +2568,7 @@ impl App { .await } - /// All recorded per-source calls for a subject + DNA type (for display / audit). + /// Every recorded call, from every source, for a subject and DNA type, to show or to audit. pub async fn haplogroup_calls( &self, biosample_guid: SampleGuid, @@ -2437,8 +2577,8 @@ impl App { Ok(haplogroup_call::list_for(self.store.pool(), biosample_guid, dna_type).await?) } - /// Like [`assign_mtdna_haplogroup`](Self::assign_mtdna_haplogroup) but with the tree - /// JSON supplied directly (no network) — the testable core. + /// Like [`assign_mtdna_haplogroup`](Self::assign_mtdna_haplogroup), but the caller supplies the + /// tree JSON directly, with no network. This is the core that a test can drive. pub async fn assign_mtdna_haplogroup_with_tree( &self, mtdna_id: i64, @@ -2477,23 +2617,30 @@ impl App { self.fetch_tree(&url, "decodingus-ytree.json").await } - /// DecodingUs mtDNA tree-with-variants JSON from our AppView (`/api/v1/mt-tree/full`), host - /// from [`decodingus_appview_url`]. Same schema as the Y tree; coordinates are keyed by build, - /// but the mt tree currently carries only `hs1` (CHM13 `chrM`) positions — a *rotation* of rCRS - /// (~577, plus local indels), so callers must remap onto rCRS via [`mt_tree_rcrs`]. On-disk - /// cached like the other trees. + /// DecodingUs mtDNA tree-with-variants JSON from our AppView (`/api/v1/mt-tree/full`), with the + /// host from [`decodingus_appview_url`]. The schema is the same as the Y tree, and the build + /// keys the coordinates. + /// + /// But the mt tree now carries `hs1` (CHM13 `chrM`) positions alone. Those are a *rotation* of + /// rCRS, by about 577, plus local indels. So a caller must remap onto rCRS through + /// [`mt_tree_rcrs`]. The on-disk cache holds it, as it holds the other trees. pub(crate) async fn fetch_decodingus_mt_tree(&self) -> Result { let url = self.appview_url("mt-tree/full"); self.fetch_tree(&url, "decodingus-mttree.json").await } - /// The **mtDNA placement tree in rCRS coordinates**, with the provider tag. Honors the - /// configured Y-tree provider (the Preferences toggle / `NAVIGATOR_Y_TREE_PROVIDER`): when it is - /// set to FTDNA, use the FTDNA mt tree (already rCRS) directly. Otherwise prefer the DecodingUs - /// mt tree remapped from its native `hs1` (CHM13 `chrM`) positions onto rCRS — so it drops - /// straight into the existing rCRS mt pipeline (FASTA/chip sources and the `chrM` genotyper all - /// speak rCRS) — and still fall back to FTDNA when the DecodingUs tree or the CHM13 `chrM` needed - /// to build the remap is unavailable. + /// The **mtDNA placement tree in rCRS coordinates**, with the provider tag. + /// + /// It obeys the configured Y-tree provider, which is the Preferences toggle or + /// `NAVIGATOR_Y_TREE_PROVIDER`. When that names FTDNA, use the FTDNA mt tree directly, because + /// it is already rCRS. + /// + /// If not, prefer the DecodingUs mt tree, remapped from its native `hs1` (CHM13 `chrM`) + /// positions onto rCRS. It then drops straight into the rCRS mt pipeline, where the FASTA and + /// chip sources and the `chrM` genotyper all use rCRS. + /// + /// Fall back to FTDNA when the DecodingUs tree is absent, or when the CHM13 `chrM` that the + /// remap needs is absent. pub(crate) async fn mt_tree_rcrs(&self) -> Result<(navigator_analysis::haplo::HaploTree, &'static str), AppError> { if !matches!(y_tree_provider(), YTreeProvider::Ftdna) { if let Some(tree) = self.decodingus_mt_tree_rcrs().await { @@ -2505,15 +2652,17 @@ impl App { Ok((tree, "ftdna")) } - /// The DecodingUs mt tree parsed and remapped from `hs1` (CHM13 `chrM`) coordinates onto rCRS. - /// `None` (→ FTDNA fallback) when the tree can't be fetched, or the CHM13 reference is not cached - /// to build the `hs1`↔rCRS map. Best-effort so an offline / reference-less workspace still works. + /// The DecodingUs mt tree, parsed and remapped from `hs1` (CHM13 `chrM`) coordinates onto rCRS. + /// It gives `None`, and the caller falls back to FTDNA, in two cases: the fetch fails, or the + /// cache has no CHM13 reference to build the `hs1`↔rCRS map from. It is best-effort, so a + /// workspace that is offline, or that has no reference, still works. async fn decodingus_mt_tree_rcrs(&self) -> Option { let json = self.fetch_decodingus_mt_tree().await.ok()?; let mut tree = navigator_analysis::haplo::parse_decodingus_json(&json, "hs1").ok()?; let hs1_to_rcrs = self.hs1_to_rcrs_mt_map().await?; - // Remap each defining locus from hs1 (CHM13 chrM) to rCRS; drop any that do not map (indel - // regions near the rotation wrap). An emptied node still exists in the topology. + // Remap each locus of a node from hs1 (CHM13 chrM) to rCRS. Drop any that does not map, + // which happens in the indel regions near the rotation wrap. A node that loses every locus + // still exists in the topology. for node in tree.nodes.values_mut() { node.loci.retain_mut(|l| match hs1_to_rcrs.get(&l.position) { Some(&r) => { @@ -2527,7 +2676,8 @@ impl App { } /// The `hs1` (CHM13 `chrM`, 1-based) → rCRS (1-based) position map, memoized for the process. - /// Built by aligning the bundled rCRS to the cached CHM13 reference's `chrM` (rotation-aware). + /// The code builds it with an alignment of the bundled rCRS to the cached CHM13 reference's + /// `chrM`, and it knows about the rotation. /// `None` when the CHM13 reference is not cached (never forces a multi-GB download for this). async fn hs1_to_rcrs_mt_map(&self) -> Option> { static MAP: std::sync::OnceLock>> = std::sync::OnceLock::new(); @@ -2560,18 +2710,19 @@ impl App { .await } - /// The AppView's full instrument→lab map (`GET /api/v1/sequencer/lab-instruments`), on-disk - /// cached like the trees (7-day TTL + offline fallback). Looked up locally so a batch import - /// makes one network call, not one per sample. + /// The AppView's full instrument→lab map (`GET /api/v1/sequencer/lab-instruments`). The on-disk + /// cache holds it, as it holds the trees, with a 7-day TTL and an offline fallback. The lookup + /// is local, so a batch import makes one network call, and not one for each sample. async fn fetch_lab_instruments(&self) -> Result, AppError> { let url = self.appview_url("sequencer/lab-instruments"); let json = self.fetch_tree(&url, "sequencer-lab-instruments.json").await?; serde_json::from_str(&json).map_err(|e| AppError::Import(format!("parsing lab-instruments: {e}"))) } - /// Resolve an instrument id to a lab display name via the AppView (cached). Normalizes the - /// returned name to the local [`labs`] catalog's canonical display name when it matches. - /// `None` if the instrument has no association or the AppView is unreachable (best-effort). + /// Resolve an instrument id to a lab display name through the AppView, from the cache. It + /// normalizes the returned name to the canonical display name of the local [`labs`] catalog + /// when the two match. It gives `None` when the instrument has no association, or when the + /// AppView is unreachable. It is best-effort. pub async fn lookup_lab_by_instrument(&self, instrument_id: &str) -> Option { let id = instrument_id.trim(); if id.is_empty() { @@ -2586,11 +2737,14 @@ impl App { ) } - /// Resolve the FTDNA Big Y **generation** for a generic Targeted-Y run from its callable chrY - /// footprint: a Big Y-500 covers ≤ ~10 Mb of callable chrY, and only the newer Big Y-700 - /// consistently exceeds it. Only acts on an FTDNA `TARGETED_Y` run — a header `@RG LB` label - /// already pins the generation at import (those are `BIG_Y_500`/`BIG_Y_700`, never `TARGETED_Y`, - /// so they are never second-guessed here), and a non-FTDNA targeted-Y stays generic. Idempotent. + /// Resolve the FTDNA Big Y **generation** for a generic Targeted-Y run, from its callable chrY + /// footprint. A Big Y-500 covers about 10 Mb of callable chrY or less, and only the newer Big + /// Y-700 goes above that consistently. + /// + /// It acts on an FTDNA `TARGETED_Y` run alone. A header `@RG LB` label already fixes the + /// generation at import time, and gives `BIG_Y_500` or `BIG_Y_700`, never `TARGETED_Y`. So this + /// never questions such a run. A targeted-Y run that is not FTDNA stays generic. The function + /// is idempotent. pub(crate) async fn refine_big_y_generation(&self, run: &SequenceRun, callable_chr_y: u64) -> Option<&'static str> { const BIG_Y_500_MAX_CALLABLE: u64 = 10_000_000; if run.test_type != "TARGETED_Y" { @@ -2614,9 +2768,10 @@ impl App { Some(code) } - /// [`Self::refine_big_y_generation`] keyed off an alignment's freshly computed (or cached) - /// coverage — the callable-chrY base count is the discriminator. Called after coverage runs. - /// Returns the new code when the generation changed (so the caller can refresh the run card). + /// [`Self::refine_big_y_generation`], keyed off an alignment's coverage, which the app has just + /// computed or read from the cache. The count of callable chrY bases is what separates the two + /// generations. The caller runs this after coverage. It returns the new code when the + /// generation changed, so that the caller can refresh the run card. pub async fn refine_big_y_generation_for_alignment( &self, alignment_id: i64, @@ -2629,12 +2784,14 @@ impl App { Ok(None) } - /// Resolve the sequencing lab for every run that has an inferred `instrument_id` but no facility - /// yet, via the AppView (one cached fetch). Best-effort; returns how many were filled. Run after - /// import and on startup so pre-existing runs pick up newly-seeded associations. + /// Resolve the sequencing lab for every run that has an inferred `instrument_id` but no + /// facility yet, through the AppView, with one cached fetch. It is best-effort, and returns how + /// many it filled. Run it after an import and at startup, so an older run takes up an + /// association that the app has just seeded. pub async fn backfill_run_labs(&self) -> Result { - // One network/cache fetch (empty when offline — the FTDNA test-type normalization below is - // local and still runs for runs whose facility was resolved earlier, e.g. subject 103589). + // One fetch, from the network or the cache. It is empty when the app is offline. The FTDNA + // test-type normalization below is local, and still runs for a run whose facility the app + // resolved earlier, such as subject 103589. let list = self.fetch_lab_instruments().await.unwrap_or_default(); let by_instrument: HashMap<&str, &str> = list .iter() @@ -2660,9 +2817,10 @@ impl App { None => None, }, }; - // A run we now know is FTDNA but typed as the generic TARGETED_Y is a Big Y — pick - // its generation (500/700) from cached coverage when it is already been analyzed, so - // pre-existing runs get corrected on startup without a re-analysis. + // A run that the app now knows is FTDNA, but that carries the generic TARGETED_Y + // type, is a Big Y. Pick its generation, 500 or 700, from the cached coverage when + // an analysis has already run. So an older run corrects itself at startup, with no + // second analysis. let _ = facility; // (resolved above; the refine reads facility off the run record) if run.test_type == "TARGETED_Y" { if let Ok(Some(cov)) = self.cached_coverage_for_run(run.id).await { @@ -2677,8 +2835,8 @@ impl App { Ok(filled) } - /// Cached coverage for a run, via its first alignment that has one (Big Y runs have a single - /// alignment). `None` when the run has not been analyzed yet. + /// Cached coverage for a run, through the first alignment that has one. A Big Y run has one + /// alignment. It gives `None` when no analysis has run yet. async fn cached_coverage_for_run(&self, run_id: i64) -> Result, AppError> { for aln in alignment::list_for_run(self.store.pool(), run_id).await? { if let Some(cov) = self.cached_coverage(aln.id).await? { @@ -2688,16 +2846,22 @@ impl App { Ok(None) } - /// A cached-or-downloaded haplotree JSON. The on-disk cache has a **7-day life** (see - /// [`TREE_CACHE_TTL`]): a fresh cache short-circuits the network; a stale or missing cache - /// triggers a re-download (and refresh). If the re-download fails (e.g. the AppView is - /// unreachable) but a stale copy exists, the stale copy is used rather than failing — so the - /// app keeps working offline, just on an older tree. (A server-side ETag/version would let us - /// revalidate without a full re-download; tracked as an AppView backlog item.) - /// Force a fresh pull of the haplotrees on the next placement: clear the session memo AND delete - /// the on-disk tree caches, so a corrected AppView tree (e.g. a polarity fix) is picked up without - /// an app restart. Observation-first profiles then re-interpret against the new tree on read — no - /// re-genotyping. Returns the number of cache files removed. + /// A haplotree JSON, from the cache or from a download. The on-disk cache has a **7-day life** + /// (see [`TREE_CACHE_TTL`]). A fresh cache keeps the code off the network. A cache that is + /// stale or absent starts a download, which also refreshes the cache. + /// + /// If that download fails, but a stale copy exists, the code uses the stale copy and does not + /// fail. An unreachable AppView is one such failure. So the app still runs offline, on an older + /// tree. + /// + /// A server-side ETag or version would let the app revalidate with no full download. That is an + /// AppView backlog item. + /// + /// Force a fresh pull of the haplotrees on the next placement. It clears the session memo AND + /// deletes the on-disk tree caches. So the app takes up a corrected AppView tree, such as a + /// polarity fix, with no restart. An observation-first profile then interprets against the new + /// tree on read, with no second genotype pass. Returns the count of cache files that it + /// removed. pub async fn refresh_trees(&self) -> Result { tree_memo().lock().unwrap().clear(); let mut removed = 0usize; @@ -2715,15 +2879,18 @@ impl App { } async fn fetch_tree(&self, url: &str, cache_file: &str) -> Result { - // Session memo: the Y/mt haplotrees are 4–121 MB and each placement consults them several - // times (per alignment, per vendor set, and for the polarity map). A single genome-consensus - // build alone would otherwise re-read/re-validate them repeatedly, and a *stale*-cache refresh - // blocks on the network. Resolve each tree at most once per process and serve every later call - // from memory — trees are effectively static within a session, so this is the batch's biggest - // win (a project pass was spending minutes per subject re-fetching the 121 MB FTDNA tree). - // Keyed by the *resolved* path, not the bare file name: `NAVIGATOR_TREE_DIR` can point the - // same `cache_file` at different trees, and a name-keyed memo would serve the first one for - // the rest of the process. + // Session memo. The Y and mt haplotrees are 4–121 MB, and one placement consults them + // many times: for each alignment, for each vendor set, and for the polarity map. Without + // this memo, one genome-consensus build would read and check them again and again, and a + // refresh of a *stale* cache would block on the network. + // + // So resolve each tree once for each process at most, and serve every later call from + // memory. A tree does not change inside a session, so this is the biggest gain in the + // batch. A project pass took minutes for each subject to fetch the 121 MB FTDNA tree again. + // + // The key is the *resolved* path, and not the bare file name. `NAVIGATOR_TREE_DIR` can + // point the same `cache_file` at a different tree, and a memo keyed by name would serve the + // first one for the rest of the process. let path = tree_cache_path(cache_file); let key = path.to_string_lossy().into_owned(); let memo = tree_memo(); @@ -2737,13 +2904,15 @@ impl App { let json = if fresh { cached.expect("fresh implies present") } else { - // Stale or absent → *conditional*, time-bounded refresh. When we have a cached copy and - // its stored ETag we send `If-None-Match`: an unchanged tree comes back as a tiny `304` - // (a few bytes) instead of re-streaming the full ~60–127 MB body — the curated tree - // changes only every week or so, so most refreshes are 304s. Any failure (connect/ - // timeout, a non-2xx/304 status, or a body read cut short — the whole-request timeout - // also covers streaming the body, see [`TREE_DOWNLOAD_TIMEOUT`]) falls back to the cached - // copy when present; only a first-ever fetch with no cache errors. + // A stale or absent cache starts a *conditional* refresh with a time bound. With a + // cached copy and its stored ETag, the code sends `If-None-Match`. An unchanged tree + // then comes back as a small `304`, a few bytes, instead of the full 60–127 MB body. + // The curated tree changes about once a week, so most refreshes are 304s. + // + // Any failure falls back to the cached copy when there is one. Such a failure is a + // connect or a timeout, a status that is not 2xx or 304, or a body read that stops + // short. The whole-request timeout also covers the body read, see + // [`TREE_DOWNLOAD_TIMEOUT`]. Only a first fetch with no cache gives an error. enum TreeFetch { NotModified, Modified { body: String, etag: Option }, @@ -2828,9 +2997,10 @@ impl App { pub async fn assign_mtdna_haplogroup_from_alignment(&self, alignment_id: i64) -> Result { let bio = self.biosample_of_alignment(alignment_id).await.ok(); - // Prefer an external (sidecar-GVCF) mt call over re-walking the CRAM — same rationale as the - // Y path (see `assign_y_haplogroup`); this is the guard the unguarded single-alignment - // "Full Analysis" was missing, so an internal re-run no longer overwrites the GATK4 mt call. + // Prefer an external mt call, from the sidecar GVCF, over a second walk of the CRAM. The + // reason is the same as on the Y path, see `assign_y_haplogroup`. This is the guard that + // the single-alignment "Full Analysis" did not have. So a second internal run no longer + // overwrites the GATK4 mt call. if let Some(guid) = bio { if let Some(call) = self.preferred_external_call(guid, DnaType::Mt, alignment_id).await? { return Ok(assignment_from_call(&call)); @@ -2840,10 +3010,13 @@ impl App { self.assign_mtdna_haplogroup_walk(alignment_id, bio).await } - /// The internal-caller mtDNA placement: place chrM against the FTDNA mt tree and record under the - /// walk key (`aln:{id}:mt`, `NavigatorWalk`), skipping the re-score when the fingerprint is - /// unchanged. Split out of [`assign_mtdna_haplogroup_from_alignment`] so [`compare_callers`] can - /// force the internal walk even when an external call is preferred. + /// The internal-caller mtDNA placement. Place chrM against the FTDNA mt tree, and record it + /// under the walk key (`aln:{id}:mt`, `NavigatorWalk`). It does not score again when the + /// fingerprint is unchanged. + /// + /// This is split out of [`assign_mtdna_haplogroup_from_alignment`], so that + /// [`compare_callers`] can force the internal walk even when an external call is the preferred + /// one. pub(crate) async fn assign_mtdna_haplogroup_walk( &self, alignment_id: i64, @@ -2852,7 +3025,7 @@ impl App { let source_key = format!("aln:{alignment_id}:mt"); let tree_json = self.fetch_ftdna_mt_tree().await?; - // Cache: skip re-scoring when the file and the mt tree are unchanged. + // Cache: do not score again when the file and the mt tree are unchanged. let fingerprint = self .alignment_content_hash(alignment_id) .await @@ -2888,7 +3061,8 @@ impl App { Ok(assignment) } - /// mtDNA assignment + per-SNP lineage evidence (for exact GRCh38-vs-CHM13 comparison). + /// The mtDNA assignment, plus the lineage evidence at each SNP, for an exact comparison of + /// GRCh38 against CHM13. pub async fn assign_mtdna_haplogroup_detail( &self, alignment_id: i64, @@ -2897,10 +3071,10 @@ impl App { self.assign_haplogroup_detail(alignment_id, "chrM", &tree_json).await } - /// Scan an alignment's chrM pileup for heteroplasmic positions — sites where a second - /// mitochondrial allele coexists above the noise floor. A screening pass for the - /// reconciliation view (a curator judges real heteroplasmy vs. artefacts); ascending - /// by position. Requires a chrM-bearing BAM. + /// Scan an alignment's chrM pileup for heteroplasmic positions. Those are the sites where a + /// second mitochondrial allele is present above the noise floor. This is a first pass for the + /// reconciliation view, where a curator then separates real heteroplasmy from an artefact. The + /// output is in order of position, lowest first. It needs a BAM that holds chrM. pub async fn mtdna_heteroplasmy(&self, alignment_id: i64) -> Result, AppError> { // Resolve the reference for decode (see alignment_reference_for_decode): required for a CRAM, // None for a BAM. The chrM pileup finds a second allele from reads; it needs no reference base. @@ -2914,15 +3088,19 @@ impl App { .map_err(Into::into) } - /// Estimate the donor's ancestry for an alignment by the allele-frequency likelihood: load - /// the (build-matched) AIMs panel, genotype the sample at its sites with the GL caller, and - /// score each super-population's binomial likelihood. Persists the result; returns it for - /// display. Requires a recorded BAM/CRAM and a resolvable reference (CRAM/genotyping). - /// Estimate autosomal ancestry from the subject's **consensus** — no BAM genotyping. Reads the - /// cached autosomal [`DiploidProfile`] (reconciled 0/1/2 dosages over the probe panel, pooled - /// across all WGS + chip sources), bridges it to genotypes, and runs the same estimators as the - /// per-alignment path used to. Persisted under the consensus pseudo-source - /// ([`CONSENSUS_SOURCE_ID`]). Errors if the autosomal consensus has not been built yet. + /// Estimate the donor's ancestry for an alignment by the allele-frequency likelihood. Load the + /// AIMs panel that matches the build, genotype the sample at its sites with the GL caller, and + /// score the binomial likelihood of each super-population. It stores the result, and returns it + /// to show. It needs a recorded BAM/CRAM, and a reference that the code can resolve, for the + /// CRAM and the genotype step. + /// + /// Estimate autosomal ancestry from the subject's **consensus**, with no BAM genotyping. + /// + /// It reads the cached autosomal [`DiploidProfile`], which holds reconciled 0/1/2 dosages over + /// the probe panel, pooled across every WGS and chip source. It bridges that to genotypes, and + /// runs the same estimators that the per-alignment path ran. The store keeps the result under + /// the consensus pseudo-source ([`CONSENSUS_SOURCE_ID`]). It gives an error when no build of + /// the autosomal consensus has run yet. pub async fn estimate_ancestry_from_consensus( &self, biosample_guid: SampleGuid, @@ -2932,11 +3110,13 @@ impl App { })?; let genotypes = consensus_genotypes(&profile); - // The consensus is canonical CHM13; the AIM freq / PCA assets are keyed by (contig,pos) there. + // The consensus uses canonical CHM13. The AIM freq and PCA assets key on (contig,pos) + // there. let build = ReferenceBuild::Chm13v2; let reference_version = "chm13v2.0".to_string(); - // Auto-download the prebuilt panels on first use (no `panelbuild`). The super-pop panel is - // required; PCA + fine frequencies are optional (best-effort — the feature degrades if absent). + // Download the prebuilt panels on first use, so nobody has to run `panelbuild`. The + // super-pop panel is necessary. The PCA and the fine frequencies are optional, and + // best-effort: the feature does less when they are absent. self.ensure_ancestry_asset(build, &ancestry_panel_path(build)).await?; let _ = self.ensure_ancestry_asset(build, &ancestry_pca_path(build)).await; let _ = self diff --git a/crates/navigator-app/src/lib.rs b/crates/navigator-app/src/lib.rs index acb770e9..5359a534 100644 --- a/crates/navigator-app/src/lib.rs +++ b/crates/navigator-app/src/lib.rs @@ -77,8 +77,8 @@ pub struct HaploAssignment { pub ranked: Vec, pub branches: Vec, /// The evidence of each SNP along the lineage, from the root to the terminal node. The list - /// holds each mutation that defines a node, and the state of the sample at that mutation. The - /// three states are Derived, Ancestral, and NoCall. + /// holds each mutation that defines a node, and the state of the sample at that mutation. A + /// state has one of three values: Derived, Ancestral, or NoCall. /// /// The variant **profile**, which pools many sources, reconciles this set. /// @@ -476,8 +476,8 @@ impl PublishGate { g } - /// Shows whether a variant passes the gate. Such a variant must be new and have no name, must - /// be in unique sequence, must have almost no second allele, and must have enough reads. + /// Shows whether a variant passes the gate. Such a variant must be new and have no name. It + /// must also be in unique sequence, have almost no second allele, and have enough reads. pub fn admits(&self, v: &PrivateVariant) -> bool { v.class == PrivateClass::Novel && v.region.is_none() @@ -514,8 +514,8 @@ mod publish_gate_tests { assert!(!g.admits(&var(PrivateClass::OffPathKnown("M269".into()), None, 30, 1.0))); // Paralog-prone structural region → rejected even when deep/homozygous. assert!(!g.admits(&var(PrivateClass::Novel, Some(YRegionClass::Palindrome), 30, 1.0))); - // The alleles are mixed. The placement caller accepts a fraction of 0.5, and a publish - // must not. + // This site has mixed alleles. The placement caller accepts a fraction of 0.5, and a + // publish must not. assert!(!g.admits(&var(PrivateClass::Novel, None, 30, 0.6))); // A short-read sample needs more reads than this call holds. assert!(!g.admits(&var(PrivateClass::Novel, None, 4, 1.0))); @@ -676,8 +676,7 @@ pub struct IbdSuggestion { /// stranger and the user. /// /// This code is beside [`IbdSuggestion`] and not in the card that draws it. So the rule has one -/// home, and a later change to it is a change to the reading of the evidence and not to a -/// widget. +/// home. A later change to it changes the reading of the evidence, and not a widget. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MatchStrength { /// The signals agree strongly; presented as a likely relative. @@ -857,8 +856,8 @@ pub struct MatchingEntry { pub result: Option, } -/// A pulled relay envelope: the opaque ciphertext `blob` plus its routing (`from_did`/`seq`) and the -/// broker `id` to ack. From `GET /api/v1/exchange/relay/pull`. +/// A pulled relay envelope: the opaque ciphertext `blob`, its route fields (`from_did`/`seq`), and +/// the broker `id` to ack. From `GET /api/v1/exchange/relay/pull`. #[derive(Debug, Clone, PartialEq)] pub struct RelayEnvelope { pub id: i64, @@ -867,8 +866,8 @@ pub struct RelayEnvelope { pub blob: String, } -/// A live exchange session with a derived shared key, ready to seal/open payloads. Holds key -/// material, so it is deliberately not `Debug`/`Serialize` and should be kept in memory only. +/// A live exchange session with a derived shared key, ready to seal/open payloads. It holds key +/// material. So it is deliberately not `Debug`/`Serialize`, and you must keep it in memory only. #[derive(Clone)] pub struct EstablishedSession { pub session_id: String, @@ -876,16 +875,17 @@ pub struct EstablishedSession { key: [u8; 32], } -/// Jetstream-ingest retry budget for a freshly-published device key: a 403 right after -/// publishing means the AppView has not ingested our `deviceKey` record yet. Exponential -/// backoff 1+2+4+8 s ≈ 15 s total before giving up. +/// Jetstream-ingest retry budget for a device key that the app published a moment ago. A 403 +/// immediately after the publish means the AppView has not ingested our `deviceKey` record yet. +/// Exponential backoff of 1+2+4+8 s, about 15 s in total, before the app stops. const DEVICE_KEY_INGEST_RETRIES: u32 = 4; /// Poll rounds (≈1s each) an IBD exchange waits for the partner's dosages / attestation. const EXCHANGE_POLL_ROUNDS: u32 = 30; -/// The fed-record collections this client publishes — a PULL reconcile scans each (mirrors the -/// `publish_*` NSIDs). Derived-summary collections are tracked but not overwritten locally. +/// The fed-record collections this client publishes. A PULL reconcile scans each one, and the set +/// mirrors the `publish_*` NSIDs. The app tracks the derived-summary collections, but it does not +/// overwrite them locally. const PUBLISHED_COLLECTIONS: &[&str] = &[ NS_BIOSAMPLE, NS_ALIGNMENT, @@ -895,14 +895,17 @@ const PUBLISHED_COLLECTIONS: &[&str] = &[ HAPLOGROUP_RECONCILIATION_COLLECTION, ]; -/// PDS collection NSID for a published IBD match attestation (the AppView indexes these via Jetstream). +/// PDS collection NSID for a published IBD match attestation (the AppView indexes these through +/// Jetstream). const IBD_ATTESTATION_COLLECTION: &str = "com.decodingus.atmosphere.ibdAttestation"; -/// Above this many sites, the exchanged dosage vector is decimated to fit the relay's 1 MiB envelope. +/// Above this many sites, the app decimates the exchanged dosage vector to fit the relay's 1 MiB +/// envelope. const EXCHANGE_SITE_BUDGET: usize = 100_000; -/// Decimation stride when over budget: keep sites at `position % N == 0`. A **position-based** rule -/// (not index) so both peers keep the *same physical sites* — preserving the IBD intersection — even -/// when their panels differ in size (WGS vs chip). Yields ~1/N of the canonical panel. +/// Decimation stride when over budget: keep sites at `position % N == 0`. The rule is +/// **position-based** and not index-based, so both peers keep the *same physical sites*. This keeps +/// the IBD intersection even when their panels differ in size (WGS against chip). The result is +/// about 1/N of the canonical panel. const EXCHANGE_DECIMATE: i64 = 16; /// Downsample a dosage vector to fit the relay envelope, deterministically + cross-peer-aligned. @@ -917,9 +920,9 @@ fn decimate_for_exchange(sites: Vec) -> Vec { .collect() } -/// Parse the AppView's `/api/v1/ibd/suggestions` body into [`IbdSuggestion`]s. Lenient on -/// field casing (camel/snake) and on the `signals` shape (object map or array) so a minor -/// contract drift degrades gracefully rather than dropping every candidate. +/// Parse the AppView's `/api/v1/ibd/suggestions` body into [`IbdSuggestion`]s. The parser accepts +/// both field casings (camel and snake) and both `signals` shapes (an object map or an array). A +/// small change to the contract then loses only some fields, and it does not drop every candidate. fn parse_ibd_suggestions(body: &serde_json::Value) -> Vec { let Some(items) = body .get("items") @@ -944,8 +947,8 @@ fn parse_ibd_suggestions(body: &serde_json::Value) -> Vec { .and_then(|v| v.as_str()) .unwrap_or("unknown") .to_string(); - // Optional: older AppViews omit it, and the row is still usable for everything but - // attesting, so a missing value must not drop the candidate. + // Optional: older AppViews omit it, and the row is still usable for everything except + // an attestation. So a missing value must not drop the candidate. let target_sample_guid = it .get("targetSampleGuid") .or_else(|| it.get("target_sample_guid")) @@ -969,9 +972,9 @@ fn parse_ibd_suggestions(body: &serde_json::Value) -> Vec { .collect() } -/// Signal names. The AppView emits an array of plain strings -/// (`["POPULATION_OVERLAP", "HAPLOGROUP"]`); also tolerate an array of `{name|source}` -/// objects or an object map (keys) so a contract tweak degrades gracefully. +/// Signal names. The AppView emits an array of plain strings, as in +/// `["POPULATION_OVERLAP", "HAPLOGROUP"]`. The parser also accepts an array of `{name|source}` +/// objects, or an object map whose keys are the names. A change to the contract then loses less. fn parse_ibd_signals(v: &serde_json::Value) -> Vec { if let Some(arr) = v.as_array() { arr.iter() @@ -1035,7 +1038,7 @@ pub mod sync_reconcile; pub use settings::AppSettings; pub use update::UpdateInfo; -/// Artifact kind for de-novo calls, keyed per contig so different contigs do not +/// Artifact kind for de-novo calls, keyed by contig so different contigs do not /// overwrite each other in the cache. fn denovo_kind(contig: &str) -> String { format!("denovo_snps:{contig}") @@ -1051,31 +1054,33 @@ fn tree_cache_path(file: &str) -> PathBuf { dir.join(file) } -/// Sidecar path holding the HTTP `ETag` of a cached haplotree (`.etag`). [`App::fetch_tree`] -/// sends it back as `If-None-Match` on a refresh, so an unchanged tree returns a tiny `304` instead -/// of re-streaming the full ~60–127 MB body. +/// Sidecar path that holds the HTTP `ETag` of a cached haplotree (`.etag`). +/// [`App::fetch_tree`] sends it back as `If-None-Match` on a refresh. An unchanged tree then +/// returns a small `304` instead of the full 60 to 127 MB body. fn tree_etag_path(cache_path: &Path) -> PathBuf { let mut p = cache_path.as_os_str().to_owned(); p.push(".etag"); PathBuf::from(p) } -/// How long a cached haplotree is trusted before [`App::fetch_tree`] re-downloads it. The -/// AppView's curated tree changes slowly (curator review, periodic builds), so a weekly -/// refresh keeps placements current without hitting the network on every run. Override with +/// How long the app trusts a cached haplotree before [`App::fetch_tree`] downloads it again. The +/// AppView's curated tree changes slowly (curator review, periodic builds). So a weekly refresh +/// keeps placements current, and it does not touch the network on every run. Override with /// `NAVIGATOR_TREE_TTL_DAYS` (0 = always refetch). const TREE_CACHE_TTL_DAYS_DEFAULT: u64 = 7; /// Whole-request timeout for a haplotree download. reqwest's `.timeout()` bounds the *entire* -/// request, streaming the body included — and the trees are large (the DecodingUs Y tree is ~60 MB, -/// the FTDNA Y tree ~127 MB), so a short cap aborts the body read partway (surfacing as reqwest's -/// "error decoding response body") and a refresh can then *never* complete, leaving the cache -/// permanently stale. Generous on purpose: a present cache makes any failure fall back instantly -/// (see [`App::fetch_tree`]), so only a first-ever fetch with no cache can wait this long. +/// request, and it includes the body read. The trees are large: the DecodingUs Y tree is about +/// 60 MB, and the FTDNA Y tree about 127 MB. A short cap stops the body read part of the way +/// through. reqwest reports that as "error decoding response body". A refresh can then never +/// complete, and the cache stays stale for ever. +/// +/// The value is large on purpose. A cache that is present makes any failure fall back immediately +/// (see [`App::fetch_tree`]). So only a first fetch with no cache can wait this long. const TREE_DOWNLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); /// Is the cached tree at `path` still within its TTL (default 7 days; `NAVIGATOR_TREE_TTL_DAYS` -/// overrides)? Unknown mtime / unreadable metadata → not fresh (forces a refresh attempt). +/// overrides)? Unknown mtime or unreadable metadata → not fresh, which forces a refresh. fn tree_cache_is_fresh(path: &Path) -> bool { let days = std::env::var("NAVIGATOR_TREE_TTL_DAYS") .ok() @@ -1087,21 +1092,30 @@ fn tree_cache_is_fresh(path: &Path) -> bool { /// Score a tree against the sample calls and attach the terminal's child-branch evidence. /// -/// The Kulczynski `score` ranks the candidates by proportional similarity (and supplies the -/// alternatives list), but the *reported terminal* is chosen in two steps: (1) the best-ranked -/// candidate the path-supported parsimony guard admits — i.e. whose lineage does not tunnel -/// through a branch the sample contradicts (the distal-Y paralog artifact); then (2) -/// [`haplo::deepen_terminal`] descends further into any child the sample clearly entered, -/// correcting under-calls at **unsplit tree nodes** (a half-ancestral SNP block scores below -/// its parent). The chosen node is moved to the front so every `ranked.first()` consumer -/// transparently gets it. See `documents/design/PangenomeExpansion.md`. -/// Pool every source's vote into one consensus map by a `SourceType`-weighted majority, keyed by -/// `K` (SNP **name** for Y — build-portable; rCRS **position** for mt) over value `V` (a per-SNP -/// **state** for Y — strand-/build-independent, since CHM13 vs GRCh38 can flip a base but not -/// "carries the derived allele"; a **base** for mt, which has one coordinate system). The weight -/// matches the variant reconcile's [`navigator_domain::consensus::obs_weight`] `SourceType` term; -/// the highest-weight value wins per key. The pooled set is placed on the tree **once** (genome- -/// level placement) instead of voting among per-run terminal labels. +/// The Kulczynski `score` ranks the candidates by proportional similarity, and it supplies the +/// alternatives list. But two steps choose the *reported terminal*. +/// +/// The first step takes the best-ranked candidate that the path-supported parsimony guard admits. +/// The guard rejects a candidate whose lineage tunnels through a branch that the sample +/// contradicts, which is the distal-Y paralog artifact. +/// +/// The second step calls [`haplo::deepen_terminal`], which descends into any child that the sample +/// clearly entered. This corrects an under-call at an **unsplit tree node**, where a half-ancestral +/// SNP block scores below its parent. +/// +/// The function moves the chosen node to the front, so every `ranked.first()` consumer gets it. +/// See `documents/design/PangenomeExpansion.md`. +/// Pool every source's vote into one consensus map by a `SourceType`-weighted majority. +/// +/// The key `K` is the SNP **name** for Y, which is portable across builds. For mt it is the rCRS +/// **position**. The value `V` is a **state** for each Y SNP, which is independent of strand and +/// build. CHM13 and GRCh38 can flip a base, but neither changes whether the sample carries the +/// derived allele. For mt the value is a **base**, because mt has one coordinate system. +/// +/// The weight matches the `SourceType` term of the variant reconcile, in +/// [`navigator_domain::consensus::obs_weight`]. The value with the highest weight wins for each +/// key. The function places the pooled set on the tree **once**, at the genome level. It does not +/// vote among the terminal labels of each run. fn pool_votes(sources: &[(SourceType, HashMap)]) -> HashMap where K: std::hash::Hash + Eq + Clone, @@ -1119,9 +1133,10 @@ where .filter_map(|(k, votes)| { votes .into_iter() - // Highest weight wins; on a tie break by the allele itself so the pooled call is - // deterministic (a `HashMap` iteration order otherwise picked the winner at random, - // which flipped the placed terminal between runs over identical genotypes). + // The highest weight wins. On a tie, break by the allele itself, so the pooled + // call is deterministic. Without that tie-break the `HashMap` iteration order + // picked the winner at random, which flipped the placed terminal between runs + // over identical genotypes. .max_by(|a, b| { a.1.partial_cmp(&b.1) .unwrap_or(std::cmp::Ordering::Equal) @@ -1159,15 +1174,20 @@ fn assemble_assignment(tree: &navigator_analysis::haplo::HaploTree, calls: &Hash } } -/// Terminal selection for **named Y-SNP panel** data (BISDNA chip), as opposed to the -/// alignment-tuned [`assemble_assignment`]. Such panels give confident but sparse genotype -/// calls: a handful of recurrent or mis-probed ancestral calls on backbone nodes can make the -/// strict `path_admissible` guard (designed to kill distal tunnel artifacts in *coverage- -/// limited* alignment data) veto the genuine deep lineage, dropping the call to a shallow node -/// (e.g. A1). With confident chip calls that failure mode dominates, so here we trust the -/// proportional Kulczynski top — robust to a few stray calls — then [`deepen_terminal`] into -/// clearly-entered children. (Validated: this kit's chromo2 export → R-S1121 on both the -/// DecodingUs/hs1 and FTDNA/GRCh38 trees, on the lineage to its WGS-confirmed R-FGC29071.) +/// Terminal selection for **named Y-SNP panel** data (BISDNA chip), against the alignment-tuned +/// [`assemble_assignment`]. +/// +/// Such panels give confident but sparse genotype calls. A few recurrent or mis-probed ancestral +/// calls on backbone nodes can make the strict `path_admissible` guard veto the genuine deep +/// lineage. The call then drops to a shallow node such as A1. That guard exists to kill distal +/// tunnel artifacts in alignment data with limited coverage. +/// +/// With confident chip calls that failure mode dominates. So here the code trusts the top of the +/// proportional Kulczynski rank, which survives a few stray calls. It then calls +/// [`deepen_terminal`] on the children that the sample clearly entered. +/// +/// Checked: this kit's chromo2 export gives R-S1121 on both the DecodingUs/hs1 tree and the +/// FTDNA/GRCh38 tree, on the lineage to its WGS-confirmed R-FGC29071. fn assemble_assignment_robust( tree: &navigator_analysis::haplo::HaploTree, calls: &HashMap, @@ -1176,9 +1196,10 @@ fn assemble_assignment_robust( let mut ranked = haplo::score(tree, calls); if let Some(top_id) = ranked.first().map(|r| r.id) { let terminal_id = haplo::deepen_terminal(tree, calls, top_id); - // Parsimony back-off: do not report a deeper terminal than the evidence supports. Trim any - // net-contradicted tail of the lineage (sparse-panel / damaged-aDNA over-deepening) while - // a lone contradiction outweighed by deeper derived support still reaches the deep terminal. + // Parsimony back-off: do not report a terminal deeper than the evidence supports. Trim any + // net-contradicted tail of the lineage, which a sparse panel or damaged aDNA can make too + // deep. A lone contradiction that deeper derived support outweighs still reaches the deep + // terminal. let chosen_id = support_backoff_terminal(tree, calls, terminal_id); if let Some(idx) = ranked.iter().position(|r| r.id == chosen_id) { if idx != 0 { @@ -1238,14 +1259,19 @@ fn lineage_names(tree: &navigator_analysis::haplo::HaploTree, name: &str) -> Vec .collect() } -/// Back off an over-deepened terminal to the node that maximizes running support along its -/// lineage. Walking root→terminal, each node contributes `(covered derived − covered ancestral)` -/// over its defining SNPs the sample has a call for; the chosen terminal is the deepest node at -/// which that running balance peaks. A net-contradicted tail (more ancestral than derived calls — -/// a sparse chip or degraded aDNA sample tunnelling into a wrong sub-clade) is trimmed, but a tail -/// whose deeper derived calls outweigh a shallow contradiction is kept (ties favour the deeper -/// node, preserving the robust "survive a lone backbone contradiction" behaviour). Returns -/// `terminal_id` unchanged when its lineage can't be traced. +/// Back off an over-deep terminal to the node with the maximum cumulative support along its +/// lineage. +/// +/// The walk goes from the root to the terminal. Each node contributes +/// `(covered derived − covered ancestral)` over the SNPs that define it and that the sample has a +/// call for. The chosen terminal is the deepest node at which that cumulative balance peaks. +/// +/// The function trims a net-contradicted tail, which has more ancestral than derived calls. A +/// sparse chip or a degraded aDNA sample makes such a tail when it tunnels into a wrong sub-clade. +/// But the function keeps a tail whose deeper derived calls outweigh a shallow contradiction. A +/// tie favours the deeper node, which keeps the "survive a lone backbone contradiction" behaviour. +/// +/// Returns `terminal_id` unchanged when the code can not trace its lineage. fn support_backoff_terminal( tree: &navigator_analysis::haplo::HaploTree, calls: &HashMap, @@ -1275,9 +1301,10 @@ fn support_backoff_terminal( } } // Deepen on strictly more support, or on a tie *only* when this node is itself - // derived-supported. So a contradiction recovered by a deeper derived call still reaches - // the deep terminal, while a net-negative tail or a flat run of marker-less nodes (the - // sparse-panel / aDNA tunnel) is trimmed back to the last positively-supported node. + // derived-supported. A contradiction that a deeper derived call recovers then still + // reaches the deep terminal. A net-negative tail, or a flat run of nodes with no marker, + // goes back to the last node with positive support. That flat run is the sparse-panel or + // aDNA tunnel. if balance > best_balance || (balance == best_balance && node_derived) { best_balance = balance; best_id = id; @@ -1286,14 +1313,18 @@ fn support_backoff_terminal( best_id } -/// Reconcile chip genotype calls to a haplotree's strand. Consumer arrays report alleles on the -/// reference plus strand, but a subset of sites sit on the opposite strand from the tree's -/// ancestral/derived convention. For each call at a tree position: keep the observed base if it -/// already equals the ancestral or derived allele; else substitute its complement when *that* -/// matches; else keep it (a genuine no-match the scorer will count against the branch). Positions -/// absent from the tree pass through unchanged (they do not affect scoring). This is a no-op for -/// dictionary-reconciled BISDNA calls (their base is always the derived allele), so it is safe to -/// apply on the shared chip-placement path. +/// Reconcile chip genotype calls to a haplotree's strand. +/// +/// Consumer arrays report alleles on the reference plus strand. But some sites sit on the opposite +/// strand from the ancestral/derived convention of the tree. +/// +/// For each call at a tree position, keep the observed base when it already equals the ancestral +/// or the derived allele. If not, use its complement when *that* base matches. If neither matches, +/// keep the observed base, and the score counts it against the branch. A position that the tree +/// does not have passes through unchanged, and it changes no score. +/// +/// This does nothing for BISDNA calls that the dictionary reconciled, because their base is always +/// the derived allele. So it is safe on the shared chip-placement path. fn strand_reconcile_to_tree( tree: &navigator_analysis::haplo::HaploTree, calls: HashMap, @@ -1324,11 +1355,15 @@ fn strand_reconcile_to_tree( .collect() } -/// Map GVCF-decoded bases at *lifted* positions back to tree positions (the GVCF-sourced -/// analogue of [`App::build_calls_from_lifted`]). A variant base wins; otherwise a callable -/// hom-ref lifted site takes the **reference base** at that lifted position — both reverse- -/// complemented for a minus-strand lift; otherwise the position is a no-call. `ref_base` is -/// keyed by lifted position (the GVCF/reference coordinate), not the tree position. +/// Map GVCF-decoded bases at *lifted* positions back to tree positions. This is the GVCF form of +/// [`App::build_calls_from_lifted`]. +/// +/// A variant base wins. If there is none, a callable hom-ref lifted site takes the **reference +/// base** at that lifted position. A minus-strand lift reverse-complements both. Any other +/// position is a no-call. +/// +/// `ref_base` uses the lifted position as its key, which is the GVCF or reference coordinate, and +/// not the tree position. fn assemble_calls_lifted( called: &gvcf::CalledBases, lifted: &[LiftedPos], @@ -1350,12 +1385,15 @@ fn assemble_calls_lifted( calls } -/// Minimum callable/calling depth adapted to read technology. The default (4) is a -/// short-read assumption — ~4 reads to call a base confidently. Long, accurate reads (HiFi, -/// mean read length > 1 kb) make a confident haploid observation from a *single* read, so a -/// ~4× HiFi sample is callable at 1×; clamping the floor at 2 needlessly threw away half its -/// already-shallow coverage. (ONT long reads are less accurate — revisit if we ever adapt by -/// platform rather than read length.) +/// Minimum callable depth, adapted to the read technology. The default of 4 is a short-read +/// assumption: about 4 reads to call a base with confidence. +/// +/// Long, accurate reads (HiFi, with a mean read length above 1 kb) give a confident haploid +/// observation from a *single* read. So a HiFi sample at about 4x is callable at 1x. A +/// floor clamped at 2 threw away half of its already shallow coverage for no gain. +/// +/// ONT long reads are less accurate. Look at this again if the code ever adapts by platform and +/// not by read length. fn adaptive_min_depth(base: u32, read_len: f64) -> u32 { if read_len > 1000.0 { 1 @@ -1364,9 +1402,9 @@ fn adaptive_min_depth(base: u32, read_len: f64) -> u32 { } } -/// Haploid-caller params adapted to the sample's read tech (see [`adaptive_min_depth`]). -/// Sampled from the BAM head; falls back to defaults on any error. Blocking (reads the BAM) -/// — call inside `spawn_blocking`. +/// Haploid-caller params adapted to the sample's read technology (see [`adaptive_min_depth`]). +/// The function samples the head of the BAM, and falls back to the defaults on any error. It +/// blocks, because it reads the BAM, so call it inside `spawn_blocking`. fn adaptive_haploid_params(bam_path: &Path, reference: Option<&Path>) -> HaploidCallerParams { let mut params = HaploidCallerParams::default(); if let Ok((read_len, _)) = coverage::estimate_molecule_lengths(bam_path, reference) { @@ -1376,7 +1414,7 @@ fn adaptive_haploid_params(bam_path: &Path, reference: Option<&Path>) -> Haploid } /// Minimum genotyped sites for a reliable AIMs ancestry estimate (Scala `minSnpsAims`). -/// Overridable via `$NAVIGATOR_ANCESTRY_MIN_SNPS` (tests use a small panel). +/// `$NAVIGATOR_ANCESTRY_MIN_SNPS` overrides it (tests use a small panel). fn ancestry_min_snps() -> usize { std::env::var("NAVIGATOR_ANCESTRY_MIN_SNPS") .ok() @@ -1385,8 +1423,8 @@ fn ancestry_min_snps() -> usize { } /// Resolve an ancestry/IBD asset path under `/ancestry/`: an `$` override -/// (when non-empty) wins, else `/ancestry/_.`. The per-asset wrappers below -/// delegate here so the override+join+format pattern lives in one place. +/// (when non-empty) wins, else `/ancestry/_.`. The wrapper for each asset +/// below delegates here, so the override, join, and format pattern lives in one place. fn ancestry_asset_path(env_var: &str, stem: &str, build: ReferenceBuild, ext: &str) -> PathBuf { if !env_var.is_empty() { if let Ok(p) = std::env::var(env_var) { @@ -1406,47 +1444,49 @@ fn ancestry_panel_path(build: ReferenceBuild) -> PathBuf { } /// Where the PCA loadings for `build` live: `$NAVIGATOR_ANCESTRY_PCA` (override), else -/// `/ancestry/ancestry_pca_.bin`. Optional — absent means the -/// AF-likelihood estimate runs without PCA coordinates. +/// `/ancestry/ancestry_pca_.bin`. Optional. When it is absent, the +/// AF-likelihood estimate runs with no PCA coordinates. fn ancestry_pca_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_PCA", "ancestry_pca", build, "bin") } /// The fine-population frequency asset path (`$NAVIGATOR_ANCESTRY_FREQ` override, else -/// `/ancestry/ancestry_freq_global_.bin`). Optional — fine admixture is skipped if absent. +/// `/ancestry/ancestry_freq_global_.bin`). Optional. Without it, the app skips fine +/// admixture. fn ancestry_freq_global_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_FREQ", "ancestry_freq_global", build, "bin") } /// The phased-haplotype reference asset (`$NAVIGATOR_ANCESTRY_HAPS` override, else /// `/ancestry/ancestry_haps_.bin`): the phased 1000G haplotypes the statistical phaser -/// copies from, for the parent-split chromosome painter. Optional — when absent, the painter falls -/// back to the unphased diploid path (two arbitrary sorted copies rather than parental sides). +/// copies from, for the parent-split chromosome painter. Optional. When it is absent, the painter +/// falls back to the unphased diploid path, which gives two sorted copies and not parental sides. fn ancestry_haps_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_HAPS", "ancestry_haps", build, "bin") } /// The **ancient** deep-source frequency asset (`$NAVIGATOR_ANCESTRY_FREQ_ANCIENT` override, else -/// `/ancestry/ancestry_freq_ancient_.bin`): per-site WHG/ANF/Steppe alt-allele -/// frequencies, built by `panelbuild ancient-panel` from the AADR. Optional — deep ancestry is -/// skipped if absent. Built over the AIM panel's own sites, so the single genotyping pass covers it. +/// `/ancestry/ancestry_freq_ancient_.bin`): WHG/ANF/Steppe alt-allele frequencies for +/// each site, which `panelbuild ancient-panel` builds from the AADR. Optional. Without it, the app +/// skips deep ancestry. It covers the AIM panel's own sites, so one genotyping pass supplies it. /// -/// This supersedes the old `ancestry_pca_ancient_.bin`, which is no longer read by anything: -/// PCA-projected ancient centroids collapse onto the modern cloud and carry no ancient signal. +/// This replaces the old `ancestry_pca_ancient_.bin`, which nothing reads now. Ancient +/// centroids that a PCA projects collapse onto the modern cloud and carry no ancient signal. fn ancestry_freq_ancient_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_FREQ_ANCIENT", "ancestry_freq_ancient", build, "bin") } /// The qpAdm deep-ancestry panel asset path (`$NAVIGATOR_ANCESTRY_QPADM` override, else -/// `/ancestry/ancestry_qpadm_.bin`). The full-1240k Patterson-2022 config (WHG/EEF/Steppe -/// sources + sister outgroups) — see documents/design/ancient-ancestry-rebuild.md §7.14. +/// `/ancestry/ancestry_qpadm_.bin`). The full-1240k Patterson-2022 config, with +/// WHG/EEF/Steppe sources and sister outgroups. See +/// documents/design/ancient-ancestry-rebuild.md §7.14. fn ancestry_qpadm_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_QPADM", "ancestry_qpadm", build, "bin") } /// The archaic (Neanderthal / Denisovan) marker panel asset path /// (`$NAVIGATOR_ARCHAIC_MARKERS` override, else `/ancestry/archaic_markers_.bin`). -/// Built by `panelbuild archaic-panel` — see documents/design/ArchaicAncestry_Design.md §4. +/// `panelbuild archaic-panel` builds it. See documents/design/ArchaicAncestry_Design.md §4. fn archaic_markers_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_MARKERS", "archaic_markers", build, "bin") } @@ -1457,20 +1497,20 @@ fn archaic_marker_dist_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_DIST", "archaic_marker_dist", build, "bin") } -/// Tier B: positions variable in the African outgroup, for stripping shared variants. -/// Cache signature for a Tier B segment result: the alignment it came from plus the caller's -/// genotype version, so re-calling with a newer caller invalidates it. +/// Tier B: positions variable in the African outgroup, which let the code remove shared variants. +/// Cache signature for a Tier B segment result: the alignment it came from, plus the genotype +/// version of the caller. So a newer caller invalidates the result. pub(crate) fn archaic_segment_sig(alignment_id: i64, called_contigs: &[String]) -> String { // Three things make a result stale, and all three are in the key. // - // The METHOD version, because Tier B was rebuilt from a private-variant density model to - // archaic-genome matching; without it a workspace carrying output from the withdrawn caller - // would keep serving it. + // The METHOD version, because Tier B changed from a private-variant density model to a match + // against the archaic genomes. Without it, a workspace that holds output from the withdrawn + // caller would continue to serve that output. // - // The CONTIGS ACTUALLY CALLED, because the result covers only those. A subject called on chr21 - // alone and later called genome-wide would otherwise keep the two-chromosome answer forever — - // observed doing exactly that during genome-wide validation, reporting 1.94 Mb over 2 contigs - // while 22 sat cached and ready. + // The CONTIGS THE CODE CALLED, because the result covers only those. Take a subject called on + // chr21 alone, and later called genome-wide. Without this term the subject would keep the + // two-chromosome answer for ever. A genome-wide check showed exactly that: a report of 1.94 Mb + // over 2 contigs, while 22 more sat in the cache and ready. let mut contigs: Vec<&str> = called_contigs.iter().map(String::as_str).collect(); contigs.sort_unstable(); format!( @@ -1498,12 +1538,13 @@ fn archaic_outgroup_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_OUTGROUP", "archaic_outgroup_af", build, "bin") } -/// Tier B: genome-wide archaic diagnostic sites, for labelling called segments. +/// Tier B: genome-wide archaic diagnostic sites, which give a label to each called segment. fn archaic_classify_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_CLASSIFY", "archaic_classify", build, "bin") } -/// Tier B: callable bases per window. Without this the segment HMM calls mapping artifacts. +/// Tier B: the count of callable bases in each window. Without this the segment HMM calls mapping +/// artifacts. fn archaic_callable_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_CALLABLE", "archaic_callable", build, "bin") } @@ -1514,8 +1555,9 @@ fn ibd_panel_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_IBD_PANEL", "ibd_panel", build, "bin") } -/// The ancestry/IBD reference assets for the analysis build (CHM13), each with presence + manifest -/// verification — the "data sources" transparency line. Pure filesystem inspection (no analysis). +/// The ancestry/IBD reference assets for the analysis build (CHM13). Each one carries its presence +/// and its manifest check, which is the "data sources" transparency line. This looks only at the +/// file system, and does no analysis. pub fn ancestry_asset_status() -> Vec { let build = ReferenceBuild::Chm13v2; let manifest = load_asset_manifest(build); @@ -1555,10 +1597,11 @@ pub struct SeedSummary { pub skipped: usize, } -/// Copy every regular file in `src_dir` into `dest_dir` that is not already present there. Never -/// overwrites an existing file — a CDN-refreshed asset must win over the bundled one. Creates -/// `dest_dir`. A missing/unreadable `src_dir` is a no-op (returns the empty summary). Pure over the -/// two directories (no globals) so it is unit-testable. +/// Copy every regular file in `src_dir` into `dest_dir` that is not already present there. It never +/// overwrites a file that exists, because an asset refreshed from the CDN must win over the bundled +/// one. It creates `dest_dir`. A `src_dir` that is missing or unreadable does nothing, and returns +/// the empty summary. The function is pure over the two directories, with no globals, so a unit +/// test can drive it. pub fn seed_assets_from(src_dir: &Path, dest_dir: &Path) -> std::io::Result { let mut summary = SeedSummary::default(); let Ok(entries) = std::fs::read_dir(src_dir) else { @@ -1571,8 +1614,8 @@ pub fn seed_assets_from(src_dir: &Path, dest_dir: &Path) -> std::io::Result