diff --git a/crates/navigator-align/src/batch.rs b/crates/navigator-align/src/batch.rs index 2fdf614a..6e5568f0 100644 --- a/crates/navigator-align/src/batch.rs +++ b/crates/navigator-align/src/batch.rs @@ -132,7 +132,7 @@ impl BatchSize { /// Why [`BatchSize::for_this_machine`] chose what it did, for a log line or a UI tooltip. /// - /// A realignment is a multi-hour job whose memory profile the user cannot see; when it is + /// A realignment is a multi-hour job whose memory profile the user can not see; when it is /// sized automatically, the sizing has to be inspectable rather than a mystery. pub fn explain() -> String { if let Some(bases) = env_override() { @@ -269,7 +269,7 @@ mod tests { /// Detection has to work on whatever machine this runs on — that is the entire point of taking /// the dependency. The assertions are about plausibility rather than a specific number, since - /// the test cannot know the host. + /// the test can not know the host. #[test] fn the_machine_reports_its_own_memory() { let memory = detect_memory().expect("every desktop target sysinfo supports reports memory"); diff --git a/crates/navigator-align/src/index.rs b/crates/navigator-align/src/index.rs index ee48dfd2..1614a759 100644 --- a/crates/navigator-align/src/index.rs +++ b/crates/navigator-align/src/index.rs @@ -40,9 +40,9 @@ pub type ProgressFn<'a> = &'a mut dyn FnMut(usize, u64); /// Deliberately the same answer `navigator-refgenome::cache::base_dir` gives, reached the same way /// — through `navigator_domain::paths::decodingus_dir`, the one definition of the cache root — so /// `minimap2_index/` lands beside `references/` and `liftover/` rather than in a second location -/// that only this crate knows about. This crate is a leaf and cannot depend on `navigator-refgenome` +/// that only this crate knows about. This crate is a leaf and can not depend on `navigator-refgenome` /// (that would invert the layering), which is why the resolution is repeated rather than imported; -/// the shared *definition* is what stops the two drifting. +/// the shared *definition* is what keeps the two the same. pub fn cache_root() -> PathBuf { if let Some(dir) = std::env::var_os("NAVIGATOR_REFGENOME_DIR") { return PathBuf::from(dir); diff --git a/crates/navigator-align/src/map.rs b/crates/navigator-align/src/map.rs index bc96fed5..aadf6e04 100644 --- a/crates/navigator-align/src/map.rs +++ b/crates/navigator-align/src/map.rs @@ -29,7 +29,7 @@ //! ## Why not the upstream file-level entry points //! //! `minimap2-pure-rs` ships `map_file_sam_split` and friends, which look like exactly this. They -//! cannot be used: they write to **stdout** (unusable from a desktop app) and they take +//! can not be used: they write to **stdout** (unusable from a desktop app) and they take //! `parts: &[MmIdx]`, holding every part resident — giving up the entire memory bound this design //! exists to buy. What is reused is the per-part record format and the merge; the loop is ours. //! @@ -113,7 +113,7 @@ pub struct MapStats { /// Cancellation, as a callback rather than a shared token type. /// -/// This crate is a leaf — it deliberately does not depend on `navigator-analysis`, so it cannot +/// This crate is a leaf — it deliberately does not depend on `navigator-analysis`, so it can not /// take that crate's `CancelToken` without inverting the layering. A closure lets the caller wire /// whatever cancellation it already has, and costs this crate no dependency. pub type CancelFn<'a> = &'a dyn Fn() -> bool; diff --git a/crates/navigator-align/src/map/tests.rs b/crates/navigator-align/src/map/tests.rs index 3044308c..385a0f18 100644 --- a/crates/navigator-align/src/map/tests.rs +++ b/crates/navigator-align/src/map/tests.rs @@ -590,7 +590,7 @@ fn the_output_format_can_be_read_off_the_path() { assert_eq!(F::from_path(Path::new("x")), F::Bam, "BAM is the default"); } -/// CRAM cannot be written without the reference it is compressed against, and saying so up front +/// CRAM can not be written without the reference it is compressed against, and saying so up front /// beats failing partway through a multi-hour job. #[test] fn cram_without_a_reference_is_refused_before_any_work() { diff --git a/crates/navigator-align/src/output.rs b/crates/navigator-align/src/output.rs index 72e063e2..0d8f18da 100644 --- a/crates/navigator-align/src/output.rs +++ b/crates/navigator-align/src/output.rs @@ -143,7 +143,7 @@ impl AlignmentWriter { /// Parse one SAM line from the mapper and hand it to `edit` before writing. /// - /// `edit` is where paired fields get set. It sees a typed record, so it cannot write a value + /// `edit` is where paired fields get set. It sees a typed record, so it can not write a value /// into the wrong column — which was the entire failure mode this module removes. pub fn write_line_with( &mut self, diff --git a/crates/navigator-align/src/pe.rs b/crates/navigator-align/src/pe.rs index 909e216e..a73fb61a 100644 --- a/crates/navigator-align/src/pe.rs +++ b/crates/navigator-align/src/pe.rs @@ -707,7 +707,7 @@ fn primary(result: &MapResult) -> Option<&AlignReg> { /// Fill in the paired half of a record: flags, `RNEXT`, `PNEXT`, `TLEN`. /// /// The single-end writer produced everything else. This used to patch the formatted SAM text by -/// column position; it now mutates a typed [`RecordBuf`], so a mate position cannot end up in the +/// column position; it now mutates a typed [`RecordBuf`], so a mate position can not end up in the /// template-length field however the formatter's layout changes. /// /// `own` is this record's region (`None` for an unmapped read) and `mate` is the mate's primary. @@ -822,7 +822,7 @@ impl PairReader { /// read is the same length. /// /// A file genuinely ending before the other is still an error rather than a truncation: R1/R2 - /// that have drifted out of step would pair every later read with the wrong mate, which is far + /// that have lost their order would pair every later read with the wrong mate, which is far /// worse than refusing to run. fn next_batch(&mut self, chunk: i64) -> Result>, AlignError> { let mut batch = Vec::new(); diff --git a/crates/navigator-align/src/pe/tests.rs b/crates/navigator-align/src/pe/tests.rs index d15e9d9b..6da7145b 100644 --- a/crates/navigator-align/src/pe/tests.rs +++ b/crates/navigator-align/src/pe/tests.rs @@ -372,7 +372,7 @@ fn a_split_index_pairs_reads_exactly_as_a_whole_index_does() { } } -/// R1/R2 that have drifted out of step would pair every later read with the wrong mate — a +/// R1/R2 that have lost their order would pair every later read with the wrong mate — a /// corruption that produces confident, wrong alignments. It must refuse rather than truncate. #[test] fn mismatched_read_counts_are_refused() { diff --git a/crates/navigator-analysis/examples/archaic_callable_dump.rs b/crates/navigator-analysis/examples/archaic_callable_dump.rs index 7b50bdcb..ab498389 100644 --- a/crates/navigator-analysis/examples/archaic_callable_dump.rs +++ b/crates/navigator-analysis/examples/archaic_callable_dump.rs @@ -1,8 +1,9 @@ -//! Dump the Tier B callability mask as BED, so what the segment caller can and cannot see is -//! checkable against an external callset rather than assumed. +//! Write out the Tier B callability mask as a BED file. Somebody can then check what the segment +//! caller can see, and what it can not, against an external callset. Nobody has to assume it. //! -//! Windows below `min_frac` of `window_bp` callable are excluded by the caller itself, so the same -//! threshold is applied here — the output is the territory a segment could actually be called in. +//! The caller itself removes a window whose callable fraction is below `min_frac` of `window_bp`. +//! This tool applies the same threshold. So its output is the territory where a segment could get +//! a call. //! //! ```sh //! cargo run --release -p navigator-analysis --example archaic_callable_dump -- \ diff --git a/crates/navigator-analysis/examples/archaic_classify_dump.rs b/crates/navigator-analysis/examples/archaic_classify_dump.rs index 3307c2af..7f12bd11 100644 --- a/crates/navigator-analysis/examples/archaic_classify_dump.rs +++ b/crates/navigator-analysis/examples/archaic_classify_dump.rs @@ -1,10 +1,13 @@ //! Dump the archaic **diagnostic** sites (position, derived base, lineage class) as TSV. //! -//! Written to test a different observable for the Tier B HMM. The current model counts *all* -//! private variants per window, and that signal is weak: measured on a real European, archaic -//! tracts carry only 2.89x the background density while the background itself varies 5.3x between -//! its 10th and 90th percentile. Restricting the observable to sites where the derived allele is -//! actually known to be archaic should be far more specific. +//! Somebody wrote this to test a different observable for the Tier B HMM. +//! +//! The current model counts *all* of the private variants in each window, and that signal is weak. +//! A measurement on a real European showed it. An archaic tract carries only 2.89x the background +//! density. And the background itself changes by 5.3x, between its 10th and 90th percentile. +//! +//! Take an observable that holds only the sites where the derived allele is archaic, and that +//! nobody doubts. That observable must be far more specific. //! //! ```sh //! cargo run --release -p navigator-analysis --example archaic_classify_dump -- \ diff --git a/crates/navigator-analysis/examples/archaic_match_probe.rs b/crates/navigator-analysis/examples/archaic_match_probe.rs index d8552eab..cde77ecf 100644 --- a/crates/navigator-analysis/examples/archaic_match_probe.rs +++ b/crates/navigator-analysis/examples/archaic_match_probe.rs @@ -1,9 +1,11 @@ //! Run the reference-based archaic tract caller ([`archaic_match`]) on real cached calls. //! -//! The unit tests prove the model behaves on synthetic runs; this is what shows whether it finds -//! REAL tracts. Emits segments as JSON for `scripts/archaic-validation/compare_locations.py`, which -//! scores them against an external callset and — critically — against the random-placement null the -//! density caller failed. +//! The unit tests show that the model behaves on a synthetic run. This tool shows whether it finds +//! a REAL tract. +//! +//! It writes the segments as JSON, for `scripts/archaic-validation/compare_locations.py`. That +//! script scores them against an external callset, and, most important, against the null from +//! random placement that the density caller failed. //! //! ```sh //! cargo run --release -p navigator-analysis --example archaic_match_probe -- \ @@ -46,8 +48,8 @@ fn main() -> Result<(), Box> { let mut observations: BTreeMap> = BTreeMap::new(); let mut lengths: Vec<(String, i32)> = Vec::new(); for (contig, pos_map) in &by_contig { - // The reference base decides which diagnostic sites are informative at all, so it is read - // rather than assumed (see `observations_for_contig`). + // The reference base decides which diagnostic sites carry information at all. So the code + // reads it, and it does not assume it. See `observations_for_contig`. let seq = read_contig_sequence(&reference, contig)?; let obs = observations_for_contig( contig, @@ -83,10 +85,13 @@ fn main() -> Result<(), Box> { GeneticMap::from_bytes(&std::fs::read(&a[3])?).map_err(|e| e.to_string())? }; - // `ARCHAIC_RATIOS=2.0,2.5,3.04` sweeps the emission ratio in one process. It cannot be swept - // post-hoc like the three thresholds — it changes the emissions, so the HMM must be re-decoded — - // but the expensive part (reading the reference, walking the diagnostic sites) is per sample, - // not per ratio, so doing it here costs one pass instead of one per value. + // `ARCHAIC_RATIOS=2.0,2.5,3.04` sweeps the emission ratio inside one process. Nobody can sweep + // that ratio after the run, as they can the three thresholds, because it changes the emissions + // and the HMM must decode again. + // + // But the costly part happens once for each sample, and not once for each ratio. That part is + // the read of the reference, and the walk over the diagnostic sites. So a sweep here costs one + // pass, and not one pass at every value. if let Ok(spec) = std::env::var("ARCHAIC_RATIOS") { let mut out = serde_json::Map::new(); for tok in spec.split(',').filter(|t| !t.trim().is_empty()) { diff --git a/crates/navigator-analysis/examples/archaic_outgroup_density.rs b/crates/navigator-analysis/examples/archaic_outgroup_density.rs index 08a26637..8388bbda 100644 --- a/crates/navigator-analysis/examples/archaic_outgroup_density.rs +++ b/crates/navigator-analysis/examples/archaic_outgroup_density.rs @@ -1,15 +1,20 @@ -//! Per-window counts of African-outgroup segregating sites — a candidate local mutation-rate proxy. +//! The count of sites that vary in the African outgroup, in each window. It is a candidate proxy +//! for the local mutation rate. //! -//! The Tier B emission model assumes one background rate genome-wide. Measured, the background -//! private-variant density varies 5.3x between its 10th and 90th percentile and is 14.6x -//! overdispersed relative to the Poisson it is modelled with, which is larger than the 2.89x -//! enrichment inside real archaic tracts — so the model calls its own upper tail archaic. hmmix -//! avoids this with a mutation-rate map; we have no such asset. +//! The Tier B emission model takes one background rate over the whole genome. A measurement shows +//! otherwise. The background density of private variants changes by 5.3x between its 10th and 90th +//! percentile. It is also 14.6x more spread out than the Poisson distribution that the model gives +//! it. //! -//! The density of sites segregating in Africans is already in `archaic_outgroup_af_.bin` and -//! is a direct measure of how variable a region is, for reasons that have nothing to do with -//! archaic introgression (mutation rate, reference quality, mappability). This dumps it so that -//! proxy can be tested as a normalizer before an asset is built for the purpose. +//! That spread is larger than the 2.89x enrichment inside a real archaic tract. So the model calls +//! its own upper tail archaic. The hmmix tool avoids this with a map of the mutation rate, and +//! this project has no such asset. +//! +//! `archaic_outgroup_af_.bin` already holds the density of the sites that vary in Africans. +//! That density measures directly how much a region varies. Its reasons have nothing to do with +//! archaic introgression. They are the mutation rate, the quality of the reference, and how well +//! reads map there. This tool writes it out, so that somebody can test it as a normalizer before anybody +//! builds an asset for the purpose. //! //! ```sh //! cargo run --release -p navigator-analysis --example archaic_outgroup_density -- \ diff --git a/crates/navigator-analysis/examples/archaic_panel_dump.rs b/crates/navigator-analysis/examples/archaic_panel_dump.rs index 52805f8c..018548b7 100644 --- a/crates/navigator-analysis/examples/archaic_panel_dump.rs +++ b/crates/navigator-analysis/examples/archaic_panel_dump.rs @@ -1,16 +1,19 @@ -//! Dump the Tier A marker panel with its **per-archaic-genome** calls, as TSV. +//! Write out the Tier A marker panel as a TSV, with the calls of **each archaic genome**. //! -//! This is the independent evidence for arbitrating Tier B calls. The segment caller -//! ([`navigator_analysis::archaic_match`]) reads only `ArchaicClassify` — a derived base and a -//! lineage class per site — and never sees which archaic genome carries what. So the per-genome -//! pattern is information the caller cannot have fitted to, which is what makes it usable as a -//! referee. +//! This is the independent evidence that decides a Tier B call. The segment caller +//! ([`navigator_analysis::archaic_match`]) reads `ArchaicClassify` alone, which holds a derived +//! base and a lineage class at each site. It never sees which archaic genome carries what. The +//! pattern over the genomes is information that nobody could have fitted the caller to, and that is +//! what makes it a referee. //! -//! Why a referee is needed: precision has been measured against hmmix's callset, but a call absent -//! from hmmix is not necessarily wrong — hmmix's own tracts are enriched only 1.84x for their own -//! archaic SNPs, so that callset is incomplete by an unknown amount. Scoring a segment against the -//! archaic genomes directly asks whether it looks like an inherited archaic haplotype, without -//! asking another caller's opinion. +//! Here is why a referee is necessary. A measurement gave the precision against the callset of +//! hmmix. But a call that hmmix does not hold is not wrong by that fact alone. The own tracts of +//! hmmix show an enrichment of only 1.84x for their own archaic SNPs. So that callset is +//! incomplete by an amount that nobody knows. +//! +//! A score of a segment against the archaic genomes asks a direct question. Does this segment look +//! like an archaic haplotype that came down from an ancestor? It asks no other caller for an +//! opinion. //! //! ```sh //! cargo run --release -p navigator-analysis --example archaic_panel_dump -- \ @@ -29,9 +32,11 @@ fn main() -> Result<(), Box> { let panel = ArchaicMarkerPanel::from_bytes(&std::fs::read(&path)?).map_err(|e| e.to_string())?; eprintln!("panel: {} sites, build {}", panel.sites.len(), panel.build); - // One column per archaic genome: D = carries the derived allele, A = positively called - // homozygous-ancestral, . = no call. The A/. distinction is load-bearing — treating a no-call as - // ancestral is the error that produced ~19 % Denisovan for a European in an earlier pass. + // One column for each archaic genome. D means that the genome carries the derived allele. A + // means that the caller positively called it homozygous-ancestral. A `.` means no call. + // + // The difference between A and `.` carries weight. To read a no-call as ancestral is the error + // that gave about 19% Denisovan for a European, in an earlier pass. println!("contig\tposition\tderived\tclass\t{}", ARCHAIC_GENOMES.join("\t")); let mut n = 0usize; for s in &panel.sites { diff --git a/crates/navigator-analysis/examples/archaic_private_dump.rs b/crates/navigator-analysis/examples/archaic_private_dump.rs index 7454bd4e..b4d39cc6 100644 --- a/crates/navigator-analysis/examples/archaic_private_dump.rs +++ b/crates/navigator-analysis/examples/archaic_private_dump.rs @@ -1,11 +1,12 @@ -//! Dump the **private** variant positions the Tier B HMM actually sees — the subject's derived -//! variants after the African-outgroup strip — so the input to the model can be checked against an -//! external truth set independently of the model. +//! Write out the **private** variant positions that the Tier B HMM sees. Those are the derived +//! variants of the subject, after the code removes the ones that the African outgroup also carries. +//! Somebody can then check the input of the model against an external truth set, and that check +//! does not depend on the model. //! -//! The segment caller is a density model over exactly these positions. If they are not enriched -//! inside known archaic tracts, no amount of HMM tuning can help, and the fault is upstream in the -//! variant calls or the outgroup strip rather than in the model. That question is unanswerable from -//! the caller's own output, which is why this exists. +//! The segment caller is a density model over exactly these positions. If they show no enrichment +//! inside a known archaic tract, then no change to the HMM can help. The fault would lie earlier, +//! in the variant calls or in the removal of the outgroup sites, and not in the model. The own +//! output of the caller can not answer that question, and that is why this tool exists. //! //! ```sh //! cargo run --release -p navigator-analysis --example archaic_private_dump -- \ @@ -27,14 +28,15 @@ fn main() -> Result<(), Box> { let calls: Vec = serde_json::from_str(&std::fs::read_to_string(&calls_path)?)?; let og = ArchaicOutgroup::from_bytes(&std::fs::read(&og_path)?).map_err(|e| e.to_string())?; - // Group by contig, mirroring what the caller does before it strips. + // Put them into groups by contig, as the caller does before it removes the outgroup sites. let mut by_contig: std::collections::BTreeMap> = Default::default(); for c in &calls { by_contig.entry(c.contig.clone()).or_default().push(c); } - // Quality columns come out too: whether the background's excess variance is real biology or - // this caller's own error rate varying by region is not answerable without them. + // The quality columns come out too. Without them, nobody can answer one question. Is the + // excess variance of the background real biology, or is it the own error rate of this caller, + // which changes from region to region? println!("contig\tposition\tdosage\tgq\tdepth"); for (contig, mut sites) in by_contig { sites.sort_by_key(|s| s.position); diff --git a/crates/navigator-analysis/examples/archaic_segments_probe.rs b/crates/navigator-analysis/examples/archaic_segments_probe.rs index 5fc3b67f..0390ace9 100644 --- a/crates/navigator-analysis/examples/archaic_segments_probe.rs +++ b/crates/navigator-analysis/examples/archaic_segments_probe.rs @@ -1,8 +1,12 @@ -//! Development harness: run the Tier B segment caller on cached diploid calls. +//! A harness for development. It runs the Tier B segment caller on cached diploid calls. //! -//! The threshold sweep this also carried has been removed — its output is recorded in the design -//! doc (M3 calibration) and the winning values are the `ArchaicConfig` defaults. -//! archaic_segments_probe +//! This tool once also swept the thresholds, and that sweep is gone. Its output sits in the design +//! document, under the M3 calibration, and the values that won are the defaults of +//! `ArchaicConfig`. +//! +//! ```text +//! archaic_segments_probe +//! ``` use navigator_analysis::archaic::{ArchaicCallable, ArchaicClassify, ArchaicOutgroup}; use navigator_analysis::archaic_segments::{call_archaic_segments, ArchaicConfig}; use navigator_analysis::caller::SiteGenotype; diff --git a/crates/navigator-analysis/examples/cram_query_probe.rs b/crates/navigator-analysis/examples/cram_query_probe.rs index b6ef6a35..75fed177 100644 --- a/crates/navigator-analysis/examples/cram_query_probe.rs +++ b/crates/navigator-analysis/examples/cram_query_probe.rs @@ -1,6 +1,7 @@ -//! Time the phases of an indexed region read, so a slow BAM/CRAM path can be attributed to the -//! part actually responsible (open, header, first query, warm query, bulk region iteration) rather -//! than to whichever call the wall-clock happened to land in. +//! Time each phase of a region read that uses an index. A slow path over a BAM or a CRAM then goes +//! to the part that caused it. Those parts are the open, the header, the first query, a warm +//! query, and the walk over the whole region. Without this, the blame lands on whichever call the +//! wall clock happened to be inside. //! //! Written to diagnose a CRAM that took ~500x longer than an equivalent BAM for one region query. //! @@ -29,7 +30,8 @@ fn main() { let (header, mut reader) = open_indexed(path, Some(refp)).expect("open"); println!("open + header : {:>8.2?}", t0.elapsed()); - // A single-base region: cost here is per-query overhead, not per-record work. + // A region of one base. The cost here is the overhead of one query, and not the work at each + // record. let one = |p: usize| -> Region { format!("{contig}:{p}-{p}").parse().expect("region") }; let t = Instant::now(); @@ -64,10 +66,13 @@ fn main() { el / (span as u32 / 1_000_000).max(1) ); - // VERIFY=1 checks the container-skipping query against noodles' own (whole-contig) Query on - // REAL data. The checked-in fixture is a single container, so only a large multi-container CRAM - // can catch a container wrongly skipped -- which would present as a faster caller, not a broken - // one. Slow by construction: the oracle is the implementation we replaced. + // With `VERIFY=1` the probe checks the query that skips containers against the own `Query` of + // noodles, which walks the whole contig, on REAL data. + // + // The fixture in the repo holds one container. Only a large CRAM with many containers can + // catch a container that the code skips wrongly. Such a fault would look like a faster caller, + // and not a broken one. This check is slow by construction, because the oracle is the code + // that we replaced. if std::env::var("VERIFY").is_ok_and(|v| v == "1") { use noodles::cram; diff --git a/crates/navigator-analysis/examples/denovo_profile.rs b/crates/navigator-analysis/examples/denovo_profile.rs index 7a4931c4..14baaeac 100644 --- a/crates/navigator-analysis/examples/denovo_profile.rs +++ b/crates/navigator-analysis/examples/denovo_profile.rs @@ -1,5 +1,6 @@ -//! Standalone de-novo caller profiling harness — runs [`caller::call_denovo`] on one contig -//! of a BAM/CRAM with no async/test wrapper, so a sampling profiler sees only the hot path. +//! A profile harness for the de-novo caller, on its own. It runs [`caller::call_denovo`] over one +//! contig of a BAM or a CRAM. There is no async wrapper and no test wrapper, so a profiler that +//! samples sees the hot path alone. //! //! ```sh //! BAM=/Users/jkane/Genomics/WGS229/WGS229.bwa-mem.chm13v2.cram \ diff --git a/crates/navigator-analysis/examples/flag_census.rs b/crates/navigator-analysis/examples/flag_census.rs index 2ddf4476..f29a3d47 100644 --- a/crates/navigator-analysis/examples/flag_census.rs +++ b/crates/navigator-analysis/examples/flag_census.rs @@ -1,8 +1,9 @@ -//! Count records by SAM flag class in a BAM — primary, secondary, supplementary, unmapped. +//! Count the records of a BAM by their class of SAM flag: primary, secondary, supplementary, and +//! unmapped. //! -//! A realigned alignment should hold roughly one primary per input read. When it holds several -//! times that, the extra records are the mapper's alternative placements, and the census says so -//! in a line instead of leaving a suspicious record count to be argued about. +//! A realigned alignment must hold about one primary record for each input read. When it holds some +//! times that count, the extra records are the alternative placements that the mapper chose. This +//! census says so in one line. Without it, a record count that looks wrong leads to an argument. //! //! ```sh //! BAM=~/.decodingus/realigned/alignment-8.chm13v2.0.bam \ diff --git a/crates/navigator-analysis/examples/place_y.rs b/crates/navigator-analysis/examples/place_y.rs index e07da64b..5f017236 100644 --- a/crates/navigator-analysis/examples/place_y.rs +++ b/crates/navigator-analysis/examples/place_y.rs @@ -1,7 +1,11 @@ -//! One-off placement debug: genotype a BAM at every Y-tree position and dump the ranked -//! placement + what `deepen_terminal` returns + the calls along a named lineage. +//! A one-off debug of a placement. It genotypes a BAM at every position of the Y tree. It then +//! writes out the ranked placement, what `deepen_terminal` returns, and the calls along a lineage +//! that you name. //! -//! cargo run --release --example place_y -p navigator-analysis -- [FOCUS_NODE] +//! ```text +//! cargo run --release --example place_y -p navigator-analysis -- \ +//! [FOCUS_NODE] +//! ``` use std::collections::HashSet; use std::path::Path; @@ -75,7 +79,8 @@ fn main() { println!("\nfirst admissible start = {sname}; deepen_terminal -> {tname}"); } - // Walk root->focus and print the call state of each node's defining SNPs. + // Walk from the root to the focus node. At each node, print the call state of the SNPs that + // define it. let byname: std::collections::HashMap<&str, i64> = tree.nodes.values().map(|n| (n.name.as_str(), n.id)).collect(); if let Some(&fid) = byname.get(focus.as_str()) { let mut parent = std::collections::HashMap::new(); diff --git a/crates/navigator-analysis/examples/place_y_vcf.rs b/crates/navigator-analysis/examples/place_y_vcf.rs index 6992e75a..aa2c78e2 100644 --- a/crates/navigator-analysis/examples/place_y_vcf.rs +++ b/crates/navigator-analysis/examples/place_y_vcf.rs @@ -1,8 +1,13 @@ -//! One-off placement debug for a vendor Y VCF (FTDNA Big Y / YSEQ / Full Genomes): parse the VCF's -//! chrY calls, place them on the FTDNA tree, and print the terminal + lineage. Mirrors the app's -//! `place_chip_panel` (score → path_admissible → deepen_terminal) without the store/network. +//! A one-off debug of a placement, for a Y VCF from a vendor: an FTDNA Big Y, a YSEQ test, or a +//! Full Genomes test. //! -//! cargo run --release --example place_y_vcf -p navigator-analysis -- +//! It parses the chrY calls of the VCF, places them on the FTDNA tree, and prints the terminal and +//! the lineage. It follows the same path as `place_chip_panel` in the app, which is `score`, then +//! `path_admissible`, then `deepen_terminal`. It uses no store and no network. +//! +//! ```text +//! cargo run --release --example place_y_vcf -p navigator-analysis -- +//! ``` use std::collections::HashMap; use std::path::Path; diff --git a/crates/navigator-analysis/examples/profile_analysis.rs b/crates/navigator-analysis/examples/profile_analysis.rs index 2d13fdf8..81ae216c 100644 --- a/crates/navigator-analysis/examples/profile_analysis.rs +++ b/crates/navigator-analysis/examples/profile_analysis.rs @@ -1,9 +1,15 @@ -//! Profile the per-step analysis cost on a real BAM/CRAM to find batch-analysis hotspots. -//! Times the walkers the deep-analyze pipeline drives, whole-genome vs targeted-Y scoped, and the -//! sequential vs the parallel coverage path — plus a chrY region-query genotyping pass (the -//! haplogroup step). Read-only; nothing persisted. +//! Profile the cost of each analysis step, on a real BAM or CRAM, to find where a batch analysis +//! spends its time. //! -//! cargo run --release --example profile_analysis -p navigator-analysis -- +//! It times the walkers that the deep-analyze pipeline drives, over the whole genome and over a +//! targeted-Y scope. It times the sequential coverage path against the parallel one. And it times +//! a genotype pass over chrY through a region query, which is the haplogroup step. +//! +//! It only reads. It stores nothing. +//! +//! ```text +//! cargo run --release --example profile_analysis -p navigator-analysis -- +//! ``` use std::collections::HashSet; use std::path::Path; use std::time::Instant; @@ -56,8 +62,9 @@ fn main() { .err() }); - // chrY haplogroup genotyping pass: a region query over chrY tallying ~200k target sites - // (representative of the Y tree's chrY loci) — the deep-analyze Y step's read pattern. + // The genotype pass of the chrY haplogroup step. It is a region query over chrY, and it + // tallies about 200k target sites. That count stands in for the chrY loci of the Y tree. The + // pattern of the reads matches the Y step of the deep analysis. let hp = HaploidCallerParams::default(); let targets: HashSet = (1..=200_000u32).map(|i| i as i64 * 300).collect(); timed("chrY genotyping call_bases_at (200k sites)", || { diff --git a/crates/navigator-analysis/examples/profile_metrics.rs b/crates/navigator-analysis/examples/profile_metrics.rs index 387ea46d..8d96cadb 100644 --- a/crates/navigator-analysis/examples/profile_metrics.rs +++ b/crates/navigator-analysis/examples/profile_metrics.rs @@ -1,10 +1,14 @@ -//! Profiling harness for the unified quality-metrics walker's hot per-read loop. +//! A profile harness for the hot loop of the unified quality-metrics walker, which runs at each +//! read. //! -//! cargo build --release --example profile_metrics -p navigator-analysis -//! ./target/release/examples/profile_metrics +//! ```text +//! cargo build --release --example profile_metrics -p navigator-analysis +//! ./target/release/examples/profile_metrics +//! ``` //! -//! Times one contig in three passes (raw decode / +RecordBuf copy / +metrics) so the per-read -//! cost splits out, and completes in ~a minute instead of walking the whole genome for hours. +//! It times one contig in three passes: the raw decode, then that plus the `RecordBuf` copy, then +//! that plus the metrics. The cost at each read then separates into its parts. It finishes in +//! about a minute, and it does not walk the whole genome for hours. use std::path::Path; @@ -24,7 +28,8 @@ fn main() { } let params = navigator_analysis::coverage::CallableLociParams::default(); - // "FULL" → run the real production walker on the whole BAM with per-contig timestamps. + // "FULL" runs the real production walker over the whole BAM, and it prints a timestamp at + // each contig. if args[3].eq_ignore_ascii_case("full") { let start = std::time::Instant::now(); let progress = move |done: usize, total: usize| { diff --git a/crates/navigator-analysis/examples/reassembly_probe.rs b/crates/navigator-analysis/examples/reassembly_probe.rs index 643a3119..d44e4dbc 100644 --- a/crates/navigator-analysis/examples/reassembly_probe.rs +++ b/crates/navigator-analysis/examples/reassembly_probe.rs @@ -1,25 +1,45 @@ -//! Prove-out for the reassembly caller (Option B) — pure Rust (bio), no external tools, Windows-clean. +//! A proof of the reassembly caller, which is Option B. It is pure Rust, over the `bio` crate. It +//! needs no external tool, and it is clean on Windows. //! -//! Purpose: (1) validate the pure-Rust library stack end-to-end on a real CRAM, and (2) scope what -//! the resolver actually needs. It builds a reference and an alternate haplotype over a window around -//! `pos`, realigns every spanning read to both (`bio::alignment::pairwise`), drops reads that align -//! poorly to both (misaligned paralog junk), POA-assembles the survivors as a check -//! (`bio::alignment::poa`), and — the load-bearing step — scores each spanning read against both -//! haplotypes with a **base-quality-aware PairHMM** (`bio::stats::pairhmm`) and genotypes by the -//! aggregate log-likelihood ratio. +//! It has two purposes. The first is to check the pure-Rust library stack from end to end, on a +//! real CRAM. The second is to find out what the resolver needs. //! -//! Finding (validated on the real WGS229 CHM13 CRAM vs GATK's own gVCF, which calls all five DERIVED): -//! plain realignment resolves the *clean* controls but TIES the marginal misaligned-ref sites the pileup -//! caller misses (e.g. 4284195: crude score 10/10). The base-quality-aware PairHMM breaks those ties and -//! recovers **4 of 5** — 3318203/16652092 (controls), 4284195 (GATK AD 9,10 / GQ44 / MQRankSum -3.55, the -//! textbook misaligned-ref case) and 11191589 all → DERIVED, matching GATK. The one miss (20973395) has -//! paralog reference reads that pass the MQ≥20 gate; GATK drops them via active-region fragment/read -//! selection (its own DP falls 7→5 there) — the remaining ingredient the full caller needs. So the Option -//! B caller = active-region detection + POA assembly + **read-vs-haplotype PairHMM** + genotyping; this -//! probe proves the whole pure-Rust stack compiles/runs on a real CRAM and that PairHMM is the tie-breaker. +//! It builds a reference haplotype and an alternate one, over a window around `pos`. It realigns +//! every read that covers the window to both, with `bio::alignment::pairwise`. It drops a read +//! that aligns poorly to both, which is paralog junk that the aligner put in the wrong place. It +//! then assembles the reads that stay with a POA, as a check, through `bio::alignment::poa`. //! -//! cargo run --release --example reassembly_probe -p navigator-analysis -- \ -//! chrY [window=40] +//! The step that carries the weight comes last. It scores each read that covers the window +//! against both haplotypes, with a **PairHMM that knows the base qualities**, from +//! `bio::stats::pairhmm`. It then takes a genotype from the log-likelihood ratio over all of the +//! reads. +//! +//! **What it found.** The check ran on the real WGS229 CHM13 CRAM, against the own gVCF of GATK, +//! which calls all five sites DERIVED. +//! +//! A plain realignment resolves the *clean* control sites. But it gives a TIE at the marginal +//! misaligned-reference sites that the pileup caller misses. At 4284195 the crude score is 10 +//! against 10. +//! +//! The PairHMM over base qualities breaks those ties, and it recovers **4 of the 5**. Those are +//! 3318203 and 16652092, which are the controls; 4284195, where GATK gives AD 9,10, GQ 44 and +//! MQRankSum -3.55, and which is the textbook misaligned-reference case; and 11191589. All four +//! come out DERIVED, and they match GATK. +//! +//! The one that it misses is 20973395. There, paralog reference reads pass the MQ≥20 gate. GATK +//! drops them in its selection of the fragments and reads of the active region, and its own DP +//! falls from 7 to 5 there. That selection is the one part that the full caller still needs. +//! +//! So the Option B caller has four parts. Find the active region. Assemble with a POA. Score each +//! **read against each haplotype with a PairHMM**. Then genotype. +//! +//! This probe shows two things. The whole pure-Rust stack builds and runs on a real CRAM. And the +//! PairHMM is what breaks the tie. +//! +//! ```text +//! cargo run --release --example reassembly_probe -p navigator-analysis -- \ +//! chrY [window=40] +//! ``` use std::path::Path; @@ -45,7 +65,8 @@ fn base_index(b: u8) -> Option { /// segmental-duplication / ampliconic loci masquerade as high-base-quality reference support. const MIN_MAPQ: u8 = 20; -/// A spanning read's window sequence, its per-base Phred qualities, mapping quality, and site cover. +/// One read that covers the window. It holds the sequence in the window frame, the Phred quality +/// of each of its bases, its mapping quality, and whether it covers the site. struct WinRead { bases: Vec, quals: Vec, @@ -53,7 +74,7 @@ struct WinRead { covers_pos: bool, } -/// Reads overlapping `[lo, hi]` as [`WinRead`]s, plus the raw A/C/G/T pileup at `pos`. +/// The reads inside `[lo, hi]`, as [`WinRead`] values, and the raw A/C/G/T pileup at `pos`. fn window_reads(cram: &Path, refp: &Path, contig: &str, pos: i64, lo: i64, hi: i64) -> (Vec, [u32; 4]) { let (header, mut reader) = open_indexed(cram, Some(refp)).expect("open cram"); let region: Region = format!("{contig}:{lo}-{hi}").parse().expect("region"); @@ -103,7 +124,8 @@ fn window_reads(cram: &Path, refp: &Path, contig: &str, pos: i64, lo: i64, hi: i } (true, false) => ref_pos += len as i64, (false, true) => { - // keep insertion bases inside the window so an indel haplotype is preserved + // Keep the bases of an insertion that lie inside the window, so that an + // indel haplotype survives. if kind == noodles::sam::alignment::record::cigar::op::Kind::Insertion && ref_pos > lo && ref_pos <= hi @@ -143,8 +165,9 @@ fn poa_consensus(reads: &[Vec]) -> Vec { aligner.consensus() } -/// Base the consensus carries at reference coordinate `pos`, by semiglobally aligning the consensus -/// (query) to the reference window `win_ref` (which starts at reference coordinate `win_start`). +/// The base that the consensus carries at the reference coordinate `pos`. It aligns the consensus, +/// as the query, to the reference window `win_ref`, in a semiglobal way. That window starts at the +/// reference coordinate `win_start`. fn consensus_base_at(consensus: &[u8], win_ref: &[u8], win_start: i64, pos: i64) -> char { let score = |a: u8, b: u8| if a == b { 1i32 } else { -4i32 }; let mut aligner = PwAligner::new(-5, -1, score); @@ -198,18 +221,23 @@ fn consensus_base_at(consensus: &[u8], win_ref: &[u8], win_start: i64, pos: i64) // ---- base-quality-aware PairHMM: P(read | haplotype) ------------------------------------------ // -// This is the tie-breaker the crude alignment score lacks. Each read base votes for ref-vs-alt at -// `pos` weighted by its Phred quality: a Q40 base mismatching a haplotype costs ~10^-4, a Q10 base -// costs only ~10^-1, so noisy bases can't outvote clean ones. Aggregating the log-likelihood ratio -// over all spanning reads is exactly how GATK's HaplotypeCaller resolves the misaligned-ref pileups. +// This is what breaks a tie, and the crude alignment score can not. Each base of a read votes for +// the ref allele, or for the alt allele, at `pos`. The Phred quality of that base sets the weight +// of its vote. A Q40 base that does not match a haplotype costs about 10^-4, and a Q10 base costs +// only about 10^-1. A noisy base can then not out-vote a clean one. +// +// The sum of the log-likelihood ratio, over every read that covers the site, is how GATK does +// this. Its HaplotypeCaller resolves a pileup with a wrong reference alignment that way. -/// Phred score → error probability, clamped to a sane band (Q>0, and never a certain match/mismatch). +/// The error probability that a Phred score gives, clamped to a correct band: Q>0, and never a +/// match or a mismatch that is sure. fn phred_err(q: u8) -> f64 { let q = q.clamp(2, 60) as f64; 10f64.powf(-q / 10.0) } -/// Emission: `x` = read (carries per-base quality), `y` = candidate haplotype. +/// The emission. `x` is the read, which carries a quality at each base. `y` is the candidate +/// haplotype. struct ReadHapEmission<'a> { read: &'a [u8], quals: &'a [u8], @@ -256,7 +284,8 @@ impl GapParameters for GapParams { } } -/// Semiglobal in the read: free leading/trailing offset so window-edge trimming isn't penalised. +/// Semiglobal in the read. The offset at the start and at the end is free, so a cut at the edge of +/// the window costs nothing. struct Semiglobal; impl StartEndGapParameters for Semiglobal { fn free_start_gap_x(&self) -> bool { @@ -267,7 +296,8 @@ impl StartEndGapParameters for Semiglobal { } } -/// Log-probability that `read` (with `quals`) was produced by `hap`, marginalised over alignments. +/// The log-probability that `hap` gave `read`, which carries `quals`. It marginalises over the +/// alignments. fn hap_likelihood(hmm: &mut PairHMM, read: &[u8], quals: &[u8], hap: &[u8]) -> LogProb { hmm.prob_related(&ReadHapEmission { read, quals, hap }, &Semiglobal, None) } @@ -319,9 +349,12 @@ fn main() { win_alt[off] = alt_base as u8; } - // Realign each spanning read to both haplotypes; drop reads that align poorly to *both* - // (they carry mismatches beyond the site → misaligned paralog / junk, not from this locus). - // For the survivors, also score P(read|ref-hap) vs P(read|alt-hap) with the PairHMM. + // Realign each read that covers the window to both haplotypes. Drop a read that aligns + // poorly to *both*, because it carries mismatches beyond the site. That read is a paralog + // that the aligner put in the wrong place, or junk, and it did not come from this locus. + // + // For the reads that stay, also score P(read|ref-hap) against P(read|alt-hap), with the + // PairHMM. let mut hmm = PairHMM::new(&GapParams); let (mut ref_supp, mut alt_supp, mut dropped) = (0u32, 0u32, 0u32); let (mut hmm_ref, mut hmm_alt) = (0u32, 0u32); @@ -352,7 +385,7 @@ fn main() { std::cmp::Ordering::Less => ref_supp += 1, std::cmp::Ordering::Equal => {} } - // Base-quality-aware likelihood ratio (the load-bearing tie-breaker). + // The likelihood ratio that knows the base qualities. It is what breaks the tie. let lp_ref = hap_likelihood(&mut hmm, win, &r.quals, &win_ref); let lp_alt = hap_likelihood(&mut hmm, win, &r.quals, &win_alt); logodds += *lp_alt - *lp_ref; @@ -362,7 +395,7 @@ fn main() { _ => {} } } - // POA consensus of the surviving reads as a cross-check (informational). + // The POA consensus of the reads that stay, as a cross-check. It is for information. let cons_note = if kept.len() >= 2 { let cons = poa_consensus(&kept); let cb = consensus_base_at(&cons, &win_ref, lo, pos); diff --git a/crates/navigator-analysis/examples/reassembly_validate.rs b/crates/navigator-analysis/examples/reassembly_validate.rs index 59d63cb8..b07cd6d2 100644 --- a/crates/navigator-analysis/examples/reassembly_validate.rs +++ b/crates/navigator-analysis/examples/reassembly_validate.rs @@ -1,9 +1,14 @@ -//! Validate the phase-2 reassembly wiring end-to-end on a real CRAM: for each position, run the -//! bounded de-novo caller with reassembly OFF (pileup only) vs ON, and report whether the position -//! is now called. Confirms the misaligned-ref truth privates the paralog gate drops are recovered. +//! Check the phase-2 reassembly path from end to end, on a real CRAM. At each position it runs the +//! bounded de-novo caller twice: once with the reassembly OFF, which is the pileup alone, and once +//! with it ON. It then reports whether the position now has a call. //! -//! cargo run --release --example reassembly_validate -p navigator-analysis -- \ -//! chrY +//! That confirms one thing: the code recovers the true private variants, at a misaligned reference, +//! that the paralog gate drops. +//! +//! ```text +//! cargo run --release --example reassembly_validate -p navigator-analysis -- \ +//! chrY +//! ``` use std::path::Path; diff --git a/crates/navigator-analysis/examples/site_reads.rs b/crates/navigator-analysis/examples/site_reads.rs index e324257b..0c8078f1 100644 --- a/crates/navigator-analysis/examples/site_reads.rs +++ b/crates/navigator-analysis/examples/site_reads.rs @@ -1,9 +1,14 @@ -//! Dump per-read detail at a site so we can see *why* a marginal pileup is ambiguous — the input to -//! designing v2 active-region read selection. For each spanning read: query name, mate/pair flags, -//! MAPQ, the base + base-quality it carries at the site, and its mismatch count vs the reference -//! window (excluding the site itself). +//! Write out the detail of each read at a site, so that a person can see *why* a marginal pileup is +//! ambiguous. That is the input to the design of the v2 selection of reads in an active region. //! -//! cargo run --release --example site_reads -p navigator-analysis -- chrY [win=40] +//! At each read that covers the site it gives six things. The query name. The flags of the mate +//! and the pair. The MAPQ. The base at the site, and the quality of that base. And the count of +//! mismatches against the reference window, which leaves the site itself out. +//! +//! ```text +//! cargo run --release --example site_reads -p navigator-analysis -- \ +//! chrY [win=40] +//! ``` use std::collections::HashMap; use std::path::Path; @@ -53,7 +58,8 @@ fn main() { .unwrap_or_default(); let mapq = rec.mapping_quality().map_or(255, |m| m.get()); - // Walk the CIGAR: capture the base at `pos` and count mismatches vs ref (excluding `pos`). + // Walk the CIGAR. Take the base at `pos`, and count the mismatches against the reference. + // That count leaves `pos` out. let mut ref_pos = start; let mut qoff = 0usize; let mut site_base = '.'; diff --git a/crates/navigator-analysis/examples/str_calibrate.rs b/crates/navigator-analysis/examples/str_calibrate.rs index 3fc9c9b0..5ffe42a0 100644 --- a/crates/navigator-analysis/examples/str_calibrate.rs +++ b/crates/navigator-analysis/examples/str_calibrate.rs @@ -1,7 +1,13 @@ -//! STR convention calibration: run the caller on a corpus of Big Y kits (BAM + FTDNA DYS CSV each) -//! and tabulate, per marker, the offset (ftdna − caller) distribution across kits — to classify each -//! marker reliable / convention-offset / variable-exclude and measure callability. -//! Usage: cargo run --release -p navigator-analysis --example str_calibrate -- +//! The calibration of the STR convention. It runs the caller over a corpus of Big Y kits, where +//! each kit holds a BAM and an FTDNA DYS CSV. At each marker it tabulates the distribution of the +//! offset, which is `ftdna − caller`, across the kits. +//! +//! That distribution puts each marker into a class: reliable, an offset of the convention, or too +//! variable to use. It also measures how often the caller can call each marker. +//! +//! ```text +//! cargo run --release -p navigator-analysis --example str_calibrate -- +//! ``` use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; @@ -9,8 +15,9 @@ use navigator_analysis::strcaller::{genotype_str_loci, StrCallerParams, StrConfi use navigator_analysis::strmarker::{to_ftdna, MarkerStatus}; use navigator_analysis::strref::{load_hipstr_contig, StrLocus}; -/// Find (csv, alignment) under a kit folder: a `*_DYS_Results*.csv` + an alignment, preferring the -/// CHM13-realigned `.cram` (chrYM) over the original `.bam`. +/// Find the `(csv, alignment)` pair under the folder of one kit. That is a `*_DYS_Results*.csv`, +/// and an alignment. It takes the `.cram` that somebody realigned to CHM13, which holds chrYM, +/// before the original `.bam`. fn kit_pair(dir: &Path) -> Option<(PathBuf, PathBuf)> { let mut csv = None; let mut cram = None; @@ -47,7 +54,8 @@ fn walkdir(dir: &Path) -> Vec { out } -/// Parse the wide FTDNA DYS CSV (row1 names, row2 values; quoted, leading spaces, "-" = no call). +/// Parse the wide DYS CSV of FTDNA. Row 1 holds the names, and row 2 holds the values. The fields +/// carry quotation marks and spaces in front, and a `-` means a no-call. fn parse_ftdna(csv: &Path) -> HashMap { let text = std::fs::read_to_string(csv).unwrap_or_default(); let mut lines = text.lines(); @@ -129,9 +137,11 @@ fn main() { .filter(|g| g.confidence != StrConfidence::Low && g.alleles.len() == 1) .collect(); - // QC: end-user-provided data may have a swapped BAM↔CSV. Fingerprint the kit on the markers - // strmarker already calls Reliable (offset 0): if ≥10 are comparable but <70% match this - // kit's own CSV, the alignment and the CSV are likely different people — exclude it. + // A QC step. Data that an end user gives can hold a BAM and a CSV that belong to two + // different people. So fingerprint the kit on the markers that strmarker already calls + // Reliable, at an offset of 0. Take a kit where 10 or more of those are comparable, and + // where fewer than 70% match its own CSV. The alignment and the CSV of that kit probably + // come from two people, so leave it out. let (mut rel_ok, mut rel_tot) = (0, 0); for g in &single { let cm = to_ftdna(&g.name, g.alleles[0]); @@ -166,7 +176,7 @@ fn main() { } eprintln!("calibrated on {n_kits} kits ({n_skipped_swap} skipped as likely swaps, {n_panic} CRAM-decode panics)"); - // Per-marker classification. + // The class of each marker. println!("\n# marker n callable_kits modal_offset agreement% class"); let mut markers: Vec<&String> = offsets.keys().collect(); markers.sort(); diff --git a/crates/navigator-analysis/examples/validate_coverage.rs b/crates/navigator-analysis/examples/validate_coverage.rs index de7f08d1..5cb012d5 100644 --- a/crates/navigator-analysis/examples/validate_coverage.rs +++ b/crates/navigator-analysis/examples/validate_coverage.rs @@ -1,11 +1,17 @@ -//! End-to-end validation of the **standalone** coverage path (`collect_coverage_callable`, the -//! sequential walker that now consumes lazy `bam::Record` via `records_lazy`) against the trusted -//! per-contig **parallel** walker, on a real BAM. Both produce a `CoverageResult`; the invariant is -//! exact equality (the same one the `unified_matches_standalone_walkers` unit test asserts on a -//! fixture — this runs it on a whole WGS). +//! An end-to-end check of the **separate** coverage path against the **parallel** walker over the +//! contigs, which this project trusts, on a real BAM. //! -//! cargo build --release --example validate_coverage -p navigator-analysis -//! ./target/release/examples/validate_coverage +//! The separate path is `collect_coverage_callable`. That sequential walker now reads a lazy +//! `bam::Record`, through `records_lazy`. +//! +//! Both give a `CoverageResult`, and the invariant is that the two are exactly equal. The +//! `unified_matches_standalone_walkers` unit test asserts the same thing on a fixture. This tool +//! runs it over a whole WGS. +//! +//! ```text +//! cargo build --release --example validate_coverage -p navigator-analysis +//! ./target/release/examples/validate_coverage +//! ``` use std::path::Path; use std::time::Instant; @@ -50,7 +56,8 @@ fn main() { eprintln!("\nstandalone coverage done in {standalone_dur:.1?}"); summarize("standalone", &standalone); - // 2. The oracle: trusted per-contig parallel walker on the same file. + // 2. The oracle. It is the parallel walker over the contigs, which this project trusts, on + // the same file. let t1 = Instant::now(); let progress2 = |_done: usize, _total: usize| {}; let unified = match unified::collect_unified_metrics_parallel_with_progress( @@ -71,7 +78,7 @@ fn main() { eprintln!("\nparallel walker done in {parallel_dur:.1?}"); summarize("parallel ", &unified.coverage); - // 3. The invariant: byte-for-byte identical coverage. + // 3. The invariant. The two coverage results must match to the last byte. eprintln!("\n=== comparison ==="); if standalone == unified.coverage { eprintln!("PASS — standalone == parallel (field-for-field identical)"); diff --git a/crates/navigator-analysis/src/ancestry.rs b/crates/navigator-analysis/src/ancestry.rs index 3f3e33c7..1f72e1f9 100644 --- a/crates/navigator-analysis/src/ancestry.rs +++ b/crates/navigator-analysis/src/ancestry.rs @@ -1,11 +1,12 @@ -//! Ancestry estimation — the genotype → population-proportion path, Navigator-side. +//! Ancestry estimation. This is the path from genotypes to population proportions, on the +//! Navigator side. //! -//! Phase 1 is the allele-frequency likelihood (no PCA, no GATK): the bundled [`AncestryPanel`] -//! carries per-(super-)population alt-allele frequencies at a set of ancestry-informative -//! sites; we genotype the sample there with the GL caller ([`crate::caller::genotype_sites`]), -//! then score each population by the binomial likelihood of the observed diploid genotypes -//! under its allele frequencies. The panel is built offline by `navigator-panelbuild` from the -//! 1000G-on-CHM13 VCFs. +//! Phase 1 is the allele-frequency likelihood. It uses no PCA and no GATK. The bundled +//! [`AncestryPanel`] carries alt-allele frequencies for each (super-)population at a set of +//! ancestry-informative sites. The code genotypes the sample at those sites with the GL caller +//! ([`crate::caller::genotype_sites`]). It then scores each population by the binomial +//! likelihood of the observed diploid genotypes under the allele frequencies of that +//! population. `navigator-panelbuild` builds the panel offline from the 1000G-on-CHM13 VCFs. //! //! The result is a [`navigator_domain::ancestry::AncestryResult`]. PCA projection //! ([`AncestryResult::pca_coordinates`]) is phase 2. @@ -22,7 +23,7 @@ use serde::{Deserialize, Serialize}; use crate::caller::SiteGenotype; use crate::AnalysisError; -/// One ancestry-informative site with its per-population alt-allele frequencies. `freqs[i]` +/// One ancestry-informative site with the alt-allele frequency for each population. `freqs[i]` /// aligns with [`AncestryPanel::populations`]`[i]`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PanelSite { @@ -39,7 +40,7 @@ pub struct PanelSite { pub struct AncestryPanel { /// Canonical reference build the site coordinates are in (e.g. "chm13v2.0"). pub build: String, - /// Population codes, defining the axis order of every `PanelSite::freqs`. + /// Population codes. They give the axis order of every `PanelSite::freqs`. pub populations: Vec, pub sites: Vec, } @@ -55,9 +56,9 @@ impl AncestryPanel { bincode::serialize(self).map_err(|e| AnalysisError::Message(format!("panel encode: {e}"))) } - /// A panel restricted to `codes` (those present, in `codes` order), projecting each site's - /// per-population frequencies down to the kept columns. Used to run a well-conditioned - /// admixture EM over a curated subset of a large fine-frequency panel. + /// A panel that keeps only `codes`, those that are present, in `codes` order. It projects + /// the frequencies of each site down to the columns that stay. Use it to run a + /// well-conditioned admixture EM over a curated subset of a large fine-frequency panel. pub fn subset(&self, codes: &[&str]) -> AncestryPanel { let keep: Vec = codes .iter() @@ -90,16 +91,18 @@ impl AncestryPanel { } } -/// PCA loadings for projecting a sample onto the reference populations' principal-component -/// space (Phase 2). Built offline by `navigator-panelbuild` from the 1000G genotype matrix: -/// per-SNP loadings + means (for centering), plus each population's centroid and diagonal -/// variance in PC space (for the Mahalanobis/Gaussian assignment and the scatter plot). +/// PCA loadings. They project a sample onto the principal-component space of the reference +/// populations (Phase 2). `navigator-panelbuild` builds them offline from the 1000G genotype +/// matrix. They hold a loading and a mean for each SNP, and the mean centres the data. They +/// also hold the centroid and the diagonal variance of each population in PC space, for the +/// Mahalanobis/Gaussian assignment and for the scatter plot. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PcaLoadings { pub build: String, - /// (contig, 1-based pos) per row, aligned with `means` and the rows of `loadings`. + /// A (contig, 1-based pos) for each row, aligned with `means` and the rows of `loadings`. pub sites: Vec<(String, i64)>, - /// Mean dosage per site (reference panel) — used to centre the sample before projecting. + /// The mean dosage at each site, from the reference panel. It centres the sample before + /// the projection. pub means: Vec, pub n_components: usize, /// Row-major `sites.len() × n_components`. @@ -142,18 +145,24 @@ pub struct HapSite { pub alternate_allele: char, } -/// A reference panel of **phased haplotypes** at the painting loci — the substrate for statistical -/// phasing (the Li & Stephens copying model) and, later, copying-model local-ancestry inference. +/// A reference panel of **phased haplotypes** at the painting loci. It is the substrate for +/// statistical phasing, which uses the Li & Stephens copying model, and, later, for +/// local-ancestry inference with the copying model. /// -/// Distinct from [`AncestryPanel`], which stores per-population allele *frequencies*: this carries -/// each individual reference *haplotype*'s allele at every site plus its population label, so a -/// sample can be phased as a mosaic of these haplotypes. Built offline by `navigator-panelbuild` -/// from the **phased** 1000G-on-CHM13 VCFs. Only modern, phased references enter it (1000G super + -/// fine populations); pseudo-haploid / unphased sources (ancient, AADR continental groups) are -/// excluded — those contribute to the frequency panel used by the two-tier fine-resolution step. +/// It is different from [`AncestryPanel`], which stores allele *frequencies* for each +/// population. This panel carries the allele of each individual reference *haplotype* at every +/// site, and the population label of that haplotype. The code can then phase a sample as a +/// mosaic of these haplotypes. `navigator-panelbuild` builds it offline from the **phased** +/// 1000G-on-CHM13 VCFs. /// -/// Alleles are bit-packed row-major: haplotype `h`'s allele at site `s` is bit `h * n_sites + s` -/// of [`alleles`](Self::alleles). This keeps a ~5000-haplotype × ~20k-site reference near ~12 MB. +/// Only modern, phased references go into it: the 1000G super populations and fine populations. +/// Pseudo-haploid and unphased sources, which are the ancient data and the AADR continental +/// groups, stay out. Those sources go into the frequency panel that the two-tier +/// fine-resolution step uses. +/// +/// The alleles are bit-packed row-major. The allele of haplotype `h` at site `s` is bit +/// `h * n_sites + s` of [`alleles`](Self::alleles). This keeps a reference of about 5000 +/// haplotypes by about 20k sites near 12 MB. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct HaplotypeReference { /// Canonical reference build the site coordinates are in (e.g. "chm13v2.0"). @@ -162,7 +171,7 @@ pub struct HaplotypeReference { pub sites: Vec, /// Distinct population codes; `hap_pop[h]` indexes into this axis. pub populations: Vec, - /// One entry per haplotype: the index into [`populations`](Self::populations) of its label. + /// One entry for each haplotype: the index of its label in [`populations`](Self::populations). pub hap_pop: Vec, /// `n_haplotypes × n_sites` alleles, bit-packed row-major (see the type doc). `1` = alt. pub alleles: Vec, @@ -181,8 +190,9 @@ impl HaplotypeReference { bincode::serialize(self).map_err(|e| AnalysisError::Message(format!("hap reference encode: {e}"))) } - /// Pack per-haplotype allele rows (`rows[h][s]` = 0/1) into the bit-packed form. `hap_pop[h]` - /// is the population index of haplotype `h`. Used by the offline builder and by tests. + /// Pack the allele row of each haplotype (`rows[h][s]` = 0/1) into the bit-packed form. + /// `hap_pop[h]` is the population index of haplotype `h`. The offline builder and the tests + /// use this. pub fn from_rows( build: String, sites: Vec, @@ -230,10 +240,10 @@ impl HaplotypeReference { self.n_sites == 0 || self.n_haplotypes == 0 } - /// The same reference with the haplotypes in `drop` removed (sites, population axis and every - /// other haplotype unchanged). Leave-one-out validation of the copying painter needs it: a test - /// individual taken *from* the reference would otherwise be painted by copying itself, which - /// measures nothing. Unknown indices are ignored. + /// The same reference with the haplotypes in `drop` removed. The sites, the population axis + /// and every other haplotype do not change. The leave-one-out check of the copying painter + /// needs this. A test individual taken *from* the reference would else get its paint from a + /// copy of itself, which measures nothing. The code ignores an index that it does not know. pub fn without_haplotypes(&self, drop: &[usize]) -> Self { let dropped: std::collections::HashSet = drop.iter().copied().collect(); let keep: Vec = (0..self.n_haplotypes).filter(|h| !dropped.contains(h)).collect(); @@ -258,10 +268,11 @@ impl HaplotypeReference { } } - /// The same reference thinned to every `step`-th site (`step` <= 1 returns a clone). Marker - /// density is the copying model's binding constraint — a shared tract only identifies whose - /// haplotype it is if enough markers fall inside it — so measuring accuracy against density - /// needs the same panel at several densities, which this produces without rebuilding the asset. + /// The same reference thinned to every `step`-th site. A `step` of 1 or less returns a + /// clone. Marker density is the constraint that limits the copying model. A shared tract + /// identifies whose haplotype it is only if enough markers fall inside it. To measure + /// accuracy against density you need the same panel at more than one density. This + /// function makes those panels, and it does not build the asset again. pub fn thin_sites(&self, step: usize) -> Self { if step <= 1 { return self.clone(); @@ -290,11 +301,11 @@ impl HaplotypeReference { } } -/// Project a sample's genotypes onto the reference PCA space: centre each site by its panel -/// mean and accumulate `centered · loading` into each component. A missing genotype contributes -/// 0 (mean-imputed), then the projection is rescaled by `total_sites / sites_used` so a sample -/// with missing genotypes isn't shrunk toward the origin (which would pull it off its true -/// cluster). Returns the sample's coordinate in each principal component. +/// Project the genotypes of a sample onto the reference PCA space. Centre each site by its +/// panel mean, then add `centered · loading` into each component. A missing genotype adds 0, +/// which imputes the mean. The code then scales the projection by `total_sites / sites_used`. +/// Without that scale, a sample with missing genotypes moves toward the origin and away from +/// its true cluster. Returns the coordinate of the sample in each principal component. pub fn project_pca(genotypes: &[SiteGenotype], pca: &PcaLoadings) -> Vec { let dosage: HashMap<(&str, i64), i32> = genotypes .iter() @@ -311,14 +322,14 @@ pub fn project_pca(genotypes: &[SiteGenotype], pca: &PcaLoadings) -> Vec { }) } -/// The PCA projection kernel: accumulate `centered · loading` into each component over the sites -/// the sample actually has, then un-shrink by `n_sites / used` so a sample with missing genotypes -/// isn't pulled toward the origin (see [`project_pca`]). +/// The PCA projection kernel. It adds `centered · loading` into each component, over the sites +/// that the sample has. It then scales up by `n_sites / used`, so that a sample with missing +/// genotypes does not move toward the origin. See [`project_pca`]. /// -/// `centered` yields `(site index, dosage − site mean)` for each present site; `loading` reads the -/// `(site, component)` basis entry. Both are supplied by the caller because the runtime projector -/// and the offline basis builder hold their basis in different layouts (`PcaLoadings` vs a -/// `DMatrix`) — the scaling policy, which has to agree between them, lives here. +/// `centered` gives `(site index, dosage − site mean)` for each site that is present. `loading` +/// reads the `(site, component)` basis entry. The caller gives both, because the runtime +/// projector and the offline basis builder hold their basis in different layouts: `PcaLoadings` +/// against a `DMatrix`. The scale policy must agree between the two, so it lives here. pub fn project_centered( n_sites: usize, n_components: usize, @@ -333,7 +344,8 @@ pub fn project_centered( *coord += value * loading(i, c); } } - // Un-shrink: reference coords were built from all sites; scale up for the missing fraction. + // The reference coordinates come from all of the sites. Scale up for the fraction that is + // missing. if used > 0 { let scale = n_sites as f64 / used as f64; for coord in &mut coords { @@ -346,16 +358,20 @@ pub fn project_centered( /// Parameters for [`paint_local_ancestry`]. #[derive(Debug, Clone)] pub struct PaintParams { - /// Per-bp ancestry-switch rate (segment-length knob): switch prob over distance `d` bp is - /// `1 - exp(-d·rate)`. Smaller → longer segments. Default ≈ one switch per 20 Mb. + /// The ancestry-switch rate for each bp, which controls the segment length. The switch + /// probability over a distance of `d` bp is `1 - exp(-d·rate)`. A smaller rate gives longer + /// segments. The default is about one switch in 20 Mb. pub rate: f64, - /// Runs shorter than this many markers are merged into the neighbouring segment. + /// The code merges a run of fewer markers than this into the segment that is next to it. pub min_segment_sites: usize, - /// Global-composition gate: a super-population whose genome-wide `prior` weight is below this - /// fraction is dropped from the HMM's state set entirely (the dominant ancestry is always kept), - /// so a 99%-European donor can't be *locally* painted East-Asian/South-Asian by a handful of - /// noise loci. AF-based local ancestry over coarse super-pops is prone to inventing a globally - /// absent continent; anchoring the states to the global estimate suppresses that. `0.0` disables. + /// The gate on the global composition. The HMM drops a super-population from its state set + /// if the genome-wide `prior` weight of that population is below this fraction. It always + /// keeps the dominant ancestry. A donor who is 99% European can then not get a *local* + /// East-Asian or South-Asian paint from a few noise loci. `0.0` turns the gate off. + /// + /// Local ancestry from allele frequencies over coarse super-populations can show a continent + /// that the genome does not contain at all. To hold the states to the global estimate stops + /// that. pub min_ancestry: f64, } @@ -369,9 +385,10 @@ impl Default for PaintParams { } } -/// Diploid genotype log-likelihood when the two genome copies draw their alt allele from frequencies -/// `fa` and `fb` (one copy per ancestry): `P(0)=(1-fa)(1-fb)`, `P(1)=fa(1-fb)+(1-fa)fb`, `P(2)=fa·fb`. -/// Missing dosage → uniform. This is the proper diploid (two-copy) emission the pair-state HMM needs. +/// The log-likelihood of a diploid genotype. The two genome copies draw their alt allele from +/// the frequencies `fa` and `fb`, one copy for each ancestry: `P(0)=(1-fa)(1-fb)`, +/// `P(1)=fa(1-fb)+(1-fa)fb`, `P(2)=fa·fb`. A missing dosage gives a uniform value. This is the +/// correct diploid emission, over two copies, that the pair-state HMM needs. fn emit_diploid_ln(g: i32, fa: f64, fb: f64) -> f64 { let fa = fa.clamp(1e-4, 1.0 - 1e-4); let fb = fb.clamp(1e-4, 1.0 - 1e-4); @@ -384,16 +401,21 @@ fn emit_diploid_ln(g: i32, fa: f64, fb: f64) -> f64 { p.max(1e-300).ln() } -/// Paint each chromosome with local ancestry: an HMM over the panel sites whose hidden states are -/// the super-populations, emissions are the diploid genotype likelihood under each population's -/// allele frequency, and transitions penalise ancestry switches by physical distance. Viterbi -/// gives the segment path; forward-backward gives per-site posteriors (segment confidence). +/// Paint each chromosome with local ancestry. The model is an HMM over the panel sites. Its +/// hidden states are the super-populations. Its emissions are the diploid genotype likelihood +/// under the allele frequency of each population. Its transitions penalise an ancestry switch +/// by physical distance. Viterbi gives the segment path, and forward-backward gives the +/// posterior at each site, which is the segment confidence. +/// +/// `prior` is the genome-wide composition `(population_code, weight)`, which this function rolls +/// up to super-populations. It is the stationary and switch distribution of the HMM, and it +/// holds the painting to the global estimate. /// -/// `prior` is the genome-wide composition `(population_code, weight)` (rolled to super-populations -/// here) — the HMM's stationary/switch distribution, anchoring the painting to the global estimate. -/// **Diploid pair-state HMM**: the hidden state is an ancestry pair (both genome copies), so a region -/// where the two copies differ (e.g. EUR/SAS) is shown, not collapsed. Output is two sorted, unphased -/// copies per chromosome (segments tagged `copy` 0/1) — not maternal/paternal (no phasing). +/// **The HMM has diploid pair states**: one hidden state is a pair of ancestries, for both +/// genome copies. A region where the two copies are different, for example EUR and SAS, stays +/// visible and does not collapse. The output is two sorted, unphased copies for each chromosome, +/// with the segments tagged `copy` 0 or 1. The copies are not maternal and paternal, because +/// there is no phasing. pub fn paint_local_ancestry( genotypes: &[SiteGenotype], panel: &AncestryPanel, @@ -416,8 +438,9 @@ pub fn paint_local_ancestry( return Vec::new(); } - // Prior π over the full state set (roll the global composition up to super-pops; normalize; - // uniform fallback when no prior is supplied). + // The prior π over the full state set. Roll the global composition up to the + // super-populations and normalize it. Fall back to a uniform π when the caller gives no + // prior. let mut full_pi = vec![0.0f64; all_states.len()]; for (code, w) in prior { let sp = population_super(code).unwrap_or(code); @@ -432,9 +455,10 @@ pub fn paint_local_ancestry( full_pi.iter_mut().for_each(|p| *p = 1.0 / all_states.len() as f64); } - // Global-composition gate: keep only ancestries present genome-wide (>= min_ancestry), always - // retaining the dominant one, so local painting can't invent a globally absent continent. With a - // uniform (no-prior) π every state clears the default threshold, so gating is a no-op there. + // The gate on the global composition. Keep only an ancestry that the genome shows + // (>= min_ancestry), and always keep the dominant one. The local painting can then not show + // a continent that the genome does not contain at all. With a uniform π, which is the + // no-prior case, every state clears the default threshold, and the gate does nothing. let argmax = full_pi .iter() .enumerate() @@ -463,7 +487,8 @@ pub fn paint_local_ancestry( .map(|g| ((g.contig.as_str(), g.position), g.dosage)) .collect(); - // Per-contig sites with a genotype: (pos, per-state super-pop AF, dosage). Sorted by pos. + // The sites with a genotype, for each contig: (pos, super-pop AF for each state, dosage). + // Sorted by pos. let mut by_contig: BTreeMap, i32)>> = BTreeMap::new(); for site in &panel.sites { if site.freqs.len() != panel.populations.len() { @@ -496,8 +521,9 @@ pub fn paint_local_ancestry( if sites.is_empty() { continue; } - // Diploid MAP path: one ancestry PAIR per locus, canonicalized (min,max) into two sorted, - // coherent copies (copy 0 = lower-index ancestry, copy 1 = higher). Unphased. + // The diploid MAP path. It gives one ancestry PAIR at each locus. The code puts the + // pair into a canonical (min,max) order. That makes two sorted, coherent copies. Copy 0 + // is the ancestry with the lower index, and copy 1 is the higher. There is no phasing. let pairs = diploid_viterbi(&sites, &pi, params.rate, k); let copy0: Vec = pairs.iter().map(|&(a, b)| a.min(b)).collect(); let copy1: Vec = pairs.iter().map(|&(a, b)| a.max(b)).collect(); @@ -521,10 +547,14 @@ pub fn paint_local_ancestry( segments } -/// The HMM state scaffold shared by the diploid (unphased) and haploid (phased-side) painters: the -/// kept super-population states, each panel population's super-pop, and the prior π over the kept -/// states. Selection + global-composition gating are identical to [`paint_local_ancestry`], so both -/// painters anchor to the same global estimate and can't invent a globally-absent continent. +/// The HMM state scaffold that the two painters share. One painter is diploid and unphased, and +/// the other is haploid and works on a phased side. The scaffold holds the super-population +/// states that stay, the super-population of each panel population, and the prior π over the +/// states that stay. +/// +/// The selection and the gate on the global composition are the same as in +/// [`paint_local_ancestry`]. Both painters hold to the same global estimate, and neither +/// can show a continent that the genome does not contain at all. struct PaintStates { states: Vec, pop_state: Vec, @@ -582,8 +612,9 @@ fn build_paint_states(panel: &AncestryPanel, prior: &[(String, f64)], params: &P Some(PaintStates { states, pop_state, pi }) } -/// Per-state alt-allele frequency at one site: the mean fine-pop frequency within each kept -/// super-population state (`0.5` when a state has no contributing population). `states` order. +/// The alt-allele frequency of each state at one site. It is the mean fine-population frequency +/// in each super-population state that stays. A state with no population in it gets `0.5`. The +/// order is the `states` order. fn per_state_af(freqs: &[f32], pop_state: &[String], states: &[String]) -> Vec { let k = states.len(); let mut sum = vec![0.0f64; k]; @@ -606,10 +637,11 @@ fn emit_haploid_ln(a: u8, f: f64) -> f64 { p.max(1e-300).ln() } -/// Haploid Viterbi: the MAP super-population per site for **one** phased haplotype. Hidden state = -/// the super-population being copied; emission is [`emit_haploid_ln`] on the side's allele; -/// transitions penalise ancestry switches by physical distance (same `switch_prob`/`ln_trans` as -/// the diploid painter). `sites` are `(pos, per-state AF, allele 0/1)`, position-sorted. +/// Haploid Viterbi. It gives the MAP super-population at each site for **one** phased +/// haplotype. The hidden state is the super-population that the model copies. The emission is +/// [`emit_haploid_ln`] on the allele of that side. The transitions penalise an ancestry switch +/// by physical distance, with the same `switch_prob` and `ln_trans` as the diploid painter. +/// `sites` are `(pos, AF for each state, allele 0/1)`, in position order. fn haploid_viterbi(sites: &[(i64, Vec, u8)], pi: &[f64], rate: f64, k: usize) -> Vec { let n = sites.len(); let lnpi: Vec = (0..k).map(|s| pi[s].max(1e-300).ln()).collect(); @@ -643,11 +675,14 @@ fn haploid_viterbi(sites: &[(i64, Vec, u8)], pi: &[f64], rate: f64, k: usiz path } -/// Paint local ancestry from **phased** genotypes: a haploid ancestry HMM run independently on each -/// of the two phased sides, so the two output tracks are internally-consistent parental sides -/// (segment `copy` = phased side 0/1, consistent across the whole genome) — the parent-split the -/// unphased [`paint_local_ancestry`] cannot produce. `prior` is the genome-wide composition -/// (anchors the state set); `panel` supplies per-super-pop allele frequencies. +/// Paint local ancestry from **phased** genotypes. A haploid ancestry HMM runs on each of the +/// two phased sides, and the two runs are independent. The two output tracks are then parental +/// sides that agree with themselves. The segment `copy` is the phased side, 0 or 1, and it keeps +/// that sense across the whole genome. The unphased [`paint_local_ancestry`] can not make that +/// parent split. +/// +/// `prior` is the genome-wide composition, which holds the state set in place. `panel` gives the +/// allele frequency of each super-population. pub fn paint_local_ancestry_phased( phased: &crate::phasing::PhasedGenotypes, panel: &AncestryPanel, @@ -659,7 +694,8 @@ pub fn paint_local_ancestry_phased( }; let k = states.len(); - // Per-site per-state AF, keyed by (contig, pos) — computed once, shared by both sides. + // The AF of each state at each site, keyed by (contig, pos). The code computes this once, + // and both sides use it. let site_af: HashMap<(&str, i64), Vec> = panel .sites .iter() @@ -692,7 +728,8 @@ pub fn paint_local_ancestry_phased( continue; } let path = haploid_viterbi(&sites, &pi, params.rate, k); - // collapse_copy needs (pos, _, dosage-ish); the AF/allele payload is unused there. + // collapse_copy needs (pos, _, a dosage-like value). It does not read the AF and + // allele payload. let collapse_sites: Vec<(i64, Vec, i32)> = sites.iter().map(|s| (s.0, Vec::new(), s.2 as i32)).collect(); segments.extend(collapse_copy( @@ -708,13 +745,14 @@ pub fn paint_local_ancestry_phased( segments } -/// Tuning for [`resolve_fine_populations`] (the two-tier super→fine step). +/// The controls of [`resolve_fine_populations`], which is the two-tier super-to-fine step. #[derive(Debug, Clone)] pub struct FineResolveParams { - /// Minimum informative sites in a segment before a fine call is attempted. + /// The count of informative sites that a segment needs before the code tries a fine call. pub min_sites: usize, - /// Minimum average per-site ln-likelihood advantage of the best fine population over the - /// runner-up to accept the call (otherwise the segment keeps `fine_population_code = None`). + /// The mean ln-likelihood advantage at each site that the best fine population needs over + /// the second one before the code accepts the call. Below this, the segment keeps + /// `fine_population_code = None`. pub min_margin_per_site: f64, } @@ -727,12 +765,17 @@ impl Default for FineResolveParams { } } -/// Two-tier fine resolution: for each already-painted super-population segment, pick the most likely -/// **fine** population *within that super-population* from the fine-frequency panel, scoring the -/// segment's phased-side alleles by the haploid likelihood under each candidate fine population. -/// Sets [`AncestrySegment::fine_population_code`] in place; leaves it `None` when the segment is too -/// short or the best fine call isn't clearly ahead of the runner-up (mirrors the super→fine admixture -/// hierarchy). `fine_panel.populations` are fine-pop codes; each site's `freqs` are per-fine-pop AF. +/// Two-tier fine resolution. For each super-population segment that the painter made, this +/// takes the most likely **fine** population *inside that super-population* from the +/// fine-frequency panel. It scores the phased-side alleles of the segment by the haploid +/// likelihood under each candidate fine population. +/// +/// It sets [`AncestrySegment::fine_population_code`] in place. It leaves the code `None` when +/// the segment is too short, or when the best fine call is not clearly ahead of the second one. +/// This is the same shape as the super-to-fine admixture hierarchy. +/// +/// `fine_panel.populations` are fine-population codes. The `freqs` of each site are the AF of +/// each fine population. pub fn resolve_fine_populations( segments: &mut [AncestrySegment], phased: &crate::phasing::PhasedGenotypes, @@ -765,10 +808,11 @@ pub fn resolve_fine_populations( allele.insert((s.contig.as_str(), 1, s.position), s.side1); } - // Sites grouped by contig in position order, so each segment binary-searches its own window. - // Scanning `phased.sites` per segment instead is O(segments × sites) — at genome scale (hundreds - // of segments over ~1M sites) that dominates the whole fine-resolution pass. Same grouped-by- - // contig shape as `paint_local_ancestry`. + // The sites, in groups by contig and in position order. Each segment can then do a binary + // search for its own window. A scan of `phased.sites` for each segment instead is + // O(segments × sites). At genome scale, which is hundreds of segments over about 1M sites, + // that scan controls the time of the whole fine-resolution pass. The shape of the groups by + // contig is the same as in `paint_local_ancestry`. let mut by_contig: HashMap<&str, Vec<&crate::phasing::PhasedSite>> = HashMap::new(); for s in &phased.sites { by_contig.entry(s.contig.as_str()).or_default().push(s); @@ -779,8 +823,9 @@ pub fn resolve_fine_populations( for seg in segments.iter_mut() { let sp = seg.population_code.as_str(); - // Candidate fine columns in this segment's super-population, excluding a fine code identical - // to the super code (no extra resolution to offer, e.g. MEA/CAS/OCE). + // The candidate fine columns in the super-population of this segment. A fine code that + // is the same as the super code stays out, because it offers no more resolution. MEA, + // CAS and OCE are examples. let candidates: Vec = col_super .iter() .enumerate() @@ -791,7 +836,8 @@ pub fn resolve_fine_populations( continue; } - // Per-candidate summed ln-likelihood over the segment's informative sites on this side. + // The sum of the ln-likelihood of each candidate, over the informative sites of the + // segment on this side. let mut ll = vec![0.0f64; candidates.len()]; let mut n = 0usize; let contig_sites = by_contig.get(seg.contig.as_str()).map(Vec::as_slice).unwrap_or(&[]); @@ -814,7 +860,7 @@ pub fn resolve_fine_populations( continue; } - // Best and runner-up; accept only with a clear per-site margin. + // The best and the second. Accept only with a clear margin at each site. let mut order: Vec = (0..candidates.len()).collect(); order.sort_by(|&a, &b| ll[b].total_cmp(&ll[a])); let best = order[0]; @@ -838,10 +884,11 @@ fn switch_prob(d: i64, rate: f64) -> f64 { (1.0 - (-(d.max(0) as f64) * rate).exp()).clamp(0.0, 0.999) } -/// Diploid Viterbi: the MAP ancestry **pair** `(a1, a2)` per site. Hidden state = an ordered pair of -/// ancestries (the two genome copies, independent Markov chains), so transitions factorize as -/// `ln_trans(a1,b1) + ln_trans(a2,b2)` and the emission is the two-copy [`emit_diploid_ln`]. Returns -/// one `(a1, a2)` per site (state index `a1*k + a2`). +/// Diploid Viterbi. It gives the MAP ancestry **pair** `(a1, a2)` at each site. The hidden state +/// is an ordered pair of ancestries, one for each of the two genome copies, and the two copies +/// are independent Markov chains. The transitions then factorize as +/// `ln_trans(a1,b1) + ln_trans(a2,b2)`, and the emission is the two-copy [`emit_diploid_ln`]. +/// Returns one `(a1, a2)` for each site, at the state index `a1*k + a2`. fn diploid_viterbi(sites: &[(i64, Vec, i32)], pi: &[f64], rate: f64, k: usize) -> Vec<(usize, usize)> { let n = sites.len(); let ns = k * k; @@ -855,8 +902,9 @@ fn diploid_viterbi(sites: &[(i64, Vec, i32)], pi: &[f64], rate: f64, k: usi } for i in 1..n { let sw = switch_prob(sites[i].0 - sites[i - 1].0, rate); - // Per-chain best predecessor for each target chain-state (factorized, so the pair step is - // O(k²) not O(k⁴)): for chain value b, max over a of v_chain[a] + ln_trans(a,b). + // The best predecessor in each chain, for each target chain-state. The step factorizes, + // so the pair step is O(k²) and not O(k⁴). For a chain value b, take the maximum over a + // of v_chain[a] + ln_trans(a,b). for b1 in 0..k { for b2 in 0..k { let (mut best, mut arg) = (f64::NEG_INFINITY, 0usize); @@ -887,9 +935,10 @@ fn diploid_viterbi(sites: &[(i64, Vec, i32)], pi: &[f64], rate: f64, k: usi path } -/// Collapse one copy's per-site ancestry path into segments, merging runs shorter than `min_sites` -/// into the previous segment (keeping its ancestry). Each segment is tagged with the `copy` index. -/// `posterior` is set to 1.0 (the MAP path; per-copy posterior shading is a future refinement). +/// Collapse the ancestry path of one copy, which has a value at each site, into segments. A run +/// of fewer sites than `min_sites` merges into the segment before it and takes the ancestry of +/// that segment. Each segment carries the `copy` index. The `posterior` is 1.0, because this is +/// the MAP path. A posterior for each copy is a later improvement. fn collapse_copy( contig: &str, sites: &[(i64, Vec, i32)], @@ -933,10 +982,11 @@ fn collapse_copy( use navigator_domain::seq::complement_base as revcomp_base; -/// Alt-allele dosage (0/1/2) for a chip diploid call `(a1,a2)` against a panel site's -/// `ref_allele`/`alt_allele`. When the call's alleles don't both lie in `{ref,alt}`, retry once on -/// the **reverse-complemented** call (the array reported the other strand); `None` if it still -/// doesn't match (no-call / multi-allelic mismatch). The minimal strand-flip logic chip→panel needs. +/// The alt-allele dosage (0/1/2) for a diploid chip call `(a1,a2)`, against the `ref_allele` and +/// `alt_allele` of a panel site. If the two alleles are not both in `{ref,alt}`, the code tries +/// once more on the **reverse-complement** of the call. The array can report the other strand. It returns `None` if the call still does not match, which is a no-call or a +/// multi-allelic mismatch. This is the small amount of strand-flip logic that the path from a +/// chip to the panel needs. pub fn dosage_from_alleles(a1: char, a2: char, ref_allele: char, alt_allele: char) -> Option { let (r, alt) = (ref_allele.to_ascii_uppercase(), alt_allele.to_ascii_uppercase()); let count = |x: char, y: char| -> Option { @@ -949,18 +999,20 @@ pub fn dosage_from_alleles(a1: char, a2: char, ref_allele: char, alt_allele: cha const PIPELINE_VERSION: &str = "1.0.0-af"; -/// Estimate ancestry by the per-population binomial allele-frequency likelihood. +/// Estimate ancestry by the binomial allele-frequency likelihood of each population. /// -/// For each population, the log-likelihood sums `ln P(genotype | f)` over genotyped sites, -/// where `f` is that population's alt-allele frequency (clamped to [0.001, 0.999]) and the -/// diploid genotype probability is `(1-f)²` (hom-ref), `2f(1-f)` (het), or `f²` (hom-alt). -/// Likelihoods are exponentiated relative to the best population and normalized to percentages. +/// For each population, the log-likelihood is the sum of `ln P(genotype | f)` over the sites +/// with a genotype. `f` is the alt-allele frequency of that population, clamped to +/// [0.001, 0.999]. The diploid genotype probability is `(1-f)²` for hom-ref, `2f(1-f)` for het, +/// or `f²` for hom-alt. The code then takes the exponential of each likelihood against the best +/// population, and normalizes the results to percentages. pub fn estimate_by_allele_frequency( genotypes: &[SiteGenotype], panel: &AncestryPanel, reference_version: &str, ) -> AncestryResult { - // (contig, position) -> dosage; missing/no-call dosages (< 0) are dropped. + // A map from (contig, position) to a dosage. The code drops a missing or no-call dosage, + // which is a value less than 0. let dosage: HashMap<(&str, i64), i32> = genotypes .iter() .filter(|g| g.dosage >= 0) @@ -1012,16 +1064,19 @@ pub fn estimate_by_allele_frequency( ) } -/// Estimate the sample's **admixture proportions** over the panel populations by supervised -/// ADMIXTURE: the reference allele frequencies `P` (the panel) are fixed and we estimate the -/// mixture vector `Q` (on the simplex, summing to 1) that maximizes the genotype likelihood -/// `∏_j P(g_j | f_j)`, where the mixed alt-allele frequency at site `j` is `f_j = Σ_k q_k·p_{k,j}` -/// and `P(g_j|f_j)` is the diploid binomial under HWE. +/// Estimate the **admixture proportions** of the sample over the panel populations, by +/// supervised ADMIXTURE. The reference allele frequencies `P`, which are the panel, stay fixed. +/// The code estimates the mixture vector `Q`, which lies on the simplex and sums to 1, that +/// maximizes the genotype likelihood `∏_j P(g_j | f_j)`. The mixed alt-allele frequency at site +/// `j` is `f_j = Σ_k q_k·p_{k,j}`, and `P(g_j|f_j)` is the diploid binomial under HWE. +/// +/// The frappe/ADMIXTURE EM does the fit. Each allele copy has a latent source population. The +/// E-step gives the posterior of that population, given ref or alt. The M-step estimates `q_k` +/// again, as the mean posterior. /// -/// Fitted by the frappe/ADMIXTURE EM: each allele copy has a latent source population; the E-step -/// is its posterior given ref/alt, the M-step re-estimates `q_k` as the mean posterior. Unlike -/// [`estimate_by_allele_frequency`] (which picks the single best-fitting population), this yields -/// a 100%-summing composition — the shape of a consumer ancestry report. +/// [`estimate_by_allele_frequency`] takes the one population that fits best. This function +/// instead gives a composition that sums to 100%, which is the shape of a consumer ancestry +/// report. pub fn estimate_admixture( genotypes: &[SiteGenotype], panel: &AncestryPanel, @@ -1034,7 +1089,7 @@ pub fn estimate_admixture( .collect(); let k = panel.populations.len(); - // Informative sites: (dosage 0/1/2, clamped per-pop alt frequencies). + // The informative sites: (dosage 0/1/2, the clamped alt frequency of each population). let sites: Vec<(f64, Vec)> = panel .sites .iter() @@ -1050,7 +1105,8 @@ pub fn estimate_admixture( let mut q = vec![1.0 / k.max(1) as f64; k]; if snps_with_data > 0 { - // EM to convergence (monotone in the likelihood); cheap — O(sites·k) per iteration. + // Run the EM until it converges. It is monotone in the likelihood, and it costs only + // O(sites·k) in each iteration. for _ in 0..500 { let mut acc = vec![0.0f64; k]; for (g, freqs) in &sites { @@ -1089,11 +1145,14 @@ pub fn estimate_admixture( ) } -/// Fine-population admixture: the same supervised EM as [`estimate_admixture`], run over a curated -/// **modern subset** of a large fine-frequency panel (the `freq_global` asset carries all reference -/// populations incl. ancient; a flat 173-way EM is ill-posed, so we restrict to `modern_codes`). -/// Reuses the *same* genotypes (the fine panel shares the AIM panel's sites). The result is labeled -/// `FINE_ADMIXTURE`; its components roll up to the super-pops via the domain `population_super` map. +/// Fine-population admixture. It is the same supervised EM as [`estimate_admixture`], over a +/// curated **modern subset** of a large fine-frequency panel. The `freq_global` asset holds all +/// of the reference populations, and that includes the ancient ones. A flat 173-way EM is +/// ill-posed, so the code keeps only `modern_codes`. +/// +/// It uses the *same* genotypes, because the fine panel and the AIM panel share their sites. The +/// result carries the label `FINE_ADMIXTURE`. Its components roll up to the super-populations +/// through the `population_super` map in the domain crate. pub fn estimate_fine_admixture( genotypes: &[SiteGenotype], fine_panel: &AncestryPanel, @@ -1106,71 +1165,88 @@ pub fn estimate_fine_admixture( result } -/// The `ANCIENT_ADMIXTURE` method label — deep (pre-historic) source proportions. +/// The `ANCIENT_ADMIXTURE` method label. It marks deep, pre-historic source proportions. pub const ANCIENT_ADMIXTURE: &str = "ANCIENT_ADMIXTURE"; /// Below this many genotyped panel sites the three-way fit is too noisy to report at all. const ANCIENT_MIN_SITES: usize = 500; -/// Dispersion above which the sample is **outside the span of the ancient sources** and we report -/// nothing. Under a correct model the dispersion is ≈1 by construction (see [`ancient_dispersion`]). +/// The dispersion above which the sample is **outside the span of the ancient sources**, and the +/// code reports nothing. Under a correct model the dispersion is about 1 by construction. See +/// [`ancient_dispersion`]. /// -/// Calibrated on simulated reference individuals (`panelbuild validate-ancient`), worst case per -/// population: GBR 1.65 · CEU 1.58 · FIN 1.78 · TSI 2.38 · **IBS 3.65** ‖ CHB 13.1 · JPT 12.4 · -/// YRI 175 · LWK 158. So 4.0 sits in the wide, empty gap between "every European individual" and -/// "the closest East Asian" — it is not a knob tuned to taste, it is the middle of a real gap. +/// The calibration ran on simulated reference individuals +/// (`panelbuild validate-ancient`). The worst case for each population was: +/// GBR 1.65 · CEU 1.58 · FIN 1.78 · TSI 2.38 · **IBS 3.65** ‖ CHB 13.1 · JPT 12.4 · +/// YRI 175 · LWK 158. 4.0 sits in the wide, empty gap between "every European individual" +/// and "the closest East Asian". It is the middle of a real gap, and it is not a value that +/// somebody chose because it looked correct. /// -/// It deliberately does **not** try to separate South Asians (PJL 3.3–4.0), who overlap the European -/// tail: no dispersion threshold can do that, which is why there is a second guard, -/// [`ANCIENT_MIN_WEST_EURASIAN`]. +/// This threshold does **not** try to separate South Asians, at PJL 3.3 to 4.0, who lie in the +/// same range as the European tail. No dispersion threshold can do that. That is why there is a +/// second guard, [`ANCIENT_MIN_WEST_EURASIAN`]. const ANCIENT_MAX_DISPERSION: f64 = 4.0; -/// Minimum European share (by the modern super-population admixture) for the deep three-way model to -/// apply at all. +/// The smallest European share, from the modern super-population admixture, at which the deep +/// three-way model applies at all. /// -/// WHG / Anatolian Farmer / Steppe is a **West-Eurasian** model: those three sources are the ones -/// that actually compose modern Europeans. It has no term for Ancestral South Indian, no term for -/// East Asian, and no term for Sub-Saharan African, so for a person who carries a lot of any of -/// those, a three-way decomposition of their *whole genome* is not an approximation — it is a -/// category error. A Punjabi fits at Steppe 67% here; their real Steppe ancestry is nearer 20–30%, -/// with the rest Iranian-Neolithic and AASI that this model simply cannot see, so it piles the -/// unexplained ancestry onto whichever source is least unlike it. +/// WHG / Anatolian Farmer / Steppe is a **West-Eurasian** model. Those three sources are the +/// ones that compose modern Europeans. The model has no term for Ancestral South Indian, no term +/// for East Asian, and no term for Sub-Saharan African. For a person who carries much of any of +/// those, a three-way decomposition of the *whole genome* is not an approximation. It is a +/// category error. /// -/// Dispersion alone cannot catch that (South Asians overlap the European tail), but the *modern* -/// estimate — which is well validated and independent of this panel — separates them cleanly. So -/// deep ancestry only runs for samples the modern model already calls predominantly European. +/// A Punjabi fits at Steppe 67% here. The real Steppe ancestry of that person is nearer to 20 or +/// 30%, and the remainder is Iranian-Neolithic and AASI, which this model can not see. The model +/// then puts all of the ancestry that it can not explain onto the source that is least unlike +/// it. +/// +/// The dispersion alone can not catch that, because South Asians lie in the same range as the +/// European tail. But the *modern* estimate separates them cleanly, and that estimate is well +/// checked and independent of this panel. Deep ancestry runs only for a sample that the modern +/// model already calls mostly European. const ANCIENT_MIN_WEST_EURASIAN: f64 = 50.0; /// qpAdm model-fit acceptance: report the deep breakdown only when the model is **not rejected** at /// this tail probability (documents/design/ancient-ancestry-rebuild.md §7.14). The garbage fits the gate /// exists to suppress reject at p ≈ 1e-13; a real British WGS/chip accepts at p ≈ 0.15–0.21. const QPADM_MIN_P: f64 = 0.05; -/// Tolerance for the "weights are valid proportions" check — a fit that needs a source weight outside -/// `[0,1]` is the model failing, not a small numerical overshoot. +/// The tolerance of the check that the weights are correct proportions. A fit that needs a +/// source weight outside `[0,1]` shows that the model does not hold. It is not a small numerical +/// overshoot. const QPADM_WEIGHT_TOL: f64 = 0.02; -/// Estimate **deep ancestral (ancient) source proportions** — the Western Hunter-Gatherer / -/// Anatolian Farmer / Steppe pastoralist decomposition — by the same supervised allele-frequency -/// admixture EM as [`estimate_admixture`], over the dedicated ancient frequency panel -/// (`ancestry_freq_ancient_.bin`, built by `panelbuild ancient-panel` from the AADR). +/// Estimate **deep ancestral (ancient) source proportions**. This is the decomposition into +/// Western Hunter-Gatherer, Anatolian Farmer and Steppe pastoralist. It uses the same supervised +/// allele-frequency admixture EM as [`estimate_admixture`], over the dedicated ancient frequency +/// panel `ancestry_freq_ancient_.bin`, which `panelbuild ancient-panel` builds from the +/// AADR. +/// +/// This *replaces* an earlier PCA-centroid classifier. That classifier was wrong in two ways. +/// It asked which ancient population this sample **is**, which is a membership posterior, when +/// the question is what **mixture** of ancient sources the sample is. It also ran against the +/// wrong centroids. The projection had pulled those centroids in on top of the modern European +/// cloud, so they carried no ancient signal at all. /// -/// This *replaces* an earlier PCA-centroid classifier, which was wrong twice over: it asked "which -/// ancient population **is** this sample?" (a membership posterior) where the question is "what -/// **mixture** of ancient sources is this sample?", and it ran against centroids that had been -/// shrunk on top of the modern European cloud by projection, so they carried no ancient signal at -/// all. A modern European is not a *member* of WHG; they are a *mixture*. Allele frequencies keep -/// the sources genuinely distinct (WHG↔ANF Fst ≈ 0.07) where the projected PCA did not. +/// A modern European is not a *member* of WHG. That person is a *mixture*. /// -/// `modern` is the sample's **modern** super-population admixture ([`estimate_admixture`] over the -/// super-pop panel) — an independent, already-validated estimate, used only to decide whether this -/// West-Eurasian model applies to this person at all (see [`ANCIENT_MIN_WEST_EURASIAN`]). +/// Allele frequencies keep the sources truly separate, at a WHG-to-ANF Fst of about 0.07, where +/// the projected PCA did not. /// -/// Returns `None` whenever the model does not apply: too few genotyped sites, too little European -/// ancestry for a WHG/ANF/Steppe decomposition to mean anything, or a fit dispersion above -/// [`ANCIENT_MAX_DISPERSION`] (the sample's ancestry lies outside the span of the three sources — a -/// Yoruba is not *any* mixture of them). Reporting nothing is the entire point: the EM will always -/// return *some* simplex vector, and presenting that vector for a sample the model cannot express is -/// precisely the failure this rebuild exists to prevent. +/// `modern` is the **modern** super-population admixture of the sample, which is +/// [`estimate_admixture`] over the super-population panel. It is an independent estimate that is +/// already checked. It has one use here: to decide if this West-Eurasian model applies to this +/// person at all. See [`ANCIENT_MIN_WEST_EURASIAN`]. +/// +/// Returns `None` when the model does not apply. There are three such cases. The run genotyped +/// too few sites. Or the sample has too little European ancestry for a WHG/ANF/Steppe +/// decomposition to say anything. Or the fit dispersion is above +/// [`ANCIENT_MAX_DISPERSION`], which puts the ancestry of the sample outside the span of the +/// three sources. A Yoruba is not *any* mixture of them. +/// +/// To report nothing is the whole point. The EM always returns *some* simplex vector. To show +/// that vector for a sample that the model can not express is the exact failure that this +/// rebuild prevents. pub fn estimate_ancient_admixture( genotypes: &[SiteGenotype], ancient_panel: &AncestryPanel, @@ -1185,12 +1261,12 @@ pub fn estimate_ancient_admixture( (dispersion.is_finite() && dispersion <= ANCIENT_MAX_DISPERSION).then_some(result) } -/// The sample's European share (%) according to a modern super-population estimate — the scope -/// check for the deep three-way model. Reads the `EUR` rollup, so it works whether `modern` came -/// from the 5-way super-pop panel or a finer panel that rolls up to it. +/// The European share (%) of the sample, from a modern super-population estimate. It is the +/// scope check for the deep three-way model. It reads the `EUR` rollup. It works for the +/// 5-way super-population panel, and also for a finer panel that rolls up to that one. /// -/// `SuperPopulationSummary::super_population` carries the *display name*, not the code, so the -/// lookup goes through the catalog rather than hard-coding either spelling. +/// `SuperPopulationSummary::super_population` carries the *display name* and not the code. The +/// lookup goes through the catalog, and neither form of the name goes into the code. pub fn west_eurasian_share(modern: &AncestryResult) -> f64 { let eur = population_name("EUR"); modern @@ -1200,14 +1276,17 @@ pub fn west_eurasian_share(modern: &AncestryResult) -> f64 { .map_or(0.0, |s| s.percentage) } -/// The ancient mixture fit **without** the applicability threshold: the EM result with its -/// dispersion attached as `fit_distance`, for any sample with enough genotyped sites. +/// The ancient mixture fit **without** the threshold that decides if the model applies. It is +/// the EM result with its dispersion attached as `fit_distance`, for any sample that has enough +/// genotyped sites. +/// +/// [`estimate_ancient_admixture`] is this function plus the [`ANCIENT_MAX_DISPERSION`] gate, and +/// the app calls that one. This one exists so that the offline checker can *report* the +/// dispersion of the samples that the gate refuses. You can defend the threshold only if you can +/// see the separation that it stands on. /// -/// [`estimate_ancient_admixture`] is this plus the [`ANCIENT_MAX_DISPERSION`] gate, and is what the -/// app calls. This variant exists so the offline validator can *report* the dispersion of samples -/// that the gate rejects — the threshold is only defensible if you can see the separation it rests -/// on. Do not use it on a user's data: its components are exactly the numbers the gate exists to -/// suppress. +/// Do not use this function on the data of a user. Its components are the exact numbers that the +/// gate exists to hold back. pub fn ancient_admixture_fit( genotypes: &[SiteGenotype], ancient_panel: &AncestryPanel, @@ -1218,7 +1297,8 @@ pub fn ancient_admixture_fit( return None; } - // Recover the fitted mixture on the panel's axis order (`components` are sorted by percentage). + // Get the fitted mixture again, on the axis order of the panel. The `components` come in + // the order of their percentage. let q: Vec = ancient_panel .populations .iter() @@ -1237,19 +1317,27 @@ pub fn ancient_admixture_fit( Some(result) } -/// The **app-facing deep-ancestry estimator** (documents/design/ancient-ancestry-rebuild.md §7.14): fit -/// `target = Σ wᵢ · sourcesᵢ` by qpAdm f4 and return it as an [`AncestryResult`] over the source -/// components (WHG / EEF / Steppe), or `None` when the deep model does not apply. +/// The **deep-ancestry estimator that the app calls**. See +/// documents/design/ancient-ancestry-rebuild.md §7.14. It fits `target = Σ wᵢ · sourcesᵢ` by +/// qpAdm f4. It returns the fit as an [`AncestryResult`] over the source components, which are +/// WHG, EEF and Steppe. It returns `None` when the deep model does not apply. /// -/// `sources` and `outgroups` are indices into `panel.populations` — the committed Patterson-2022 -/// config: sources first (WHG/EEF/Steppe), then the sister outgroups. Gates, all of which must pass: -/// the sample is West-Eurasian (`modern`, [`ANCIENT_MIN_WEST_EURASIAN`]); enough sites were genotyped -/// ([`ANCIENT_MIN_SITES`]); the qpAdm model is **not rejected** (`p ≥` [`QPADM_MIN_P`]); and the -/// weights are feasible proportions ([`QPADM_WEIGHT_TOL`]). `None` must stay `None` all the way to the -/// UI/PDS — an inapplicable or rejected fit is reported as nothing, never as confident percentages. +/// `sources` and `outgroups` are indices into `panel.populations`. That order is the committed +/// Patterson-2022 configuration: the sources first, which are WHG, EEF and Steppe, and then the +/// sister outgroups. /// -/// This supersedes [`estimate_ancient_admixture`] (the frequency-mixture EM, which failed the -/// WGS-vs-chip stability gate); the model-fit **p-value** rides out on `fit_distance`. +/// There are four gates, and the fit must pass all of them. The sample is West-Eurasian, which +/// `modern` and [`ANCIENT_MIN_WEST_EURASIAN`] decide. The run genotyped enough sites, which is +/// [`ANCIENT_MIN_SITES`]. The qpAdm model is **not rejected**, at `p ≥` [`QPADM_MIN_P`]. The +/// weights are proportions that can occur, within [`QPADM_WEIGHT_TOL`]. +/// +/// A `None` must stay a `None` all the way out to the UI and the PDS. A fit that does not apply, +/// or that the test rejects, goes out as nothing. It never goes out as percentages that look +/// sure. +/// +/// This function replaces [`estimate_ancient_admixture`], which is the frequency-mixture EM. +/// That EM did not pass the stability gate between WGS and chip data. The **p-value** of the +/// model fit goes out on `fit_distance`. pub fn estimate_qpadm_ancestry( genotypes: &[SiteGenotype], panel: &AncestryPanel, @@ -1285,16 +1373,18 @@ pub fn estimate_qpadm_ancestry( Some(result) } -/// Goodness of fit of a fitted ancient mixture `q`, as a **variance-ratio dispersion**. +/// How well a fitted ancient mixture `q` agrees with the data, as a **variance-ratio +/// dispersion**. /// -/// At each genotyped site the mixture predicts an alt-allele frequency `f = Σ q_k·p_k`, so under -/// the model's own HWE assumption the observed dosage `g` has mean `2f` and variance `2f(1-f)`. -/// Averaging `(g − 2f)² / 2f(1-f)` over sites therefore gives ≈1 **when the model is right**, and -/// grows without bound as the sample's true ancestry moves outside the span of the sources — the -/// mixture is then forced to predict frequencies the genotypes keep contradicting. +/// At each genotyped site the mixture predicts an alt-allele frequency `f = Σ q_k·p_k`. Under +/// the HWE assumption of the model, the observed dosage `g` then has a mean of `2f` and a +/// variance of `2f(1-f)`. The mean of `(g − 2f)² / 2f(1-f)` over the sites is then about 1 +/// **when the model is correct**. It grows without a limit as the true ancestry of the sample +/// moves outside the span of the sources. The mixture must then predict frequencies that the +/// genotypes continue to deny. /// -/// It is a *ratio*, so it does not drift with panel size or the sample's coverage — which is what -/// makes it usable as a fixed applicability threshold rather than a tuned magic number. +/// It is a *ratio*, so it does not change with the panel size or with the coverage of the +/// sample. You can use it as a fixed threshold, and it is not a number that somebody tuned. fn ancient_dispersion(genotypes: &[SiteGenotype], panel: &AncestryPanel, q: &[f64]) -> f64 { let dosage: HashMap<(&str, i64), i32> = genotypes .iter() @@ -1326,27 +1416,32 @@ fn ancient_dispersion(genotypes: &[SiteGenotype], panel: &AncestryPanel, q: &[f6 // ── f-statistics core (Lever 2 / qpAdm) ───────────────────────────────────────────────────────── // -// `f4(A,B;C,D) = mean_site (a−b)(c−d)` over per-population alt-allele frequencies. It is the -// ascertainment-robust primitive qpAdm is built on (documents/design/ancient-ancestry-rebuild.md §7): a -// difference-of-differences against outgroups that cancels drift shared across the whole set, and is -// **unbiased from *pooled* frequencies** — the estimation noise in each of the four slots is -// independent, so the cross-terms vanish in expectation (no per-sample hzcorr, unlike f2/f3). The -// genotyped sample enters as its own "population" with frequency `dosage/2 ∈ {0, 0.5, 1}`. +// `f4(A,B;C,D) = mean_site (a−b)(c−d)`, over the alt-allele frequency of each population. It is +// the primitive that qpAdm stands on, and it is robust to ascertainment. See +// documents/design/ancient-ancestry-rebuild.md §7. It is a difference of differences against the +// outgroups, and it cancels the drift that the whole set shares. +// +// It is also **unbiased from *pooled* frequencies**. The estimation noise in each of the four +// slots is independent, so the cross-terms go to zero in expectation. There is no hzcorr for +// each sample, which f2 and f3 need. The genotyped sample goes in as its own "population", +// with the frequency `dosage/2 ∈ {0, 0.5, 1}`. // -// This module is the primitive: a jointly-estimated **vector** of f4 statistics with its -// block-jackknife covariance. The qpAdm GLS solve (§7.2) is assembled on top of it in a later step. +// This module is the primitive. It gives a **vector** of f4 statistics that the code estimates +// together, with its block-jackknife covariance. A later step puts the qpAdm GLS solve (§7.2) on +// top of it. -/// Genome block size (bp) for the f-statistic block jackknife. ~5 Mb ≫ the LD range, so blocks are -/// effectively independent — the assumption the jackknife variance rests on. +/// The genome block size, in bp, for the block jackknife of the f-statistics. About 5 Mb is much +/// more than the LD range, so the blocks are independent for this purpose. That is the +/// assumption that the jackknife variance stands on. pub const F4_BLOCK_BP: i64 = 5_000_000; /// A population slot in an f-statistic: either a reference population (index into /// [`AncestryPanel::populations`]) or the genotyped sample. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Pop { - /// Reference population `i` (its per-site frequency is `PanelSite::freqs[i]`). + /// The reference population `i`. Its frequency at each site is `PanelSite::freqs[i]`. Ref(usize), - /// The sample being placed (per-site frequency `dosage/2`). + /// The sample that the code places. Its frequency at each site is `dosage/2`. Target, } @@ -1365,18 +1460,20 @@ impl Quartet { } } -/// A jointly-estimated vector of f4 statistics with its block-jackknife covariance — the input the -/// qpAdm GLS solve consumes. +/// A vector of f4 statistics that the code estimates together, with its block-jackknife +/// covariance. The qpAdm GLS solve reads this. #[derive(Clone, Debug)] pub struct F4Estimate { /// Full-sample f4 point estimates, parallel to the requested quartets (ADMIXTOOLS reports the /// full-sample estimate as the statistic; the jackknife supplies only the covariance). pub values: Vec, - /// `d×d` delete-one-block jackknife covariance of `values` (Busing et al. 1999, unequal blocks). + /// The `d×d` delete-one-block jackknife covariance of `values`. See Busing and others, 1999, + /// for blocks that are not equal. pub cov: Vec>, - /// Sites contributing (target genotyped ∧ every referenced population present). + /// The count of sites that count. The target has a genotype, and every population that the + /// statistic names is present. pub n_sites: usize, - /// Genome blocks with ≥1 contributing site. + /// The count of genome blocks that hold one such site or more. pub n_blocks: usize, } @@ -1393,17 +1490,23 @@ impl F4Estimate { } } -/// Point estimate of a single `f4(a,b;c,d)` over the genotyped sites (no covariance) — a thin -/// convenience over [`f4_vector`]. `None` if fewer than two genome blocks carry a contributing site. +/// The point estimate of one `f4(a,b;c,d)` over the genotyped sites, with no covariance. It is a +/// thin wrapper over [`f4_vector`]. It returns `None` if fewer than two genome blocks hold a +/// site that counts. pub fn f4(genotypes: &[SiteGenotype], panel: &AncestryPanel, q: Quartet, block_bp: i64) -> Option { f4_vector(genotypes, panel, &[q], block_bp).map(|e| e.values[0]) } -/// A jointly-estimated f4 vector over `quartets`, with the Busing et al. (1999) unequal-block -/// jackknife covariance. Every statistic is measured over the **same** informative site set (target -/// genotyped ∧ all referenced populations present at the site), which is what makes the covariance a -/// valid joint covariance for a downstream GLS. `None` if any quartet references a non-existent -/// population, or fewer than two blocks carry a site (a jackknife needs ≥2 blocks). +/// An f4 vector over `quartets` that the code estimates together, with the unequal-block +/// jackknife covariance of Busing and others (1999). +/// +/// The code measures every statistic over the **same** set of informative sites. A site counts +/// when the target has a genotype there, and every population that the statistic names is +/// present there. That is what makes the covariance a correct joint covariance for a GLS later +/// in the chain. +/// +/// It returns `None` if a quartet names a population that does not exist, or if fewer than two +/// blocks hold a site. A jackknife needs two blocks or more. pub fn f4_vector( genotypes: &[SiteGenotype], panel: &AncestryPanel, @@ -1415,7 +1518,8 @@ pub fn f4_vector( if d == 0 || block_bp <= 0 { return None; } - // Reject out-of-range population indices up front — a mis-built quartet must not panic mid-scan. + // Refuse a population index that is out of range, at the start. A quartet that somebody + // built wrongly must not panic in the middle of the scan. let ref_ok = |p: Pop| matches!(p, Pop::Target) || matches!(p, Pop::Ref(i) if i < k); if !quartets .iter() @@ -1430,7 +1534,8 @@ pub fn f4_vector( .map(|g| ((g.contig.as_str(), g.position), g.dosage)) .collect(); - // Accumulate per genome block over the informative sites: Σ x and site count, plus the totals. + // Add up over the informative sites, in each genome block: Σ x, the count of sites, and the + // totals. let mut block_index: HashMap<(&str, i64), usize> = HashMap::new(); let mut block_sum: Vec> = Vec::new(); let mut block_n: Vec = Vec::new(); @@ -1470,8 +1575,9 @@ pub fn f4_vector( let n = n_sites as f64; let theta: Vec = total.iter().map(|&s| s / n).collect(); - // Delete-one-block estimates θ̂_(j) and per-block weights h_j = n/m_j (Busing et al. 1999, for - // unequal block sizes). With g ≥ 2 and every block non-empty, n − m_j ≥ 1 and h_j > 1. + // The delete-one-block estimates θ̂_(j), and the weight of each block, h_j = n/m_j. See + // Busing and others, 1999, for block sizes that are not equal. With g ≥ 2, and with no empty + // block, n − m_j ≥ 1 and h_j > 1. let h: Vec = block_n.iter().map(|&m| n / m as f64).collect(); let theta_j: Vec> = (0..g) .map(|j| { @@ -1516,25 +1622,28 @@ pub fn f4_vector( /// test. See [`qpadm_fit`] and documents/design/ancient-ancestry-rebuild.md §7.2. #[derive(Clone, Debug)] pub struct QpAdmFit { - /// Weights over the sources, **in the order they were passed** (sums to 1). `weights[0]` is the - /// base source's weight, recovered as `1 − Σ others`. + /// The weights over the sources, **in the order that the caller gave them**. They sum to 1. + /// `weights[0]` is the weight of the base source, which the code gets as `1 − Σ others`. pub weights: Vec, /// Standard error of each weight (from the GLS normal-equations covariance). pub std_errors: Vec, - /// Model-fit χ² — the minimized GLS objective (residual not explained by the source span). + /// The χ² of the model fit. It is the minimized GLS objective, which is the residual that + /// the span of the sources does not explain. pub chi2: f64, /// Degrees of freedom `= (#outgroups − 1) − (#sources − 1) = #outgroups − #sources`. pub dof: usize, - /// Tail probability `P(χ²_dof ≥ chi2)`. The model is **rejected** when this is small (< ~0.05): - /// the sources can't express the target's allele-sharing with the outgroups. + /// The tail probability `P(χ²_dof ≥ chi2)`. A small value, below about 0.05, **rejects** the + /// model. The sources can then not express how the target shares alleles with the + /// outgroups. pub p_value: f64, pub n_sites: usize, pub n_blocks: usize, } impl QpAdmFit { - /// Whether every weight is a valid proportion (within `tol` of `[0,1]`). qpAdm accepts a model - /// only when it is not rejected **and** the weights are feasible. + /// True when every weight is a correct proportion, within `tol` of `[0,1]`. qpAdm accepts a + /// model only when two things hold. The test does not reject it, **and** the weights can + /// occur. pub fn weights_feasible(&self, tol: f64) -> bool { self.weights.iter().all(|&w| w >= -tol && w <= 1.0 + tol) } @@ -1571,20 +1680,24 @@ fn qpadm_residual_cov(cov: &[Vec], n: usize, l: usize, w: &[f64]) -> DMatri sigma } -/// Fit `target = Σ wᵢ · sourcesᵢ` by the qpAdm f4 method (docs §7.2). The weights are estimated from -/// the target's *allele-sharing against outgroups* — differences-of-differences that cancel drift -/// and SNP ascertainment — not from its raw frequencies, which is the property §3's frequency-EM -/// lacked. `sources` and `outgroups` are indices into `panel.populations`; the target enters through -/// `genotypes` (dosage/2 per site). +/// Fit `target = Σ wᵢ · sourcesᵢ` by the qpAdm f4 method. See the design document, §7.2. +/// +/// The weights come from how the target *shares alleles against the outgroups*. Those are +/// differences of differences, and they cancel drift and SNP ascertainment. The weights do not +/// come from the raw frequencies of the target. That is the property that the frequency-EM of §3 +/// did not have. `sources` and `outgroups` are indices into `panel.populations`. The target comes +/// in through `genotypes`, which holds `dosage/2` at each site. /// -/// Method: for each left population `X ∈ {target, S₂..Sₙ}` form the vector -/// `φ_X = [f4(X, S₁; R₁, Rⱼ)]_{j=2..m}`; the admixture identity is `φ_target = Σ_{i≥2} wᵢ φ_{Sᵢ}`. -/// Solve the weights by iteratively-reweighted GLS against the block-jackknife covariance (the -/// residual covariance depends on the weights, since the sources are themselves estimated), then -/// read the model-fit χ²/p-value from the weighted residual. +/// The method is this. For each left population `X ∈ {target, S₂..Sₙ}`, make the vector +/// `φ_X = [f4(X, S₁; R₁, Rⱼ)]_{j=2..m}`. The admixture identity is then +/// `φ_target = Σ_{i≥2} wᵢ φ_{Sᵢ}`. Solve for the weights by GLS against the block-jackknife +/// covariance, and reweight at each iteration. That iteration is necessary because the residual +/// covariance depends on the weights, since the code also estimates the sources. Last, read the +/// χ² and the p-value of the model fit from the weighted residual. /// -/// Returns `None` when `sources.len() < 2`, `outgroups.len() < sources.len()`, the f4 vector can't be -/// formed (too few blocks/sites), or the GLS system is singular. +/// Returns `None` in four cases: `sources.len() < 2`; `outgroups.len() < sources.len()`; the code +/// can not make the f4 vector, because there are too few blocks or sites; or the GLS system is +/// singular. pub fn qpadm_fit( genotypes: &[SiteGenotype], panel: &AncestryPanel, @@ -1602,8 +1715,9 @@ pub fn qpadm_fit( let s1 = Pop::Ref(sources[0]); let r1 = outgroups[0]; - // Left populations relative to the base source: target, then S₂..Sₙ. Group order in the f4 - // vector is [target, S₂, …, Sₙ], each contributing `l` statistics over the non-base outgroups. + // The left populations against the base source: the target, then S₂..Sₙ. The group order in + // the f4 vector is [target, S₂, …, Sₙ]. Each group adds `l` statistics over the outgroups + // that are not the base. let lefts: Vec = std::iter::once(Pop::Target) .chain(sources[1..].iter().map(|&i| Pop::Ref(i))) .collect(); @@ -1664,8 +1778,8 @@ pub fn qpadm_fit( }) } -/// Upper tail of the χ² distribution, `P(χ²_k ≥ x)`, via the regularized upper incomplete gamma -/// `Q(k/2, x/2)`. Used for the qpAdm model-fit p-value. +/// The upper tail of the χ² distribution, `P(χ²_k ≥ x)`, through the regularized upper +/// incomplete gamma `Q(k/2, x/2)`. The qpAdm model-fit p-value uses this. fn chi2_sf(x: f64, k: usize) -> f64 { if k == 0 { return if x <= 0.0 { 1.0 } else { 0.0 }; @@ -1676,7 +1790,7 @@ fn chi2_sf(x: f64, k: usize) -> f64 { gammq(k as f64 / 2.0, x / 2.0) } -/// `ln Γ(x)` via the Lanczos approximation (g=7), with the reflection formula for `x < 0.5`. +/// `ln Γ(x)` by the Lanczos approximation (g=7), with the reflection formula for `x < 0.5`. fn ln_gamma(x: f64) -> f64 { const C: [f64; 9] = [ 0.999_999_999_999_809_9, @@ -1757,9 +1871,9 @@ fn gammq(a: f64, x: f64) -> f64 { } } -/// Build an [`AncestryResult`] from raw per-population probabilities (need not be normalized). -/// With the phase-1 super-population panel each component *is* a super-population, so the -/// super-population summary is 1:1 with the components. +/// Build an [`AncestryResult`] from the raw probability of each population. The caller does not +/// have to normalize them. With the phase-1 super-population panel, each component *is* a +/// super-population, so the super-population summary is 1:1 with the components. fn from_probabilities( method: &str, panel_type: &str, @@ -1794,9 +1908,9 @@ fn from_probabilities( }) .collect(); - // Roll components up into super-population summaries. With a super-population panel each - // component is its own super-population; with a fine-grained (26-pop) panel several - // components aggregate into one super-population. + // Roll the components up into super-population summaries. With a super-population panel, + // each component is its own super-population. With a fine-grained 26-population panel, more + // than one component goes into a single super-population. let mut by_super: BTreeMap)> = BTreeMap::new(); for (code, p) in &pct { let sp = population_super(code).unwrap_or(code.as_str()).to_string(); @@ -1818,7 +1932,7 @@ fn from_probabilities( .unwrap_or(std::cmp::Ordering::Equal) }); - // Touch the catalog color path so the API stays cohesive; color is consumed by the UI. + // Touch the color path of the catalog, to keep the API coherent. The UI reads the color. debug_assert!(!population_color("EUR").is_empty()); AncestryResult { @@ -1871,7 +1985,8 @@ fn confidence_from_completeness(snps_with_data: usize, total_snps: usize) -> f64 mod tests { use super::*; - /// Dropping haplotypes keeps every remaining row's alleles and population label intact. + /// When the code drops haplotypes, every row that stays keeps its alleles and its population + /// label. #[test] fn without_haplotypes_preserves_the_kept_rows() { let sites: Vec = (0..7) @@ -1905,7 +2020,7 @@ mod tests { } } - /// Thinning keeps every step-th site's alleles intact on every haplotype. + /// When the code thins the panel, every step-th site keeps its alleles on every haplotype. #[test] fn thin_sites_keeps_every_nth_column() { let sites: Vec = (0..9) @@ -2035,8 +2150,9 @@ mod tests { assert_eq!(panel, back); } - /// A 1-component PCA where the loading is +1 at every site and the panel mean is 1.0 - /// (a het reference): a hom-alt sample projects to +n_sites, a hom-ref sample to −n_sites. + /// A 1-component PCA where the loading is +1 at every site and the panel mean is 1.0, which + /// is a het reference. A hom-alt sample then projects to +n_sites, and a hom-ref sample to + /// −n_sites. #[test] fn project_pca_centres_and_accumulates() { let sites: Vec<(String, i64)> = (1..=4).map(|p| ("chr1".to_string(), p)).collect(); @@ -2139,8 +2255,9 @@ mod tests { #[test] fn fine_admixture_restricts_to_modern_subset_and_labels_method() { - // A fine panel with two modern pops + one ancient (Steppe). The modern subset must drop the - // ancient column, and the result is labeled FINE_ADMIXTURE rolling up to super-pops. + // A fine panel with two modern populations and one ancient population, which is Steppe. + // The modern subset must drop the ancient column. The result carries the label + // FINE_ADMIXTURE, and it rolls up to the super-populations. let sites: Vec = (1..=40) .map(|pos| PanelSite { contig: "chr1".into(), @@ -2188,8 +2305,9 @@ mod tests { } } - /// Ancestry-HOMOZYGOUS sample: hom-alt (→ both copies A) first half, hom-ref (→ both copies B) - /// second half. Diploid painting emits two copies, each switching A→B at the midpoint. + /// A sample that is HOMOZYGOUS in its ancestry. The first half is hom-alt, so both copies are + /// A. The second half is hom-ref, so both copies are B. The diploid painting gives two + /// copies, and each one goes from A to B at the middle point. #[test] fn painting_diploid_homozygous_switch() { let n = 80; @@ -2211,7 +2329,7 @@ mod tests { /// Ancestry-HETEROZYGOUS sample: every site het (one copy A, one copy B). Diploid painting must /// put A on one copy and B on the other across the whole chromosome (the case a single-track - /// painter cannot express). + /// painter can not express). #[test] fn painting_diploid_heterozygous_copies_differ() { let n = 60; @@ -2228,10 +2346,12 @@ mod tests { assert_eq!(copy1[0].population_code, "B"); } - /// Global-composition gate: a sample that is overwhelmingly A with a short hom-ref run that, - /// ungated, paints as B — but B is only a 1% trace globally, below the 2% gate. The gate must - /// drop B from the state set so local painting can't invent a globally absent ancestry (the - /// "99%-European donor shown East-Asian on one chromosome arm" bug). Also exercises the k=1 path. + /// The gate on the global composition. The sample is almost all A, with a short hom-ref run + /// that paints as B when there is no gate. But B is only a 1% trace across the genome, which + /// is below the 2% gate. The gate must drop B from the state set, so that the local painting + /// can not show an ancestry that the genome does not contain. That was the bug that showed a + /// 99%-European donor as East-Asian on one chromosome arm. This test also covers the k=1 + /// path. #[test] fn painting_gate_suppresses_globally_absent_ancestry() { let n = 80; @@ -2272,9 +2392,10 @@ mod tests { ); } - /// Phased painting: two genuine parental sides. Side 0 is ancestry A (alt) on the first half, - /// B (ref) on the second; side 1 is the mirror. Each side must paint as a consistent two-segment - /// track (A→B on side 0, B→A on side 1) — the parent-split the unphased painter cannot express. + /// Phased painting, with two true parental sides. Side 0 is ancestry A (alt) on the first + /// half and B (ref) on the second. Side 1 is the mirror of that. Each side must paint as a + /// coherent two-segment track: A to B on side 0, and B to A on side 1. That is the parent + /// split that the unphased painter can not express. #[test] fn painting_phased_two_consistent_sides() { use crate::phasing::{PhasedGenotypes, PhasedSite}; @@ -2312,9 +2433,10 @@ mod tests { ); } - /// Two-tier fine resolution: a EUR segment whose phased side is alt-rich must resolve to the - /// alt-rich fine population (GBR), not the alt-poor one (TSI). A fine code equal to the super - /// code offers no extra resolution and is skipped. + /// Two-tier fine resolution. A EUR segment with an alt-rich phased side must resolve to the + /// alt-rich fine population, GBR, and not to the alt-poor one, TSI. A fine + /// code that is the same as the super code offers no more resolution, and the code skips + /// it. #[test] fn fine_resolution_picks_alt_rich_fine_pop() { use crate::phasing::{PhasedGenotypes, PhasedSite}; @@ -2379,11 +2501,11 @@ mod tests { // ── deep (ancient) ancestry ───────────────────────────────────────────────────────────────── // - // The three-source model is the one that previously shipped fabricated numbers, so these tests - // pin the two properties whose absence made that possible: it must recover a mixture it was - // never told, and it must refuse a sample its sources cannot express. + // The three-source model is the one that once gave invented numbers to a user. These tests + // hold the two properties whose absence made that possible. The model must find a mixture + // that nobody told it. And it must refuse a sample that its sources can not express. - /// A deterministic LCG — the simulations below must give the same answer on every run. + /// A deterministic LCG. The simulations below must give the same answer on every run. struct Lcg(u64); impl Lcg { fn next_f64(&mut self) -> f64 { @@ -2399,21 +2521,23 @@ mod tests { } } - /// A 3-source panel over `n` sites whose frequencies differ sharply between sources (so the - /// mixture is well-conditioned), plus an "outsider" frequency track standing in for a sample - /// from outside the sources' span (a Yoruba against WHG/ANF/Steppe). + /// A 3-source panel over `n` sites. The frequencies differ sharply between the sources, so + /// the mixture is well-conditioned. The panel also holds an "outsider" frequency track, which + /// represents a sample from outside the span of the sources. A Yoruba against WHG, ANF and + /// Steppe is such a sample. /// - /// What makes the outsider genuinely unreachable is the last site pattern, where **all three - /// sources agree** at 0.10: a mixture can only ever predict a value inside the convex hull of - /// its sources, so at those sites every possible `q` predicts 0.10 while the outsider carries the - /// allele at 0.95. No mixture can absorb that, which is exactly the situation the applicability - /// gate exists to detect. Without such sites, a near-pure single source approximates the outsider - /// well enough to slip under the threshold. + /// The last site pattern is what makes the outsider truly unreachable. There, **all three + /// sources agree** at 0.10. A mixture can predict only a value inside the convex hull of its + /// sources, so at those sites every possible `q` predicts 0.10. The outsider carries the + /// allele at 0.95. No mixture can absorb that, and that is the exact case that the gate must + /// detect. Without such sites, one nearly pure source comes close enough to the outsider to + /// pass below the threshold. fn ancient_panel(n: i64) -> (AncestryPanel, Vec) { let mut sites = Vec::new(); let mut outsider = Vec::new(); for pos in 1..=n { - // Cycle through contrasting frequency patterns so every source is identifiable. + // Go around a set of frequency patterns that differ, so that the code can find + // every source. let (a, b, c, out) = match pos % 6 { 0 => (0.90, 0.10, 0.50, 0.02), 1 => (0.10, 0.90, 0.50, 0.98), @@ -2450,17 +2574,18 @@ mod tests { .map_or(0.0, |c| c.percentage) } - /// A stand-in modern super-population estimate that is `eur`% European — the scope input the - /// deep model gates on. + /// A modern super-population estimate for a test, which is `eur`% European. It is the scope + /// input that the deep model gates on. fn modern_eur(eur: f64) -> AncestryResult { let probs = [("EUR".to_string(), eur), ("AFR".to_string(), 100.0 - eur)]; from_probabilities("ADMIXTURE", "aims", 1000, 1000, &probs, 0.9, "t") } - /// The estimator recovers a mixture it was never given: simulate a 20/30/50 individual from the - /// panel's own source frequencies and the EM must return ~20/30/50, with a dispersion near the - /// model's noise floor of 1. This is the property the PCA-centroid classifier never had — it - /// answered "which source *is* this?", so a genuine mixture came back as a single population. + /// The estimator finds a mixture that nobody gave it. The test simulates a 20/30/50 + /// individual from the source frequencies of the panel itself. The EM must then return about + /// 20/30/50, with a dispersion near 1, which is the noise floor of the model. The + /// PCA-centroid classifier never had this property. It answered which source the sample + /// *is*, so a true mixture came back as one population. #[test] fn ancient_admixture_recovers_a_known_mixture() { let (panel, _) = ancient_panel(4000); @@ -2487,9 +2612,9 @@ mod tests { assert!(d > 0.5 && d < 1.5, "dispersion of a well-specified sample = {d}"); } - /// A sample from outside the sources' span is **rejected**, not decomposed. The EM will always - /// return *some* simplex vector — that vector is exactly what the old implementation printed as - /// a result — so the applicability gate, not the EM, is what makes this safe. + /// The code **refuses** a sample from outside the span of the sources. It does not decompose + /// it. The EM always returns *some* simplex vector, and that vector is exactly what the old + /// implementation printed as a result. The gate, and not the EM, is what makes this safe. #[test] fn ancient_admixture_rejects_a_sample_outside_the_sources() { let (panel, outsider) = ancient_panel(4000); @@ -2501,7 +2626,7 @@ mod tests { .map(|(s, &f)| sg("chr1", s.position, rng.dosage(f))) .collect(); - // The raw fit still produces a confident-looking breakdown … + // The raw fit still gives a breakdown that looks sure … let raw = ancient_admixture_fit(&genos, &panel, "t").expect("enough sites to fit"); let total: f64 = raw.components.iter().map(|c| c.percentage).sum(); assert!((total - 100.0).abs() < 1e-6, "the EM always returns a full simplex"); @@ -2509,12 +2634,13 @@ mod tests { raw.fit_distance.unwrap() > ANCIENT_MAX_DISPERSION, "an out-of-span sample must be driven above the dispersion threshold" ); - // … and the shipping estimator refuses to report it, even for a sample the modern model - // calls European (so this is the dispersion gate doing the work, not the scope gate). + // … and the estimator that the app calls refuses to report it. That holds even for a + // sample that the modern model calls European. The dispersion gate does the work here, + // and not the scope gate. assert!(estimate_ancient_admixture(&genos, &panel, &modern_eur(95.0), "t").is_none()); } - /// Too few genotyped sites → no estimate at all, rather than a noisy one. + /// With too few genotyped sites, the code gives no estimate at all, and not a noisy one. #[test] fn ancient_admixture_needs_enough_sites() { let (panel, _) = ancient_panel(4000); @@ -2528,9 +2654,10 @@ mod tests { assert!(estimate_ancient_admixture(&genos, &panel, &modern_eur(95.0), "t").is_none()); } - /// A WHG/ANF/Steppe decomposition is a *West-Eurasian* model. A sample the modern estimate calls - /// mostly non-European is out of scope and gets nothing — even if its genotypes happen to fit the - /// three sources well, because "fits the arithmetic" is not the same as "means anything". + /// A WHG/ANF/Steppe decomposition is a *West-Eurasian* model. A sample that the modern + /// estimate calls mostly non-European is out of scope, and it gets nothing. That holds even + /// when its genotypes fit the three sources well, because a fit to the arithmetic does not + /// make the result true. #[test] fn ancient_admixture_is_scoped_to_european_samples() { let (panel, _) = ancient_panel(4000); @@ -2545,20 +2672,22 @@ mod tests { }) .collect(); - // Same genotypes, same perfect fit — only the scope differs. + // The genotypes and the fit are the same. Only the scope is different. assert!(estimate_ancient_admixture(&genos, &panel, &modern_eur(95.0), "t").is_some()); assert!(estimate_ancient_admixture(&genos, &panel, &modern_eur(20.0), "t").is_none()); } // ── f-statistics core (Lever 2 / qpAdm) ───────────────────────────────────────────────────── // - // Three properties pin the f4 primitive against known-value graphs: the exact f4-ratio algebra - // qpAdm rests on, the antisymmetries of f4, and a *calibrated* block-jackknife SE (a symmetric - // tree reads f4 ≈ 0 within noise, while a real internal edge reads many SE from zero). - - /// A panel over `pops` whose per-site frequency rows are `freqs[site][pop]`, laid out one site - /// per 100 kb so the 5 Mb block jackknife sees ~50 sites/block. Plus a target genotyped - /// (dosage 0) at every site, so every site is informative for reference-only quartets. + // Three properties test the f4 primitive against graphs with known values. They are the + // exact f4-ratio algebra that qpAdm stands on, the antisymmetries of f4, and a *calibrated* + // block-jackknife SE. For that last one, a symmetric tree reads f4 ≈ 0 within the noise, + // while a real internal edge reads many SE away from zero. + + /// A panel over `pops`. Its frequency row at each site is `freqs[site][pop]`. The sites lie + /// one in each 100 kb, so the 5 Mb block jackknife sees about 50 sites in each block. The + /// panel also holds a target with a genotype (dosage 0) at every site, so every site counts + /// for a quartet that names only references. fn f4_panel(pops: &[&str], freqs: &[Vec]) -> (AncestryPanel, Vec) { let sites: Vec = freqs .iter() @@ -2580,10 +2709,11 @@ mod tests { (panel, genos) } - /// The f4-ratio identity qpAdm generalizes: if `X = α·P + (1−α)·Q` (a frequency mixture), then - /// `f4(X,P;O1,O2) = (1−α)·f4(Q,P;O1,O2)` **exactly** — per site the two are proportional. So the - /// ratio recovers `1−α` regardless of the outgroups. This is the core arithmetic the whole method - /// stands on; a wrong sign or a transposed index would blow the recovered α far past f32 noise. + /// The f4-ratio identity that qpAdm makes more general. If `X = α·P + (1−α)·Q`, which is a + /// frequency mixture, then `f4(X,P;O1,O2) = (1−α)·f4(Q,P;O1,O2)` **exactly**. At each site + /// the two are proportional. The ratio then gives back `1−α` for any outgroups. This is the + /// core arithmetic that the whole method stands on. A wrong sign, or two indices in the wrong + /// order, would push the recovered α far outside the f32 noise. #[test] fn f4_ratio_recovers_the_mixture_weight() { let alpha = 0.3_f64; @@ -2593,8 +2723,9 @@ mod tests { let (r1, r2) = (rng.next_f64(), rng.next_f64()); let p = 0.1 + 0.8 * r1; let q = 0.1 + 0.8 * r2; - // Outgroups tied to the sources so f4(Q,P;O1,O2) is large and clean (no small-denom - // amplification): O1−O2 = 0.7(r1−r2) tracks −(Q−P), giving a firmly non-zero f4. + // Tie the outgroups to the sources, to keep f4(Q,P;O1,O2) large and clean. A + // small denominator then does not amplify it. O1−O2 = 0.7(r1−r2) follows + // −(Q−P), which makes f4 clearly different from zero. let o1 = 0.15 + 0.7 * r1; let o2 = 0.15 + 0.7 * r2; let x = alpha * p + (1.0 - alpha) * q; @@ -2621,8 +2752,9 @@ mod tests { ); } - /// f4's exact symmetries (pure f64 arithmetic over one fixed site set): swapping either pair - /// negates it, and swapping the two pairs leaves it unchanged. + /// The exact symmetries of f4, in pure f64 arithmetic over one fixed set of sites. An + /// exchange of the two members of either pair negates it. An exchange of the two pairs leaves + /// it the same. #[test] fn f4_obeys_its_antisymmetries() { let mut rng = Lcg(7); @@ -2651,11 +2783,12 @@ mod tests { assert!(est.se(0) >= 0.0); } - /// A symmetric tree `((A,B),(C,D))` has `f4(A,B;C,D) = 0` in expectation (the A–B and C–D drift - /// paths don't overlap), while `f4(A,C;B,D)` sits on the shared internal edge and is non-zero. - /// Simulate exactly that and require the jackknife SE to *tell them apart*: the null within a few - /// SE of zero, the real edge many SE away. This is the test that the covariance is calibrated — - /// the property §5.4 needs and simulation-of-frequencies alone can't fake. + /// A symmetric tree `((A,B),(C,D))` has `f4(A,B;C,D) = 0` in expectation, because the A–B + /// and C–D drift paths do not overlap. But `f4(A,C;B,D)` lies on the shared internal edge, + /// and it is not zero. This test simulates that tree, and the jackknife SE must *separate* + /// the two. The null must lie within a few SE of zero, and the real edge many SE away. Only + /// this test shows that the covariance carries the correct scale. §5.4 needs that property, + /// and a simulation of frequencies alone can not produce it. #[test] fn f4_jackknife_se_separates_a_null_from_a_real_edge() { let mut rng = Lcg(2024); @@ -2697,7 +2830,8 @@ mod tests { ); } - /// The χ² upper tail against textbook critical points — the model-fit p-value depends on it. + /// The χ² upper tail against critical points from a textbook. The model-fit p-value depends + /// on this. #[test] fn chi2_sf_matches_known_critical_values() { assert!((chi2_sf(3.841, 1) - 0.05).abs() < 2e-3); @@ -2716,11 +2850,14 @@ mod tests { sd * (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos() } - /// Simulate frequencies on a small admixture graph: three sources S1/S2/S3, each carrying a - /// distinct deep component (iA/iB/iC); six outgroups differentially related to those components - /// (R0 pure near-root; R1,R4→A; R2,R5→B; R3→C); and a target that is an exact per-site frequency - /// mixture of the three sources, drawn as one diploid genome. Under Brownian drift the f4 - /// tree-identities hold, so this is a graph qpAdm is entitled to decompose. + /// Simulate frequencies on a small admixture graph. It has three sources, S1, S2 and S3, and + /// each one holds a distinct deep component, iA, iB or iC. It has six outgroups, and each one + /// relates to those components in a different way. R0 is pure and near the root, R1 and R4 go + /// to A, R2 and R5 go to B, and R3 goes to C. + /// + /// The graph also has a target, which is an exact frequency mixture of the three sources at + /// each site, drawn as one diploid genome. Under Brownian drift the f4 tree identities hold, + /// so qpAdm may decompose this graph. fn qpadm_graph(n_sites: usize, weights: [f64; 3], seed: u64) -> (AncestryPanel, Vec) { let mut rng = Lcg(seed); let clamp = |x: f64| x.clamp(0.02, 0.98); @@ -2761,9 +2898,10 @@ mod tests { (panel, genos) } - /// qpAdm recovers the true mixture weights from allele-sharing against the outgroups, accepts the - /// well-specified model, and **rejects** a model missing a needed source — the property §3's - /// frequency-EM never had (it always returned a confident simplex). + /// qpAdm finds the true mixture weights from how the target shares alleles against the + /// outgroups. It accepts the model that names the correct sources. It **rejects** a model + /// that leaves out a source that the target needs. The frequency-EM of §3 never had that last + /// property, because it always returned a simplex that looked sure. #[test] fn qpadm_recovers_a_known_mixture_and_rejects_a_deficient_model() { let truth = [0.5, 0.3, 0.2]; @@ -2801,8 +2939,10 @@ mod tests { ); } - /// The app-facing `estimate_qpadm_ancestry`: reports the source weights for a well-specified - /// European fit, and gates on both scope (non-European → None) and model-fit (deficient → None). + /// `estimate_qpadm_ancestry`, which is the function that the app calls. It reports the source + /// weights for a European fit that names the correct sources. It gates on two things. The + /// first is the scope, where a non-European sample gives `None`. The second is the model fit, + /// where a model that lacks a source gives `None`. #[test] fn estimate_qpadm_ancestry_reports_european_and_gates_the_rest() { let truth = [0.5, 0.3, 0.2]; @@ -2830,7 +2970,7 @@ mod tests { // Scope gate: a mostly-non-European sample gets nothing, even though the arithmetic fits. assert!(estimate_qpadm_ancestry(&genos, &panel, &[0, 1, 2], &outgroups, &modern_eur(20.0), "t").is_none()); - // Model-fit gate: a source-deficient model is rejected by the p-value. + // The gate on the model fit. The p-value rejects a model that lacks a source. assert!(estimate_qpadm_ancestry(&genos, &panel, &[0, 1], &outgroups, &modern_eur(95.0), "t").is_none()); } } diff --git a/crates/navigator-analysis/src/archaic.rs b/crates/navigator-analysis/src/archaic.rs index bb3b1df6..888948cb 100644 --- a/crates/navigator-analysis/src/archaic.rs +++ b/crates/navigator-analysis/src/archaic.rs @@ -1,18 +1,20 @@ -//! Archaic (Neanderthal / Denisovan) ancestry — panel types. +//! Archaic ancestry, from Neanderthals and Denisovans. This module holds the panel types. //! -//! Design: `documents/design/ArchaicAncestry_Design.md`. This module carries the **Tier A** asset -//! types (the archaic-informative marker panel and the percentile reference); the counting routine -//! and the Tier B segment HMM land on top of them in later milestones. +//! The design is in `documents/design/ArchaicAncestry_Design.md`. This module carries the asset +//! types of **Tier A**, which are the panel of archaic-informative markers and the percentile +//! reference. The routine that counts, and the segment HMM of Tier B, go on top of them in later +//! milestones. //! -//! The panel is *computed by us* rather than ingested (design §3a): the Sankararaman 2014 list -//! 23andMe used is not publicly downloadable, and every openly-licensed alternative is either the -//! wrong build or per-individual probabilistic. We therefore derive sites from the EVA archaic -//! VCFs, Ensembl-75 EPO ancestral alleles, and a 1kGP AFR outgroup, redistributing only the -//! derived sites. +//! This project *computes* the panel, and does not take it from elsewhere. See design §3a. The +//! Sankararaman 2014 list that 23andMe used is not available to download. Every alternative with +//! an open licence is either on the wrong build, or probabilistic for each individual. So the +//! code makes the sites from the EVA archaic VCFs, the Ensembl-75 EPO ancestral alleles, and a +//! 1kGP AFR outgroup. It distributes only the derived sites. //! -//! A consequence worth remembering when validating: **the panel is ours, so its marker count is -//! panel-relative and is not comparable to a vendor's count by equality** — compare the per-site -//! rate on the intersection instead (design §10, M2 validation gate). +//! Keep one consequence in mind during a check. **The panel belongs to this project, so its +//! marker count is relative to the panel.** You can not compare it to the count of a vendor by +//! equality. Compare the rate at each site on the intersection instead. See design §10, the M2 +//! gate. use serde::{Deserialize, Serialize}; @@ -24,19 +26,20 @@ use crate::ibd_panel::Locus; /// [`ArchaicSite::calls`] array uses. pub const ARCHAIC_GENOMES: [&str; 4] = ["AltaiNeanderthal", "Vindija33.19", "Chagyrskaya8", "Denisova3"]; -/// Index into [`ARCHAIC_GENOMES`] / [`ArchaicSite::calls`] for the sole Denisovan genome. The other -/// three are Neanderthals, which is what [`classify_diagnostic`] keys off. +/// The index into [`ARCHAIC_GENOMES`] and [`ArchaicSite::calls`] of the one Denisovan genome. The +/// other three are Neanderthals, and [`classify_diagnostic`] uses that fact. pub const DENISOVA: usize = 3; -/// One archaic genome's state at a site, expressed **relative to the site's derived allele** rather -/// than to ref/alt — so it survives the ref/alt swap that CHM13 orientation can apply. +/// The state of one archaic genome at a site. It is **against the derived allele of the site**, +/// and not against ref and alt. It stays correct through the exchange of ref and alt that the +/// CHM13 orientation can apply. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ArchaicCall { /// Homozygous for the ancestral allele. HomAncestral, /// One derived copy. Het, - /// Homozygous derived — the introgression donor state (design §4, step 2). + /// Homozygous derived. This is the state of the introgression donor. See design §4, step 2. HomDerived, /// Missing / filtered out by the genome's own quality mask. NoCall, @@ -49,33 +52,37 @@ impl ArchaicCall { } } -/// Which archaic lineage a site's derived allele points to. +/// The archaic lineage that the derived allele of a site points to. /// -/// The HMM in Tier B cannot itself separate Neanderthal from Denisovan (they coalesce before either -/// meets modern humans, design §3); this classification is what lets called segments be labelled -/// downstream, so it is stored per site at build time. +/// The HMM in Tier B can not separate Neanderthal from Denisovan on its own, because the two +/// coalesce before either one meets modern humans. See design §3. This classification is what +/// lets a later step put a label on a called segment, so the build stores it at each site. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum DiagnosticClass { /// Derived in ≥1 Neanderthal, and Denisova positively **called ancestral**. Neanderthal, /// Derived in Denisova, and ≥1 Neanderthal positively **called ancestral**. Denisovan, - /// Not attributable to one lineage: derived in both, or the other lineage had no call so its - /// absence cannot be established. + /// The code can not attribute this site to one lineage. Either both lineages are derived, or + /// the other lineage had no call, so nothing shows that it is absent. SharedArchaic, } -/// Classify a site from its per-genome calls. +/// Classify a site from the calls of each genome. /// -/// Lineage-specificity requires **positive evidence of absence** in the other lineage — that lineage -/// must be *called* homozygous-ancestral, not merely missing. Treating `NoCall` as absence is what -/// an earlier version did, and it was the dominant error in Tier B attribution: a site where the -/// Neanderthals happened to be masked out and Denisova was called read as Denisovan-*specific*, -/// inflating Denisovan-diagnostic sites to 18,551 against 24,077 Neanderthal on chr21+22 and -/// producing ~19 % Denisovan for a European, where design §7 expects approximately zero. +/// A site belongs to one lineage only with **positive evidence that the allele is absent** in the +/// other lineage. The code must *call* that other lineage homozygous-ancestral. A missing call is +/// not enough. /// -/// Sites that cannot be attributed fall to [`DiagnosticClass::SharedArchaic`], which therefore means -/// "archaic but not attributable" rather than strictly "derived in both". +/// An earlier version read `NoCall` as absence, and that was the largest error in the Tier B +/// attribution. A site where a mask had removed the Neanderthals, and where the caller did call +/// Denisova, read as *specific* to Denisova. That raised the Denisovan-diagnostic sites to 18,551 +/// against 24,077 Neanderthal ones on chr21 and chr22. It gave about 19% Denisovan for a +/// European, where design §7 expects about zero. +/// +/// A site that the code can not attribute falls to [`DiagnosticClass::SharedArchaic`]. So that +/// class means "archaic, but the code can not attribute it". It does not mean strictly "derived +/// in both". pub fn classify_diagnostic(calls: &[ArchaicCall; 4]) -> DiagnosticClass { let nea_derived = calls .iter() @@ -88,9 +95,10 @@ pub fn classify_diagnostic(calls: &[ArchaicCall; 4]) -> DiagnosticClass { let den_derived = calls[DENISOVA].carries_derived(); let den_ancestral = calls[DENISOVA] == ArchaicCall::HomAncestral; - // Derived in BOTH lineages short-circuits to shared. Without this the Denisovan branch fires - // whenever some *other* Neanderthal is ancestral at a site both lineages carry — the four - // genomes disagree with each other constantly, so this is common, not a corner case. + // A site that is derived in BOTH lineages goes straight to shared. Without this test, the + // Denisovan branch fires whenever some *other* Neanderthal is ancestral at a site that both + // lineages carry. The four genomes disagree with each other all the time, so that is common + // and not a rare case. if nea_derived && den_derived { DiagnosticClass::SharedArchaic } else if nea_derived && den_ancestral { @@ -104,11 +112,13 @@ pub fn classify_diagnostic(calls: &[ArchaicCall; 4]) -> DiagnosticClass { /// One archaic-informative marker. /// -/// Coordinates are CHM13 and **oriented**: `reference_allele` is the actual CHM13 base at -/// `position`. That orientation is not optional — the pipeline lifts GRCh37→CHM13 with `CrossMap -/// bed`, which is not allele-aware, so roughly a third of sites arrive with ref/alt reversed -/// relative to CHM13. Shipping them unoriented is the bug that cost the ancient-ancestry build a -/// full rebuild (that design's §7.16). +/// The coordinates are CHM13, and they are **oriented**: `reference_allele` is the true CHM13 +/// base at `position`. +/// +/// That orientation is necessary. The pipeline lifts GRCh37 to CHM13 with `CrossMap bed`, which +/// does not know about alleles. So about a third of the sites arrive with ref and alt the wrong +/// way round against CHM13. To send them out without the orientation is the bug that cost the +/// ancient-ancestry build a full rebuild. See §7.16 of that design. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ArchaicSite { pub contig: String, @@ -117,10 +127,11 @@ pub struct ArchaicSite { /// The actual base on the panel's build (post-orientation). pub reference_allele: char, pub alternate_allele: char, - /// The archaic-derived allele — always one of `reference_allele` / `alternate_allele`. Stored as - /// a base, not a ref/alt flag, so a later orientation pass cannot silently invert its meaning. + /// The archaic-derived allele. It is always `reference_allele` or `alternate_allele`. The + /// field holds a base and not a ref-or-alt flag. A later orientation pass can then not turn + /// it around where nobody looks. pub archaic_derived_allele: char, - /// Per-genome state, indexed by [`ARCHAIC_GENOMES`]. + /// The state of each genome, at the index that [`ARCHAIC_GENOMES`] gives. pub calls: [ArchaicCall; 4], pub diagnostic_class: DiagnosticClass, /// Derived-allele frequency in the African outgroup, kept for transparency and so the panel can @@ -130,12 +141,14 @@ pub struct ArchaicSite { /// alleles, so there is no liftover and no strand risk on this build. #[serde(default)] pub grch37: Option, - /// GRCh38 locus, when the site lifted cleanly and could be oriented against an hg38 reference. + /// The GRCh38 locus. It is present when the site lifted cleanly, and when the code could + /// orient it against an hg38 reference. #[serde(default)] pub grch38: Option, } -/// The complement of a base, for comparing alleles across builds that may differ in strand. +/// The complement of a base. Use it to compare alleles across builds that can hold different +/// strands. fn complement(b: char) -> char { match b.to_ascii_uppercase() { 'A' => 'T', @@ -147,7 +160,8 @@ fn complement(b: char) -> char { } impl ArchaicSite { - /// The locus for a build name, mirroring [`crate::ibd_panel::IbdPanelSite::locus`]. + /// The locus for a build name. It has the same shape as + /// [`crate::ibd_panel::IbdPanelSite::locus`]. pub fn locus(&self, build: &str) -> Option<&Locus> { let b = build.to_ascii_lowercase(); if b.contains("38") || b == "hg38" { @@ -162,10 +176,12 @@ impl ArchaicSite { /// Re-express a dosage measured against some build's `alt` allele as a dosage against this /// site's **CHM13** alternate allele. /// - /// Genotyping a GRCh37/38 alignment tallies that build's alleles, which may be ref/alt-swapped - /// or strand-flipped relative to CHM13. Feeding such a dosage into the count unchanged would - /// invert those sites silently — the same class of error the CHM13 orientation pass exists to - /// prevent. Returns `None` when the measured allele corresponds to neither CHM13 allele. + /// A genotype run on a GRCh37 or GRCh38 alignment counts the alleles of that build. Those + /// alleles can have ref and alt the other way round against CHM13, or sit on the other + /// strand. To put such a dosage into the count without a change would turn those sites + /// around, and nobody would see it. That is the same class of error that the CHM13 + /// orientation pass prevents. Returns `None` when the measured allele matches neither CHM13 + /// allele. pub fn rekey_dosage(&self, measured_alt: char, dosage: i32, ploidy: u8) -> Option { let m = measured_alt.to_ascii_uppercase(); let (r, a) = ( @@ -183,19 +199,20 @@ impl ArchaicSite { /// Copies of the archaic-derived allele (0–2) in a diploid observation. /// - /// Takes the subject's two alleles as bases rather than a dosage, because dosage is defined - /// against ref/alt whereas the derived allele may be either one. + /// It takes the two alleles of the subject as bases, and not as a dosage. A dosage counts + /// against ref and alt, and the derived allele can be either one of those. pub fn derived_copies(&self, allele_a: char, allele_b: char) -> u8 { let d = self.archaic_derived_allele.to_ascii_uppercase(); u8::from(allele_a.to_ascii_uppercase() == d) + u8::from(allele_b.to_ascii_uppercase() == d) } } -/// The site-selection thresholds a panel was built with, carried in the asset itself. +/// The site-selection thresholds that made a panel. The asset itself carries them. /// -/// Recorded because these are the panel's scientific content: the site list *is* the product, every -/// downstream number inherits it, and a count is meaningless without knowing which filter produced -/// it. Design §10 fixes them by calibration (M1 checkpoint A), not by the illustrative values in §4. +/// The asset records them because they are the scientific content of the panel. The site list +/// *is* the product, and every number after it inherits that list. A count says nothing unless +/// you know which filter made it. Design §10 sets these thresholds by calibration, at M1 +/// checkpoint A. It does not set them from the example values in §4. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct ArchaicPanelThresholds { /// Maximum derived-allele frequency in the African outgroup (the introgression signature: rare @@ -233,10 +250,10 @@ impl ArchaicMarkerPanel { self.sites.is_empty() } - /// The denominator a Tier-A count is reported against: two copies per site. + /// The denominator of a Tier-A count: two copies at each site. /// - /// 23andMe reports exactly this shape — the ground-truth report for the project's validation - /// sample reads "191 of 7,462", and 7,462 is 2 × 3,731 assayed sites (design §10). + /// 23andMe reports exactly this shape. The ground-truth report of the check sample of this + /// project reads "191 of 7,462", and 7,462 is 2 × 3,731 assayed sites. See design §10. pub fn possible_copies(&self) -> usize { self.sites.len() * 2 } @@ -244,21 +261,26 @@ impl ArchaicMarkerPanel { /// The Tier-A result: archaic-derived allele copies carried, over copies assayed. /// -/// Deliberately a **count over what was actually called**, not a "percent Neanderthal" — the same -/// shape a consumer report uses ("191 of 7,462" = copies of 2 × 3,731 assayed sites, design §1). -/// Because the denominator is whatever subset of the panel the subject's data covered, chip and WGS -/// each get an honest headline with no cross-data-type comparison involved. +/// This is a **count over the sites that the caller called**, and that choice is deliberate. It +/// is not a "percent Neanderthal". It is the same shape that a consumer report uses: "191 of +/// 7,462" is the copies out of 2 × 3,731 assayed sites. See design §1. +/// +/// The denominator is whatever subset of the panel the data of the subject covered. So a chip +/// and a WGS run each get an honest headline, and no comparison between the two data types +/// occurs. +/// +/// [`percentile`](Self::percentile) is the one field that *does* need such a comparison, so it is +/// an `Option`. The caller fills it only when the coverage of the subject is comparable to the +/// reference cohort. See design §10. /// -/// [`percentile`](Self::percentile) is the one field that *does* require comparability, so it is an -/// `Option` the caller fills only when the subject's coverage is comparable to the reference cohort -/// (design §10). A chip covers ~3–4 % of the panel and is biased toward its common tail, so ranking -/// a chip count against a WGS-scored cohort is meaningless — it stays `None` there rather than -/// rendering a number that looks authoritative and is not. +/// A chip covers about 3 to 4% of the panel, and it leans toward the common tail of the panel. To +/// rank a chip count against a cohort that a WGS run scored says nothing. The field stays `None` +/// there, and the report does not show a number that looks authoritative and is not. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ArchaicMarkerResult { /// Archaic-derived allele copies carried across called panel sites. pub total_copies: u32, - /// 2 × called sites — the denominator the count is reported against. + /// 2 × the count of called sites. This is the denominator of the report. pub possible_copies: u32, /// Panel sites with a usable genotype for this subject. pub called_sites: usize, @@ -268,27 +290,29 @@ pub struct ArchaicMarkerResult { pub call_rate: f32, /// Copies at Neanderthal-diagnostic sites. pub neanderthal_copies: u32, - /// Copies at Denisovan-diagnostic sites. For a European this should be near the noise floor; - /// §7 forbids presenting a small value here as a positive Denisovan finding. + /// The copies at Denisovan-diagnostic sites. For a European this must lie near the noise + /// floor. §7 does not let the report show a small value here as Denisovan ancestry. pub denisovan_copies: u32, - /// Copies at sites derived in both lineages — archaic, but not attributable. + /// The copies at sites that are derived in both lineages. They are archaic, but the code can + /// not attribute them. pub shared_copies: u32, /// Percentile within `cohort`, when the comparison is valid (see the type docs). pub percentile: Option, - /// The cohort `percentile` was computed against (e.g. "EUR"). + /// The cohort that `percentile` counts against, for example "EUR". pub cohort: Option, - /// Indices (panel order) of the sites that were actually called for this subject. + /// The indices, in panel order, of the sites that the caller called for this subject. /// - /// Needed to score the reference cohort over the *same* sites — the whole basis of an honest - /// percentile for sparse input. Deliberately **not serialized**: it is ~300 k integers on a WGS - /// subject, and the cached result is stored as JSON. + /// The code needs these to score the reference cohort over the *same* sites. That is the + /// whole basis of an honest percentile when the input is sparse. The field is **not + /// serialized**, and that is deliberate. It holds about 300k integers on a WGS subject, and + /// the cache holds the result as JSON. #[serde(skip)] pub called_indices: Vec, } impl ArchaicMarkerResult { - /// Copies carried as a fraction of copies assayed — the rate that is comparable *within* one - /// data type. Returns 0.0 when nothing was called. + /// The copies that the subject carries, as a fraction of the copies assayed. You can compare + /// this rate *inside* one data type. Returns 0.0 when the caller called nothing. pub fn rate(&self) -> f32 { if self.possible_copies == 0 { 0.0 @@ -300,11 +324,14 @@ impl ArchaicMarkerResult { /// Count a subject's archaic-derived allele copies across the panel. /// -/// Pure dosage arithmetic over the consensus genotypes, so it works identically for chip and WGS -/// input. The one subtlety: `SiteGenotype::dosage` counts the **alternate** allele, while the -/// panel's derived allele may sit on either side, so the dosage is re-expressed against the derived -/// *base*. Reading dosage directly as "archaic copies" would invert every site where CHM13 -/// orientation left the derived allele on REF — 3 % of the panel. +/// This is dosage arithmetic over the consensus genotypes and nothing more, so it works the same +/// way for chip input and for WGS input. +/// +/// There is one thing to watch. `SiteGenotype::dosage` counts the **alternate** allele, and the +/// derived allele of the panel can sit on either side. So the code expresses the dosage again, +/// against the derived *base*. To read the dosage directly as "archaic copies" would turn around +/// every site where the CHM13 orientation left the derived allele on REF. That is 3% of the +/// panel. pub fn count_archaic_markers(genotypes: &[SiteGenotype], panel: &ArchaicMarkerPanel) -> ArchaicMarkerResult { let by_pos: std::collections::HashMap<(&str, i64), &SiteGenotype> = genotypes.iter().map(|g| ((g.contig.as_str(), g.position), g)).collect(); @@ -328,8 +355,8 @@ pub fn count_archaic_markers(genotypes: &[SiteGenotype], panel: &ArchaicMarkerPa } else if matches(&g.reference_allele) { (g.ploidy as i32 - g.dosage).max(0) as u32 } else { - // The subject's alleles at this position disagree with the panel's — skip rather than - // guess, and do not count it toward the denominator either. + // The alleles of the subject at this position disagree with those of the panel. + // Skip the site, do not guess, and do not count it in the denominator. continue; }; called += 1; @@ -368,60 +395,67 @@ pub struct CohortCounts { pub population: String, /// Super-population the fine code rolls up to (e.g. "EUR"). pub super_population: String, - /// Per-sample archaic-derived copy totals, ascending. + /// The total of archaic-derived copies for each sample, from the lowest up. pub counts: Vec, } /// The percentile reference asset. /// -/// Stored **per population** rather than pre-reduced to super-populations. v1 renders the percentile -/// against a super-population cohort (design §9 Q3 — keying it to the user's inferred fine ancestry -/// would let an ancestry error silently move the archaic headline), but keeping fine-grained counts -/// means a fine-pop percentile is a later re-keying rather than an asset rebuild. +/// The asset stores the counts **for each population**, and does not reduce them to +/// super-populations first. v1 shows the percentile against a super-population cohort. See design +/// §9 Q3. A key on the fine ancestry of the user would let an error in that ancestry move the +/// archaic headline where nobody sees it. The code infers that fine ancestry. +/// +/// The fine-grained counts stay in the asset. A percentile for a fine population is then a change +/// of key later, and not a rebuild of the asset. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ArchaicCountDistribution { pub build: String, - /// The panel these counts were produced from — a percentile is only meaningful against the same - /// site list, so a mismatch must be detected rather than silently rendered. + /// The panel that made these counts. A percentile is correct only against the same site + /// list. So the code must find a mismatch, and it must not show a percentile that came from + /// the wrong list. pub panel_sites: usize, pub cohorts: Vec, /// Super-population codes indexing [`site_freqs`](Self::site_freqs) and /// [`variance_inflation`](Self::variance_inflation). #[serde(default)] pub populations: Vec, - /// Per-population derived-allele frequency at each panel site, in **panel order**: + /// The derived-allele frequency of each population at each panel site, in **panel order**: /// `site_freqs[pop][site_index]`. /// - /// This is what lets a *sparse* subject get an honest percentile. Per-sample totals can only - /// rank someone scored on the whole panel; frequencies let the cohort's expected count and - /// variance be computed over exactly the sites a given subject called, whatever those are — - /// which is the only valid comparison when a chip covers ~3 % of the panel and those sites are - /// its common tail. + /// This is what lets a *sparse* subject get an honest percentile. A total for each sample can + /// rank only a person whom the code scored on the whole panel. Frequencies instead give the + /// expected count and the variance of the cohort over exactly the sites that a given subject + /// called, whatever those sites are. That is the only correct comparison when a chip covers + /// about 3% of the panel, and those sites are its common tail. #[serde(default)] pub site_freqs: Vec>, - /// Per-population variance inflation measured at a **ladder of site densities**: - /// `variance_inflation[pop] = [(density, inflation), …]`, densest first. + /// The variance inflation of each population, measured at a **ladder of site densities**: + /// `variance_inflation[pop] = [(density, inflation), …]`, with the densest first. /// - /// Archaic alleles travel in linked haplotype blocks, so sites are not independent and the - /// binomial sum understates the spread — but crucially the inflation is **not a constant**. It - /// was measured at 52.4× on the full panel and 5.3× on a 2.6 % subset of the same panel, because - /// a sparse subset samples fewer sites per linked block. Applying a single full-panel factor to - /// a chip would over-widen the deviation ~3× and squash every percentile toward 50. + /// Archaic alleles travel in linked haplotype blocks, so the sites are not independent, and + /// the binomial sum gives a spread that is too small. But the inflation is **not a constant**, + /// and that is the important part. A measurement gave 52.4x on the full panel, and 5.3x on a + /// 2.6% subset of the same panel. A sparse subset takes fewer sites from each linked block. To + /// apply one full-panel factor to a chip would make the deviation about 3x too wide, and it + /// would push every percentile toward 50. /// - /// No simple block model fits the two measurements (solving for a common block size gives a - /// negative size), so this is measured empirically at several densities and interpolated in log - /// space at runtime rather than modelled. + /// No simple block model agrees with the two measurements. A solution for one common block + /// size gives a negative size. So the code measures the inflation at some densities, and + /// interpolates between them in log space at run time. It does not model it. #[serde(default)] pub variance_inflation: Vec>, - /// SHA-256 of the panel asset these frequencies were computed against. A percentile is only - /// rendered when this matches the loaded panel — site_freqs is indexed by panel position, so a - /// mismatched panel would silently score against the wrong sites. + /// The SHA-256 of the panel asset that these frequencies count against. The UI shows a + /// percentile only when this value matches the panel that the app loaded. An index into + /// site_freqs is a panel position. A panel that does not match would then score against the + /// wrong sites, and nobody would see it. #[serde(default)] pub panel_fingerprint: String, } -/// Standard-normal CDF via the Abramowitz & Stegun 7.1.26 error-function approximation -/// (|error| < 1.5e-7) — enough for a percentile rendered to whole numbers. +/// The standard-normal CDF, through the error-function approximation of Abramowitz and Stegun +/// 7.1.26, whose |error| is less than 1.5e-7. That is accurate enough for a percentile that the +/// UI shows as a whole number. fn normal_cdf(z: f64) -> f64 { let sign = if z < 0.0 { -1.0 } else { 1.0 }; let x = z.abs() / std::f64::consts::SQRT_2; @@ -442,8 +476,9 @@ impl ArchaicCountDistribution { bincode::serialize(self).map_err(|e| AnalysisError::Message(format!("archaic dist encode: {e}"))) } - /// Percentile (0–100) of `count` within a super-population cohort, or `None` when that cohort is - /// absent. Reported as the fraction of reference samples scoring **strictly below** `count`. + /// The percentile (0 to 100) of `count` inside a super-population cohort, or `None` when the + /// asset does not hold that cohort. It is the fraction of the reference samples whose score is + /// **below** `count`, and not equal to it. pub fn percentile_in_super(&self, super_population: &str, count: u32) -> Option { let mut below = 0usize; let mut total = 0usize; @@ -454,18 +489,21 @@ impl ArchaicCountDistribution { (total > 0).then(|| below as f32 * 100.0 / total as f32) } - /// Percentile of `observed_copies` for a subject who called exactly `called_sites` (indices into - /// the panel, in panel order), against `super_population`. + /// The percentile of `observed_copies`, against `super_population`, for a subject who called + /// exactly `called_sites`. Those are indices into the panel, in panel order. /// - /// Works at any coverage — a chip calling 3 % of the panel is compared against the cohort's - /// expected count *over those same sites*, so the call-rate artefact that would otherwise pin - /// every chip user near the 0th percentile disappears. + /// This works at any coverage. For a chip that calls 3% of the panel, the code compares + /// against the expected count of the cohort *over those same sites*. The artifact of the call + /// rate, which would otherwise hold every chip user near the 0th percentile, then goes away. /// - /// Modelled as a sum of independent per-site binomials under Hardy-Weinberg (mean `2f`, variance - /// `2f(1-f)`), normal-approximated — with thousands of sites the CLT is comfortable — then the - /// variance scaled by the measured LD inflation. Returns `None` when the asset predates the - /// frequency data, the panel fingerprint disagrees, the population is unknown, or too few sites - /// were called for the approximation to mean anything. + /// The model is a sum of independent binomials, one at each site, under Hardy-Weinberg. Each + /// has a mean of `2f` and a variance of `2f(1-f)`. The code takes the normal approximation of + /// that sum, which is safe with thousands of sites. It then scales the variance by the + /// measured LD inflation. + /// + /// Returns `None` in four cases. The asset is older than the frequency data. The fingerprint + /// of the panel disagrees. The code does not know the population. Or the caller called too + /// few sites for the approximation to say anything. pub fn percentile_for_called( &self, super_population: &str, @@ -499,9 +537,10 @@ impl ArchaicCountDistribution { } impl ArchaicCountDistribution { - /// Variance inflation for a population at a given site density, log-interpolated between the - /// measured rungs and clamped to the ends (never extrapolated — an extrapolated inflation is - /// exactly the kind of confident-but-unfounded number this whole feature avoids). + /// The variance inflation of a population at a given site density. The code interpolates in + /// log space between the measured rungs, and it clamps the result to the two ends. It never + /// goes outside the measured range. A value from outside that range is exactly the kind of + /// number that looks sure and has no basis, which this whole feature avoids. fn inflation_at(&self, pop: usize, density: f32) -> f32 { let Some(ladder) = self.variance_inflation.get(pop) else { return 1.0; @@ -533,7 +572,8 @@ impl ArchaicCountDistribution { } } -/// Below this many called sites the normal approximation is not worth rendering as a percentile. +/// Below this count of called sites, the normal approximation is not good enough to show as a +/// percentile. pub const MIN_SITES_FOR_PERCENTILE: usize = 200; #[cfg(test)] @@ -562,7 +602,7 @@ mod tests { classify_diagnostic(&calls(HomDerived, HomAncestral, HomAncestral, Het)), DiagnosticClass::SharedArchaic ); - // A heterozygous Neanderthal still counts as carrying the derived allele. + // A heterozygous Neanderthal still counts, because it carries the derived allele. assert_eq!( classify_diagnostic(&calls(Het, NoCall, NoCall, HomAncestral)), DiagnosticClass::Neanderthal @@ -673,9 +713,10 @@ mod tests { #[test] fn counting_re_expresses_dosage_against_the_derived_base() { - // Site 1: derived is the ALT, so dosage counts it directly. - // Site 2: derived is the REF, so the archaic copies are ploidy - dosage. Reading dosage - // straight through here would invert the site — the failure this test exists to catch. + // At site 1 the derived allele is the ALT, so the dosage counts it directly. + // At site 2 the derived allele is the REF, so the archaic copies are ploidy - dosage. To + // read the dosage straight through here would turn the site around. That is the failure + // that this test catches. let panel = ArchaicMarkerPanel { build: "chm13v2.0".into(), thresholds: ArchaicPanelThresholds { @@ -725,7 +766,7 @@ mod tests { assert_eq!(s.locus("GRCh37").map(|l| l.position), Some(42)); assert_eq!(s.locus("hg19").map(|l| l.position), Some(42)); assert_eq!(s.locus("b37").map(|l| l.position), Some(42)); - // Not populated / not a per-build lookup. + // The field is empty, and it is not a lookup by build. assert!(s.locus("GRCh38").is_none()); assert!(s.locus("chm13v2.0").is_none()); } @@ -745,7 +786,8 @@ mod tests { site(400, 'A', 'G', 'G', DiagnosticClass::Neanderthal), ], }; - // Panel site 400 is absent from the genotypes entirely — the fourth way a site goes uncalled. + // Panel site 400 is not in the genotypes at all. That is the fourth way that a site gets + // no call. let genotypes = vec![ gt("chr1", 100, "A", "G", 2), // counted gt("chr1", 200, "A", "G", -1), // explicit no-call @@ -757,7 +799,8 @@ mod tests { assert_eq!(r.total_copies, 2); assert_eq!(r.panel_sites, 4); assert!((r.call_rate - 0.25).abs() < 1e-6); - // Never fabricated by the counter — the caller fills it only when comparable. + // The counter never invents this value. The caller fills it in only when a comparison is + // correct. assert_eq!(r.percentile, None); } @@ -785,15 +828,15 @@ mod tests { min_allele_count: 1, contigs: vec![PositionStream::encode("chr21", &[100, 200, 300, 400])], }; - // 200 and 400 are shared with Africans -> stripped; the rest are private. + // Africans also carry 200 and 400, so the code removes those two. The rest are private. assert_eq!( og.retain_private("chr21", &[50, 200, 250, 400, 500]), vec![50, 250, 500] ); // Exact-boundary behaviour: first and last outgroup entries. assert_eq!(og.retain_private("chr21", &[100, 400]), Vec::::new()); - // A contig with no outgroup data yields NOTHING rather than everything — stripping nothing - // would declare the whole contig archaic. + // A contig with no outgroup data gives NOTHING, and not everything. To remove no site at + // all would call the whole contig archaic. assert_eq!(og.retain_private("chr7", &[1, 2, 3]), Vec::::new()); } @@ -817,9 +860,10 @@ mod tests { #[test] fn subset_percentile_compares_against_the_same_sites() { - // Two populations. At every site the "high" cohort carries the derived allele at 50% and the - // "low" cohort at 5%. A subject calling only 1000 sites is scored against the cohort's - // expectation OVER THOSE SITES, so sparse coverage no longer drags them to the bottom. + // Two populations. At every site the "high" cohort carries the derived allele at 50%, and + // the "low" cohort at 5%. The code scores a subject who calls only 1000 sites against the + // expectation of the cohort OVER THOSE SITES. Sparse coverage then no longer pulls that + // subject to the bottom. let n = 2000usize; let dist = ArchaicCountDistribution { build: "chm13v2.0".into(), @@ -832,7 +876,8 @@ mod tests { }; let called: Vec = (0..1000).collect(); - // Expected copies over 1000 sites at f=0.5 is 1000; landing exactly there is the median. + // The expected copies over 1000 sites at f=0.5 is 1000. A subject at exactly that value + // sits at the median. let p = dist .percentile_for_called("HIGH", &called, 1000, "fp") .expect("percentile"); @@ -842,11 +887,12 @@ mod tests { assert!(dist.percentile_for_called("HIGH", &called, 1200, "fp").unwrap() > 95.0); assert!(dist.percentile_for_called("HIGH", &called, 800, "fp").unwrap() < 5.0); - // The SAME raw count is unremarkable for HIGH but extraordinary for LOW — which is the whole - // point of scoring against the right cohort rather than a single pooled distribution. + // The SAME raw count is usual for HIGH and very rare for LOW. That is the whole point of a + // score against the correct cohort, and not against one pooled distribution. assert!(dist.percentile_for_called("LOW", &called, 1000, "fp").unwrap() > 99.0); - // Guards: wrong panel, unknown population, too few sites -> no number rather than a wrong one. + // The guards are a wrong panel, a population that the code does not know, and too few + // sites. Each one gives no number at all, and not a wrong one. assert_eq!(dist.percentile_for_called("HIGH", &called, 1000, "other-panel"), None); assert_eq!(dist.percentile_for_called("NOPE", &called, 1000, "fp"), None); assert_eq!(dist.percentile_for_called("HIGH", &called[..10], 5, "fp"), None); @@ -886,12 +932,14 @@ mod tests { // ─────────────────────────── Tier B assets (design §4, assets 2 and 3) ─────────────────────────── -/// Variable-length integer encoding for a sorted position stream: 7 bits per byte, high bit = more. +/// A variable-length integer encoding for a stream of sorted positions. Each byte holds 7 bits, +/// and the high bit says that more bytes follow. /// -/// Both Tier B assets store **gaps between consecutive positions**, not the positions themselves. -/// The African-outgroup track alone is ~67 M positions genome-wide; as raw `u32`s that is ~270 MB, -/// which is too much to hold and too much to ship. Typical gaps are tens of bases, so a varint delta -/// costs one byte for most sites. +/// Both Tier B assets store the **gap between one position and the next**, and not the positions +/// themselves. The African-outgroup track alone holds about 67M positions across the genome. As +/// raw `u32` values that is about 270 MB, which is too much to hold in memory and too much to +/// send to a user. A usual gap is tens of bases, so a varint delta costs one byte at most +/// sites. fn push_varint(out: &mut Vec, mut v: u64) { while v >= 0x80 { out.push((v as u8) | 0x80); @@ -900,7 +948,7 @@ fn push_varint(out: &mut Vec, mut v: u64) { out.push(v as u8); } -/// Decode the varint starting at `i`, returning the value and the next index. +/// Decode the varint that starts at `i`. Returns the value and the next index. fn read_varint(bytes: &[u8], mut i: usize) -> Option<(u64, usize)> { let (mut v, mut shift) = (0u64, 0u32); loop { @@ -942,10 +990,11 @@ impl PositionStream { } } - /// Stream the positions back in ascending order. + /// Stream the positions back, from the lowest up. /// - /// An iterator rather than a `Vec` on purpose: the caller merge-joins it against the subject's - /// own (much smaller) sorted variants, so neither side is ever fully materialised. + /// This gives an iterator and not a `Vec`, and that is deliberate. The caller does a merge + /// join of it against the sorted variants of the subject, which are much fewer. Neither side + /// has to be in memory in full. pub fn iter(&self) -> impl Iterator + '_ { let mut i = 0usize; let mut pos = 0i64; @@ -958,15 +1007,17 @@ impl PositionStream { } } -/// Asset 2 — positions **variable in the African outgroup**, used to strip shared variants before -/// the segment HMM (design §5 step 1). +/// Asset 2: the positions that are **variable in the African outgroup**. The code uses them to +/// remove shared variants before the segment HMM. See design §5, step 1. /// -/// Anything a modern African population also carries is not evidence of archaic introgression; what -/// survives the strip is the "private" derived variation the HMM looks for a density excess in. +/// A variant that a modern African population also carries is not evidence of archaic +/// introgression. What stays after the removal is the "private" derived variation, and the HMM +/// looks in that for an excess of density. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ArchaicOutgroup { pub build: String, - /// Minimum allele count in the outgroup for a site to be considered variable there. + /// The allele count that the outgroup needs at a site before the code calls that site + /// variable there. pub min_allele_count: u32, pub contigs: Vec, } @@ -984,14 +1035,15 @@ impl ArchaicOutgroup { self.contigs.iter().find(|c| c.contig == contig) } - /// Retain only those `sorted_positions` **absent** from the outgroup — the private set. + /// Keep only the `sorted_positions` that are **not** in the outgroup. That is the private + /// set. /// - /// A linear merge over both sorted streams, so cost is O(subject + outgroup) with no index and - /// no allocation proportional to the outgroup. + /// This is a linear merge over both sorted streams. It costs O(subject + outgroup), it needs + /// no index, and it allocates nothing that grows with the outgroup. pub fn retain_private(&self, contig: &str, sorted_positions: &[i64]) -> Vec { let Some(stream) = self.contig(contig) else { - // No outgroup data for this contig: stripping nothing would call the whole contig - // archaic, so refuse rather than guess. + // There is no outgroup data for this contig. To remove no site at all would call the + // whole contig archaic. So refuse, and do not guess. return Vec::new(); }; let mut out = Vec::new(); @@ -1008,11 +1060,12 @@ impl ArchaicOutgroup { } } -/// Asset 3 — genome-wide archaic diagnostic sites, for labelling a called segment Neanderthal vs -/// Denisovan (design §5 step 3). +/// Asset 3: the archaic diagnostic sites across the genome. They put a Neanderthal or Denisovan +/// label on a called segment. See design §5, step 3. /// -/// The HMM itself cannot tell the lineages apart — they coalesce before either meets modern humans -/// (§3) — so attribution is a downstream count of derived-allele matches against these sites. +/// The HMM alone can not separate the two lineages, because they coalesce before either one meets +/// modern humans. See §3. The attribution is instead a later count of derived-allele matches +/// against these sites. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ArchaicClassify { pub build: String, @@ -1023,9 +1076,9 @@ pub struct ArchaicClassify { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ClassifyContig { pub positions: PositionStream, - /// Derived base per site, same order as `positions`. + /// The derived base at each site, in the same order as `positions`. pub derived: Vec, - /// Diagnostic class per site: 0 = Neanderthal, 1 = Denisovan, 2 = shared archaic. + /// The diagnostic class at each site: 0 = Neanderthal, 1 = Denisovan, 2 = shared archaic. pub classes: Vec, } @@ -1061,21 +1114,25 @@ impl ArchaicClassify { } } -/// Callable-region track: **callable bases per fixed-width window**, per contig. +/// The track of callable regions: the count of **callable bases in each fixed-width window**, for +/// each contig. +/// +/// The asset holds a count for each window, and not a list of intervals. That is what the segment +/// HMM needs, and it is much smaller. The archaic masks break into hundreds of thousands of +/// intervals below one kb on each chromosome. A grid of 1 kb windows across the genome is about +/// 3.1M `u16` values, which is about 6 MB. /// -/// Stored as per-window counts rather than intervals because that is what the segment HMM actually -/// needs and it is far smaller: the archaic masks are fragmented into hundreds of thousands of -/// sub-kb intervals per chromosome, while a genome-wide 1 kb window grid is ~3.1 M `u16`s (~6 MB). +/// This track is necessary. Without it, the HMM finds an excess of private-variant density in +/// repetitive regions and reports those regions as archaic. A measurement gave 4,000 to 9,700 +/// variants/Mb there, against 50 to 200/Mb for a true introgressed tract. /// -/// This is not a nicety. Without it the HMM finds private-variant density excesses in repetitive -/// regions — measured at 4,000–9,700 variants/Mb against 50–200/Mb for a real introgressed tract — -/// and reports them as archaic. Those regions are exactly where the archaic genomes' own quality -/// masks exclude data, so a region callable in all four archaic genomes is the region where a -/// private-variant excess can be interpreted at all. +/// Those regions are exactly where the quality masks of the archaic genomes remove the data. So +/// a region that is callable in all four archaic genomes is the only region where an excess of +/// private variants says anything at all. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ArchaicCallable { pub build: String, - /// Window width the counts are binned at. + /// The window width that the counts go into. pub window_bp: i64, pub contigs: Vec, } @@ -1085,7 +1142,7 @@ pub struct CallableContig { pub contig: String, /// Genomic start of window 0. pub start: i64, - /// Callable bases in each window (saturating at `window_bp`). + /// The count of callable bases in each window. It stops at `window_bp`. pub callable_bp: Vec, } @@ -1102,9 +1159,10 @@ impl ArchaicCallable { self.contigs.iter().find(|c| c.contig == contig) } - /// Callable fraction (0.0–1.0) of the window containing `position`, or 0.0 when the contig or - /// window is absent — an unknown region is treated as **not** callable, so the HMM skips it - /// rather than interpreting density it cannot trust. + /// The callable fraction (0.0 to 1.0) of the window that holds `position`. It is 0.0 when the + /// asset holds neither the contig nor the window. A region that the code does not know counts + /// as **not** callable. The HMM then skips it, and it does not read a density that it can not + /// trust. pub fn callable_fraction(&self, contig: &str, position: i64) -> f64 { let Some(c) = self.contig(contig) else { return 0.0 }; if position < c.start { @@ -1117,7 +1175,8 @@ impl ArchaicCallable { } } - /// Total callable megabases across all contigs — the honest denominator for "% of genome". + /// The total callable megabases over all of the contigs. It is the honest denominator of a + /// "% of genome" figure. pub fn callable_mb(&self) -> f64 { self.contigs .iter() diff --git a/crates/navigator-analysis/src/archaic_match.rs b/crates/navigator-analysis/src/archaic_match.rs index 77f9c2f2..9d12645e 100644 --- a/crates/navigator-analysis/src/archaic_match.rs +++ b/crates/navigator-analysis/src/archaic_match.rs @@ -1,47 +1,51 @@ -//! Tier B, second attempt — archaic tracts by **matching the archaic genomes**, not by counting -//! mutations. +//! Tier B, the second try. It finds archaic tracts by a **match against the archaic genomes**, +//! and not by a count of mutations. //! //! # Why this replaces the density caller //! -//! [`crate::archaic_segments`] follows Skov 2018 (hmmix): strip variants Africans also carry, then -//! look for regions dense in what remains. That method exists for people who do **not** have archaic -//! reference genomes and must infer them indirectly. We have all four, and already ship -//! [`ArchaicClassify`] — 2,031,406 sites where the archaics carry a derived allele. +//! [`crate::archaic_segments`] follows Skov 2018 (hmmix). It removes the variants that Africans +//! also carry, and then looks for a region that is dense in what stays. That method exists for a +//! person who does **not** have archaic reference genomes, and who must infer them indirectly. +//! This project has all four, and it already ships [`ArchaicClassify`], which holds 2,031,406 +//! sites where the archaics carry a derived allele. //! -//! Measured on a real European against hmmix's own calls for the same person, the difference is not -//! subtle. Both observables carry the same ~3x contrast, but they differ 30-fold in how much -//! evidence one tract holds, and that is what decides whether a tract can be called at all: +//! A measurement on a real European, against the hmmix calls for the same person, shows a large +//! difference. Both observables carry the same contrast of about 3x. But they differ 30-fold in +//! how much evidence one tract holds, and that is what decides whether the code can call a tract +//! at all: //! -//! | observable | evidence per 36 kb tract | sensitivity at 5 % false positives | +//! | observable | evidence in a 36 kb tract | sensitivity at 5 % false positives | //! |---|---|---| //! | private-variant density | ~1 variant | 14.3 % | //! | archaic-allele matching (this) | ~30 sites | 95.1 % | //! -//! Density does not reach 80 % sensitivity at **500 kb**; matching reaches 95 % at the real median -//! tract of 36 kb. See `documents/design/ArchaicAncestry_Design.md` § *Why it failed*. +//! Density does not get to 80 % sensitivity at **500 kb**. Matching gets to 95 % at the real +//! median tract of 36 kb. See `documents/design/ArchaicAncestry_Design.md`, § *Why it failed*. //! //! # The model //! -//! An introgressed tract is a haplotype inherited intact from an archaic ancestor, so it carries the -//! archaic allele at a large share of the diagnostic sites it spans; elsewhere the subject carries -//! them only at the background rate. That is a two-state HMM whose observation is one **bit per -//! diagnostic site** — carried or not — with Bernoulli emissions, indexed **by site rather than by -//! base pair**. +//! An introgressed tract is a haplotype that came down whole from an archaic ancestor. So it +//! carries the archaic allele at a large share of the diagnostic sites that it covers. Elsewhere +//! the subject carries those alleles at the background rate alone. //! -//! Indexing by site is what makes this robust where the density model was not. Diagnostic sites -//! become the denominator, so their uneven density cancels out: the mutation-rate map the density -//! model needed (and which no available proxy supplied — the best explained 38 % of a 14.6x -//! overdispersion) is simply not required here. +//! That is a two-state HMM. Its observation is one **bit at each diagnostic site**: the subject +//! carries the allele, or does not. The emissions are Bernoulli. The index runs **over the sites +//! and not over the base pairs**. //! -//! Transitions stay recombination-scaled between consecutive sites, as in [`crate::roh`] and the -//! chromosome painter. +//! The index over sites is what makes this robust where the density model was not. The diagnostic +//! sites become the denominator, so their uneven density cancels. The density model needed a map +//! of the mutation rate, and no available proxy gave one. The best proxy explained 38 % of a 14.6x +//! overdispersion. This model does not need that map at all. //! -//! # Validation +//! The transitions scale with recombination between one site and the next, as they do in +//! [`crate::roh`] and in the chromosome painter. //! -//! ## Genome-wide (the shipping configuration) +//! # The checks //! -//! Three Europeans called across all 22 autosomes and scored against hmmix's genome-wide callset for -//! the same individuals: +//! ## Across the genome, which is the configuration that ships +//! +//! Three Europeans, called across all 22 autosomes, and scored against the genome-wide hmmix +//! callset for the same individuals: //! //! | | ours | hmmix | ratio | sensitivity | precision | null (max of 400 draws) | //! |---|---|---|---|---|---|---| @@ -49,116 +53,137 @@ //! | HG00102 | 83.9 Mb | 89.3 | 0.94 | 42.4 % | 45.1 % | 4.9 % | //! | HG00112 | 82.1 Mb | 91.0 | 0.90 | 42.9 % | 47.5 % | 5.1 % | //! -//! All three sit above the *entire* random-placement null. Both sensitivity and precision are -//! **better** genome-wide than on chr21+22 (40–43 % against 31.6 %, ~46 % against 34.9 %), so the -//! two-chromosome figures below are conservative rather than optimistic — worth stating because the -//! previous caller's design was burned by the opposite, extrapolating a chr21+22 target 6 % low. +//! All three sit above the *entire* null from random placement. Both the sensitivity and the +//! precision are **better** across the genome than they are on chr21 and chr22: 40 to 43 % against +//! 31.6 %, and about 46 % against 34.9 %. +//! +//! The two-chromosome figures below are careful, and not optimistic. Say that here, +//! because the opposite burned the design of the caller before this one. That design took a +//! chr21+22 target and went outside the measured range, to a value 6 % too low. //! -//! ## chr21+22, with a train/test split +//! ## chr21 and chr22, with a split into train and test //! -//! Scored against hmmix's own calls for the same individuals, 60 Europeans on chr21+22, split 30 -//! **train** / 30 **test** on a fixed seed. Thresholds were fitted on train only; every figure below -//! is the held-out half. The split exists because the previous caller was tuned until a cohort -//! statistic matched and the statistic was then reported as evidence. +//! 60 Europeans on chr21 and chr22, scored against the hmmix calls for the same individuals. A +//! fixed seed split them into 30 for **train** and 30 for **test**. The fit of the thresholds used +//! the train half alone, and every figure below comes from the half that the fit did not see. +//! +//! The split exists because of what happened before. Somebody tuned the caller before this one +//! until a cohort statistic agreed, and then reported that statistic as evidence. //! //! | | density caller | this, uncalibrated | this, calibrated | //! |---|---|---|---| -//! | base-level F1 | — | 27.9 % | **34.5 %** | +//! | base-level F1 | n/a | 27.9 % | **34.5 %** | //! | precision | 1.5 % | 20.2 % | **34.9 %** | //! | extent ratio ours/theirs | 1.45 | 2.23 | **0.98** | -//! | per-individual extent `r` | −0.018 (p = 0.94) | +0.520 | **+0.710 (p < 0.0001)** | +//! | extent `r` over the individuals | −0.018 (p = 0.94) | +0.520 | **+0.710 (p < 0.0001)** | //! -//! The extent ratio of 0.98 is the one to notice: the caller is no longer systematically -//! over-calling, which the emission-ratio sweep is what fixed. +//! Look at the extent ratio of 0.98. The caller no longer calls too much everywhere, and the sweep +//! over the emission ratio is what fixed that. //! -//! On locations, all 20 individuals of an earlier cohort scored above their own random-placement -//! null (mean 45.3 % sensitivity against a 7.1 % null); the density caller scored 2.1 % against a -//! 5.0 % null, i.e. below chance. +//! Now the locations. All 20 individuals of an earlier cohort scored above their own null from +//! random placement, at a mean sensitivity of 45.3 % against a null of 7.1 %. The density caller +//! scored 2.1 % against a null of 5.0 %, which is below chance. //! -//! ## Cross-population: transfers per individual, but the reported number does not +//! ## Across populations: the detection transfers, and the reported number does not //! -//! Run on 30 East Asians with the parameters **frozen** at the European fit, nothing refitted: +//! A run on 30 East Asians, with the parameters **frozen** at the European fit. The run fitted +//! nothing again: //! //! | | Europe (fitted) | East Asia (new) | //! |---|---|---| //! | above own random-placement null | 60/60 | **30/30** | //! | sensitivity | 31.6 % | **31.6 %** | //! | precision | 32.2 % | **41.9 %** | -//! | per-individual extent `r` | +0.620 | **+0.545** | +//! | extent `r` over the individuals | +0.620 | **+0.545** | +//! +//! The detection transfers. The sensitivity is the same, and the precision is *better*, on a +//! population that the thresholds never saw. The calibration learned archaic structure, and not +//! European structure. //! -//! Detection transfers: identical sensitivity and *better* precision on a population the thresholds -//! never saw, so the calibration learned archaic structure rather than European structure. +//! **But the reported extent puts the two populations in the wrong order.** The truth puts the +//! archaic extent of East Asia at **1.217x** that of Europe. The extent that this caller reports +//! is **0.937x**. A user would read that an East Asian carries *less* archaic ancestry than a +//! European. That is the wrong way round, and it is the one reason that this module stays gated. //! -//! **But the reported extent orders the populations backwards.** The truth puts East Asian archaic -//! extent at **1.217x** Europe's; our called extent is **0.937x**. A user would be told an East -//! Asian carries *less* archaic ancestry than a European, which is the wrong way round and is the -//! single reason this is still gated. +//! Here is the cause. The reported extent is the true positives *plus* the false positives. The +//! load of false positives depends on the population, at a precision of 32.2 % against 41.9 %. +//! Europeans then collect more extent that is not real. //! -//! The cause is that reported extent is true positives *plus* false positives, and the false-positive -//! load is population-dependent (precision 32.2 % against 41.9 %), so Europeans accumulate more -//! spurious extent. Note that "detected sequence reproduces 1.22x" is **not** evidence to the -//! contrary: detected = sensitivity x truth, and sensitivity is equal across the two populations, so -//! that ratio matches by construction. It restates the invariance, it does not test the ordering. +//! One thing is **not** evidence against this: that the detected sequence reproduces the 1.22x +//! ratio. The detected extent is the sensitivity times the truth, and the sensitivity is equal +//! across the two populations. So that ratio agrees by construction. It repeats the invariance, +//! and it does not test the order. //! -//! Ruled out as causes, each measured rather than argued: background contamination of `p_background` -//! (carrying rates 11.9 % vs 12.2 %, and both states scale together), tract length (median 29 kb in -//! both; East Asians simply have more tracts, 54 vs 46 per person), and panel ascertainment -//! (in-tract contrast 2.99x vs 3.04x, ratio 1.014 — the panel is equally informative in both). +//! Three causes are out, and a measurement rules out each one. No argument does. //! -//! ## How much of the "false positive" rate is really ours +//! Contamination of `p_background`: the rates at which the two populations carry the allele are +//! 11.9 % against 12.2 %, and both states scale together. Tract length: the median is 29 kb in +//! both, and East Asians only have more tracts, at 54 against 46 for each person. Panel +//! ascertainment: the contrast inside a tract is 2.99x against 3.04x, a ratio of 1.014, so the +//! panel carries equal information in both. //! -//! Precision is measured against hmmix, but a call they did not make is not automatically wrong. -//! An independent arbiter settles this without asking another caller: the Tier A panel records, per -//! site, which of the four archaic genomes carries the derived allele, and **this caller never sees -//! that** — it reads only a derived base and a lineage class. So per-genome concordance is evidence -//! it cannot have been fitted to. +//! ## How much of the "false positive" rate belongs to this caller //! -//! Of the sites where a given archaic genome is derived, what fraction does the subject carry -//! (best-matching genome): +//! The precision counts against hmmix, but a call that hmmix did not make is not wrong by that +//! fact alone. There is an independent arbiter, and it needs no second caller. The Tier A panel +//! records, at each site, which of the four archaic genomes carries the derived allele. **This +//! caller never sees that**: it reads a derived base and a lineage class alone. The concordance +//! with each genome is evidence that nobody could have fitted it to. +//! +//! Of the sites where a given archaic genome is derived, this is the fraction that the subject +//! carries, for the genome that matches best: //! //! | | true positive | false positive | background | //! |---|---|---|---| //! | Europe | 93.6 % | **81.3 %** | 59.0 % | //! | East Asia | 93.5 % | **72.9 %** | 45.5 % | //! -//! Our "false positives" sit **64 % / 57 %** of the way from background to true positive. They are a -//! mixture: real tracts hmmix missed, plus genuine noise, plus calls that are correctly placed but -//! over-extended. So precision against hmmix **understates** this caller — though not enough to -//! dismiss it, and F1 remains a usable objective. -//! -//! Note the background rates differ by population (59.0 % against 45.5 %): Europeans carry -//! archaic-derived alleles more often *outside* tracts. That is a candidate mechanism for the -//! population-varying false-positive load, and hence for the ordering inversion above. -//! -//! ## A concordance filter fixes precision, and exposes a harder limit -//! -//! Scoring each called segment against the archaic genomes and dropping the poor matches raises -//! **precision from 54 % to 90 %**. The filter is sound: with Denisova held out of it entirely, kept -//! segments score 74.9 % on Denisova concordance against 21.5 % for dropped ones — a 3.5x separation -//! on a genome the filter never saw. -//! -//! It does **not** fix the population ordering, and tightening it makes the ordering worse. At 90 % -//! precision the reported extent is mostly true positives, and it still orders the populations -//! backwards, so the cause is no longer false positives. What remains is a difference in *recovery*: -//! roughly 46 % of European truth against 38 % of East Asian. -//! -//! The reason is visible in the concordance itself. East Asian tracts match our archaic genomes less -//! well than European ones (83.4 % against 89.2 %), and Denisova is the best match for **32.2 % of -//! East Asian tracts against 11.2 % of European** ones. That 2.9x is the known Denisovan ancestry -//! East Asians carry and Europeans essentially lack — the data reproduces it — but it also means our -//! four sequenced archaic genomes **under-represent East Asian archaic diversity**. Any -//! reference-based filter therefore under-calls East Asians, and holding Denisova out (the first -//! design here) makes it markedly worse. -//! -//! That is a limit of the approach, not a threshold to tune: it would take archaic genomes closer to -//! the populations that introgressed into East Asia, which do not exist. **A cross-population -//! comparable number is therefore not currently achievable this way** — the caller is defensible -//! within a population and not between them. -//! -//! **Still not enough to re-enable.** Beyond the ordering: precision is 34.9 % unfiltered on -//! held-out Europeans, the cohort is **chr21+22 only**, and the reference callset is itself weakly -//! supported (hmmix's own tracts are enriched just 1.84x for their own archaic SNPs), so agreement -//! with it caps well below 100 % even for a correct caller — F1 alone cannot say when this is done. +//! The "false positives" of this caller sit **64 %** and **57 %** of the way from the background +//! to a true positive. They are a mixture: real tracts that hmmix missed, plus true noise, plus +//! calls that sit in the correct place and reach too far. So the precision against hmmix +//! **understates** this caller. That is not enough to dismiss the figure, and F1 stays a usable +//! objective. +//! +//! Note the background rates in the two populations: 59.0 % against 45.5 %. Europeans carry an +//! archaic-derived allele more often *outside* a tract. That is a candidate mechanism for the +//! load of false positives that changes with the population, and so for the inverted order +//! above. +//! +//! ## A concordance filter fixes the precision, and shows a harder limit +//! +//! Score each called segment against the archaic genomes, and drop the poor matches. That raises +//! the **precision from 54 % to 90 %**. The filter is sound. With Denisova held out of it +//! completely, the segments that stay score 74.9 % on Denisova concordance, against 21.5 % for the +//! segments that go. That is a separation of 3.5x on a genome that the filter never saw. +//! +//! It does **not** fix the order of the populations, and a tighter filter makes that order worse. +//! At 90 % precision the reported extent is mostly true positives. It still puts the populations +//! in the wrong order, so false positives are no longer the cause. What remains is a difference in +//! *recovery*: about 46 % of the European truth against 38 % of the East Asian truth. +//! +//! The concordance itself shows the reason. East Asian tracts match our archaic genomes less well +//! than European tracts do, at 83.4 % against 89.2 %. And Denisova is the best match for **32.2 % +//! of the East Asian tracts, against 11.2 % of the European** ones. +//! +//! That 2.9x is the known Denisovan ancestry that East Asians carry and Europeans almost do not, +//! and the data reproduces it. But it also means that our four sequenced archaic genomes +//! **under-represent the archaic diversity of East Asia**. Any filter that uses those references +//! under-calls East Asians. To hold Denisova out, which was the first design here, makes it much +//! worse. +//! +//! That is a limit of the approach, and not a threshold to tune. To fix it would need archaic +//! genomes nearer to the populations that introgressed into East Asia, and those do not exist. **A +//! number that you can compare across populations is not possible this way at present.** The +//! caller is defensible inside one population, and not between two. +//! +//! **This is still not enough to turn the module on.** Beyond the order of the populations, there +//! are three more reasons. +//! +//! The precision is 34.9 % without the filter, on held-out Europeans. The cohort is **chr21 and +//! chr22 alone**. And the reference callset itself has weak support: the tracts of hmmix show an +//! enrichment of only 1.84x for their own archaic SNPs. Agreement with that callset then stops +//! well below 100 %, even for a caller that is correct. F1 alone can not tell you when this work +//! reaches its end. use std::collections::BTreeMap; @@ -169,19 +194,21 @@ use crate::archaic_segments::{ArchaicSegment, ArchaicSegmentResult, ArchaicSourc use crate::caller::SiteGenotype; use crate::ibd::GeneticMap; -/// Bumped whenever a change would alter the segments this module produces. +/// Raise this after a change that would alter the segments that this module makes. /// -/// Persisted results are keyed on it (`archaic_segment_sig`), so a workspace holding output from an -/// earlier method — notably the withdrawn private-variant density caller — re-derives instead of -/// serving answers the current code would never produce. +/// A stored result carries it as part of its key, in `archaic_segment_sig`. A workspace that holds +/// output from an earlier method derives that output again. The private-variant density +/// caller, which this project withdrew, is the case that matters. Without the key, such a +/// workspace would serve answers that the current code would never make. pub const METHOD_VERSION: u32 = 1; -/// Concordance a segment must reach to be kept — measured, from the threshold sweep where precision -/// plateaus (54 % -> 90 % at 0.70, and no better above it while recall keeps falling). +/// The concordance that a segment must reach before the code keeps it. This is a measured value, +/// from the sweep over the threshold where the precision stops to rise. It goes from 54 % to 90 % +/// at 0.70. Above 0.70 the precision gets no better, and the recall continues to fall. pub const MIN_CONCORDANCE: f64 = 0.70; -/// Sites a genome needs in a segment before its concordance is trusted. Without a floor a genome -/// called at a single site scores 1.0 and wins every segment. +/// The count of sites that a genome needs in a segment before the code trusts its concordance. +/// Without a floor, a genome with a call at one site alone scores 1.0 and wins every segment. pub const MIN_CONCORDANCE_SITES: usize = 3; /// One diagnostic site, reduced to what the HMM consumes. @@ -193,37 +220,40 @@ pub struct SiteObs { pub class: DiagnosticClass, } -/// Tuning knobs. +/// The controls of this module. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct MatchConfig { /// Rate at which a **non-introgressed** genome carries the archaic allele at a diagnostic site. /// `None` estimates it from the subject's own genome-wide rate, which absorbs coverage, call /// behaviour and ancestry. /// - /// Estimated rather than fixed because it is the denominator of the whole inference — but - /// estimated *directly*, not by EM. Unconstrained Baum-Welch on the previous caller diverged to - /// a degenerate fit (a 22x emission ratio and 9 kb tracts, calling 7x the truth), so parameters - /// here are measured, not fitted. + /// The code estimates this rate, and does not hold it fixed, because it is the denominator of + /// the whole inference. But it estimates it *directly*, and not by EM. + /// + /// Baum-Welch with no constraint, on the caller before this one, went to a degenerate fit. It + /// gave an emission ratio of 22x, tracts of 9 kb, and 7x the true extent. So a measurement + /// gives every parameter here, and nobody fits one. pub p_background: Option, /// Rate inside an introgressed tract. `None` derives it as `p_background * archaic_ratio`. pub p_archaic: Option, - /// Multiple of the background rate expected inside a tract when `p_archaic` is `None`. - /// Measured at 3.04x (39.5 % inside real tracts against 13.0 % elsewhere). + /// How many times the background rate the code expects inside a tract, when `p_archaic` is + /// `None`. A measurement gave 3.04x: 39.5 % inside a real tract, against 13.0 % elsewhere. pub archaic_ratio: f64, - /// Expected state switches per centimorgan. + /// The count of state switches that the model expects in one centimorgan. pub switches_per_cm: f64, /// Discard tracts whose mean posterior is below this. pub min_posterior: f64, - /// Minimum diagnostic sites in a tract. A tract resting on one or two sites is exactly the - /// failure mode of the density caller, restated in a new observable. + /// The smallest count of diagnostic sites in a tract. A tract that stands on one site or two + /// is exactly how the density caller failed, in a new observable. pub min_sites: usize, /// Discard tracts shorter than this. pub min_segment_bp: i64, - /// Minimum callable fraction for a site's window to be used at all. + /// The callable fraction that the window of a site needs before the code uses that site at + /// all. pub min_callable_fraction: f64, - /// Whether to attempt per-segment Neanderthal/Denisovan attribution. Default `false`, unchanged - /// from the density caller: the lineage signal has not been shown to work, and this module does - /// not by itself change that. + /// True to try a Neanderthal or Denisovan attribution at each segment. The default is + /// `false`, as it was for the density caller. Nobody has shown that the lineage signal works, + /// and this module alone does not change that. pub attribute_lineage: bool, } @@ -232,25 +262,31 @@ impl Default for MatchConfig { MatchConfig { p_background: None, p_archaic: None, - // FITTED (not measured): the observed enrichment inside real tracts is 3.04x, but the - // model separates best at 4.5x. That is not a contradiction — 3.04x is the *average* - // over an external tract set that is itself only weakly supported, while the emission - // ratio is what makes the HMM selective enough to place boundaries. Fitted on 30 - // Europeans, reported on 30 held-out ones; it is the parameter that removed the - // over-calling (extent ratio 2.23 -> 0.98). + // A fit gives this value, and no measurement does. The observed enrichment inside a + // real tract is 3.04x, but the model separates best at 4.5x. The two do not + // disagree. + // 3.04x is the *mean* over a tract set from outside this project, and that set itself + // has weak support. The emission ratio is instead what makes the HMM selective enough + // to place a boundary. The fit used 30 Europeans, and the report uses 30 held-out + // ones. This is the parameter that stopped the over-call, and it moved the extent + // ratio from 2.23 to 0.98. archaic_ratio: 4.5, switches_per_cm: 1.0, - // All three CALIBRATED on train, reported on held-out test (see the module docs). - // Objective was base-level F1: sensitivity alone is bought by calling more sequence, and - // the uncalibrated caller over-called 2.2x while still scoring 45 %. + // The calibration of all three used the train half, and the report uses the held-out + // test half. See the module documentation. The objective was the F1 at the base + // level. Sensitivity alone comes from a call over more sequence: the caller before + // the calibration called 2.2x too much extent, and it still scored 45 %. min_posterior: 0.98, min_sites: 16, - // 5 kb, though the grid's argmax preferred 10 kb. Within the plateau the two differ by - // 0.1 F1 points, the 5 kb floor is slightly BETTER on per-individual extent correlation - // (+0.710 vs +0.706), and it discards half as many real tracts (8 % of the truth under - // 5 kb against 16 % under 10 kb). An earlier sweep wanted 40 kb, which would have - // discarded 61 %; the design records the same trap once before at 50 kb. Structural - // exclusion of real tracts is not worth a tenth of a point. + // 5 kb, although the argmax over the grid took 10 kb. Inside the flat part of the + // curve the two differ by 0.1 F1 points. The 5 kb floor is a little BETTER on the + // correlation of the extent of each individual, at +0.710 against +0.706. And it + // throws away half as many real tracts: 8 % of the truth lies below 5 kb, against + // 16 % below 10 kb. + // + // An earlier sweep wanted 40 kb, which would have thrown away 61 %. The design + // records the same trap once before, at 50 kb. To exclude real tracts by + // construction is not worth a tenth of a point. min_segment_bp: 5_000, min_callable_fraction: 0.5, attribute_lineage: false, @@ -260,16 +296,18 @@ impl Default for MatchConfig { /// Reduce one contig's diagnostic sites to observations. /// -/// `ref_base` supplies the reference base at a position; sites where the archaic-derived allele -/// **is** the reference base are dropped. At such a site every reference-matching genome trivially -/// "carries" the derived allele, so it separates nothing and would dilute the contrast — and, -/// because the caller emits only variant records, a no-call there means the subject *does* carry it, -/// the opposite of what a no-call means everywhere else. +/// `ref_base` gives the reference base at a position. The code drops a site where the +/// archaic-derived allele **is** the reference base. /// -/// A site with no variant record is hom-reference, hence **not** carrying. Restricting instead to -/// sites where the subject happens to have a call is the trap that made an early version of this -/// analysis report an 80 % carrying rate against a known 4.3 % background: it samples only sites -/// where a variant already exists. +/// There are two reasons. At such a site every genome that matches the reference holds the derived +/// allele, which separates nothing and would weaken the contrast. And the caller emits a variant +/// record alone, so a no-call there means that the subject *does* carry the allele. That is the +/// opposite of what a no-call means everywhere else. +/// +/// A site with no variant record is hom-reference, so the subject does **not** carry the allele +/// there. Do not instead keep the sites where the subject happens to have a call. That is the trap +/// that made an early version of this analysis report a rate of 80 %, against a known background +/// of 4.3 %. It takes only the sites where a variant already exists. pub fn observations_for_contig( contig: &str, classify: &ArchaicClassify, @@ -325,8 +363,9 @@ fn ln_sum_exp(a: f64, b: f64) -> f64 { /// Posterior probability of the archaic state at each observation. /// -/// Log-space forward/backward with recombination-scaled transitions, as in [`crate::roh`]. Exposed -/// so the decoding can be tested against hand-computed posteriors without constructing assets. +/// This is forward-backward in log space, with the transitions scaled by recombination, as in +/// [`crate::roh`]. It is public so that a test can check the decoding against a posterior computed +/// by hand, and build no asset. pub fn posteriors( obs: &[SiteObs], contig: &str, @@ -354,8 +393,9 @@ pub fn posteriors( (1.0 - (-switches_per_cm * cm.max(0.0)).exp()).clamp(1e-9, 0.5) }; - // Prior: the stationary share of the archaic state, from the rates themselves rather than a - // tuned constant — with p_arch > p_bg the algebra puts it at a few percent, matching reality. + // The prior is the stationary share of the archaic state. It comes from the rates themselves, + // and it is not a tuned constant. With p_arch > p_bg, the algebra puts it at a few percent, + // which agrees with reality. let prior_arch = ((p_bg - (1.0 - p_arch) * 0.0) / p_arch).clamp(0.001, 0.5) * 0.1; let mut fwd = vec![[f64::NEG_INFINITY; 2]; n]; let e0 = emit(0); @@ -389,8 +429,9 @@ pub fn posteriors( /// Call archaic tracts for one subject by matching the archaic genomes. /// -/// `observations` is per contig, already reduced by [`observations_for_contig`], so this function -/// does no I/O and no asset decoding — it is the model, and is unit-testable as such. +/// `observations` holds one entry for each contig, and [`observations_for_contig`] has already +/// reduced them. This function does no I/O, and it decodes no asset. It is the model alone, and a +/// unit test can cover it as such. pub fn call_from_observations( observations: &BTreeMap>, gmap: &GeneticMap, @@ -415,8 +456,8 @@ pub fn call_from_observations( }, }; } - // The genome-wide rate is dominated by non-archaic sequence (archaic tracts are a few percent - // of it), so it estimates the background directly. + // Sequence that is not archaic controls the genome-wide rate, because the archaic tracts are + // a few percent of the genome. That rate gives the background directly. let p_bg = cfg .p_background .unwrap_or((carried as f64 / total as f64).clamp(0.001, 0.5)); @@ -498,14 +539,17 @@ pub fn call_from_observations( ArchaicSegmentResult { segments, summary } } -/// How well a segment matches each archaic genome: of the sites where a given archaic genome -/// carries the derived allele, what fraction does the subject also carry. The best genome wins. +/// How well a segment matches each archaic genome. Take the sites where a given archaic genome +/// carries the derived allele, and ask what fraction of those the subject also carries. The genome +/// with the highest fraction wins. +/// +/// The condition is on the **genome**, and not on the subject, and that is what makes the measure +/// separate anything. The other way is to take the sites that the subject carries, and ask how +/// many an archaic genome shares. That scores about 100 % everywhere, and the background too, +/// because at an informative site some archaic holds the derived allele by construction. /// -/// Conditioning on the **genome**, not on the subject, is what makes this discriminate. The -/// intuitive version — over the sites the subject carries, how many does an archaic genome share — -/// scores ~100 % everywhere including background, because at an informative site some archaic -/// carries the derived allele by construction. Read this way, background sits at the subject's -/// genome-wide carrying rate and an inherited haplotype sits far above it. +/// Read the measure the way this function reads it. The background then sits at the genome-wide +/// rate of the subject, and a haplotype that came down from an ancestor sits far above that. /// /// Returns `None` when no genome has enough called sites in the span to judge. pub fn segment_concordance( @@ -538,8 +582,9 @@ pub fn segment_concordance( /// Which sites of the Tier A panel the subject carries the archaic-derived allele at. /// -/// A site with no variant record is hom-reference and therefore **not** a carrier; sites where the -/// derived allele is the reference base are excluded upstream by the panel's own orientation. +/// A site with no variant record is hom-reference, so the subject does **not** carry the allele +/// there. The orientation of the panel itself removes the sites where the derived allele is the +/// reference base, before the data reaches here. pub fn carried_panel_sites<'a>( panel: &'a ArchaicMarkerPanel, calls: &'a [SiteGenotype], @@ -559,16 +604,19 @@ pub fn carried_panel_sites<'a>( /// Drop segments that do not look like an inherited archaic haplotype. /// -/// This is the single largest quality lever measured: **precision 54 % -> 90 %** on real data. It -/// works because it consults evidence the segment caller never sees — which archaic genome carries -/// what — so it is genuinely new information rather than a re-reading of the same signal. +/// This is the largest measured gain in quality: the **precision goes from 54 % to 90 %** on real +/// data. It works because it reads evidence that the segment caller never sees, which is the +/// question of which archaic genome carries what. That is truly new information, and not a second +/// reading of the same signal. /// -/// A segment with too few judgable sites is **kept**: absence of evidence is not evidence of a bad -/// call, and dropping on it would silently penalise sparse regions. +/// The code **keeps** a segment with too few sites to judge. The absence of evidence is not +/// evidence of a bad call, and to drop such a segment would punish a sparse region where nobody +/// looks. /// -/// Note the population caveat in the module docs: East Asian tracts match the four sequenced archaic -/// genomes less well than European ones, so this filter removes proportionally more of them. It -/// improves precision everywhere and makes extent **less** comparable between populations. +/// Read the note about the populations in the module documentation. East Asian tracts match the +/// four sequenced archaic genomes less well than European tracts do, so this filter removes a +/// larger share of them. It raises the precision everywhere, and it makes the extent **less** +/// comparable between two populations. pub fn filter_by_concordance( result: ArchaicSegmentResult, panel: &ArchaicMarkerPanel, @@ -641,8 +689,8 @@ mod tests { .collect() } - /// A run of carried sites against a background of non-carried ones is what a real tract looks - /// like, and is the thing this model exists to find. + /// A real tract looks like a run of carried sites, against a background of sites that the + /// subject does not carry. That is the thing that this model exists to find. #[test] fn finds_a_run_of_carried_sites() { let mut sites: Vec<(i64, bool)> = (0..60).map(|i| (10_000 + i * 500, false)).collect(); @@ -651,10 +699,10 @@ mod tests { } let mut m = BTreeMap::new(); m.insert("chr21".to_string(), obs(&sites)); - // Thresholds pinned rather than inherited: this test is about whether the model finds a - // run at all, and should not move when the calibrated defaults do. (It broke once when - // `min_posterior` rose to 0.98 and trimmed the run's edges — correct behaviour, wrong - // thing for this test to be sensitive to.) + // This test fixes the thresholds, and it does not take them from the defaults. It asks + // whether the model finds a run at all, and it must not move when a calibrated default + // moves. It broke once, when `min_posterior` rose to 0.98 and cut the edges off the run. + // That was correct behaviour, and this test must not react to it. let cfg = MatchConfig { p_background: Some(0.13), p_archaic: Some(0.40), @@ -694,8 +742,8 @@ mod tests { ); } - /// Scattered carried sites at the background rate must not accumulate into a tract — the - /// density caller's defining failure, restated in this observable. + /// Carried sites that lie apart, at the background rate, must not add up into a tract. That + /// is how the density caller failed, in this observable. #[test] fn scattered_background_carriers_do_not_form_a_tract() { // 13 % carried, evenly spread: exactly the background rate, no run. @@ -720,7 +768,8 @@ mod tests { } /// A site whose derived allele IS the reference base separates nothing, and a no-call there - /// means the opposite of what it means elsewhere. Such sites must be dropped, not counted. + /// means the opposite of what it means elsewhere. The code must drop such a site, and must not + /// count it. #[test] fn observations_drop_sites_where_reference_is_derived() { let classify = ArchaicClassify { @@ -780,10 +829,11 @@ mod tests { } } - /// Concordance must be read per ARCHAIC GENOME, not per carried site. The intuitive version — - /// over the sites the subject carries, how many does some archaic share — scores ~100 % - /// everywhere including background, because at an informative site some archaic is derived by - /// construction. That version was written first and separated nothing. + /// Read the concordance over each ARCHAIC GENOME, and not over each carried site. The other + /// way is to take the sites that the subject carries, and ask how many some archaic + /// shares. That scores about 100 % everywhere, and the background too, because at an + /// informative site some archaic is derived by construction. Somebody wrote that version + /// first, and it separated nothing. #[test] fn concordance_conditions_on_the_genome_not_the_subject() { use ArchaicCall::{HomAncestral as A, HomDerived as D}; @@ -814,9 +864,9 @@ mod tests { assert!((c - 0.75).abs() < 1e-9, "expected Altai's 0.75, got {c}"); } - /// A segment with too few judgable sites must be KEPT. Absence of evidence is not evidence of a - /// bad call, and dropping on it would quietly penalise sparse regions — which are exactly the - /// regions where a caller most needs the benefit of the doubt. + /// The code must KEEP a segment with too few sites to judge. The absence of evidence is not + /// evidence of a bad call. To drop such a segment would punish a sparse region. Those are + /// exactly the regions where a caller most needs the doubt to go in its favour. #[test] fn filter_keeps_segments_it_cannot_judge() { let panel = ArchaicMarkerPanel { @@ -908,8 +958,9 @@ mod tests { ); } - /// A no-call is hom-reference, i.e. NOT carrying. Conditioning on "has a call" instead is what - /// made an early version of this analysis report ~80 % carrying against a 4.3 % background. + /// A no-call is hom-reference, so the subject does NOT carry the allele. A condition on "has + /// a call" instead is what made an early version of this analysis report about 80 %, against a + /// background of 4.3 %. #[test] fn a_missing_call_is_not_a_carrier() { let classify = ArchaicClassify { diff --git a/crates/navigator-analysis/src/archaic_segments.rs b/crates/navigator-analysis/src/archaic_segments.rs index c09f933e..b7043c9b 100644 --- a/crates/navigator-analysis/src/archaic_segments.rs +++ b/crates/navigator-analysis/src/archaic_segments.rs @@ -1,17 +1,21 @@ -//! Tier B — archaic **segment** calling (design `documents/design/ArchaicAncestry_Design.md` §5). +//! Tier B: the call of archaic **segments**. The design is in +//! `documents/design/ArchaicAncestry_Design.md`, §5. //! -//! Where Tier A counts marker copies, this finds the actual introgressed **tracts**: an hmmix-style -//! (Skov et al. 2018) two-state HMM over the density of *private* derived variants — the subject's -//! variants that no African outgroup individual carries. Anything Africans also carry is not -//! evidence of introgression, so stripping them is what makes the remaining density informative. +//! Tier A counts marker copies. This module instead finds the introgressed **tracts** themselves. +//! It is a two-state HMM in the style of hmmix (Skov and others, 2018), over the density of +//! *private* derived variants. A private variant is a variant of the subject that no individual +//! in the African outgroup carries. A variant that Africans also carry is not evidence of +//! introgression, so the removal of those is what makes the density that stays informative. //! -//! The HMM **cannot tell Neanderthal from Denisovan** — the two lineages coalesce before either -//! meets modern humans (§3) — so it finds segments and a downstream pass labels them by counting -//! derived-allele matches against the archaic genomes (`ArchaicClassify`). +//! The HMM **can not separate Neanderthal from Denisovan**, because the two lineages coalesce +//! before either one meets modern humans (§3). So it finds the segments, and a later pass puts a +//! label on each one. That pass counts the matches of the derived allele against the archaic +//! genomes, in `ArchaicClassify`. //! -//! Log-space Viterbi + forward/backward, cM-scaled transitions: the same idiom as -//! [`crate::roh`] and the chromosome painter. Only the emission differs — a Poisson point process -//! over private-variant counts per window, rather than het/hom. +//! The model uses Viterbi and forward-backward in log space, with transitions that scale in cM. +//! That is the same idiom as [`crate::roh`] and the chromosome painter. Only the emission is +//! different. Here it is a Poisson point process over the count of private variants in each +//! window, and not het against hom. use std::collections::BTreeMap; @@ -21,14 +25,15 @@ use crate::archaic::{ArchaicCallable, ArchaicClassify, ArchaicOutgroup, Diagnost use crate::caller::SiteGenotype; use crate::ibd::GeneticMap; -/// Which archaic lineage a called segment was attributed to. +/// The archaic lineage that a called segment belongs to. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ArchaicSource { Neanderthal, Denisovan, - /// Archaic by density, but the diagnostic sites in it do not favour either lineage — the - /// honest label for a segment we cannot attribute, and a substantial share in real data - /// (Skov 2020 reported ~12 % unknown on Icelanders). + /// The density says archaic, but the diagnostic sites inside the segment do not point to + /// either lineage. This is the honest label for a segment that the code can not attribute, and + /// real data holds a large share of them. Skov 2020 reported about 12 % unknown on + /// Icelanders. Unknown, } @@ -41,10 +46,11 @@ pub struct ArchaicSegment { pub end: i64, /// Mean posterior probability of the archaic state across the segment's windows. pub posterior: f64, - /// Private derived variants inside the segment — the evidence the call rests on. + /// The count of private derived variants inside the segment. That is the evidence that the + /// call stands on. pub n_private: usize, pub source: ArchaicSource, - /// Diagnostic-site matches supporting each lineage (the basis for `source`). + /// The count of diagnostic-site matches for each lineage. `source` comes from these. pub neanderthal_matches: usize, pub denisovan_matches: usize, } @@ -59,9 +65,10 @@ impl ArchaicSegment { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ArchaicSummary { pub total_mb: f64, - /// Archaic share of the **callable** span, as a percentage. Reported against callable rather - /// than nominal genome length: a partially-covered genome would otherwise read as having less - /// archaic ancestry purely because it was sequenced less. + /// The archaic share of the **callable** span, as a percentage. It counts against the callable + /// length, and not against the nominal length of the genome. A genome with partial coverage + /// would else read as one with less archaic ancestry, for one reason alone: the run sequenced + /// less of it. pub pct_callable: f64, pub callable_mb: f64, pub neanderthal_mb: f64, @@ -76,60 +83,68 @@ pub struct ArchaicSegmentResult { pub summary: ArchaicSummary, } -/// Tuning knobs, same shape as `RohConfig` / `PaintParams`. +/// The controls of this module. They have the same shape as `RohConfig` and `PaintParams`. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct ArchaicConfig { /// Window size in bp for the density process. pub window_bp: i64, - /// Expected private variants per window outside archaic tracts. `None` = estimate from the - /// sample's own genome-wide private rate, which adapts to depth and to how aggressively the - /// outgroup stripped. + /// The count of private variants that the model expects in one window, outside an archaic + /// tract. `None` tells the code to estimate it from the genome-wide private rate of the sample + /// itself. That estimate follows the depth, and it follows how much the outgroup removed. pub rate_background: Option, - /// Multiple of the background rate expected inside an archaic tract. Introgressed haplotypes - /// carry an excess of derived variants absent from Africans; this is that excess. + /// How many times the background rate the model expects inside an archaic tract. An + /// introgressed haplotype carries an excess of derived variants that Africans do not have, and + /// this is that excess. pub archaic_rate_multiple: f64, /// Prior probability of the archaic state. pub prior_archaic: f64, - /// Expected state switches per centimorgan — the recombination-scaled transition. + /// The count of state switches that the model expects in one centimorgan. It is the + /// transition that scales with recombination. pub switches_per_cm: f64, /// Discard called tracts shorter than this. pub min_segment_bp: i64, /// Discard tracts whose mean posterior is below this. pub min_posterior: f64, - /// A segment is attributed to a lineage only when its **enrichment** over the expected - /// carrier rate favours that lineage by at least this ratio; otherwise it is `Unknown`. + /// A segment goes to a lineage only when its **enrichment** over the expected carrier rate + /// points to that lineage by this ratio or more. Below that, the segment is `Unknown`. pub min_lineage_ratio: f64, /// Expected fraction of Neanderthal-diagnostic sites at which a non-archaic-specific genome /// carries the derived allele, and the same for Denisovan-diagnostic sites. /// - /// These base rates are the reason raw match counts cannot attribute a lineage. Measured on the - /// ground-truth European: 4.3 % at Neanderthal-diagnostic sites versus 3.9 % at - /// Denisovan-diagnostic ones — a ratio of 1.10, essentially no discrimination. Carrying a - /// "Denisovan-diagnostic" allele mostly reflects ordinary shared ancestry, not Denisovan - /// introgression, so attribution must compare observed against expected rather than - /// Neanderthal-count against Denisovan-count. + /// These base rates are why a raw count of matches can not attribute a lineage. A measurement + /// on the ground-truth European gave 4.3 % at Neanderthal-diagnostic sites, against 3.9 % at + /// Denisovan-diagnostic ones. That ratio is 1.10, which separates almost nothing. + /// + /// A "Denisovan-diagnostic" allele that a person carries mostly shows ordinary shared + /// ancestry, and not Denisovan introgression. So the attribution must compare the observed + /// rate against the expected rate. It must not compare the Neanderthal count against the + /// Denisovan count. pub base_rate_neanderthal: f64, pub base_rate_denisovan: f64, - /// Minimum diagnostic matches of the winning lineage before any attribution is made. A handful - /// of matches on a sub-megabase segment is noise at these base rates. + /// The count of diagnostic matches that the lineage in front needs before the code attributes + /// anything. At these base rates, a few matches on a segment below one megabase are noise. pub min_lineage_matches: usize, - /// Whether to attempt per-segment lineage attribution at all. **Default `false`.** + /// True to try a lineage attribution at each segment. **The default is `false`.** + /// + /// The attribution code exists, and unit tests cover it. But nobody has checked it against + /// truth. On the ground-truth European it gives the opposite of the known pattern: 0.00 Mb + /// Neanderthal against 0.48 Mb Denisovan. §7 expects almost all Neanderthal, and no Denisovan. /// - /// Attribution is built and unit-tested but is NOT validated, and on the ground-truth European - /// it produces the opposite of the known pattern: 0.00 Mb Neanderthal against 0.48 Mb - /// Denisovan, where §7 expects essentially all Neanderthal and no Denisovan. The cause is - /// visible in the base rates — a European carries the derived allele at 4.3 % of - /// Neanderthal-diagnostic sites and 3.9 % of Denisovan-diagnostic ones, a ratio of 1.10, so - /// there is almost no discriminating signal to work with at segment scale. + /// The base rates show the cause. A European carries the derived allele at 4.3 % of the + /// Neanderthal-diagnostic sites, and at 3.9 % of the Denisovan-diagnostic ones. That ratio is + /// 1.10, so at the scale of a segment there is almost no signal that separates the two. /// - /// Shipping a lineage split on that basis would manufacture exactly the Denisovan-in-Europeans - /// claim §7 forbids, so segments are reported as archaic-but-unattributed until the method is - /// validated. This is the same discipline that gated ancient ancestry. + /// To send a lineage split out on that basis would make exactly the claim of Denisovan + /// ancestry in Europeans that §7 forbids. So a segment goes out as archaic, with no lineage, + /// until somebody checks the method. That is the same discipline that gated the deep + /// ancestry. pub attribute_lineage: bool, - /// Minimum callable fraction for a window to be modelled at all. Windows below this are - /// **excluded**, not down-weighted: a mostly-uncallable window's variant density is dominated by - /// mapping error, and including it is what made the first real run call 3.62 % archaic out of - /// repetitive sequence. + /// The callable fraction that a window needs before the model uses it at all. The model + /// **removes** a window below this, and it does not give that window a lower weight. + /// + /// Mapping error controls the variant density of a window that is mostly not callable. To keep + /// such a window is what made the first real run call 3.62 % archaic, out of repetitive + /// sequence. pub min_callable_fraction: f64, } @@ -138,19 +153,20 @@ impl Default for ArchaicConfig { ArchaicConfig { window_bp: 1_000, rate_background: None, - // Calibrated against the hmmix 1000G callset (Zenodo, CC BY 4.0) on chr21+chr22 of the - // ground-truth European: 45 segments / 2.01 Mb against their EUR target of 43 / 2.09 Mb, - // both inside the p10-p90 spread (35-51 segments, 1.51-2.65 Mb). + // The calibration ran against the hmmix 1000G callset (Zenodo, CC BY 4.0), on chr21 + // and chr22 of the ground-truth European. It gave 45 segments over 2.01 Mb, against + // their EUR target of 43 segments over 2.09 Mb. Both lie inside the p10 to p90 spread, + // which is 35 to 51 segments and 1.51 to 2.65 Mb. archaic_rate_multiple: 6.0, prior_archaic: 0.02, - // 5, not 1. The transition rate IS the tract-length prior: at 1.0 with the 1 cM/Mb - // fallback a 1 kb window switches with probability ~0.001, implying ~1 Mb tracts against - // a real median of 31 kb. That single parameter was why the caller produced a third as - // many segments, each several times too long. + // 5, and not 1. The transition rate IS the prior on the tract length. At 1.0, with + // the fallback of 1 cM/Mb, a 1 kb window switches with a probability of about 0.001. + // That gives tracts of about 1 Mb, against a real median of 31 kb. This one parameter + // is why the caller gave a third as many segments, and each one was much too long. switches_per_cm: 5.0, - // 5 kb, not 50 kb. hmmix's median European tract on these chromosomes is 31 kb and its - // p10 is 7 kb, so a 50 kb floor discarded more than half of all real segments by - // construction. + // 5 kb, and not 50 kb. The median European tract of hmmix on these chromosomes is + // 31 kb, and its p10 is 7 kb. So a floor of 50 kb threw away more than half of all + // the real segments, by construction. min_segment_bp: 5_000, min_posterior: 0.70, min_lineage_ratio: 2.0, @@ -182,7 +198,8 @@ fn ln_sum_exp(a: f64, b: f64) -> f64 { m + ((a - m).exp() + (b - m).exp()).ln() } -/// cM between two bp positions, falling back to 1 cM/Mb when the map lacks the contig. +/// The cM between two bp positions. It falls back to 1 cM/Mb when the map does not hold the +/// contig. fn span_cm(gmap: &GeneticMap, chr: &str, start_bp: i64, end_bp: i64) -> f64 { gmap.interval_cm(chr, start_bp as i32, end_bp as i32) .unwrap_or_else(|| (end_bp - start_bp).max(0) as f64 / 1_000_000.0) @@ -190,8 +207,8 @@ fn span_cm(gmap: &GeneticMap, chr: &str, start_bp: i64, end_bp: i64) -> f64 { /// Call archaic tracts from a subject's genome-wide diploid calls. /// -/// `calls` should be the de-novo diploid variant calls for one alignment (Tier B is gated to -/// WGS/VCF input — a chip cannot supply the density this needs). +/// `calls` must hold the de-novo diploid variant calls of one alignment. Tier B accepts WGS and +/// VCF input alone, because a chip can not give the density that this needs. pub fn call_archaic_segments( calls: &[SiteGenotype], outgroup: &ArchaicOutgroup, @@ -200,9 +217,9 @@ pub fn call_archaic_segments( gmap: &GeneticMap, cfg: &ArchaicConfig, ) -> ArchaicSegmentResult { - // Per-contig: the subject's variant positions (carrying a non-reference allele), plus the - // callable extent. A dosage of 0 is a called reference site — real information about coverage, - // but not a variant. + // For each contig: the variant positions of the subject, which are the positions with a + // non-reference allele, and the callable extent. A dosage of 0 is a reference site with a + // call. That is real information about the coverage, but it is not a variant. let mut variants: BTreeMap> = BTreeMap::new(); let mut extent: BTreeMap = BTreeMap::new(); let mut alleles: BTreeMap<(String, i64), (char, char, i32)> = BTreeMap::new(); @@ -210,11 +227,13 @@ pub fn call_archaic_segments( if !(0..=g.ploidy as i32).contains(&g.dosage) { continue; } - // Key by the EXACT contig name, never normalized. The Tier B assets are built from the - // CHM13 VCFs and carry their naming; the subject's calls are on the same build and carry the - // same naming. Normalizing one side silently breaks the join, and the failure mode is - // invisible — retain_private finds no contig, returns nothing, and the genome reads as 0% - // archaic, which looks like a perfectly plausible answer. + // Key on the EXACT contig name, and never normalize it. The Tier B assets come from the + // CHM13 VCFs and use the names of those files. The calls of the subject are on the same + // build and use the same names. + // + // A normalization on one side breaks the join, and nobody sees it happen. retain_private + // then finds no contig, and it returns nothing. The genome reads as 0% archaic, and that + // answer looks completely reasonable. let contig = g.contig.clone(); let e = extent.entry(contig.clone()).or_insert((g.position, g.position)); e.0 = e.0.min(g.position); @@ -227,8 +246,8 @@ pub fn call_archaic_segments( } } - // Strip everything the African outgroup also carries — the step that turns raw variant density - // into an introgression signal. + // Remove everything that the African outgroup also carries. That step turns a raw variant + // density into a signal of introgression. let mut private: BTreeMap> = BTreeMap::new(); let mut total_private = 0usize; let mut total_windows = 0f64; @@ -236,8 +255,8 @@ pub fn call_archaic_segments( positions.sort_unstable(); positions.dedup(); let kept = outgroup.retain_private(&contig, &positions); - // Count only callable windows toward the background rate — otherwise the denominator is - // inflated by regions the model will not look at anyway. + // Count a callable window toward the background rate, and no other window. Else the + // regions that the model never looks at make the denominator too large. if let Some((lo, hi)) = extent.get(&contig) { let mut w = *lo; while w <= *hi { @@ -254,8 +273,9 @@ pub fn call_archaic_segments( private.insert(contig, kept); } - // Background rate: the sample's own genome-wide private density unless pinned. Adapts to depth - // and to how much the outgroup stripped, both of which vary per sample. + // The background rate. Unless the caller fixes it, it is the genome-wide private density of + // the sample itself. It follows the depth, and it follows how much the outgroup removed. Both + // of those change from one sample to the next. let background = cfg .rate_background .unwrap_or_else(|| (total_private as f64 / total_windows.max(1.0)).clamp(0.001, 10.0)); @@ -282,10 +302,10 @@ pub fn call_archaic_segments( )); } - // Both sides of the ratio must be measured in the SAME units. A segment's span includes windows - // the mask excluded, so counting span against callable megabases mixes them and inflates the - // percentage — the first run read 4.80% of "96.4 Mb callable" while the callable track was - // 44.6 Mb. Archaic extent is therefore summed over callable bases only. + // Both sides of the ratio must use the SAME units. The span of a segment holds windows that + // the mask removed. To count that span against the callable megabases mixes the two, and it + // makes the percentage too large. The first run read 4.80% of "96.4 Mb callable", while the + // callable track held 44.6 Mb. So the archaic extent adds up over the callable bases alone. let seg_callable_mb = |s: &ArchaicSegment| -> f64 { let mut bp = 0.0; let mut w = s.start; @@ -330,8 +350,8 @@ pub fn call_archaic_segments( ArchaicSegmentResult { segments, summary } } -/// The per-contig HMM: window the callable span, run Viterbi + forward/backward, stitch archaic -/// runs, then attribute each to a lineage. +/// The HMM over one contig. It puts the callable span into windows, runs Viterbi and +/// forward-backward, joins the archaic runs, and then attributes each run to a lineage. #[allow(clippy::too_many_arguments)] fn call_contig( contig: &str, @@ -358,8 +378,9 @@ fn call_contig( } } - // Callable fraction per window. A window below the floor is uninformative: it emits nothing in - // either state, so it neither supports nor breaks a segment. + // The callable fraction of each window. A window below the floor gives no information. It + // emits nothing in either state, so it does not support a segment, and it does not break + // one. let frac: Vec = (0..n_windows) .map(|i| callable.callable_fraction(contig, lo + i as i64 * cfg.window_bp)) .collect(); @@ -370,8 +391,8 @@ fn call_contig( let ln = |x: f64| x.max(1e-300).ln(); let ln_pi = [ln(1.0 - cfg.prior_archaic), ln(cfg.prior_archaic)]; - // Expected counts scale with how much of the window is actually callable, so a half-callable - // window is not mistaken for a variant-poor one. + // The expected counts scale with how much of the window is callable. A window that is half + // callable then does not read as a window with few variants. let emit = |i: usize, k: u32| -> [f64; 2] { if !usable[i] { return [0.0, 0.0]; @@ -410,7 +431,7 @@ fn call_contig( bwd[i][s] = ln_sum_exp(t[s][0] + e[0] + bwd[i + 1][0], t[s][1] + e[1] + bwd[i + 1][1]); } } - // Posterior of the archaic state per window. + // The posterior of the archaic state in each window. let post: Vec = (0..n_windows) .map(|i| { let a = fwd[i][0] + bwd[i][0]; @@ -443,8 +464,9 @@ fn call_contig( if mean_post < cfg.min_posterior { continue; } - // At least half the run's windows must be callable, so a tract cannot be carried by a - // stretch of uninformative windows riding on the transition prior. + // Half of the windows of a run must be callable, or more. A run of windows that give no + // information, and that stand on the transition prior alone, can then not hold up a + // tract. let callable_windows = usable[start_w..=end_w].iter().filter(|u| **u).count(); if callable_windows * 2 < end_w - start_w + 1 { continue; @@ -467,9 +489,9 @@ fn call_contig( /// Attribute a segment by counting the subject's derived-allele matches at diagnostic sites. /// -/// Requires a clear margin (`min_lineage_ratio`); a segment whose evidence is balanced, or which -/// has no diagnostic sites at all, is `Unknown`. Guessing here would manufacture exactly the -/// Denisovan-in-Europeans claim §7 forbids. +/// It needs a clear margin, which is `min_lineage_ratio`. A segment whose evidence is equal on the +/// two sides is `Unknown`, and so is a segment with no diagnostic site at all. To guess here would +/// make exactly the claim of Denisovan ancestry in Europeans that §7 forbids. fn attribute( contig: &str, start: i64, @@ -478,9 +500,10 @@ fn attribute( alleles: &BTreeMap<(String, i64), (char, char, i32)>, cfg: &ArchaicConfig, ) -> (ArchaicSource, usize, usize) { - // Count both MATCHES and the diagnostic sites available, per lineage: the enrichment is - // matches / (sites x base rate), and without the site counts the comparison silently favours - // whichever lineage happens to have more sites in this segment. + // Count two things for each lineage: the MATCHES, and the diagnostic sites that are there. + // The enrichment is matches / (sites x base rate). Without the site counts, the comparison + // points to whichever lineage happens to have more sites in this segment, and nobody sees + // that happen. let (mut nea, mut den) = (0usize, 0usize); let (mut nea_sites, mut den_sites) = (0usize, 0usize); for (pos, derived, class) in classify.in_range(contig, start, end) { @@ -493,7 +516,7 @@ fn attribute( continue; }; let d = derived.to_ascii_uppercase(); - // Does the subject actually carry the archaic-derived base here? + // Does the subject carry the archaic-derived base here? let carries = (a.to_ascii_uppercase() == d && dosage >= 1) || (r.to_ascii_uppercase() == d && dosage <= 1); if !carries { continue; @@ -519,8 +542,9 @@ fn attribute( } else if den >= cfg.min_lineage_matches && enr_den >= enr_nea * cfg.min_lineage_ratio { ArchaicSource::Denisovan } else { - // Not enough evidence to separate the lineages. For a European this is the expected - // outcome and the honest one — §7 forbids manufacturing a Denisovan finding. + // There is not enough evidence to separate the lineages. For a European this is the + // expected result, and the honest one. §7 does not let the code invent Denisovan + // ancestry. ArchaicSource::Unknown }; (source, nea, den) @@ -550,9 +574,9 @@ mod tests { } } - /// Background variants every 5 kb across 2 Mb, plus a dense block (every 200 bp) in - /// [1.0 Mb, 1.3 Mb] — an introgressed tract. None are in the outgroup. - /// Fully-callable track over the synthetic 2 Mb contig. + /// Background variants at every 5 kb across 2 Mb, plus a dense block at every 200 bp in + /// [1.0 Mb, 1.3 Mb], which is an introgressed tract. The outgroup holds none of them. The + /// callable track covers the whole synthetic 2 Mb contig. fn callable_all() -> ArchaicCallable { ArchaicCallable { build: "chm13v2.0".into(), @@ -612,9 +636,9 @@ mod tests { #[test] fn stripping_the_outgroup_is_what_makes_the_signal() { - // Same data, but now the outgroup carries every variant in the dense block: the density - // excess vanishes and nothing should be called. This is the step that separates - // introgression from ordinary variation. + // The same data, but now the outgroup carries every variant in the dense block. The + // excess of density then goes away, and the code must call nothing. This is the step that + // separates introgression from ordinary variation. let (calls, _) = sample(); let mut shared: Vec = (1_000_000..1_300_000).step_by(200).collect(); shared.sort_unstable(); @@ -640,8 +664,9 @@ mod tests { #[test] fn an_uncallable_dense_region_is_not_called() { - // The same dense block, but the mask says that region is not callable — the exact situation - // that made the first real run report 3.62% archaic out of repetitive sequence. + // The same dense block, but the mask says that the region is not callable. That is the + // exact case that made the first real run report 3.62% archaic, out of repetitive + // sequence. let (calls, og) = sample(); let mut track = callable_all(); for w in 950..1_350 { @@ -668,10 +693,10 @@ mod tests { #[test] fn attribution_compares_enrichment_not_raw_counts() { - // 40 Neanderthal-diagnostic sites and 40 Denisovan ones in the segment. The subject matches - // 12 of each. Raw counts say "tie"; against base rates of 4.3% vs 3.9% the expected counts - // are 1.7 and 1.6, so both are enriched almost identically -> genuinely Unknown, which is - // the honest answer rather than a coin flip. + // The segment holds 40 Neanderthal-diagnostic sites and 40 Denisovan ones. The subject + // matches 12 of each, so the raw counts give a tie. Against base rates of 4.3% and 3.9%, + // the expected counts are 1.7 and 1.6, so the enrichment of the two is almost the same. + // The result is a true Unknown, which is the honest answer, and not a coin toss. let mut positions: Vec = Vec::new(); let mut derived: Vec = Vec::new(); let mut classes: Vec = Vec::new(); diff --git a/crates/navigator-analysis/src/caller.rs b/crates/navigator-analysis/src/caller.rs index 52ced0c4..77127eec 100644 --- a/crates/navigator-analysis/src/caller.rs +++ b/crates/navigator-analysis/src/caller.rs @@ -1,28 +1,31 @@ -//! Purpose-built **haploid** variant caller (plan §4b) — the GATK replacement for -//! Y/mtDNA. There is no pure-code Scala caller to port: the legacy app shelled out to -//! GATK `HaplotypeCaller --sample-ploidy 1` (force-call at tree sites) and `Mutect2 -//! --mitochondria` / haploid `HaplotypeCaller` (de-novo discovery), then subtracted -//! known tree positions to get private variants. This module reproduces both modes by -//! **pileup-consensus calling**, which is tractable precisely because Y and mtDNA are -//! haploid (ploidy 1) — no diploid local reassembly. +//! A **haploid** variant caller built for this purpose (plan §4b). It replaces GATK for the Y +//! chromosome and the mtDNA. //! -//! Two modes: -//! 1. [`force_call_sites`] — genotype-given-alleles at known tree `Site`s (haplogroup -//! assignment): pileup, take the consensus base, report whether it is the site's -//! ref or alt allele. -//! 2. [`call_denovo`] — walk the contig, emit positions whose consensus base differs -//! from the reference (the candidate private variants). [`subtract_known`] removes -//! known tree positions to yield the private set. +//! There is no pure-code Scala caller to port. The legacy app called out to GATK +//! `HaplotypeCaller --sample-ploidy 1`, which force-calls at the tree sites, and to `Mutect2 +//! --mitochondria` or a haploid `HaplotypeCaller`, which discovers de-novo. It then subtracted +//! the known tree positions to get the private variants. This module does both of those modes by +//! a **consensus call over a pileup**. That method works here because the Y and the mtDNA are +//! haploid, at ploidy 1, so it needs no diploid local reassembly. //! -//! **v1 is SNP-only** (plan §4b): indels/homopolymers are where naive pileup calling -//! diverges from GATK (light local realignment is the planned mitigation), so indel -//! alleles are skipped here and treated as advisory until the §4c parity harness -//! validates them. Defaults are starting points the harness will tune. +//! There are two modes: +//! 1. [`force_call_sites`] gives a genotype at each known tree `Site`, from alleles that the +//! caller already has. This is for haplogroup assignment. It makes a pileup, takes the +//! consensus base, and reports whether that base is the ref allele or the alt allele of the +//! site. +//! 2. [`call_denovo`] walks the contig and emits each position whose consensus base is different +//! from the reference. Those are the candidate private variants. [`subtract_known`] then +//! removes the known tree positions to give the private set. //! -//! Memory: de-novo processes the contig in overlapping chunks (`denovo_chunk`), so the -//! dense per-position tally is bounded by the chunk, not the contig length. Both-side -//! context overlap keeps realignment windows that straddle a chunk boundary fully -//! visible. Force-call tallies only the target sites (sparse), cheap regardless of size. +//! **v1 handles a SNP alone** (plan §4b). An indel or a homopolymer is where a simple pileup call +//! moves away from GATK, and light local realignment is the planned answer. So this module skips +//! an indel allele, and it treats such an allele as advisory until the parity harness of §4c +//! checks it. The defaults are start points, and the harness will tune them. +//! +//! On memory: the de-novo path walks the contig in chunks that overlap (`denovo_chunk`). The +//! chunk, and not the length of the contig, thereby bounds the dense tally at each position. Context overlaps on both sides, so a realignment window that crosses a chunk boundary +//! stays fully visible. The force-call path tallies the target sites alone, which is sparse and +//! costs little at any size. use std::collections::{HashMap, HashSet}; use std::path::Path; @@ -41,30 +44,36 @@ use crate::reader; use crate::realign; use crate::reassembly; -/// Algorithm version for de-novo caller artifacts; bump on output-affecting changes -/// (e.g. the local-realignment addition bumped this to -2; the local-reassembly resolver to -3). +/// The algorithm version of a de-novo caller artifact. Raise it after a change that alters the +/// output. The local realignment raised it to -2, and the local-reassembly resolver to -3. pub const DENOVO_VERSION: &str = "haploid-denovo-3"; /// Algorithm version for site-genotype (panel) artifacts. pub const GENOTYPE_VERSION: &str = "genotype-1"; -/// Parameters for haploid calling. Defaults are v1 starting points (gated by §4c). +/// The parameters of a haploid call. The defaults are the v1 start points, and §4c gates them. #[derive(Debug, Clone, Copy, PartialEq)] pub struct HaploidCallerParams { - /// Minimum passing depth (reads clearing the quality filters) to make any call. + /// The smallest depth of reads that pass, which are the reads that clear the quality + /// filters, before the caller makes any call. pub min_depth: u32, - /// Reads below this MAPQ are dropped entirely. + /// The caller drops a read below this MAPQ. pub min_mapping_quality: u8, /// Bases below this quality are not counted. pub min_base_quality: u8, - /// The consensus base must be at least this fraction of passing depth to call. + /// The consensus base must hold at least this fraction of the depth that passes, before the + /// caller makes a call. pub min_allele_fraction: f64, - /// Allele-balance (paralog) filter for haploid sites. A true Y/mt-haploid site is - /// near-monoallelic; a substantial *second* allele signals paralog/mismapping (two loci - /// piled together) and the site is dropped. Tripped only when the second-most-common - /// allele has both at least `min_paralog_minor_reads` reads AND a fraction strictly above - /// `max_minor_allele_fraction` — a lone discordant read (sequencing error) does not trip - /// it. Set the fraction `>= 1.0` to disable. See PangenomeExpansion.md (Phase 1). + /// The allele-balance filter, which finds a paralog, at a haploid site. A true haploid site + /// on the Y or the mtDNA carries almost one allele alone. A large *second* allele shows a + /// paralog, or a read that the aligner put in the wrong place, which piles two loci together. + /// The caller drops such a site. + /// + /// The filter fires only when the second most common allele meets two conditions. It has + /// `min_paralog_minor_reads` reads or more, AND its fraction is above + /// `max_minor_allele_fraction`. One read that disagrees, which is a sequencing error, does + /// not fire it. Set the fraction to `1.0` or more to turn the filter off. See + /// PangenomeExpansion.md, Phase 1. pub max_minor_allele_fraction: f64, /// Minimum second-allele read count for the paralog filter to engage (guards low depth). pub min_paralog_minor_reads: u32, @@ -74,15 +83,17 @@ pub struct HaploidCallerParams { pub realign_min_indel_reads: u32, /// Padding (bp) added around indel-evidence runs to form a realignment window. pub realign_pad: i64, - /// De-novo emit chunk size (bp). The contig is processed in chunks so memory is - /// bounded; a chunk holds dense arrays for `chunk + 2*overlap` positions. + /// The chunk size, in bp, of the de-novo pass. The caller walks the contig in chunks to hold + /// the memory down. One chunk holds dense arrays for `chunk + 2*overlap` positions. pub denovo_chunk: usize, - /// Context overlap (bp) processed on each side of a chunk, so realignment windows - /// straddling a chunk boundary are still fully seen. Must exceed `realign_pad`. + /// The context overlap, in bp, that the caller walks on each side of a chunk. A realignment + /// window that crosses a chunk boundary then stays fully visible. It must be more than + /// `realign_pad`. pub denovo_overlap: usize, - /// Escalate positions the paralog gate would drop to the local-reassembly resolver - /// ([`crate::reassembly`]) instead of discarding them — recovers misaligned-ref haploid SNVs - /// (private-Y Option B). Off leaves the pileup-only behaviour unchanged. + /// Send a position that the paralog gate would drop to the local-reassembly resolver, + /// [`crate::reassembly`], and do not throw it away. This recovers a haploid SNV where the + /// reference alignment is wrong, which is Option B of the private-Y work. When this is off, + /// the caller keeps its pileup-only behaviour. pub reassembly: bool, /// Half-width (bp) of the window extracted around a reassembly candidate. Must be ≤ /// `denovo_overlap` so a boundary window stays inside the processed chunk. @@ -143,9 +154,9 @@ pub struct GenotypeCall { pub allele_fraction: f64, // alt_depth / depth } -/// A diploid/haploid genotype at a known site (genotype-likelihood model). `dosage` is -/// the alt-allele count (0..=ploidy), or -1 for a no-call — the encoding the -/// population/ancestry/IBD paths consume. +/// A diploid or haploid genotype at a known site, from the genotype-likelihood model. `dosage` is +/// the count of the alt allele, from 0 to the ploidy, or -1 for a no-call. That is the encoding +/// that the population, ancestry and IBD paths read. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SiteGenotype { pub name: String, @@ -160,12 +171,14 @@ pub struct SiteGenotype { pub ref_depth: u32, pub alt_depth: u32, pub pls: Vec, - /// Explicit VCF genotype string (e.g. `"1/2"`) for multiallelic sites. When `None`, the - /// genotype is derived from `dosage` (biallelic). Additive — old cached blobs decode to `None`. + /// The VCF genotype string, such as `"1/2"`, for a site with more than two alleles. When it + /// is `None`, the genotype comes from `dosage`, and the site has two alleles. This field is + /// additive, so an old cached blob decodes to `None`. #[serde(default)] pub gt: Option, - /// Per-allele read depths `[ref, alt1, alt2, …]` for multiallelic sites; `None` → biallelic - /// (use `ref_depth`/`alt_depth`). + /// The read depth of each allele, `[ref, alt1, alt2, …]`, at a site with more than two + /// alleles. `None` means that the site has two alleles, and you read `ref_depth` and + /// `alt_depth`. #[serde(default)] pub allele_depths: Option>, } @@ -180,9 +193,10 @@ pub struct VariantCall { pub depth: u32, // passing depth pub alt_depth: u32, // reads supporting the consensus alt pub allele_fraction: f64, - /// Phred-scaled confidence, set by the local-reassembly resolver ([`crate::reassembly`]) for - /// calls it recovered; `None` for the plain pileup/gVCF paths. Additive — old cached blobs - /// decode to `None`. + /// The confidence, on the Phred scale. The local-reassembly resolver + /// ([`crate::reassembly`]) sets it on a call that it recovered. It is `None` on the plain + /// pileup path and the gVCF path. This field is additive, so an old cached blob decodes to + /// `None`. #[serde(default)] pub quality: Option, } @@ -212,11 +226,12 @@ fn consensus(counts: &[u32; 4]) -> (usize, u32) { (bi, best) } -/// Allele-balance / paralog filter for a haploid pileup. A true haploid site is near- -/// monoallelic; when the second-most-common allele carries both enough reads -/// (`min_paralog_minor_reads`) and enough fraction (strictly above `max_minor_allele_fraction`) -/// the site looks bi-allelic — a paralog/mismapping artifact — and the caller should drop it. -/// A single discordant read (likely sequencing error) does not trip it. +/// The allele-balance filter, which finds a paralog, over a haploid pileup. A true haploid site +/// carries almost one allele alone. The site looks like it has two alleles when the second most +/// common allele has enough reads (`min_paralog_minor_reads`) and a large enough fraction (above +/// `max_minor_allele_fraction`). That is an artifact of a paralog, or of a read in the wrong +/// place. The caller must drop such a site. One read that disagrees, which is probably a +/// sequencing error, does not fire the filter. fn is_paralogous(counts: &[u32; 4], depth: u32, params: &HaploidCallerParams) -> bool { if depth == 0 { return false; @@ -253,8 +268,8 @@ pub(crate) fn contig_length(header: &noodles::sam::Header, contig: &str) -> Opti .map(|(_, map)| map.length().get()) } -/// Resolve a contig's length by opening the alignment header at `bam_path`. `reference` is -/// required for CRAM. +/// Find the length of a contig. It reads the alignment header at `bam_path`. A CRAM file needs +/// `reference`. pub(crate) fn read_contig_length( bam_path: &Path, contig: &str, @@ -264,8 +279,9 @@ pub(crate) fn read_contig_length( contig_length(&header, contig).ok_or_else(|| AnalysisError::Message(format!("contig {contig} not in BAM header"))) } -/// Load a contig's full reference sequence in one indexed-FASTA query (shared read-only across the -/// caller's chunks — each chunk slices its own window instead of re-querying). +/// Load the full reference sequence of a contig, in one query against an indexed FASTA. The +/// chunks of the caller share it read-only. Each chunk takes a slice for its own window, and it +/// does not run the query again. fn load_contig_sequence(reference_path: &Path, contig: &str, length: usize) -> Result, AnalysisError> { let mut fasta_reader = fasta::io::indexed_reader::Builder::default() .build_from_path(reference_path) @@ -281,8 +297,9 @@ fn load_contig_sequence(reference_path: &Path, contig: &str, length: usize) -> R .to_vec()) } -/// The contig (reference-sequence) names in the alignment header. `reference` is required -/// for CRAM. Used to skip lifted positions that land on contigs the alignment lacks. +/// The names of the contigs, which are the reference sequences, in the alignment header. A CRAM +/// file needs `reference`. The caller uses this to skip a lifted position that lands on a contig +/// that the alignment does not hold. pub fn header_contig_names(bam_path: &Path, reference: Option<&Path>) -> Result, AnalysisError> { let header = reader::read_header(bam_path, reference)?; Ok(header @@ -369,10 +386,12 @@ fn tally_targets( Ok((length, counts)) } -/// Call the consensus base at each 1-based `target` position on `contig` (haploid -/// genotyping for haplogroup assignment). A position is called only when it clears -/// `min_depth` passing reads and the consensus base is at least `min_allele_fraction` of -/// that depth; uncalled positions are simply absent. Returns position → uppercase base. +/// Call the consensus base at each 1-based `target` position on `contig`. This is the haploid +/// genotype for a haplogroup assignment. +/// +/// A position gets a call only when it clears `min_depth` reads that pass, and when the consensus +/// base holds at least `min_allele_fraction` of that depth. A position with no call is absent +/// from the result. Returns a map from a position to an uppercase base. pub fn call_bases_at( bam_path: &Path, contig: &str, @@ -400,10 +419,12 @@ pub fn call_bases_at( Ok(calls) } -/// Diagnostic: the raw passing A/C/G/T read tally at each `target` (1-based) — the evidence -/// **behind** [`call_bases_at`]'s consensus pick, before the depth / allele-fraction / paralog -/// filters. Returns 1-based position → `[A, C, G, T]` counts (absent = no passing read covered it). -/// For "what do the reads actually show at this tree SNP" logging. +/// A diagnostic. It gives the raw A/C/G/T tally of the reads that pass, at each 1-based +/// `target`. That is the evidence **behind** the consensus that [`call_bases_at`] takes, before +/// the depth filter, the allele-fraction filter and the paralog filter. +/// +/// Returns a map from a 1-based position to `[A, C, G, T]` counts. A position is absent when no +/// read that passes covered it. Use it to log what the reads show at a tree SNP. pub fn tally_at( bam_path: &Path, contig: &str, @@ -415,9 +436,11 @@ pub fn tally_at( Ok(counts.into_iter().map(|(pos0, c)| ((pos0 + 1) as i64, c)).collect()) } -/// The expected indel allele for a tree locus given its (VCF left-anchored) ancestral/derived -/// alleles: an insertion of the trailing bases (`A`→`ATT` ⇒ Ins("TT")) or a deletion of the length -/// difference (`TA`→`T` ⇒ Del(1)). `None` for a SNP or a complex/non-left-anchored allele. +/// The indel allele that a tree locus expects, from its ancestral and derived alleles in the +/// left-anchored VCF form. It is one of two things. It is an insertion of the bases at the end, +/// where `A`→`ATT` gives Ins("TT"). Or it is a deletion of the difference in length, where +/// `TA`→`T` gives Del(1). It is `None` for a SNP, and for an allele that is complex or not +/// left-anchored. fn expected_indel_allele(ancestral: &str, derived: &str) -> Option { let (a, d) = (ancestral.as_bytes(), derived.as_bytes()); if d.len() > a.len() && d.starts_with(a) { @@ -429,9 +452,10 @@ fn expected_indel_allele(ancestral: &str, derived: &str) -> Option } } -/// Walk one read's CIGAR from `start` (1-based), collecting each indel event as -/// `(anchor 1-based, allele)` — deletion anchor = first deleted ref base; insertion anchor = the ref -/// base the insertion precedes. Returns the events plus the read's inclusive reference end. +/// Walk the CIGAR of one read from `start`, which is 1-based, and collect each indel event as +/// `(anchor 1-based, allele)`. The anchor of a deletion is the first deleted ref base. The anchor +/// of an insertion is the ref base that comes after the insertion. Returns the events, and the +/// inclusive reference end of the read. fn read_indel_events(record: &RecordBuf, start: i64) -> (Vec<(i64, IndelAllele)>, i64) { let seq = record.sequence(); let mut ref_pos = start; @@ -463,21 +487,28 @@ fn read_indel_events(record: &RecordBuf, start: i64) -> (Vec<(i64, IndelAllele)> (events, ref_pos - 1) } -/// Targeted genotyping of tree **indel** loci. Each target is `(pos, ancestral, derived)` in VCF -/// left-anchored form (`pos` = the anchor base; e.g. `A`→`ATT` insertion, `TA`→`T` deletion). For -/// each locus, reads spanning it are examined: a read carrying the matching insertion/deletion -/// (after left-normalization into the reference repeat) supports the derived allele. +/// Genotype the **indel** loci of a tree, at given targets. Each target is +/// `(pos, ancestral, derived)`, in the left-anchored VCF form, where `pos` is the anchor base. +/// `A`→`ATT` is an insertion, and `TA`→`T` is a deletion. At each locus the code examines the +/// reads that cover it. A read that carries the matching insertion or deletion, after +/// left-normalization into the reference repeat, supports the derived allele. +/// +/// **This function only adds.** A locus with a clear derived majority over `min_depth` comes out +/// as [`haplo::INDEL_DERIVED`] at `pos`. Everything else stays a **no-call**, and it never becomes +/// an ancestral contradiction. That covers a locus with no indel support, a locus with low depth, +/// and reads that only *cover* the site cleanly. +/// +/// The reason is noise. Take an indel genotype around a homopolymer or an STR. A read that +/// covers the site cleanly is often the alternate form that the aligner chose for the same indel. +/// To call those ancestral would contradict a thin node for no reason. A node at +/// d == 0 that takes one false ancestral fires the confident-divergence guard, and that vetoes +/// the whole lineage. +/// +/// So an indel only ever *confirms* a branch. That matches the intent: cover the many DecodingUs +/// branches that an indel defines, when the sample carries them. /// -/// **Additive-only**: a locus with a clear derived majority over `min_depth` is emitted as -/// [`haplo::INDEL_DERIVED`] at `pos`; everything else — no indel support, low depth, or reads that -/// merely *span* the site cleanly — is left as **no-call**, never an ancestral contradiction. Indel -/// genotyping around homopolymers/STRs is noisy enough that a "clean-spanning" read is often just the -/// aligner's alternate representation of the same indel; calling those ancestral would spuriously -/// contradict sparse nodes (a d==0 node picking up one false ancestral trips the confident-divergence -/// guard and vetoes the whole lineage). So indels only ever *confirm* a branch, matching the intent: -/// cover the many indel-defined DecodingUs branches when the sample carries them. Requires a -/// `reference` (to left-normalize + know deleted bases); returns empty without one, or when the -/// contig isn't in the FASTA. +/// This needs a `reference`, to left-normalize and to know the deleted bases. It returns an empty +/// result without one, and also when the FASTA does not hold the contig. pub fn call_indels_at( bam_path: &Path, contig: &str, @@ -522,8 +553,10 @@ pub fn call_indels_at( ptargets.sort_by_key(|t| t.pos); let positions: Vec = ptargets.iter().map(|t| t.pos).collect(); - // Single contig-wide pass (as the SNP tally does): walk every read once, and for each target the - // read spans, accumulate matched (carries the indel) vs ref-spanning (spans it cleanly) support. + // One pass over the whole contig, as the SNP tally does. Walk every read once. At each + // target that the read covers, add up two kinds of support. One is a match, where the read + // carries the indel. The other is a clean cover, where the read covers the target and shows + // the reference there. let (header, mut reader) = reader::open_indexed(bam_path, Some(reference))?; let region: Region = contig .parse() @@ -573,8 +606,9 @@ pub fn call_indels_at( Ok(out) } -/// Dense A/C/G/T tally + per-position indel evidence for the 1-based inclusive region -/// `[lo, hi]`, indexed by `pos - lo` (the chunked de-novo path). +/// A dense A/C/G/T tally, and the indel evidence at each position, over the 1-based inclusive +/// region `[lo, hi]`. The index is `pos - lo`. The de-novo path, which works in chunks, uses +/// this. pub(crate) fn tally_region( bam_path: &Path, contig: &str, @@ -647,11 +681,16 @@ pub(crate) fn tally_region( Ok((counts, indel)) } -/// Record one read's passing `(base, qual)` at each of `targets` (sorted, 1-based) that it covers, -/// in a **single** CIGAR walk from the alignment start. Targets in a deletion/skip/insertion/clip, -/// past the read, below `min_base_quality`, or non-ACGT are skipped. This is the multi-target -/// generalization of a per-site probe — one walk feeds many sites, so a long read shared by several -/// nearby panel sites is decoded + walked once instead of once per site. +/// Record the `(base, qual)` of one read, where it passes, at each of the `targets` that the read +/// covers. `targets` comes in sorted order, and its positions are 1-based. The function does this +/// in a **single** CIGAR walk from the start of the alignment. +/// +/// The walk skips a target in a deletion, a ref skip, an insertion or a clip. It also skips a +/// target that is past the read, one below `min_base_quality`, and one whose base is not ACGT. +/// +/// This is the many-target form of a probe at one site. One walk feeds many sites. Some nearby +/// panel sites share a long read, and the code decodes and walks that read once, not once for +/// each site. fn collect_bases(record: &RecordBuf, targets: &[i64], min_base_quality: u8, obs: &mut HashMap>) { let Some(start) = record.alignment_start() else { return }; let start = start.get() as i64; @@ -687,7 +726,7 @@ fn collect_bases(record: &RecordBuf, targets: &[i64], min_base_quality: u8, obs: ref_pos = end; query_off += len as usize; } else if cr { - // Deletion / ref-skip — targets inside the gap carry no base. + // A deletion or a ref skip. A target inside the gap carries no base. let end = ref_pos + len; while ti < targets.len() && targets[ti] < end { ti += 1; @@ -699,15 +738,20 @@ fn collect_bases(record: &RecordBuf, targets: &[i64], min_base_quality: u8, obs: } } -/// Per-target-site passing `(base, qual)` observations (ACGT bases clearing the quality filters), -/// keyed by 1-based position — the input the genotype-likelihood model needs. +/// The `(base, qual)` observations that pass at each target site, keyed by a 1-based position. +/// Those are the ACGT bases that clear the quality filters. This is the input that the +/// genotype-likelihood model needs. +/// +/// The code puts the targets into runs that touch each other. It splits a run only where the gap +/// between two adjacent sites is more than one read length. It then gets each run with a +/// **single** streaming query against the index. /// -/// The targets are grouped into contiguous runs (split only where the gap between adjacent sites -/// exceeds a read length), and each run is fetched with a **single** streaming index query. So we -/// seek straight to the regions that hold targets — never scanning the whole contig — and decode -/// each read once (a point query per site re-fetches + re-converts the long HiFi reads that span -/// several nearby sites). Within a run, [`collect_bases`] distributes each read's bases to every -/// target it covers in one CIGAR walk. +/// The code then seeks straight to the regions that hold targets, and it never scans the whole +/// contig. It also decodes each read once. A point query at each site would fetch and convert +/// the long HiFi reads again, and one such read covers some nearby sites. +/// +/// Inside a run, [`collect_bases`] gives the bases of each read to every target that the read +/// covers, in one CIGAR walk. fn tally_site_observations( bam_path: &Path, contig: &str, @@ -721,8 +765,9 @@ fn tally_site_observations( return Ok(HashMap::new()); } - // Split into runs where consecutive sites are within MAX_GAP — beyond a read length no read can - // span the gap, so splitting there is free (no shared reads lost) and skips read-free spans. + // Split into runs whose consecutive sites lie within MAX_GAP. Past one read length, no read + // can cover the gap. A split there costs nothing, because no shared read goes away, and it + // skips the spans that hold no read. const MAX_GAP: i64 = 50_000; let (header, mut reader) = reader::open_indexed(bam_path, reference)?; @@ -751,9 +796,10 @@ fn tally_site_observations( Ok(obs) } -/// Genotype known SNP sites on `contig` at the given `ploidy` (1 = haploid Y/MT/male-X, -/// 2 = autosome / female-X) using the genotype-likelihood model — the panel-genotyping -/// path the population / ancestry / IBD analyses consume. Non-SNP sites are skipped. +/// Genotype the known SNP sites on `contig`, at the given `ploidy`, with the +/// genotype-likelihood model. A ploidy of 1 is a haploid Y, MT or male X. A ploidy of 2 is an +/// autosome or a female X. This is the panel-genotype path that the population, ancestry and IBD +/// analyses read. The code skips a site that is not a SNP. pub fn genotype_sites( bam_path: &Path, contig: &str, @@ -813,10 +859,12 @@ pub fn genotype_sites( Ok(out) } -/// Genotype `sites` across **every** contig they span, one contig per rayon task. The panel-genotyping -/// entry point for whole-genome panels (the per-contig [`genotype_sites`] is independent + IO-bound on -/// its own index region, so contigs parallelize cleanly). Results are concatenated (order across -/// contigs is unspecified — downstream consumers key by site, not order). +/// Genotype `sites` across **every** contig that they cover, with one rayon task for each contig. +/// This is the entry point for a whole-genome panel. [`genotype_sites`] works on one contig, and +/// it is independent and IO-bound on its own index region, so the contigs parallelize cleanly. +/// +/// The code joins the results together. The order across the contigs is not defined, because a +/// later step keys on the site and not on the order. pub fn genotype_sites_all_contigs( bam_path: &Path, sites: &[Site], @@ -831,9 +879,9 @@ pub fn genotype_sites_all_contigs( .collect::>() .into_iter() .collect(); - // Run on a decode-safe pool rather than rayon's global pool (2 MiB stacks): each task decodes - // CRAM records, which recurse deeply on CRAM 3.1 and would otherwise overflow + abort. See - // [`reader::decode_pool`]. + // Run on a pool that is safe for a decode, and not on the global rayon pool, whose stacks are + // 2 MiB. Each task decodes CRAM records. Those recurse deeply on CRAM 3.1, and the stack would + // overflow and abort. See [`reader::decode_pool`]. let pool = crate::reader::decode_pool(contigs.len().max(1).min(crate::unified::analysis_thread_count()))?; let per_contig: Result>, AnalysisError> = pool.install(|| { contigs @@ -847,15 +895,24 @@ pub fn genotype_sites_all_contigs( Ok(per_contig?.into_iter().flatten().collect()) } -/// Reconcile per-alignment force-call genotypes at a shared site set into one **consensus** diploid -/// genotype per site — the subject-level joint genotype across a person's WGS runs. Each input is -/// one alignment's [`SiteGenotype`]s at the *union* of variant sites (all on the same reference -/// build, so `(contig, position, ref, alt)` align). Per site, a depth-weighted vote over the dosage -/// classes {0,1,2}: an alignment whose depth is below `min_depth` is its no-call (excluded), so a -/// site absent-as-hom-ref in one run is a real vote (resolving "run A het vs run B hom-ref") while a -/// genuinely uncovered run abstains. Only **variant** consensus sites (het/hom-alt) are returned; -/// hom-ref / no-call consensus is not a variant. Depth/AD are summed and GQ is the max over the -/// supporting alignments; PLs are dropped (the per-run likelihoods don't compose into one PL here). +/// Reconcile the force-call genotypes of each alignment, over a shared set of sites, into one +/// **consensus** diploid genotype at each site. That is the joint genotype of the subject, across +/// the WGS runs of that person. +/// +/// Each input holds the [`SiteGenotype`] values of one alignment, at the *union* of the variant +/// sites. All of them are on the same reference build, so `(contig, position, ref, alt)` line +/// up. +/// +/// At each site the code holds a vote over the dosage classes {0,1,2}, weighted by depth. An +/// alignment whose depth is below `min_depth` casts its no-call, and the vote leaves it out. A +/// site that one run shows as hom-ref, and that is absent from the list, is then a real vote. +/// That resolves the case where run A is het and run B is hom-ref. A run that truly did not cover +/// the site abstains. +/// +/// The result holds the **variant** consensus sites alone, which are het and hom-alt. A hom-ref +/// or no-call consensus is not a variant. The code sums the depth and the AD, and it takes the +/// maximum GQ over the alignments that support the call. It drops the PLs, because the +/// likelihoods of the separate runs do not combine into one PL here. pub fn reconcile_site_genotypes(per_alignment: &[Vec], min_depth: u32) -> Vec { use std::collections::BTreeMap; struct Acc { @@ -885,8 +942,8 @@ pub fn reconcile_site_genotypes(per_alignment: &[Vec], min_depth: } let d = g.dosage; if (0..=2).contains(&d) { - // Depth-bonus weight, mirroring consensus::obs_weight's WGS term (constant method - // factor drops out of the argmax). + // The weight of the depth bonus. It has the same shape as the WGS term of + // consensus::obs_weight. The constant method factor cancels in the argmax. let weight = 1.0 + ((g.depth as f64).sqrt() / 10.0).min(1.0); w_add(&mut acc.w, &mut acc.counts, d as usize, weight); acc.depth += g.depth as u64; @@ -898,7 +955,8 @@ pub fn reconcile_site_genotypes(per_alignment: &[Vec], min_depth: } let mut out = Vec::new(); for (_, acc) in groups { - // argmax weight; tie → more raw supporting runs, then the lower dosage. + // Take the argmax of the weight. A tie goes to the higher count of raw runs that support + // the call, and then to the lower dosage. let mut best = 0usize; for d in 1..3 { if acc.w[d] > acc.w[best] || (acc.w[d] == acc.w[best] && acc.counts[d] > acc.counts[best]) { @@ -930,8 +988,9 @@ fn w_add(w: &mut [f64; 3], counts: &mut [usize; 3], d: usize, weight: f64) { counts[d] += 1; } -/// Force-call (genotype-given-alleles) at known SNP sites on `contig`. Non-SNP sites -/// (multi-base ref/alt) are skipped — v1 is SNP-only. +/// Force-call at the known SNP sites on `contig`. The caller already has the alleles. The code +/// skips a site that is not a SNP, because v1 handles a SNP alone. Such a site holds more than +/// one base in its ref allele or in its alt allele. pub fn force_call_sites( bam_path: &Path, contig: &str, @@ -1000,10 +1059,11 @@ pub fn force_call_sites( Ok(out) } -/// De-novo SNP discovery across `contig`, processed in overlapping chunks so memory is -/// bounded by the chunk (not the contig length). Emits positions whose consensus base -/// passes the depth/fraction filters and differs from the reference. Both-side context -/// overlap keeps realignment windows that straddle a chunk boundary fully visible. +/// De-novo SNP discovery across `contig`. The code walks it in chunks that overlap, so the chunk +/// bounds the memory, and not the length of the contig. It emits each position whose consensus +/// base passes the depth filter and the fraction filter, and differs from the reference. The +/// context overlaps on both sides, so a realignment window that crosses a chunk boundary stays +/// fully visible. pub fn call_denovo( bam_path: &Path, reference_path: &Path, @@ -1013,14 +1073,17 @@ pub fn call_denovo( ) -> Result, AnalysisError> { let length = read_contig_length(bam_path, contig, Some(reference_path))?; - // Load the contig's reference once, shared read-only across chunks — each chunk slices its own - // window instead of re-querying the FASTA. + // Load the reference of the contig once. The chunks share it read-only, and each chunk takes + // a slice for its own window. No chunk queries the FASTA again. let ref_seq = load_contig_sequence(reference_path, contig, length)?; - // Disjoint emit ranges, in order, each processed independently with its own indexed BAM - // region query. Chunks stay large (`denovo_chunk`, default 8 MB): a CRAM container spans - // several MB, so chunks smaller than a container would re-decode it in every overlapping - // chunk. rayon caps in-flight chunks at the pool size, so peak memory is bounded. + // The emit ranges do not overlap, and they come in order. The code walks each one on its + // own, with its own region query against the BAM index. + // + // The chunks stay large: `denovo_chunk` defaults to 8 MB. A CRAM container covers some MB. + // Take a chunk smaller than a container. Every chunk over that container decodes it again. + // The rayon pool limits how many chunks run at one time to its own size, so the peak memory + // has a bound. let threads = crate::unified::analysis_thread_count(); let chunk = params.denovo_chunk.max(1); let mut ranges: Vec<(usize, usize)> = Vec::new(); @@ -1038,21 +1101,26 @@ pub fn call_denovo( ranges .par_iter() .map(|&(lo, hi)| { - // Per chunk: each is bounded work (default 8 MB of reference), so this bounds the - // delay between a click and a stop without splitting the chunks any finer. + // One check at each chunk. The work in a chunk has a bound, at a default of 8 MB + // of reference. This bounds the delay between a click and a stop, and the chunks + // do not have to get smaller. cancel.check()?; denovo_chunk(bam_path, reference_path, contig, params, &ref_seq, length, lo, hi) }) .collect::, AnalysisError>>() })?; - // Ranges are disjoint and collected in order, so flattening preserves global position order. + // The ranges do not overlap, and the code collects them in order. A flat join then keeps the + // global position order. Ok(nested.into_iter().flatten().collect()) } -/// De-novo SNP calls restricted to `[region_lo, region_hi]` (1-based inclusive) on `contig` — the -/// same tally/realign/reassembly path as [`call_denovo`] over a single bounded emit range. Loads the -/// whole contig reference once (cheap) but only queries the region. Intended for debug/validation -/// tooling (e.g. checking recovery at specific positions) without walking the whole contig. +/// The de-novo SNP calls inside `[region_lo, region_hi]` alone, which is 1-based and inclusive, +/// on `contig`. It is the same tally, realign and reassembly path as [`call_denovo`], over one +/// bounded emit range. It loads the reference of the whole contig once, which costs little, and +/// it queries the region alone. +/// +/// Use it in a debug tool or a check tool, for example to look at the recovery at given +/// positions. It does not walk the whole contig. pub fn call_denovo_region( bam_path: &Path, reference_path: &Path, @@ -1071,10 +1139,11 @@ pub fn call_denovo_region( denovo_chunk(bam_path, reference_path, contig, params, &ref_seq, length, lo, hi) } -/// De-novo SNP calls for one emit range `[emit_lo, emit_hi]` (1-based inclusive). Tallies a -/// `denovo_overlap`-padded window so realignment windows straddling the boundary are fully -/// seen, but emits only `[emit_lo, emit_hi]`. `ref_seq` is the full contig reference (index 0 = -/// position 1). Each call opens its own BAM reader, so it is independent and thread-safe. +/// The de-novo SNP calls for one emit range `[emit_lo, emit_hi]`, which is 1-based and inclusive. +/// It tallies a window with `denovo_overlap` of padding, so that a realignment window across the +/// boundary stays fully visible. It emits `[emit_lo, emit_hi]` alone. `ref_seq` is the reference +/// of the whole contig, where index 0 is position 1. Each call opens its own BAM reader, so it is +/// independent and safe across threads. #[allow(clippy::too_many_arguments)] fn denovo_chunk( bam_path: &Path, @@ -1089,8 +1158,8 @@ fn denovo_chunk( let overlap = params.denovo_overlap; let proc_lo = emit_lo.saturating_sub(overlap).max(1); let proc_hi = (emit_hi + overlap).min(length); - // Reference window [proc_lo, proc_hi], indexed relative to proc_lo (clamped to what the - // FASTA actually returned, so a short contig tail reads as 'N' like before). + // The reference window [proc_lo, proc_hi], with its index relative to proc_lo. The code + // clamps it to what the FASTA returned, so a short contig tail reads as 'N', as before. let ref_chunk = &ref_seq[(proc_lo - 1).min(ref_seq.len())..proc_hi.min(ref_seq.len())]; let (mut counts, indel) = tally_region(bam_path, contig, params, proc_lo, proc_hi, Some(reference_path))?; @@ -1108,8 +1177,9 @@ fn denovo_chunk( } let mut out = Vec::new(); - // Stage A — positions the paralog gate would drop but that carry a real non-reference allele - // are escalated to the local-reassembly resolver instead of discarded (private-Y Option B). + // Stage A. Take the positions that the paralog gate would drop, but that carry a real + // non-reference allele. Send those to the local-reassembly resolver, and do not throw them + // away. That is Option B of the private-Y work. let mut active: Vec = Vec::new(); for pos in emit_lo..=emit_hi { let r = pos - proc_lo; // index into the chunk arrays @@ -1124,8 +1194,9 @@ fn denovo_chunk( } let ref_base = ref_chunk.get(r).copied().unwrap_or(b'N'); if is_paralogous(&c, depth, params) { - // bi-allelic at a haploid site — the pileup can't tell a true derived SNV from a - // paralog artifact. Hand it to reassembly (Stage B–E below) rather than dropping it. + // The site has two alleles, and it is haploid. The pileup can not separate a true + // derived SNV from a paralog artifact. Give it to reassembly, in stages B to E + // below, and do not drop it. if params.reassembly { if let Some(cand) = active_candidate(pos as i64, &c, ref_base, params) { active.push(cand); @@ -1152,7 +1223,8 @@ fn denovo_chunk( }); } - // Stages B–F — reassemble the escalated windows and append the recovered DERIVED calls. + // Stages B to F. Reassemble the windows that stage A sent on, and add the DERIVED calls that + // they recover. if params.reassembly && !active.is_empty() { let recovered = resolve_active(bam_path, contig, ref_seq, length, &active, params, Some(reference_path))?; out.extend(recovered); @@ -1161,8 +1233,9 @@ fn denovo_chunk( Ok(out) } -/// A reassembly candidate for a paralog-gated position: the top **non-reference** base, kept only if -/// it carries at least `min_paralog_minor_reads` reads (a real alternate, not a lone error). +/// A reassembly candidate at a position that the paralog gate held. It is the most common +/// **non-reference** base. The code keeps it only when it carries `min_paralog_minor_reads` reads +/// or more, which makes it a real alternate and not one error. fn active_candidate( pos: i64, counts: &[u32; 4], @@ -1185,9 +1258,10 @@ fn active_candidate( }) } -/// Stages B–F for a chunk's escalated candidates: group them into windows, extract the spanning -/// reads (projected onto each window's reference frame), genotype with the reassembly resolver, and -/// return the DERIVED recoveries as `VariantCall`s (paralog artifacts stay dropped). +/// Stages B to F, for the candidates that a chunk sent on. It puts them into windows. It takes +/// the reads that cover each window, and projects them onto the reference frame of that window. +/// It genotypes them with the reassembly resolver. It returns the DERIVED recoveries as +/// `VariantCall` values, and a paralog artifact stays dropped. fn resolve_active( bam_path: &Path, contig: &str, @@ -1212,8 +1286,8 @@ fn resolve_active( }; let w = params.reassembly_window.max(1); - // Merge candidates whose windows overlap into one extraction window (candidates arrive in - // ascending position order from the emit loop). + // Merge the candidates whose windows overlap into one extraction window. The candidates come + // from the emit loop in position order, from the lowest up. let mut out = Vec::new(); let mut i = 0; while i < active.len() { @@ -1249,10 +1323,13 @@ fn resolve_active( Ok(out) } -/// Project every spanning read in `[win_lo, win_hi]` onto the window's reference frame: the -/// window-frame sequence + per-base qualities (for the PairHMM) and the base/quality each read -/// carries at each candidate (for depth + dedup). One CIGAR walk per read; insertions inside the -/// window are kept so an indel haplotype survives (matching the pileup's realignment intent). +/// Project every read that covers `[win_lo, win_hi]` onto the reference frame of that window. The +/// result holds the sequence in the window frame, with the quality of each base, which the +/// PairHMM needs. It also holds the base and the quality that each read carries at each +/// candidate, which the depth and the dedup need. +/// +/// There is one CIGAR walk for each read. An insertion inside the window stays, so an indel +/// haplotype survives. That matches what the realignment in the pileup intends. fn extract_window_reads( header: &noodles::sam::Header, reader: &mut reader::IdxReader, @@ -1339,22 +1416,28 @@ fn extract_window_reads( Ok(reads) } -/// Nominal base quality for the de-novo **diploid** genotype likelihood. The chunked pileup keeps -/// only A/C/G/T counts (per-base quals would blow up WGS memory), and every counted base already -/// cleared `min_base_quality`, so the GL is evaluated at this representative phred. The resulting -/// genotype (0/1 vs 1/1 vs 0/0) is robust to the exact value; PL/GQ are approximate (the per-site -/// [`genotype_sites`] path keeps true per-read quals when exact likelihoods matter). +/// The nominal base quality of the de-novo **diploid** genotype likelihood. +/// +/// The pileup that works in chunks keeps the A/C/G/T counts alone. A quality for each base would +/// use too much memory on a WGS run. Every base that it counted already cleared +/// `min_base_quality`. So the code evaluates the GL at this one representative phred value. +/// +/// The genotype that comes out, 0/1 or 1/1 or 0/0, is robust to the exact value. The PL and the +/// GQ are approximate. The [`genotype_sites`] path, which works at one site, keeps the true +/// quality of each read where the exact likelihood matters. const DENOVO_DIPLOID_Q: u8 = 30; -/// Minimum reads supporting the alt allele before a site is even considered a candidate variant — -/// suppresses singleton sequencing-error "hets". +/// The count of reads that must support the alt allele before a site becomes a candidate variant +/// at all. It holds back a "het" that one sequencing error produced. const DENOVO_MIN_ALT_READS: u32 = 2; -/// Whole-contig **de-novo diploid** SNV calling: the same chunked, parallel pileup as -/// [`call_denovo`], but each variant site is genotyped at ploidy 2 via the genotype-likelihood model -/// ([`genotype::call_genotype`]) — emitting heterozygous (0/1) and homozygous-alt (1/1) calls, not -/// just a haploid consensus. Biallelic (REF + the top non-REF base) for v1; indels are not called -/// here. Output is in ascending position order, as [`SiteGenotype`] (ploidy 2) — feed it to -/// [`crate::vcf::write_diploid_vcf`]. +/// **De-novo diploid** SNV calling over a whole contig. It uses the same parallel pileup in +/// chunks as [`call_denovo`]. But it genotypes each variant site at ploidy 2, with the +/// genotype-likelihood model ([`genotype::call_genotype`]). So it emits heterozygous (0/1) and +/// homozygous-alt (1/1) calls, and not a haploid consensus alone. +/// +/// v1 handles two alleles: REF, and the most common non-REF base. It does not call an indel here. +/// The output comes in position order, from the lowest up, as [`SiteGenotype`] at ploidy 2. Give +/// it to [`crate::vcf::write_diploid_vcf`]. pub fn call_denovo_diploid( bam_path: &Path, reference_path: &Path, @@ -1375,7 +1458,8 @@ pub fn call_denovo_diploid( emit_lo = emit_hi + 1; } - // Decode-safe worker stack (CRAM 3.1 decode recursion — see [`reader::decode_pool`]). + // A worker stack that is safe for a decode. A CRAM 3.1 decode recurses deeply. See + // [`reader::decode_pool`]. let pool = crate::reader::decode_pool(threads)?; let nested: Vec> = pool.install(|| { ranges @@ -1432,7 +1516,8 @@ fn denovo_chunk_diploid( let Some(ref_bi) = base_index(ref_base) else { continue }; // reference N/ambiguous let ref_byte = BASES[ref_bi]; let ref_count = c[ref_bi]; - // All non-reference bases clearing the support floor are candidate alts (dominant first). + // Every non-reference base that clears the support floor is a candidate alt. The most + // common one comes first. let mut alts: Vec<(usize, u32)> = c .iter() .enumerate() @@ -1533,12 +1618,15 @@ enum IndelAllele { Del(u32), } -/// Left-align an indel within the reference repeat structure (VCF normalization): an aligner may -/// place an indel anywhere within a homopolymer/STR run, but the canonical representation is the -/// leftmost. Returns the normalized `anchor` (1-based) and allele. A deletion of `len` bases at -/// `[anchor, anchor+len-1]` shifts left while `ref[anchor-1] == ref[anchor+len-1]`; an insertion -/// before `anchor` shifts left while `ref[anchor-1]` equals its last base (rotating the bases). -/// Bounded by `proc_lo` (the loaded reference window start, 1-based). +/// Left-align an indel inside the repeat structure of the reference. This is the VCF +/// normalization. An aligner may put an indel anywhere inside a homopolymer or an STR run, but +/// the canonical form is the leftmost one. Returns the normalized `anchor`, which is 1-based, and +/// the allele. +/// +/// A deletion of `len` bases at `[anchor, anchor+len-1]` moves left while +/// `ref[anchor-1] == ref[anchor+len-1]`. An insertion before `anchor` moves left while +/// `ref[anchor-1]` equals its last base, and the bases turn around the allele. `proc_lo` bounds +/// the move, and it is the 1-based start of the reference window that the code loaded. fn left_normalize(anchor: i64, allele: &IndelAllele, ref_chunk: &[u8], proc_lo: usize) -> (i64, IndelAllele) { let at = |p: i64| -> Option { let i = p - proc_lo as i64; @@ -1566,10 +1654,12 @@ fn left_normalize(anchor: i64, allele: &IndelAllele, ref_chunk: &[u8], proc_lo: } } -/// Build a diploid indel [`SiteGenotype`] (VCF-style, left-anchored at `emit_pos`) from ref-vs-indel -/// read support, genotyped at ploidy 2 via the sentinel-byte GL (`b'R'` ref-spanning, `b'A'` -/// indel-carrying). `ref_byte` is the reference base at `emit_pos`; `deleted` is the deleted -/// reference bases (empty for an insertion). `None` for a hom-ref / no-call. +/// Build a diploid indel [`SiteGenotype`] in the VCF style, left-anchored at `emit_pos`, from the +/// read support for the reference against the indel. It genotypes at ploidy 2, with the GL over +/// sentinel bytes: `b'R'` for a read that covers the reference, and `b'A'` for a read that carries +/// the indel. `ref_byte` is the reference base at `emit_pos`. `deleted` holds the deleted +/// reference bases, and it is empty for an insertion. Returns `None` for a hom-ref call and for a +/// no-call. #[allow(clippy::too_many_arguments)] fn indel_site_genotype( contig: &str, @@ -1623,10 +1713,15 @@ fn indel_site_genotype( }) } -/// De-novo diploid **indel** calls for this chunk: over each active (indel-evidence) window, extract -/// per-read indel alleles (CIGAR I/D) + ref-spanning support, tally the dominant allele per locus -/// (biallelic v1), and genotype it at ploidy 2. Emits only loci whose VCF position is in the emit -/// range (dedup across chunk boundaries). Left-anchored at the standard VCF convention. +/// The de-novo diploid **indel** calls for this chunk. An active window is a window with indel +/// evidence. Over each of those, the code takes the indel allele of each read, from the CIGAR I +/// and D operations. It also takes the support of the reads that cover the reference. It tallies +/// the most common allele at each locus, and v1 keeps two alleles. It then genotypes that locus +/// at ploidy 2. +/// +/// It emits a locus only when the VCF position of that locus lies in the emit range, which +/// removes a duplicate across a chunk boundary. The anchor is on the left, by the standard VCF +/// convention. #[allow(clippy::too_many_arguments)] fn indels_in_chunk( bam_path: &Path, @@ -1793,7 +1888,8 @@ fn indels_in_chunk( continue; } - // Multiallelic: one common REF spanning the largest deletion, one ALT per allele. + // The site has more than two alleles. There is one common REF, which covers the + // largest deletion, and one ALT for each allele. cands.sort_by(|a, b| b.count.cmp(&a.count).then(a.anchor.cmp(&b.anchor))); // dominant first, deterministic let maxdel = cands .iter() @@ -1908,8 +2004,9 @@ fn active_windows(indel_evidence: &[u32], min_reads: u32, pad: i64) -> Vec<(usiz windows } -/// Re-fit reads in each indel-active window onto the reference and replace the tally -/// over those windows. Arrays are indexed relative to `region_lo` (1-based). +/// Fit the reads in each window with indel evidence onto the reference again, and replace the +/// tally over those windows. The index into the arrays is relative to `region_lo`, which is +/// 1-based. #[allow(clippy::too_many_arguments)] fn realign_region( bam_path: &Path, @@ -1929,11 +2026,13 @@ fn realign_region( } let mut win_counts: Vec> = windows.iter().map(|&(w0, w1)| vec![[0u32; 4]; w1 - w0 + 1]).collect(); - // ONE indexed query spanning all active windows: decode the region's reads once and route - // each read to the window(s) it overlaps. The previous code re-queried per window, which on - // a repeat-rich contig (thousands of indel windows) re-decoded the same CRAM containers over - // and over — the de-novo hot path. Reads are short, so each overlaps only a window or two, - // found by binary search over the sorted windows. + // ONE query against the index, over all of the active windows. Decode the reads of the + // region once, and send each read to the window or windows that it overlaps. + // + // The code before this one ran a query for each window. A contig with many repeats holds + // thousands of indel windows. So that code decoded the same CRAM containers again and again, + // on the hot path of the de-novo pass. The reads are short, so each one overlaps only + // one window or two, which a binary search over the sorted windows finds. let span_lo = region_lo + windows.first().unwrap().0; let span_hi = region_lo + windows.last().unwrap().1; let region: Region = format!("{contig}:{span_lo}-{span_hi}") @@ -2079,10 +2178,14 @@ mod tests { #[test] fn consensus_reconcile_resolves_homref_and_abstains_on_no_call() { - // Site 100: run A het (0/1, deep), run B hom-ref (0/0, deep) → real disagreement; depth- - // weighted vote, both deep, equal weight → tie broken by lower dosage = hom-ref → NOT emitted. - // Site 200: run A het (deep), run B no-call (depth 1 < min 4) → B abstains, A wins → het. - // Site 300: both hom-alt (1/1) → hom-alt, depths summed. + // Site 100: run A is het (0/1) and deep, run B is hom-ref (0/0) and deep. That is a real + // disagreement. In the vote by depth both are deep and their weights are equal, so the + // lower dosage breaks the tie. The result is hom-ref, and the code does NOT emit it. + // + // Site 200: run A is het and deep, run B has no call, because its depth of 1 is below the + // minimum of 4. B then abstains, A wins, and the result is het. + // + // Site 300: both are hom-alt (1/1). The result is hom-alt, with the depths added. let a = vec![ sg("chr1", 100, "G", 1, 30, 50), sg("chr1", 200, "G", 1, 30, 50), @@ -2116,7 +2219,7 @@ mod tests { // An insertion of "A" before anchor 5 (in the A-run) left-aligns to anchor 2. let (a, al) = left_normalize(5, &IndelAllele::Ins(b"A".to_vec()), refc, 1); assert_eq!((a, al), (2, IndelAllele::Ins(b"A".to_vec()))); - // A non-repeat deletion doesn't move: "ACGTC", delete the G (anchor 3). + // A non-repeat deletion does not move: "ACGTC", delete the G (anchor 3). let (a, _) = left_normalize(3, &IndelAllele::Del(1), b"ACGTC", 1); assert_eq!(a, 3); } @@ -2179,18 +2282,19 @@ mod tests { // Clean monoallelic call (HiFi-like): not paralogous. assert!(!para([11, 0, 0, 0])); - // One discordant read at low depth — a sequencing error, kept. + // One read that disagrees, at low depth. That is a sequencing error, and the site stays. assert!(!para([3, 1, 0, 0])); // second=1 (< 2 reads) - // Scattered errors across other bases, none reaching 2 reads — kept. + // Errors spread over the other bases, and none of them gets to 2 reads. The site stays. assert!(!para([18, 1, 1, 0])); // second=1 - // Genuine bi-allelic pileup (7 derived / 4 ancestral) — paralog, dropped. + // A true pileup with two alleles, at 7 derived and 4 ancestral. That is a paralog, and + // the code drops it. assert!(para([7, 4, 0, 0])); // second=4, 0.36 > 0.20 - // Boundary: 2/10 = 0.20 is not strictly above the threshold — kept. + // The boundary. 2/10 = 0.20 is not above the threshold, so the site stays. assert!(!para([8, 2, 0, 0])); - // 3/10 = 0.30 > 0.20 with 3 reads — dropped. + // 3/10 = 0.30, which is more than 0.20, with 3 reads. The code drops the site. assert!(para([7, 3, 0, 0])); // Empty pileup is never paralogous. assert!(!para([0, 0, 0, 0])); @@ -2202,7 +2306,7 @@ mod tests { max_minor_allele_fraction: 1.0, ..Default::default() }; - // Even a 50/50 split is not flagged when the filter is disabled. + // With the filter off, even a 50/50 split gets no flag. assert!(!is_paralogous(&[5, 5, 0, 0], 10, &p)); } diff --git a/crates/navigator-analysis/src/callset.rs b/crates/navigator-analysis/src/callset.rs index 07146117..28732878 100644 --- a/crates/navigator-analysis/src/callset.rs +++ b/crates/navigator-analysis/src/callset.rs @@ -1,31 +1,39 @@ -//! EIGENSTRAT (Reich-lab / `pileupCaller`) call-set reader — the external autosomal 1240K path. +//! The reader of an EIGENSTRAT call set, which comes from the Reich lab and from `pileupCaller`. +//! It is the path for an external autosomal 1240K set. //! -//! Parses a `.geno`/`.snp`/`.ind` triplet for **one** target individual into diploid allele pairs on -//! the call set's build (the AADR 1240K is GRCh37/hg19). Those pairs feed -//! [`crate::ibd_panel::IbdPanel::resolve_chip`], which re-keys them to canonical CHM13 dosages and -//! **self-orients** against the CHM13 alleles — so the EIGENSTRAT allele labelling need not match the -//! genome reference, and pseudo-haploid calls (pileupCaller emits only `0`/`2`) come through as valid -//! homozygous observations with no het synthesis. +//! It parses a `.geno`, `.snp` and `.ind` triplet, for **one** target individual, into diploid +//! allele pairs on the build of the call set. The AADR 1240K is on GRCh37, which is hg19. //! -//! EIGENSTRAT format (whitespace-delimited text): -//! - `.ind`: one line per individual — `SampleID Sex Population`. -//! - `.snp`: one line per SNP — `SNPName Chr GeneticPos PhysicalPos RefAllele VariantAllele`. -//! - `.geno`: one line per SNP (same order as `.snp`), one char per individual — -//! `0`/`1`/`2` = **count of the first (`.snp` column-5) allele**, `9` = missing. +//! Those pairs go to [`crate::ibd_panel::IbdPanel::resolve_chip`]. That function keys them to +//! canonical CHM13 dosages, and it **orients them itself** against the CHM13 alleles. So the allele +//! labels of the EIGENSTRAT set do not have to match the genome reference. And a pseudo-haploid +//! call, where pileupCaller writes `0` or `2` alone, comes through as a correct homozygous +//! observation, with no het that the code invented. +//! +//! The EIGENSTRAT format is text, with white space between its fields: +//! +//! - `.ind` holds one line for each individual: `SampleID Sex Population`. +//! - `.snp` holds one line for each SNP: +//! `SNPName Chr GeneticPos PhysicalPos RefAllele VariantAllele`. +//! - `.geno` holds one line for each SNP, in the same order as `.snp`, with one character for each +//! individual. A `0`, `1` or `2` is the **count of the first allele**, which is column 5 of +//! `.snp`. A `9` means that the value is missing. use std::io::BufRead; use std::path::Path; use crate::error::AnalysisError; -/// One target individual's genotypes from an EIGENSTRAT triplet: reference-forward diploid allele -/// pairs on `build`, ready for `IbdPanel::resolve_chip`. No-calls (`9`) are dropped, not emitted. +/// The genotypes of one target individual, from an EIGENSTRAT triplet. They are diploid allele +/// pairs on `build`, forward on the reference, ready for `IbdPanel::resolve_chip`. The code drops a +/// no-call, which is a `9`, and it emits nothing for it. pub struct CallSet { - /// Build the `.snp` positions are in. EIGENSTRAT does not encode it; the caller supplies it - /// (default GRCh37 — the AADR 1240K coordinate system). + /// The build that the `.snp` positions sit on. EIGENSTRAT does not record it, so the caller + /// gives it. The default is GRCh37, which is the coordinate system of the AADR 1240K. pub build: String, - /// `(contig, position, allele1, allele2)` per called autosomal site (contig is a bare `"1".."22"`, - /// matching the panel's GRCh37 loci). Alleles are the actual nucleotides the individual carries. + /// A `(contig, position, allele1, allele2)` at each autosomal site with a call. The contig is + /// bare, from `"1"` to `"22"`, which matches the GRCh37 loci of the panel. The alleles are the + /// nucleotides that the individual carries. pub calls: Vec<(String, i64, char, char)>, /// Count of autosomal `.snp` sites that were missing (`9`) for this individual. pub missing: usize, @@ -43,8 +51,10 @@ fn geno_to_pair(g: u8, a1: char, a2: char) -> Option<(char, char)> { } } -/// Bare autosomal contig (`"1".."22"`) for an EIGENSTRAT `Chr` field, or `None` for a sex/mt/unknown -/// contig (EIGENSTRAT uses `23`=X, `24`=Y, `90`/`91`=mt, plus `0`). Accepts an optional `chr` prefix. +/// The bare autosomal contig, from `"1"` to `"22"`, for a `Chr` field of EIGENSTRAT. It is `None` +/// for a sex contig, for mt, and for a contig that the code does not know. EIGENSTRAT writes `23` +/// for X, `24` for Y, `90` or `91` for mt, and it also uses `0`. This accepts a `chr` prefix, and +/// it does not need one. fn autosome_contig(chr: &str) -> Option { match crate::contig::bare(chr).parse::() { Ok(n @ 1..=22) => Some(n.to_string()), @@ -73,8 +83,8 @@ fn select_individual(ind_text: &str, sample: Option<&str>) -> Result( snp: S, geno: G, @@ -119,9 +129,10 @@ fn read_eigenstrat_core( }) } -/// Read an EIGENSTRAT triplet for one target individual. `sample` selects the individual (required -/// when the `.ind` lists more than one); `build` is the coordinate system of the `.snp` positions -/// (default GRCh37 for the AADR 1240K). Streams `.snp`/`.geno` so a large panel stays cheap. +/// Read an EIGENSTRAT triplet, for one target individual. `sample` selects that individual, and it +/// is necessary when the `.ind` lists more than one. `build` is the coordinate system of the `.snp` +/// positions, and the default is GRCh37, for the AADR 1240K. It streams `.snp` and `.geno`, so a +/// large panel costs little. pub fn read_eigenstrat( geno: &Path, snp: &Path, @@ -177,7 +188,7 @@ rs2 2 0.0 2000 C T rsX 23 0.0 3000 A G rs3 22 0.0 4000 T C "; - // per row, char 0 = SAMPLE_A, char 1 = SAMPLE_B. + // In each row, character 0 is SAMPLE_A and character 1 is SAMPLE_B. let geno = "\ 20 19 diff --git a/crates/navigator-analysis/src/cancel.rs b/crates/navigator-analysis/src/cancel.rs index 11b986ce..fb43b336 100644 --- a/crates/navigator-analysis/src/cancel.rs +++ b/crates/navigator-analysis/src/cancel.rs @@ -1,31 +1,37 @@ //! Cooperative cancellation for the long walks. //! -//! A whole-genome pass takes minutes, and the UI's Cancel button used to do nothing visible for -//! all of them: the flag it set lived in `navigator-ui` and was only read *between* pipeline steps, -//! while the step itself ran inside a `spawn_blocking` closure that tokio cannot interrupt. Once a -//! walk starts, the only thing that can stop it is the walk itself — so the walkers have to ask. +//! A pass over the whole genome takes minutes. For all of those minutes, the Cancel button of the +//! UI once did nothing that a user could see. The flag that it set lived in `navigator-ui`, and +//! the code read that flag only *between* the steps of the pipeline. The step itself ran inside a +//! `spawn_blocking` closure, and tokio can not interrupt one of those. Once a walk starts, the one +//! thing that can stop it is the walk itself. So a walker has to ask. //! -//! [`CancelToken`] is that question, and the rule for using it is about *where* you ask: often -//! enough that a click feels instant, rarely enough that the check does not show up in a profile. -//! Every place one is checked here sits on a path that already does real per-record or per-contig -//! work, so an atomic load is noise by comparison. Checking inside the innermost per-base loop +//! [`CancelToken`] is that question. The rule for its use is about *where* you ask. Ask often +//! enough that a click feels immediate, and rarely enough that the check does not show in a +//! profile. +//! +//! Every check in this crate sits on a path that already does real work at each record, or at each +//! contig. An atomic load is noise next to that. A check inside the innermost loop over the bases //! would not be. //! -//! A cancelled walk returns [`AnalysisError::Cancelled`] rather than a partial result. Partial -//! coverage is indistinguishable from genuinely low coverage once it is persisted, and silently -//! caching a half-finished walk as if it were complete is a far worse failure than not cancelling -//! at all — so cancellation is an error, and callers skip their persistence step on it. +//! A walk that somebody cancelled returns [`AnalysisError::Cancelled`], and not a partial result. +//! Once the store holds a partial coverage, nothing can separate it from a coverage that is truly +//! low. To cache a walk that stopped half way, as if it were complete, is a far worse failure than +//! no cancellation at all. So a cancellation is an error, and a caller skips its store step on +//! it. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use crate::error::AnalysisError; -/// A shared "stop what you're doing" flag, cheap to clone into worker threads. +/// A shared flag that says "stop what you are doing". A clone of it, into a worker thread, costs +/// almost nothing. /// -/// [`CancelToken::none`] is a token that can never be cancelled. It exists so callers with nothing -/// to cancel — tests, CLI one-shots, the non-progress convenience wrappers — pay nothing and read -/// naturally, instead of every signature growing an `Option`. +/// [`CancelToken::none`] gives a token that nobody can cancel. It exists so that a caller with +/// nothing to cancel pays nothing, and reads naturally. A test, a one-shot CLI command and the +/// wrappers that report no progress are all such callers. Without it, every signature would carry +/// an `Option`. #[derive(Clone, Debug, Default)] pub struct CancelToken(Option>); @@ -42,18 +48,19 @@ impl CancelToken { /// Request cancellation. Idempotent, and safe to call from any thread. /// - /// There is deliberately no way back to the un-cancelled state: a token covers exactly one run, - /// and reusing one across runs is what let a stale reset clobber a pending cancel before. + /// There is no way back to the state before the cancel, and that is deliberate. A token covers + /// exactly one run. To use one token across two runs is what let a stale reset write over a + /// cancel that had not yet arrived. pub fn cancel(&self) { if let Some(flag) = &self.0 { flag.store(true, Ordering::Relaxed); } } - /// Whether cancellation has been requested. + /// True when somebody has asked for a cancel. /// - /// `Relaxed` is sufficient: this guards no other memory, and the only cost of observing the - /// store one loop iteration late is one more iteration of work. + /// `Relaxed` is enough. This flag guards no other memory. To see the store one iteration of + /// the loop late costs one more iteration of work, and nothing else. pub fn is_cancelled(&self) -> bool { self.0.as_ref().is_some_and(|flag| flag.load(Ordering::Relaxed)) } @@ -89,9 +96,10 @@ mod tests { assert!(matches!(worker.check(), Err(AnalysisError::Cancelled))); } - /// `Default` has to be the inert token: a struct that gains a `CancelToken` field by - /// `..Default::default()` must not silently start out cancellable-but-never-cancelled in a way - /// that differs from `none()`. + /// `Default` must give the token that does nothing. Take a struct that gains a `CancelToken` + /// field through `..Default::default()`. That struct must not start in a state where somebody + /// can cancel it, and nobody ever does, and where it differs from `none()`. Nobody would see + /// that. #[test] fn default_is_the_inert_token() { let t = CancelToken::default(); @@ -99,9 +107,9 @@ mod tests { assert!(!t.is_cancelled()); } - /// The property the whole feature rests on: a token cancelled from *another thread* is observed - /// by a walk already in progress. This is the case the old design could not express at all — - /// the flag lived in the UI and the walk had no way to ask. + /// The property that the whole feature stands on. A walk that already runs sees a token that + /// *another thread* cancelled. The old design could not express this case at all: the flag + /// lived in the UI, and the walk had no way to ask. #[test] fn a_walk_in_progress_observes_a_cancel_from_another_thread() { let token = CancelToken::new(); @@ -122,8 +130,8 @@ mod tests { ); } - /// Cancellation must be reported as itself, never as a generic failure — the UI branches on - /// this to avoid telling the user their own click was an error. + /// A cancellation must go out as itself, and never as a general failure. The UI branches on + /// that, so it does not tell the user that their own click was an error. #[test] fn cancellation_is_its_own_error_kind() { let t = CancelToken::new(); diff --git a/crates/navigator-analysis/src/contig.rs b/crates/navigator-analysis/src/contig.rs index 1182f123..cad348ae 100644 --- a/crates/navigator-analysis/src/contig.rs +++ b/crates/navigator-analysis/src/contig.rs @@ -1,13 +1,17 @@ -//! Contig-name classification shared across walkers. Mirrors the Scala regexes: -//! autosomes `^(chr)?([1-9]|1[0-9]|2[0-2])$`, plus X / Y / M|MT. +//! The class of a contig name, which every walker shares. It follows the regular expressions of +//! the Scala code: an autosome matches `^(chr)?([1-9]|1[0-9]|2[0-2])$`, and X, Y and M or MT are +//! the others. //! -//! Prefix stripping itself lives in [`navigator_domain::contig`] (every crate needs it, including -//! ones below this one) and is re-exported here so callers have a single import for contig work. -//! Classification is case-insensitive on both the prefix and the name (`chrx`, `Chr7`, `mt`). +//! The removal of the prefix lives in [`navigator_domain::contig`], because every crate needs it, +//! and that includes crates below this one. This module exports it again, so that a caller has one +//! import for all of its work on a contig. +//! +//! The class does not depend on the case, of the prefix or of the name. `chrx`, `Chr7` and `mt` all +//! work. pub use navigator_domain::contig::{bare, bare_upper}; -/// Autosome 1-22 (no leading zeros). +/// An autosome, from 1 to 22. The number carries no zero in front. pub fn is_autosome(name: &str) -> bool { let c = bare(name); c.parse::() @@ -33,9 +37,12 @@ pub fn is_main_assembly(name: &str) -> bool { is_autosome(name) || is_chr_x(name) || is_chr_y(name) || is_chr_m(name) } -/// **Haploid** contigs: chrY and chrM/MT carry a single allele, so the diploid (het `0/1` + -/// hom-alt `1/1`) model doesn't apply — the haploid caller and Y/mt haplogroup placement own them. -/// (chrX is haploid only in a male; that's left to the sex-aware refinement, not decided here.) +/// The **haploid** contigs. chrY, and chrM or MT, each carry one allele. So the diploid model, with +/// its het `0/1` and hom-alt `1/1`, does not apply to them. The haploid caller owns them, and +/// so does the placement of a Y or mt haplogroup. +/// +/// chrX is haploid in a male alone. The refinement that knows the sex decides that, and this +/// function does not. pub fn is_haploid(name: &str) -> bool { is_chr_y(name) || is_chr_m(name) } diff --git a/crates/navigator-analysis/src/coverage.rs b/crates/navigator-analysis/src/coverage.rs index 57cd58eb..c6ffc8af 100644 --- a/crates/navigator-analysis/src/coverage.rs +++ b/crates/navigator-analysis/src/coverage.rs @@ -1,20 +1,25 @@ -//! Coverage + callable-loci walker — the Rust port of the Scala -//! `CoverageCallableWalker` (which itself replaces GATK `CollectWgsMetrics` + -//! `CallableLoci` over htsjdk). Single pass over a coordinate-sorted BAM/CRAM builds, -//! per main-assembly contig: a depth histogram, per-position callable state, and -//! samtools-style coverage stats. +//! The walker over the coverage and the callable loci. It is the Rust port of the Scala +//! `CoverageCallableWalker`, which itself replaces GATK `CollectWgsMetrics` and `CallableLoci` +//! over htsjdk. It makes one pass over a BAM or CRAM in coordinate order. It builds three things +//! for each contig on the main assembly. Those are a depth histogram, the callable state at each +//! position, and coverage statistics in the style of samtools. //! -//! Parity target is the Scala walker, not samtools — notably mean base/mapping quality -//! are averaged **per base observation** (Σ quality / Σ depth), where samtools averages -//! per read. See the crate fixture tests for hand-computed expected values. +//! The parity target is the Scala walker, and not samtools. The difference that matters is the +//! mean base quality and the mean mapping quality. This walker takes the mean **over the base +//! observations**, as Σ quality / Σ depth. samtools takes the mean over the reads. The fixture +//! tests of this crate carry the expected values, computed by hand. //! -//! Memory: a **sliding-window pileup** finalizes each position once the read frontier -//! passes it, so peak memory is the span of currently-open reads — not the contig -//! length. The only contig-sized allocation is the reference bases of the contig being -//! walked (one at a time, for N detection); whole-genome HG002 peaks ~2 GB vs the -//! ~84 GB a dense per-contig-arrays approach would need. Requires a coordinate-sorted -//! BAM. (Streaming the reference in windows too is a further optimization.) BED-interval -//! output and progress callbacks from the Scala walker are deferred. +//! On memory: a **pileup in a window that slides** finalizes each position once the read frontier +//! passes it. The peak memory is then the span of the reads that are open, and not the length of +//! the contig. +//! +//! One allocation has the size of a contig. It holds the reference bases of the contig that the +//! walker is on, one contig at a time, to find the N bases. Over the whole genome, HG002 peaks at +//! about 2 GB. A method with dense arrays for each contig would need about 84 GB. +//! +//! This walker needs a BAM in coordinate order. To stream the reference in windows as well is a +//! further improvement, and nobody has done it. The BED-interval output and the progress +//! callbacks of the Scala walker wait for later work. use std::collections::{HashMap, HashSet, VecDeque}; use std::path::Path; @@ -30,14 +35,17 @@ use crate::error::AnalysisError; use crate::reader; use crate::readview::AlnRead; -/// Algorithm version for the coverage artifact cache key; bump on any change that -/// alters output (plan §6 cache versioning). +/// The algorithm version, for the cache key of the coverage artifact. Raise it after any change +/// that alters the output. See plan §6, on the version of a cache. +/// +/// At `coverage-2`, the parallel walker attributes a record by its reference id. Before that, the +/// coverage count was too low on a CRAM whose slices hold more than one reference, such as an +/// FTDNA Big Y. /// -/// `coverage-2`: the parallel walker now attributes records by reference id, fixing under-counted -/// coverage on multi-reference-slice CRAMs (FTDNA Big Y). The bump invalidates every cached -/// `coverage-1` result — which contig sort/slice layout it was computed from is unknown per file — -/// so each alignment recomputes correctly on its next analysis (the re-run also overwrites the -/// stale read-metrics/sex from the same fused walk). +/// The new version makes every cached `coverage-1` result stale, and that is correct. Nobody knows +/// which sort order and slice layout each of those files came from. Each alignment computes the +/// correct value again at its next analysis. That run also overwrites the stale +/// read-metrics and sex from the same fused walk. pub const COVERAGE_VERSION: &str = "coverage-2"; /// Callable-loci parameters. Defaults match GATK `CallableLoci` (and the Scala walker). @@ -64,8 +72,8 @@ impl Default for CallableLociParams { } } -/// Per-position callable classification (GATK `CallableLoci` states). Hierarchical: -/// the first failing condition wins. +/// The callable class at one position. These are the states of GATK `CallableLoci`. They form a +/// hierarchy, and the first condition that fails wins. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CallableState { RefN, @@ -76,7 +84,7 @@ pub enum CallableState { Callable, } -/// Per-contig callable-state base counts. +/// The count of bases in each callable state, for each contig. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ContigCallableMetrics { pub contig: String, @@ -88,7 +96,8 @@ pub struct ContigCallableMetrics { pub poor_mapping_quality: u64, } -/// Per-contig samtools-`coverage`-style stats (averaged per base observation; see module docs). +/// The coverage statistics of each contig, in the style of samtools `coverage`. The mean is over +/// the base observations. See the module documentation. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ContigCoverageStats { pub contig: String, @@ -100,11 +109,14 @@ pub struct ContigCoverageStats { pub mean_depth: f64, pub mean_base_q: f64, pub mean_map_q: f64, - /// This contig's depth histogram (bin `d` = bases at depth `d`, clamped at index 255 — same - /// convention as the genome-wide [`CoverageResult::coverage_histogram`]). Empty for fast-path - /// (pipeline-sidecar) imports, which have no per-depth histogram. `#[serde(default)]` keeps - /// coverage blobs cached before this field was added loading (the histogram repopulates on the - /// next analysis) — so `COVERAGE_VERSION` does not need a bump. + /// The depth histogram of this contig. Bin `d` holds the count of bases at depth `d`, and the + /// index stops at 255. That is the same convention as the genome-wide + /// [`CoverageResult::coverage_histogram`]. + /// + /// It is empty for an import on the fast path, from a pipeline sidecar, because such an import + /// has no histogram over the depths. `#[serde(default)]` lets a coverage blob that the cache + /// holds from before this field still load. The histogram fills again at the next analysis, so + /// `COVERAGE_VERSION` does not need a new value. #[serde(default)] pub histogram: Vec, } @@ -124,7 +136,8 @@ pub struct CoverageResult { /// Fraction of observed bases excluded for low mapping quality (Picard PCT_EXC_MAPQ). #[serde(default)] pub pct_exc_mapq: f64, - /// Fraction of observed bases excluded for low base quality, MAPQ having passed (PCT_EXC_BASEQ). + /// The fraction of the observed bases that the walker removed for a low base quality, after + /// those bases passed the MAPQ filter. This is PCT_EXC_BASEQ. #[serde(default)] pub pct_exc_baseq: f64, /// Depth histogram, clamped at index 255. @@ -145,7 +158,7 @@ pub struct CoverageResult { const HIST_LEN: usize = 256; -/// Per-position accumulator within the sliding pileup window. +/// The accumulator at one position, inside the pileup window that slides. #[derive(Clone, Default)] struct Col { depth: u32, @@ -155,7 +168,7 @@ struct Col { low_mapq: u32, /// Bases excluded for low mapping quality (mutually exclusive with `exc_baseq`; MAPQ checked first). exc_mapq: u32, - /// Bases that passed MAPQ but were excluded for low base quality. + /// The count of bases that passed MAPQ, and that a low base quality then removed. exc_baseq: u32, } @@ -183,16 +196,19 @@ impl Globals { } } -/// Finished per-contig output (kept tiny; assembled into the result in header order). +/// The finished output of one contig. It stays very small, and the code puts it into the result +/// in header order. struct ContigOut { callable: ContigCallableMetrics, stats: ContigCoverageStats, } -/// Reference-N mask for one contig: one bit per base (set = the reference base is N), so -/// callable classification needs ~1/8 the memory of holding the raw reference bytes. That -/// matters for the parallel walker, where each concurrent contig task would otherwise pin its -/// full reference (chr1 ≈ 248 MB) for the whole pileup. +/// The mask of the N bases in the reference of one contig. It holds one bit for each base, and a +/// set bit means that the reference base is N. The callable class then needs about one eighth of +/// the memory that the raw reference bytes need. +/// +/// That matters for the parallel walker. Each contig task that runs at the same time would else +/// hold its full reference for the whole pileup, and chr1 is about 248 MB. struct NMask { bits: Vec, } @@ -215,8 +231,9 @@ impl NMask { } } -/// Streaming state for the contig currently being walked. Memory is bounded by the -/// sliding window (the span of currently-open reads), not the contig length. +/// The streaming state of the contig that the walker is on. The window that slides bounds the +/// memory, and that window is the span of the reads that are open. The length of the contig does +/// not bound it. struct CurContig { name: String, length: usize, @@ -225,23 +242,24 @@ struct CurContig { emit_cursor: usize, // 1-based position next to finalize; window front aligns here read_count: u64, cm: ContigCallableMetrics, - /// This contig's own depth histogram (kept alongside the global `Globals::hist` so the - /// per-contig histogram can be surfaced; both walker paths finalize through here). + /// The own depth histogram of this contig. It sits beside the global `Globals::hist`, so that + /// the UI can show the histogram of each contig. Both walker paths finish through here. hist: Vec, covered: u64, total_base_obs: u64, base_q_total: u64, map_q_total: u64, sum_depth: u128, - /// Count of bases dropped because they mapped *before* the finalize frontier — only possible - /// when the input isn't strictly coordinate-sorted (see [`CurContig::add`]). Surfaced as a - /// warning in [`CurContig::finish`]; stays 0 for the standard sorted layout. + /// The count of bases that the walker dropped, because they mapped *before* the finalize + /// frontier. That can happen only when the input is not strictly in coordinate order. See + /// [`CurContig::add`]. [`CurContig::finish`] gives a warning about it. It stays 0 for the + /// standard sorted layout. dropped_unsorted: u64, } impl CurContig { - /// Builds from the contig's raw reference bytes, retaining only the compact N-mask (the - /// `ref_bases` buffer can be dropped by the caller right after). + /// Build from the raw reference bytes of the contig. This keeps the compact N-mask alone, so + /// the caller can drop the `ref_bases` buffer immediately after. fn new(name: String, length: usize, ref_bases: Vec) -> Self { let cm = ContigCallableMetrics { contig: name.clone(), @@ -309,13 +327,17 @@ impl CurContig { } } - /// Add one covered base at 1-based `pos` to the window. The window only holds positions at or - /// after the finalize frontier, so a base *before* it (`pos < emit_cursor`) belongs to an - /// already-emitted column and is dropped rather than underflowing `pos - emit_cursor`. This can - /// only happen when the input isn't strictly coordinate-sorted (some vendor CRAMs, e.g. FTDNA - /// Big Y): the streaming pileup fundamentally assumes sorted input, so the few out-of-order - /// bases are counted as dropped (surfaced in `finish`) instead of crashing the walk. For the - /// standard sorted layout `pos >= emit_cursor` always holds and this guard never fires. + /// Add one covered base at the 1-based `pos` to the window. + /// + /// The window holds only the positions at the finalize frontier or after it. A base *before* + /// the frontier, where `pos < emit_cursor`, belongs to a column that the walker already + /// emitted. The code drops that base, and it does not let `pos - emit_cursor` go below zero. + /// + /// This can happen only when the input is not strictly in coordinate order, which occurs on + /// some vendor CRAMs, such as an FTDNA Big Y. The streaming pileup assumes sorted input at its + /// root. So the few bases that are out of order count as dropped, which `finish` reports, and + /// the walk does not crash. On the standard sorted layout, `pos >= emit_cursor` always holds, + /// and this guard never fires. fn add(&mut self, pos: usize, base_q: u8, mapq: u8, params: &CallableLociParams) { if pos < self.emit_cursor { self.dropped_unsorted += 1; @@ -329,7 +351,8 @@ impl CurContig { col.depth += 1; col.base_q_sum += base_q as u64; col.map_q_sum += mapq as u64; - // Mutually-exclusive exclusion attribution (MAPQ first, then base-Q), mirroring Picard. + // A base goes to one exclusion reason and no more than one: MAPQ first, then base-Q. + // Picard does the same. if mapq < params.min_mapping_quality { col.exc_mapq += 1; } else if base_q < params.min_base_quality { @@ -389,10 +412,12 @@ impl CurContig { } } -/// Single coordinate-ordered pass with a sliding pileup window: positions finalize once -/// the read frontier passes them, so peak memory is the open-read span, not the contig -/// length. Requires a coordinate-sorted BAM/CRAM (the standard genomics layout). The -/// reference is needed both to detect reference-N positions and to decode CRAM. +/// One pass in coordinate order, with a pileup window that slides. A position finalizes once the +/// read frontier passes it. The peak memory is then the span of the open reads, and not the length +/// of the contig. +/// +/// This needs a BAM or CRAM in coordinate order, which is the standard layout in genomics. It +/// needs the reference for two things: to find the N positions, and to decode a CRAM. pub fn collect_coverage_callable( bam_path: &Path, reference_path: &Path, @@ -409,9 +434,12 @@ pub fn collect_coverage_callable( ) } -/// Like [`collect_coverage_callable`], reporting `progress(contigs_done, contigs_total)` as each -/// tracked contig is finalized — so a whole-genome pass (minutes on a real WGS BAM) can drive a -/// progress bar instead of looking stalled. Assumes a coordinate-sorted BAM (contigs in order). +/// The same as [`collect_coverage_callable`], and it also reports +/// `progress(contigs_done, contigs_total)` as it finishes each contig that it tracks. +/// +/// A pass over the whole genome takes minutes on a real WGS BAM. It can then drive a progress bar, +/// and it does not look stopped. This function needs a BAM in coordinate order, with the contigs +/// in order. pub fn collect_coverage_callable_with_progress( bam_path: &Path, reference_path: &Path, @@ -435,10 +463,11 @@ pub fn collect_coverage_callable_with_progress( state.finish(progress) } -/// Streaming coverage + callable accumulator shared by the standalone walker and the fused -/// [`crate::unified`] walker, so both produce byte-identical numbers from one source of -/// truth. Feed it every record via [`CoverageState::accept`] (it applies coverage's own -/// flag/contig filtering internally), then call [`CoverageState::finish`]. +/// The streaming accumulator for the coverage and the callable state. The separate walker and the +/// fused [`crate::unified`] walker share it, so both give the same numbers, to the last digit, +/// from one source of truth. Give it every record through [`CoverageState::accept`], which applies +/// the flag filter and the contig filter of the coverage pass inside. Then call +/// [`CoverageState::finish`]. pub(crate) struct CoverageState { /// ref_id -> (name, length) for tracked (main-assembly, allowlisted) contigs; `None` elsewhere. tracked: Vec>, @@ -485,14 +514,16 @@ impl CoverageState { }) } - /// Contigs this state will walk — the progress denominator. + /// The contigs that this state walks. It is the denominator of the progress. pub(crate) fn total_tracked(&self) -> usize { self.total_tracked } - /// Feed one record. Records that coverage doesn't care about (unmapped/secondary/ - /// supplementary/duplicate/qc-fail, or off a tracked contig) are ignored, so the fused - /// walker can hand every record here unfiltered. Fires `progress` on contig finalization. + /// Give it one record. It ignores a record that the coverage pass does not want. There are + /// five of those: a record with no mapping, a secondary record, a supplementary record, a + /// duplicate, and a qc-fail. It also ignores a record on a contig that the walker does not + /// track. The fused walker can then give every record here, with no filter of its own. It + /// calls `progress` when a contig finishes. pub(crate) fn accept( &mut self, record: &impl AlnRead, @@ -574,8 +605,9 @@ impl CoverageState { } self.g.hist[0] += length as u64; self.g.n += length as u64; - // Per-contig histogram: every position at depth 0 (matches the parallel path, - // where an unseen contig finalizes all positions at depth 0 via CurContig::finish). + // The histogram of this contig: every position sits at depth 0. That matches the + // parallel path, where a contig that the walker never saw finalizes every position + // at depth 0, through CurContig::finish. let mut hist = vec![0u64; HIST_LEN]; hist[0] = length as u64; contig_callable.push(ContigCallableMetrics { @@ -615,17 +647,18 @@ impl CoverageState { } } -/// Coverage's read filter — skip unmapped / secondary / supplementary / duplicate / qc-fail. -/// Shared by the sequential [`CoverageState`] and the per-contig [`ContigCoverageAccum`] so -/// both pileups see the identical read set. +/// The read filter of the coverage pass. It skips a read with no mapping, a secondary or +/// supplementary read, a duplicate, and a qc-fail. The sequential [`CoverageState`] and the +/// [`ContigCoverageAccum`] of one contig share it, so both pileups see the same set of reads. fn coverage_passes_filter(record: &impl AlnRead) -> bool { let f = record.flags(); !(f.is_unmapped() || f.is_secondary() || f.is_supplementary() || f.is_duplicate() || f.is_qc_fail()) } -/// Feed one (already filter-passing) record into a contig's sliding-window pileup: advance -/// the finalize frontier to the read's start, then add each reference-consuming base. Shared -/// by the sequential and per-contig coverage paths so the per-base accounting is identical. +/// Give one record, which already passed the filter, to the pileup window of a contig. It moves +/// the finalize frontier to the start of the read, and then adds each base that the read takes +/// from the reference. The sequential path and the path over one contig share it, so the tally at +/// each base is the same in both. fn feed_into_contig(c: &mut CurContig, record: &impl AlnRead, params: &CallableLociParams, g: &mut Globals) { let start = match record.alignment_start() { Some(p) => p, @@ -659,9 +692,11 @@ fn feed_into_contig(c: &mut CurContig, record: &impl AlnRead, params: &CallableL }); } -/// Assemble the genome-wide [`CoverageResult`] from merged histogram/territory/depth sums and -/// per-contig outputs (already in header order). Single source of truth for the result tail, -/// used by both the sequential `finish` and the parallel `merge_coverage_partials`. +/// Build the genome-wide [`CoverageResult`]. It takes the merged sums of the histogram, the +/// territory and the depth, and the output of each contig, which is already in header order. +/// +/// It is the single source of truth for the tail of the result. The sequential `finish` and the +/// parallel `merge_coverage_partials` both use it. #[allow(clippy::too_many_arguments)] // one accumulator per metric — a coverage roll-up, not a refactor target fn assemble_coverage_result( hist: Vec, @@ -682,7 +717,7 @@ fn assemble_coverage_result( let median = median_from_hist(&hist, n); // Exclusion fractions over total observed bases (Picard PCT_EXC_{MAPQ,BASEQ}). `sum_depth` // already counts every observed base (excluded ones included), so it is the denominator. Other - // exclusion reasons (dup/unpaired/overlap/capped) aren't tallied, so these don't sum to a total. + // exclusion reasons (dup/unpaired/overlap/capped) are not tallied, so these do not sum to a total. let (pct_exc_mapq, pct_exc_baseq) = if sum_depth == 0 { (0.0, 0.0) } else { @@ -716,17 +751,19 @@ fn assemble_coverage_result( } } -/// Per-contig coverage accumulator for the parallel walker — one contig's sliding-window -/// pileup with a local copy of the genome-wide accumulators (summed across contigs at merge -/// time). Built per contig in the rayon fan-out; feed it that contig's region-query records. +/// The coverage accumulator of one contig, for the parallel walker. It holds the pileup window of +/// that contig, and a local copy of the genome-wide accumulators. The merge adds those copies +/// across the contigs. The rayon fan-out builds one for each contig. Give it the records from the +/// region query of that contig. pub(crate) struct ContigCoverageAccum { c: CurContig, g: Globals, params: CallableLociParams, } -/// One contig's finished coverage contribution: its per-contig output plus this contig's -/// share of the genome-wide histogram / territory / depth sums. +/// The finished coverage of one contig. It holds the output of that contig. It also holds the +/// share that the contig adds to the genome-wide sums of the histogram, the territory and the +/// depth. pub(crate) struct ContigCoveragePartial { ref_id: usize, callable: ContigCallableMetrics, @@ -748,18 +785,22 @@ impl ContigCoverageAccum { } } - /// Feed one record; the coverage read filter is applied internally, so off-filter records - /// are ignored and the caller can pass every record from the contig's region query. + /// Give it one record. It applies the read filter of the coverage pass inside, so it ignores a + /// record that the filter rejects. The caller can then give it every record from the region + /// query of the contig. pub(crate) fn accept(&mut self, record: &impl AlnRead) { if coverage_passes_filter(record) { feed_into_contig(&mut self.c, record, &self.params, &mut self.g); } } - /// Finalize the contig (flushing its window + uncovered tail) into a partial tagged with - /// `ref_id` for header-order reassembly. A contig that saw no reads still finalizes every - /// position at depth 0 — counting ref-N and no-coverage exactly as the sequential walker's - /// zero-coverage branch does. + /// Finalize the contig into a partial result that carries `ref_id`, so that a later step can + /// put the partials back into header order. This sends out the window of the contig, and its + /// tail that no read covered. + /// + /// A contig that saw no read still finalizes every position at depth 0. It counts the ref-N + /// bases and the bases with no coverage exactly as the zero-coverage branch of the sequential + /// walker does. pub(crate) fn finish(mut self, ref_id: usize) -> ContigCoveragePartial { let out = self.c.finish(&self.params, &mut self.g); ContigCoveragePartial { @@ -776,9 +817,10 @@ impl ContigCoverageAccum { } } -/// Merge per-contig coverage partials into the genome-wide [`CoverageResult`]: sum the -/// histogram/territory/depth accumulators and order the per-contig outputs by `ref_id` (header -/// order), so the result is byte-identical to the sequential walker's. +/// Merge the coverage partials of the contigs into the genome-wide [`CoverageResult`]. It adds up +/// the accumulators of the histogram, the territory and the depth. It then puts the output of each +/// contig into `ref_id` order, which is the header order. The result then matches that of the +/// sequential walker, to the last digit. pub(crate) fn merge_coverage_partials(mut partials: Vec) -> CoverageResult { partials.sort_by_key(|p| p.ref_id); let mut hist = vec![0u64; HIST_LEN]; @@ -810,10 +852,13 @@ pub(crate) fn merge_coverage_partials(mut partials: Vec) ) } -/// Mean read length and mean fragment (template) length, sampled from the first ~50k -/// primary mapped reads. The molecule-length proxy for the self-referential callable -/// run-length gate (long reads → long molecules → long callable runs). Fragment falls -/// back to read length when templates are unpaired (e.g. long-read single-end). +/// The mean read length and the mean fragment length, which is the template length. The code +/// samples the first 50k primary mapped reads, or about that many. +/// +/// This is the proxy for the molecule length, which the callable run-length gate needs, and that +/// gate refers to itself. Long reads mean long molecules, and long molecules mean long callable +/// runs. The fragment length falls back to the read length when the templates have no pair, as in +/// single-end long-read data. pub fn estimate_molecule_lengths(bam_path: &Path, reference: Option<&Path>) -> Result<(f64, f64), AnalysisError> { let (header, mut reader) = reader::open_seq(bam_path, reference)?; @@ -830,9 +875,10 @@ pub fn estimate_molecule_lengths(bam_path: &Path, reference: Option<&Path>) -> R } read_sum += len; n += 1; - // Fragment length only from properly-paired reads, with a sanity cap — chimeric or - // improper pairs carry enormous |TLEN| that would otherwise blow up the mean (and - // the run-length gate). Single-end / long reads have no proper pairs -> read-length. + // Take the fragment length from a read with a correct pair alone, and put an upper limit + // on it. A chimeric pair, or a pair that is not correct, carries a very large |TLEN|. + // That would move the mean far away, and the run-length gate with it. A single-end read + // and a long read have no correct pair, so those fall back to the read length. if f.is_properly_segmented() { let tlen = record.template_length().unsigned_abs() as u64; if tlen > 0 && tlen < 100_000 { @@ -856,10 +902,13 @@ pub fn estimate_molecule_lengths(bam_path: &Path, reference: Option<&Path>) -> R Ok((read_len, frag_len)) } -/// CALLABLE intervals (BED 0-based half-open) on one `contig`, coalesced and kept only -/// when the run is at least `min_run_len` bases. Reference-free: positions are classified -/// by depth / QC / MAPQ via the GATK hierarchy (reference-N regions carry no reads and -/// fall out as no-coverage). Memory is bounded by the open-read window; needs a BAM index. +/// The CALLABLE intervals on one `contig`, in BED form, which is 0-based and half-open. The code +/// joins the intervals that touch, and it keeps a run only when that run holds `min_run_len` bases +/// or more. +/// +/// It needs no reference. It classifies a position by the depth, the QC flags and the MAPQ, +/// through the GATK hierarchy. A region of N bases in the reference carries no read, and it comes +/// out as no-coverage. The window of open reads bounds the memory. This needs a BAM index. pub fn callable_intervals( bam_path: &Path, contig: &str, @@ -965,8 +1014,9 @@ pub fn callable_intervals( Ok(intervals) } -/// GATK `CallableLoci` hierarchy — first failing condition wins. Mirrors the Scala -/// `determineCallableState`. `ref_is_n` is whether the reference base is N (non-callable). +/// The hierarchy of GATK `CallableLoci`. The first condition that fails wins. It has the same +/// shape as the Scala `determineCallableState`. `ref_is_n` is true when the reference base is N, +/// and such a base is not callable. fn determine_state( ref_is_n: bool, depth: u32, @@ -1001,9 +1051,9 @@ fn pct_at_least(hist: &[u64], total: u64, min_depth: usize) -> f64 { at_least as f64 / total as f64 } -/// Median absolute deviation of depth: the median of `|depth − median|` over the depth histogram. -/// Depths are clamped at index 255 in the histogram, so deviations in that tail are lower bounds -/// (negligible for typical WGS coverage). +/// The median absolute deviation of the depth. It is the median of `|depth − median|` over the +/// depth histogram. The histogram stops the depth at index 255, so a deviation in that tail is a +/// lower bound. At a usual WGS coverage that makes no difference. fn mad_from_hist(hist: &[u64], total: u64, median: f64) -> f64 { if total == 0 { return 0.0; @@ -1045,9 +1095,10 @@ mod tests { #[test] fn add_tolerates_a_base_before_the_finalize_frontier() { - // A non-coordinate-sorted input can present a base whose position is *behind* the window's - // finalize frontier. That must be dropped (and counted), not underflow `pos - emit_cursor` - // (the regression: coverage.rs panicked "attempt to subtract with overflow"). + // An input that is not in coordinate order can give a base whose position is *behind* the + // finalize frontier of the window. The code must drop that base, and count it. It must not + // let `pos - emit_cursor` go below zero. That was the bug: coverage.rs panicked with + // `attempt to subtract with overflow`. let params = CallableLociParams::default(); let mut g = Globals::new(); let mut c = CurContig::new("chrT".into(), 100, vec![b'A'; 100]); @@ -1061,7 +1112,8 @@ mod tests { c.add(60, 30, 60, ¶ms); assert_eq!(c.dropped_unsorted, 1, "an in-order base is not dropped"); - // Finishing completes without panicking and yields the contig's stats. + // The finish runs to its end, it does not panic, and it gives the statistics of the + // contig. let out = c.finish(¶ms, &mut g); assert_eq!(out.stats.end_pos, 100); } @@ -1071,8 +1123,9 @@ mod tests { let bam = fixture("coverage.bam"); // chrM, 50 bp, well covered let params = CallableLociParams::default(); - // No run-length gate: some callable bases, all within the 50 bp contig, intervals - // sorted and non-overlapping (BED 0-based half-open). + // With no run-length gate, there are some callable bases, and all of them lie inside the + // 50 bp contig. The intervals come in sorted order, and none of them overlaps another. The + // form is BED, which is 0-based and half-open. let ivs = callable_intervals(&bam, "chrM", ¶ms, 1, None).unwrap(); assert!(!ivs.is_empty(), "expected callable intervals on the fixture"); let callable_bases: i64 = ivs.iter().map(|(s, e)| e - s).sum(); @@ -1112,8 +1165,9 @@ mod tests { fn per_contig_histograms_sum_to_genome_wide() { let params = CallableLociParams::default(); - // chrM-only and a multi-contig (autosomes + chrX) fixture, so the sum invariant is - // exercised across more than one contig. + // One fixture holds chrM alone. The other holds more than one contig, which are the + // autosomes and chrX. The test thereby covers the sum invariant across more than one + // contig. for (bam_name, ref_name) in [("coverage.bam", "ref.fa"), ("sex.bam", "sexref.fa")] { let cov = collect_coverage_callable(&fixture(bam_name), &fixture(ref_name), ¶ms, None).unwrap(); assert!( @@ -1139,8 +1193,8 @@ mod tests { } } - // The strong invariant: per-contig histograms reconstruct the genome-wide histogram - // exactly (the genome-wide one is just their bin-wise sum). + // The strong invariant. The histograms of the contigs build the genome-wide histogram + // exactly. The genome-wide one is their sum, bin by bin. assert_eq!( summed, cov.coverage_histogram, "{bam_name}: per-contig histograms must sum bin-for-bin to the genome-wide histogram" diff --git a/crates/navigator-analysis/src/error.rs b/crates/navigator-analysis/src/error.rs index ef47d261..82745d86 100644 --- a/crates/navigator-analysis/src/error.rs +++ b/crates/navigator-analysis/src/error.rs @@ -1,4 +1,4 @@ -//! Error type for the analysis layer (plan §6: one `thiserror` enum per layer). +//! The error type of the analysis layer. See plan §6: one `thiserror` enum in each layer. use std::path::PathBuf; @@ -14,11 +14,12 @@ pub enum AnalysisError { #[error("{0}")] Message(String), - /// The walk stopped because cancellation was requested (see [`crate::cancel`]). + /// The walk stopped because somebody asked for a cancel. See [`crate::cancel`]. /// - /// A distinct variant, not a `Message`, because callers must be able to tell a user-requested - /// stop from a real failure: a cancelled walk holds a *partial* result, so its caller has to - /// skip persisting it, and the UI has to report "cancelled" rather than an error. + /// This is a variant of its own, and not a `Message`. A caller must be able to separate a stop + /// that a user asked for from a real failure. A walk that somebody cancelled holds a *partial* + /// result, so its caller must not put that result into the store. And the UI must report + /// "cancelled", and not an error. #[error("cancelled")] Cancelled, } @@ -32,8 +33,8 @@ impl AnalysisError { } } -/// The text a panic carried, if any — `panic!("…")` payloads are always a `&'static str` or a -/// `String`. Used to surface *what* actually went wrong instead of guessing at a cause. +/// The text that a panic carried, when it carried one. The payload of a `panic!("…")` is always a +/// `&'static str` or a `String`. Use it to show *what* went wrong, and do not guess at a cause. pub fn panic_text(payload: &(dyn std::any::Any + Send)) -> Option<&str> { payload .downcast_ref::<&'static str>() @@ -41,20 +42,24 @@ pub fn panic_text(payload: &(dyn std::any::Any + Send)) -> Option<&str> { .or_else(|| payload.downcast_ref::().map(String::as_str)) } -/// Run a BAM/CRAM walk, converting a **panic** into a clean [`AnalysisError`] so one undecodable -/// file fails gracefully instead of unwinding into a cryptic `JoinError`/aborting a worker. The -/// motivating cases are noodles' `todo!()`/`expect()` on inputs it doesn't handle (an unimplemented -/// CRAM data series, or a decode that needs reference bases it wasn't given): without this, such a -/// file panics deep inside the decoder. `what` labels the operation/file for the surfaced message. +/// Run a walk over a BAM or CRAM, and turn a **panic** into a clean [`AnalysisError`]. One file +/// that the code can not decode then fails cleanly. It does not unwind into a `JoinError` that +/// says nothing, and it does not abort a worker. /// -/// The panic's own text is included rather than a guessed explanation — this is a last-resort net, -/// so it does not know which limitation it caught. Callers that *do* know (see -/// [`crate::index::ensure_index`]) should diagnose the specific case themselves and say what to do -/// about it; anything reaching here is genuinely unclassified. +/// The cases that led to this are the `todo!()` and `expect()` calls of noodles, on an input that +/// it does not handle. A CRAM data series that nobody implemented is one. A decode that needs +/// reference bases which nobody gave it is another. Without this net, such a file panics deep +/// inside the decoder. `what` names the operation and the file, for the message that goes out. /// -/// `AssertUnwindSafe` is sound here: on a caught panic we discard `f`'s partial state entirely and -/// return an error — no possibly-inconsistent value crosses the boundary. The default panic hook -/// still prints the original message to stderr (useful diagnostics); only the control flow changes. +/// The message holds the own text of the panic, and not an explanation that this code guessed. +/// This is a net of last resort, so it does not know which limit it caught. A caller that *does* +/// know must diagnose its own case and say what to do about it. See +/// [`crate::index::ensure_index`]. This code can put no class on anything that reaches here. +/// +/// `AssertUnwindSafe` is sound here. On a panic that this code catches, it throws away the whole +/// partial state of `f`, and it returns an error. No value that could be inconsistent crosses the +/// boundary. The default panic hook still prints the original message to stderr, which is useful, +/// and only the control flow changes. pub fn guard_walk(what: &str, f: impl FnOnce() -> Result) -> Result { std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).unwrap_or_else(|payload| { let detail = panic_text(&*payload).unwrap_or("no further detail"); diff --git a/crates/navigator-analysis/src/genotype.rs b/crates/navigator-analysis/src/genotype.rs index 28a2e63c..557ccf3d 100644 --- a/crates/navigator-analysis/src/genotype.rs +++ b/crates/navigator-analysis/src/genotype.rs @@ -1,16 +1,22 @@ -//! Genotype-likelihood model for biallelic genotyping at a known site (the foundation -//! the population / ancestry / IBD paths need — they consume dosage 0/1/2 + a quality). +//! The genotype-likelihood model, for a genotype over two alleles at a known site. The population, +//! ancestry and IBD paths all stand on it, because each of them reads a dosage of 0, 1 or 2, and a +//! quality. //! -//! Standard GATK/bcftools model. For a site with reference allele R and alternate A, -//! and per-read base observations with phred base qualities, the per-base error is -//! `e = 10^(-Q/10)` and `P(base | allele) = 1-e` on a match, `e/3` on a mismatch. For a -//! genotype carrying `g` alt copies out of `ploidy` P, the allele pool gives -//! `P(base | g) = [ (P-g)·P(base|R) + g·P(base|A) ] / P`. The genotype log-likelihoods -//! are summed over reads; the call is the argmax, with phred-scaled likelihoods (PL, -//! best = 0) and genotype quality `GQ` = the second-smallest PL. +//! This is the standard model of GATK and bcftools. Take a site with a reference allele R and an +//! alternate allele A, and base observations from the reads, each with a phred base quality. The +//! error at one base is `e = 10^(-Q/10)`. Then `P(base | allele) = 1-e` on a match, and `e/3` on a +//! mismatch. //! -//! `ploidy` is supplied by the caller (sex → `sex::ploidy_for_contig`): 2 for autosomes -//! / female chrX, 1 for chrY / chrM / male chrX. Biallelic (ref + one alt) for v1. +//! Take a genotype that carries `g` alt copies, out of a `ploidy` of P. The pool of alleles then +//! gives `P(base | g) = [ (P-g)·P(base|R) + g·P(base|A) ] / P`. +//! +//! The code adds the genotype log-likelihoods over the reads. The call is the argmax. It also gives +//! the likelihoods on the phred scale, as PL, where the best is 0, and the genotype quality `GQ`, +//! which is the second-smallest PL. +//! +//! The caller gives the `ploidy`, from the sex, through `sex::ploidy_for_contig`. It is 2 on an +//! autosome and on a female chrX. It is 1 on chrY, on chrM and on a male chrX. v1 handles two +//! alleles: the ref and one alt. /// Result of genotyping one site. #[derive(Debug, Clone, PartialEq)] @@ -21,7 +27,7 @@ pub struct GenotypeResult { pub pls: Vec, /// Genotype quality (phred), capped at 99. pub gq: u8, - /// Passing observations (ACGT bases clearing the quality filters). + /// The observations that pass, which are the ACGT bases that clear the quality filters. pub depth: u32, pub ref_depth: u32, pub alt_depth: u32, @@ -41,7 +47,7 @@ fn no_call(ploidy: u8, depth: u32, ref_depth: u32, alt_depth: u32) -> GenotypeRe } } -/// Call a biallelic genotype from passing `(base, phred_qual)` observations. +/// Call a genotype over two alleles, from the `(base, phred_qual)` observations that pass. pub fn call_genotype( observations: &[(u8, u8)], reference_allele: u8, @@ -104,17 +110,22 @@ pub struct MultiGenotype { pub gt: (usize, usize), pub gq: u8, pub depth: u32, - /// Reads supporting each allele index (0 = ref). + /// The count of reads behind each allele index, where 0 is the ref. pub allele_depths: Vec, - /// Phred-scaled per-genotype likelihoods in VCF order (`for j: for i in 0..=j`); best = 0. + /// The likelihood of each genotype, on the phred scale, in VCF order, which is + /// `for j: for i in 0..=j`. The best one is 0. pub pls: Vec, } -/// Diploid genotype likelihood over `n_alleles` (index 0 = reference). Observations are -/// `(allele_index, phred_qual)`; an index `>= n_alleles` is a non-supporting read. Same per-base -/// error model as [`call_genotype`], generalized to a pooled diploid allele pair -/// `P(obs|{i,j}) = ½·P(obs|i) + ½·P(obs|j)`. Returns the best allele pair, per-allele depths, -/// per-genotype PLs (VCF order), and GQ (second-smallest PL). +/// The diploid genotype likelihood over `n_alleles`, where index 0 is the reference. An +/// observation is `(allele_index, phred_qual)`, and an index of `n_alleles` or more marks a read +/// that supports nothing. +/// +/// The error model at each base is the same as in [`call_genotype`]. It goes to a pooled diploid +/// pair of alleles, as `P(obs|{i,j}) = ½·P(obs|i) + ½·P(obs|j)`. +/// +/// It returns four things. The best pair of alleles. The depth of each allele. The PL of each +/// genotype, in VCF order. And the GQ, which is the second-smallest PL. pub fn call_genotype_multi(observations: &[(usize, u8)], n_alleles: usize, min_depth: u32) -> MultiGenotype { let n = n_alleles.max(1); let mut allele_depths = vec![0u32; n]; diff --git a/crates/navigator-analysis/src/gvcf.rs b/crates/navigator-analysis/src/gvcf.rs index 5180972c..7e562091 100644 --- a/crates/navigator-analysis/src/gvcf.rs +++ b/crates/navigator-analysis/src/gvcf.rs @@ -1,24 +1,27 @@ //! Read called haploid bases at target positions from a GATK ploidy-1 GVCF. //! -//! The `ytree` pipeline archives a per-sample chrY/chrM GVCF (HaplotypeCaller -//! `--sample-ploidy 1 -ERC GVCF`) next to each CRAM. Those GVCFs already contain exactly -//! what [`crate::caller::call_bases_at`] would recompute by walking the (multi-GB) CRAM at -//! every haplotree position — the *observed haploid base* at each site. Reading the small -//! GVCF instead of the CRAM is the fast path for haplogroup placement. +//! The `ytree` pipeline keeps a chrY and chrM GVCF for each sample, beside each CRAM. It comes +//! from HaplotypeCaller with `--sample-ploidy 1 -ERC GVCF`. Those GVCFs already hold exactly what +//! [`crate::caller::call_bases_at`] would compute again, by a walk over the CRAM, which is some +//! GB, at every haplotree position. That is the *observed haploid base* at each site. To read the +//! small GVCF, and not the CRAM, is the fast path of a haplogroup placement. //! -//! GVCF semantics (ploidy 1, `` model): -//! - **Variant record** (`ALT` has a real allele besides ``): -//! `GT=1` → the sample carries the ALT. SNP → that base is the *derived* observation; -//! indel → confident but not a usable SNP base (skipped). `GT=0` → confident hom-ref -//! (an *ancestral* observation at a multiallelic emit site). -//! - **Ref block** (`ALT=`, `GT=0`, `END=` in INFO): every position in `[POS,END]` -//! was called hom-ref (ancestral) at the block's confidence. +//! Here is what a GVCF means, at ploidy 1, under the `` model: //! -//! This module decodes the GVCF to two facts per target: the *derived base* where one was -//! called, and whether the site was *callable* at all. [`assemble_calls`] then turns those -//! into the `position → observed base` map [`crate::haplo::score`] consumes — using the -//! tree's ancestral allele for callable-but-not-variant sites (on the native build the -//! reference base == the tree's ancestral allele, so no FASTA lookup is needed). +//! - A **variant record** has a real allele in `ALT`, beside ``. `GT=1` means that the +//! sample carries the ALT. At a SNP, that base is the *derived* observation. At an indel, the +//! call is confident, but it gives no usable SNP base, so the code skips it. `GT=0` means a +//! confident hom-ref, which is an *ancestral* observation at a site with more than two alleles. +//! - A **ref block** has `ALT=`, `GT=0`, and an `END=` in its INFO. Every position in +//! `[POS,END]` got a hom-ref, which is ancestral, at the confidence of that block. +//! +//! This module decodes the GVCF to two facts at each target. The first is the *derived base*, +//! where the caller called one. The second is whether the site was *callable* at all. +//! +//! [`assemble_calls`] then turns those into the `position → observed base` map that +//! [`crate::haplo::score`] reads. At a site that is callable and not a variant, it uses the +//! ancestral allele of the tree. On the native build the reference base equals that ancestral +//! allele, so the code needs no lookup in the FASTA. use std::collections::{HashMap, HashSet}; use std::io::BufRead; @@ -28,10 +31,11 @@ use noodles::bgzf; use crate::error::AnalysisError; -/// Confidence thresholds for trusting a GVCF call. Ref blocks are gated on `MIN_DP` -/// (falling back to `DP`) and `GQ`; variant records on `DP`/`GQ`. Defaults are permissive -/// enough for low-coverage HiFi (the pipeline's ref blocks carry GQ 70–99) while rejecting -/// genuinely unsupported sites. +/// The confidence thresholds that a GVCF call must meet. A ref block goes through a gate on +/// `MIN_DP`, or on `DP` when `MIN_DP` is absent, and on `GQ`. A variant record goes through a gate +/// on `DP` and `GQ`. The defaults are open enough for HiFi data at a low coverage, where the ref +/// blocks of the pipeline carry a GQ of 70 to 99. They still refuse a site that the data does not +/// support at all. #[derive(Debug, Clone, Copy)] pub struct GvcfReadParams { pub min_dp: u32, @@ -44,14 +48,14 @@ impl Default for GvcfReadParams { } } -/// The two facts decoded per target position from the GVCF. +/// The two facts that the code decodes from the GVCF at each target position. #[derive(Debug, Clone, Default)] pub struct CalledBases { /// SNP-derived ALT base (uppercase) at target sites where the sample carries a /// single-base ALT (`GT=1`). pub variant_bases: HashMap, - /// Target positions the GVCF confidently called (a passing ref block, a hom-ref - /// variant emit, or a confident SNP) — i.e. *not* a no-call. + /// The target positions where the GVCF made a confident call. That is a ref block that passes + /// the gate, a hom-ref variant record, or a confident SNP. None of these is a no-call. pub callable: HashSet, } @@ -66,17 +70,19 @@ pub fn read_called_bases( read_called_bases_from(bgzf::io::Reader::new(file), contig, targets, params) } -/// Decode core over any `BufRead` (plain-text VCF in tests). Streams the whole file — -/// these GVCFs are small (chrY ~3 MB, chrM ~6 KB) and the targets are a few thousand -/// scattered positions, so a single linear pass beats per-target tabix seeks. +/// The decode core, over any `BufRead`. A test gives it a plain-text VCF. It streams the whole +/// file. These GVCFs are small, at about 3 MB for chrY and about 6 KB for chrM. The targets are a +/// few thousand positions spread over the contig. So one linear pass is faster than a tabix seek +/// at each target. pub fn read_called_bases_from( mut reader: R, contig: &str, targets: &HashSet, params: &GvcfReadParams, ) -> Result { - // Sorted targets so a ref block's [POS, END] span resolves by binary search instead of - // iterating the (potentially thousands-wide) block. + // The targets come in sorted order, so a binary search resolves the [POS, END] span of a ref + // block. Without that, the code would walk the block, and a block can hold thousands of + // positions. let mut sorted: Vec = targets.iter().copied().collect(); sorted.sort_unstable(); @@ -159,7 +165,8 @@ pub fn read_called_bases_from( out.callable.insert(pos); continue; } - // GT carries an ALT. First real ALT (skip the trailing ). + // The GT carries an ALT. Take the first real ALT, and skip the at the + // end. let alt0 = alt.split(',').find(|a| *a != "").unwrap_or(""); if refa.len() == 1 && alt0.len() == 1 { let b = alt0.as_bytes()[0].to_ascii_uppercase(); @@ -168,29 +175,35 @@ pub fn read_called_bases_from( out.callable.insert(pos); } } - // An indel ALT at a (SNP) tree position is left as a no-call rather than - // asserted ancestral — conservative; avoids a false ancestral refutation. + // An indel ALT at a tree position, which is a SNP, stays a no-call. The code does not + // call it ancestral. That is the careful direction, and it prevents a false ancestral + // call that would contradict a branch. } } Ok(out) } -/// One target's diploid call from a GATK gVCF: the two alleles at a variant site, or a confident -/// hom-ref ref block (the caller supplies the reference allele — from the panel — at a hom-ref site, -/// since the gVCF's ref block only stores the base at its start position). +/// The diploid call of one target, from a GATK gVCF. It is the two alleles at a variant site, or a +/// confident hom-ref ref block. At a hom-ref site the caller gives the reference allele, and it +/// takes that from the panel. The ref block of a gVCF stores the base at its start position +/// alone. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GvcfDiploid { /// Both alleles the sample carries at a variant record (uppercase A/C/G/T), from `GT`. Genotype(char, char), - /// Covered by a passing hom-ref block — homozygous for the reference allele. + /// A hom-ref block that passes the gate covers this site. It is homozygous for the reference + /// allele. HomRef, } -/// Genotype a set of panel targets (grouped by contig, each **sorted**) from a **diploid** GATK gVCF -/// in a single linear pass — the autosomal (ploidy-2) counterpart to [`read_called_bases`]. Variant -/// records yield the `GT` alleles; passing ref blocks yield [`GvcfDiploid::HomRef`]; uncovered, -/// low-quality, indel, or ``-allele sites are left absent (no-call). Transparently reads a -/// plain or gzip/BGZF gVCF. +/// Genotype a set of panel targets from a **diploid** GATK gVCF, in one linear pass. The targets +/// come in groups by contig, and each group is **sorted**. This is the autosomal counterpart, at +/// ploidy 2, of [`read_called_bases`]. +/// +/// A variant record gives the `GT` alleles. A ref block that passes the gate gives +/// [`GvcfDiploid::HomRef`]. A site that no record covers, one of low quality, an indel, or one +/// whose allele is ``, stays absent, which is a no-call. This reads a plain gVCF, and a +/// gzip or BGZF one, and the caller sees no difference. pub fn read_diploid_calls( gvcf: &Path, targets_by_contig: &HashMap>, @@ -200,8 +213,9 @@ pub fn read_diploid_calls( read_diploid_calls_from(reader, targets_by_contig, params) } -/// Decode core over any `BufRead` (plain-text gVCF in tests). One linear pass; a whole-genome gVCF is -/// large but reading it is far cheaper than decoding the CRAM it was called from. +/// The decode core, over any `BufRead`. A test gives it a plain-text gVCF. It makes one linear +/// pass. A gVCF over the whole genome is large. But to read it costs much less than a decode of +/// the CRAM that the caller called it from. pub fn read_diploid_calls_from( mut reader: R, targets_by_contig: &HashMap>, @@ -247,7 +261,8 @@ pub fn read_diploid_calls_from( let idxs: Vec<&str> = gt.split(['/', '|']).collect(); if alt == "" { - // Ref block: confident hom-ref over [POS, END]. Require an all-ref GT (0/0). + // A ref block: a confident hom-ref over [POS, END]. The GT must be all-ref, which is + // 0/0. if idxs.iter().any(|a| *a != "0") { continue; } @@ -260,7 +275,7 @@ pub fn read_diploid_calls_from( } let end = info_end(info).unwrap_or(pos); for &t in targets_in_range(sorted, pos, end) { - // Don't overwrite a variant call (variant records are authoritative; in a well-formed + // Do not overwrite a variant call (variant records are authoritative; in a well-formed // gVCF they never overlap a ref block anyway). out.entry((chrom.to_string(), t)).or_insert(GvcfDiploid::HomRef); } @@ -288,13 +303,14 @@ pub fn read_diploid_calls_from( Some(i) => i, None => continue, }; - // A single index (a haploid emit on an autosome) is read as homozygous. + // One index alone, which is a haploid record on an autosome, counts as homozygous. let i1: usize = idxs.get(1).and_then(|s| s.parse().ok()).unwrap_or(i0); if let (Some(a), Some(b)) = (allele_at(i0), allele_at(i1)) { out.insert((chrom.to_string(), pos), GvcfDiploid::Genotype(a, b)); } - // An indel / `` allele at a target is left as a no-call (absent), never - // asserted hom-ref — conservative, same as the haploid path. + // An indel allele at a target, and a `` one, stay a no-call, and the map + // does not hold them. The code never calls them hom-ref. That is the careful + // direction, and the haploid path does the same. } } Ok(out) @@ -313,11 +329,15 @@ pub struct GvcfSnv { pub gq: u32, } -/// Stream **every** confident derived single-base SNV on `contig` from a ploidy-1 GVCF — the whole -/// chrY variant set, not just tree targets. GATK's HaplotypeCaller does local haplotype reassembly, -/// which resolves sites a pileup caller can't (misaligned ref reads → false ~50/50), so reading the -/// GVCF recovers private SNVs the de-novo pileup caller drops. Ref blocks, hom-ref, and indel records -/// are skipped; records are gated on `params.min_dp` / `params.min_gq`. +/// Stream **every** confident derived single-base SNV on `contig`, from a ploidy-1 GVCF. That is +/// the whole variant set of chrY, and not the tree targets alone. +/// +/// The HaplotypeCaller of GATK does a local reassembly of the haplotypes. That resolves a site +/// where a pileup caller can not, such as one where misaligned reference reads give a false 50/50. +/// So a read of the GVCF recovers a private SNV that the de-novo pileup caller drops. +/// +/// The code skips a ref block, a hom-ref record and an indel record. It gates the other records on +/// `params.min_dp` and `params.min_gq`. pub fn read_derived_snvs(gvcf: &Path, contig: &str, params: &GvcfReadParams) -> Result, AnalysisError> { let file = std::fs::File::open(gvcf).map_err(|e| AnalysisError::io(gvcf, e))?; read_derived_snvs_from(bgzf::io::Reader::new(file), contig, params) @@ -405,13 +425,18 @@ pub fn read_derived_snvs_from( Ok(out) } -/// Per-target genotype evidence for the branch-report tool. Unlike [`read_called_bases`] / -/// [`read_derived_snvs`] this is **not gated** on depth/quality — a spot-check report wants to show -/// low-confidence evidence too (the *call* comes from a separate, gated pass). A target covered by a -/// confident `` ref block reports `refblock: true` with the block `GQ` (DP/AD omitted — those -/// are the "full" MIN_DP columns); a variant record reports `DP`, `AD = (ref, carried-alt)`, and `GQ`. -/// A variant record overrides a ref block at the same position; positions with no covering record are -/// absent from the map (no-call). +/// The genotype evidence at each target, for the branch-report tool. +/// +/// [`read_called_bases`] and [`read_derived_snvs`] gate on the depth and the quality. This function +/// does **not**. A report that somebody uses to check one branch wants to see the evidence of low +/// confidence too. The *call* itself comes from a separate pass, and that pass does gate. +/// +/// A target that a confident `` ref block covers reports `refblock: true`, with the `GQ` +/// of that block. It leaves out the DP and the AD, because those are the "full" MIN_DP columns. A +/// variant record reports `DP`, `AD = (ref, the alt that the sample carries)`, and `GQ`. +/// +/// A variant record wins over a ref block at the same position. A position that no record covers +/// is absent from the map, which is a no-call. #[derive(Debug, Clone, Copy, PartialEq)] pub struct GvcfSiteEvidence { /// Carried GVCF allele index: `0` = hom-ref/ancestral, `≥1` = a derived ALT, `None` = missing GT. @@ -422,7 +447,8 @@ pub struct GvcfSiteEvidence { pub refblock: bool, } -/// Read per-target [`GvcfSiteEvidence`] from a ploidy-1 GVCF (ungated). See [`GvcfSiteEvidence`]. +/// Read a [`GvcfSiteEvidence`] at each target, from a ploidy-1 GVCF, with no gate. See +/// [`GvcfSiteEvidence`]. pub fn read_site_evidence( gvcf: &Path, contig: &str, @@ -521,17 +547,23 @@ pub fn read_site_evidence_from( Ok(out) } -/// Assemble the `position → observed base` map [`crate::haplo::score`] consumes from the -/// decoded GVCF facts. A variant (derived) base wins; otherwise a callable hom-ref site takes -/// the **reference genome base** at that position (`ref_base`); otherwise the position is a -/// no-call and is omitted. +/// Build the `position → observed base` map that [`crate::haplo::score`] reads, from the facts +/// that the code decoded out of the GVCF. +/// +/// A variant base, which is the derived one, wins. Otherwise a callable hom-ref site takes the +/// **base of the reference genome** at that position, which is `ref_base`. Otherwise the position +/// is a no-call, and the map leaves it out. +/// +/// `ref_base` must be the *reference* base, and not the ancestral allele of the tree. The two +/// differ wherever the reference itself carries a derived allele. +/// +/// The Y of CHM13 is HG002, which is haplogroup J1, deep in the tree. So at every backbone SNP +/// that J1 shares with the sample, the GVCF gives a ref block. There hom-ref equals the reference, +/// and it equals the *derived* allele. To read ancestral there would break the descent, and nobody +/// would see it happen. /// -/// `ref_base` must be the *reference* base, not the tree ancestral — the two differ wherever -/// the reference itself carries a derived allele. CHM13's Y is HG002 (haplogroup J1, deep in -/// the tree), so at every backbone SNP shared by J1 and the sample the GVCF emits a ref block -/// (hom-ref == reference == *derived*), and assuming ancestral there would silently break the -/// descent. This mirrors [`crate::caller::call_bases_at`], which reads the actual base off the -/// reads (== the reference base at a hom-ref site). +/// This has the same shape as [`crate::caller::call_bases_at`], which reads the base off the reads +/// themselves. At a hom-ref site that base is the reference base. pub fn assemble_calls(called: &CalledBases, ref_base: &HashMap) -> HashMap { let mut calls: HashMap = HashMap::with_capacity(called.callable.len()); for &pos in &called.callable { @@ -552,9 +584,10 @@ fn format_field<'a>(format: &str, sample: &'a str, key: &str) -> Option<&'a str> sample.split(':').nth(idx) } -/// Whether a record passes the `GQ` gate. GQ is only enforced when the record **carries** it — a -/// `bcftools mpileup` call set has no `GQ` (`GT:PL:DP:AD`), and gating an absent GQ as 0 would drop -/// every site. A GATK gVCF does carry GQ, so its low-confidence sites are still filtered. +/// True when a record passes the `GQ` gate. The gate applies only when the record **carries** a +/// GQ. A call set from `bcftools mpileup` has no `GQ`, because its format is `GT:PL:DP:AD`. To +/// read an absent GQ as 0 would drop every site. A GATK gVCF does carry a GQ, so this gate still +/// removes its sites of low confidence. fn gq_passes(format: &str, sample: &str, params: &GvcfReadParams) -> bool { match format_field(format, sample, "GQ").and_then(|s| s.parse::().ok()) { Some(gq) => gq >= params.min_gq, @@ -597,7 +630,8 @@ chrM\t100\t.\tC\tT,\t500\t.\tDP=30\tGT:AD:DP:GQ:PL\t1:0,30,0:30:99:510, #[test] fn derived_snvs_streams_snps_skips_blocks_indels_and_other_contigs() { let v = read_derived_snvs_from(SAMPLE_GVCF.as_bytes(), "chrY", &GvcfReadParams::default()).unwrap(); - // The two chrY SNVs, in order; the ref block, the A>AT indel, and the chrM SNV are skipped. + // The two chrY SNVs, in order. The code skips the ref block, the A>AT indel, and the chrM + // SNV. let got: Vec<(i64, char, char)> = v.iter().map(|s| (s.position, s.reference, s.alternate)).collect(); assert_eq!(got, vec![(2459921, 'G', 'A'), (2477255, 'C', 'T')]); // AD = 0,18,0 → alt-depth 18, af 1.0. @@ -713,11 +747,13 @@ chrM\t100\t.\tC\tT,\t500\t.\tDP=30\tGT:AD:DP:GQ:PL\t1:0,30,0:30:99:510, assert_eq!(targets_in_range(&s, 40, 100), &[40]); } - /// Real-data smoke test: decode the pipeline's actual bgzipped chrY GVCF for HG00096 - /// over a dense synthetic target grid across the non-PAR span. Validates real bgzf - /// inflation + record parsing at scale (thousands of records). No-ops when the NAS - /// file isn't mounted, so it's safe on any machine. Run with: - /// cargo test -p navigator-analysis gvcf -- --ignored --nocapture + /// A smoke test on real data. It decodes the bgzipped chrY GVCF of the pipeline, for HG00096, + /// over a dense synthetic grid of targets across the non-PAR span. It checks the real bgzf + /// inflation, and the parse of the records, at scale, over thousands of records. + /// + /// It does nothing when the NAS file is not mounted, so it is safe on any machine. Run it + /// with: + /// `cargo test -p navigator-analysis gvcf -- --ignored --nocapture` #[test] #[ignore = "reads a NAS file; run explicitly"] fn real_chr_y_gvcf_decodes() { @@ -726,7 +762,7 @@ chrM\t100\t.\tC\tT,\t500\t.\tDP=30\tGT:AD:DP:GQ:PL\t1:0,30,0:30:99:510, eprintln!("skip: {} not mounted", path.display()); return; } - // Every 50th base across the non-PAR region — a stand-in for tree positions. + // Every 50th base across the non-PAR region. It stands in for the tree positions. let t: HashSet = (2_458_321..62_122_809).step_by(50).collect(); let c = read_called_bases(path, "chrY", &t, &GvcfReadParams::default()).unwrap(); eprintln!( diff --git a/crates/navigator-analysis/src/gzio.rs b/crates/navigator-analysis/src/gzio.rs index 98087306..9f25f18e 100644 --- a/crates/navigator-analysis/src/gzio.rs +++ b/crates/navigator-analysis/src/gzio.rs @@ -1,20 +1,25 @@ //! Transparent decompression for line-oriented genomics text (VCF, BED, CompleteGenomics //! masterVar). //! -//! BGZF — the bgzip format used for `.vcf.gz` / `.bed.gz` — is a *concatenation* of -//! independent gzip members (each ≤64 KiB of payload). `flate2::read::GzDecoder` decodes -//! only the first member and then reports EOF, silently truncating any multi-block -//! bgzipped file; `MultiGzDecoder` decodes every member, so it reads plain gzip and BGZF -//! whole. bzip2 (the `.tsv.bz2` CompleteGenomics ships) is handled the same way via -//! `MultiBzDecoder`, which spans concatenated streams (pbzip2 output). Detection is by the -//! leading magic bytes, not the extension, so a compressed file is handled even when it is -//! misnamed (e.g. a `.vcf` that is really bgzf). +//! BGZF is the bgzip format of a `.vcf.gz` or a `.bed.gz`. It is a *chain* of independent gzip +//! members, and each one holds 64 KiB of payload or less. +//! +//! `flate2::read::GzDecoder` decodes the first member alone, and it then reports an end of file. +//! So it cuts short any bgzipped file of more than one block, and nobody sees it happen. +//! `MultiGzDecoder` decodes every member, so it reads a plain gzip file and a BGZF file whole. +//! +//! bzip2 gets the same treatment, through `MultiBzDecoder`, which reads a chain of streams that +//! pbzip2 wrote. CompleteGenomics ships a `.tsv.bz2`. +//! +//! The code finds the compression from the first bytes of the file, and not from the extension. So +//! it handles a compressed file even when its name is wrong, such as a `.vcf` that holds bgzf. use std::fs::File; use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom}; use std::path::Path; -/// Detected on-disk compression, by leading magic bytes. +/// The compression that the file on disk uses, which the code finds from the first bytes of that +/// file. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Compression { None, @@ -22,10 +27,11 @@ enum Compression { Bzip2, } -/// Open `path` for buffered line reading, transparently decoding gzip/BGZF when the file -/// begins with the gzip magic bytes. Plain (uncompressed) text is read directly. +/// Open `path` to read its lines through a buffer. It decodes gzip and BGZF when the file starts +/// with the first bytes of a gzip stream, and the caller sees no difference. It reads plain text +/// directly. /// -/// See [`open_maybe_compressed`] to additionally decode bzip2. +/// See [`open_maybe_compressed`] to decode bzip2 as well. pub fn open_maybe_gz(path: &Path) -> io::Result> { let mut file = File::open(path)?; match detect_compression(&mut file)? { @@ -36,9 +42,10 @@ pub fn open_maybe_gz(path: &Path) -> io::Result> { } } -/// Open `path` for buffered line reading, transparently decoding gzip/BGZF **or** bzip2 by -/// content. Plain text is read directly. Used by importers that accept the compressed dumps -/// vendors ship (e.g. a CompleteGenomics `var-*-ASM.tsv.bz2`). +/// Open `path` to read its lines through a buffer. It decodes gzip and BGZF, **or** bzip2, from +/// the content of the file, and the caller sees no difference. It reads plain text directly. An +/// importer that accepts a compressed dump from a vendor uses this, such as a CompleteGenomics +/// `var-*-ASM.tsv.bz2`. pub fn open_maybe_compressed(path: &Path) -> io::Result> { let mut file = File::open(path)?; match detect_compression(&mut file)? { @@ -48,8 +55,8 @@ pub fn open_maybe_compressed(path: &Path) -> io::Result> { } } -/// Peek the leading magic bytes for gzip (`1f 8b`) or bzip2 (`BZh`), then rewind to the start -/// so the returned reader sees the whole file. +/// Look at the first bytes of the file, for gzip (`1f 8b`) or for bzip2 (`BZh`). Then go back to +/// the start, so that the reader that comes out sees the whole file. fn detect_compression(file: &mut File) -> io::Result { let mut magic = [0u8; 3]; let n = read_up_to(file, &mut magic)?; @@ -63,8 +70,8 @@ fn detect_compression(file: &mut File) -> io::Result { } } -/// Fill `buf` from `file`, tolerating short reads; returns the number of bytes read (may be -/// fewer than `buf.len()` only at EOF). +/// Fill `buf` from `file`. It accepts a short read. It returns the count of bytes that it read, +/// and that count is below `buf.len()` only at the end of the file. fn read_up_to(file: &mut File, buf: &mut [u8]) -> io::Result { let mut filled = 0; while filled < buf.len() { @@ -115,7 +122,8 @@ mod tests { #[test] fn reads_bzip2_by_content() { - // A `.txt`-named bzip2 stream must be decoded by content (magic `BZh`), not extension. + // A bzip2 stream whose name ends in `.txt` must decode from its content, where the first + // bytes are `BZh`. The extension must not decide. let path = tmp_dir().join("cg.txt"); let mut enc = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::default()); enc.write_all(b"x\ny\nz\n").unwrap(); diff --git a/crates/navigator-analysis/src/haplo.rs b/crates/navigator-analysis/src/haplo.rs index fd4cb28c..78f7e89b 100644 --- a/crates/navigator-analysis/src/haplo.rs +++ b/crates/navigator-analysis/src/haplo.rs @@ -1,25 +1,34 @@ -//! mtDNA/Y haplogroup assignment over an FTDNA haplotree using the **Kulczynski measure** -//! (HaploGrep, Weissensteiner et al.): rank each haplogroup by the set similarity between -//! its *expected* mutations (the union of branch-defining loci from root to the node) and -//! the sample's *found* polymorphisms. Higher fidelity than a flat derived/ancestral count. +//! mtDNA and Y haplogroup assignment over an FTDNA haplotree, with the **Kulczynski measure** +//! (HaploGrep, Weissensteiner and others). It ranks each haplogroup by the set similarity +//! between two sets. The first set is the *expected* mutations of that haplogroup. That is the +//! union of the loci that define a branch, from the root down to the node. The second set is the +//! *found* polymorphisms of the sample. This measure is more accurate than a flat count of +//! derived and ancestral sites. //! -//! `score = ½·(|F∩E| / |E| + |F∩E| / |F|)` per node, equal site weights (a published -//! per-site weight table can be layered on later). Pure: callers supply the parsed tree -//! and the sample's base calls; fetching the FTDNA JSON lives in the app layer. +//! The score at each node is `score = ½·(|F∩E| / |E| + |F∩E| / |F|)`, and the site weights are +//! equal. A published table of weights for each site can go on top of this later. //! -//! **RSRS-anchored, reference-free.** Rather than diffing the sample against rCRS (which -//! would hide rCRS's own backbone mutations — the classic rCRS-vs-RSRS problem), we read -//! the sample's *actual base* at each tree position and compare it to the node's derived -//! allele. The FTDNA tree is RSRS-rooted, so a base equal to a node's derived allele is a -//! genuine carried mutation, backbone included — no reference subtraction needed. `found` -//! is then the set of tree sites where the sample carries the derived allele. (Assumes the -//! sample is on rCRS coordinates, i.e. ~16,569 bp; indels would shift later positions.) +//! This module is pure. The caller gives the parsed tree and the base calls of the sample. The +//! app layer gets the FTDNA JSON. +//! +//! **The RSRS is the anchor, and the code uses no reference.** It does not take the difference +//! of the sample against rCRS, because that would hide the backbone mutations of rCRS itself. +//! That is the classic rCRS-against-RSRS problem. It instead reads the *actual base* of the +//! sample at each tree position, and compares it to the derived allele of the node. +//! +//! The FTDNA tree has RSRS at its root. A base that equals the derived allele of a node is a +//! true mutation of the sample, and the backbone is part of that. The code needs no subtraction +//! of a reference. +//! +//! `found` is then the set of tree sites where the sample carries the derived allele. This +//! assumes that the sample is on rCRS coordinates, which is about 16,569 bp. An indel would move +//! the positions after it. use std::collections::{HashMap, HashSet}; use serde::Deserialize; -/// A branch-defining locus: a position and its ancestral/derived alleles. +/// A locus that defines a branch: a position and its ancestral and derived alleles. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Locus { pub position: i64, @@ -91,8 +100,9 @@ struct FtdnaTreeJson { all_nodes: HashMap, } -/// Parse an FTDNA haplotree JSON document into a [`HaploTree`]. Positions are abs-valued -/// (the FTDNA data carries some negatives); variants without a position are dropped. +/// Parse an FTDNA haplotree JSON document into a [`HaploTree`]. The code takes the absolute +/// value of each position, because the FTDNA data carries some negative ones. It drops a variant +/// that has no position. pub fn parse_ftdna_json(data: &str) -> Result { let raw: FtdnaTreeJson = serde_json::from_str(data).map_err(|e| e.to_string())?; let nodes = raw @@ -145,9 +155,10 @@ struct DuVariant { /// Coordinates keyed by build label (`"hs1"`, `"GRCh38"`, `"GRCh37"`). #[serde(default)] coordinates: HashMap, - /// Variant-level **authoritative** phylogenetic polarity, build-independent. Present on every - /// variant; ~1.4% of variants carry a *swapped* per-build `coordinates.ancestral/derived` (a - /// clean role swap), so this — not the coordinate alleles — is the polarity to trust. + /// The **authoritative** phylogenetic polarity at the variant level. It does not depend on + /// the build, and every variant carries it. About 1.4% of the variants carry a *swapped* + /// `coordinates.ancestral/derived` in a build, which is a clean exchange of the two roles. + /// Trust this field, and not the coordinate alleles. #[serde(default)] link_ancestral: Option, #[serde(default)] @@ -191,11 +202,14 @@ struct DuTreeJson { roots: Vec, } -/// Parse the DecodingUs AppView Y-tree (`/api/v1/y-tree/full`) into a [`HaploTree`], taking -/// each variant's coordinate for `build_key` (`"hs1"` for CHM13, `"GRCh38"`, `"GRCh37"`). -/// Because positions are read in the *alignment's own build*, no liftover is needed — -/// variants without a coordinate on `build_key` are dropped (they can't be placed there). -/// Node ids come from the AppView (unique); the nested `children` flatten into child-id lists. +/// Parse the DecodingUs AppView Y-tree (`/api/v1/y-tree/full`) into a [`HaploTree`]. It takes the +/// coordinate of each variant for `build_key`, which is `"hs1"` for CHM13, `"GRCh38"` or +/// `"GRCh37"`. +/// +/// The code reads the positions in the *build of the alignment itself*, so it needs no liftover. +/// It drops a variant that has no coordinate on `build_key`, because it can not place that +/// variant there. The node ids come from the AppView and are unique. The nested `children` +/// flatten into lists of child ids. pub fn parse_decodingus_json(data: &str, build_key: &str) -> Result { let raw: DuTreeJson = serde_json::from_str(data).map_err(|e| e.to_string())?; let mut nodes = HashMap::new(); @@ -211,8 +225,9 @@ fn flatten_du_node(n: &DuNode, is_root: bool, build_key: &str, out: &mut HashMap .iter() .filter_map(|v| { let c = v.coordinates.get(build_key)?; - // Position/contig come from the build coordinate; polarity from the authoritative - // `link_*` (the coordinate's own ancestral/derived is swapped on ~1.4% of variants). + // The position and the contig come from the build coordinate. The polarity comes + // from the authoritative `link_*`. The ancestral and derived alleles of the + // coordinate itself are in the wrong order on about 1.4% of the variants. let (ancestral, derived) = v.polarity(c); Some(Locus { position: c.position.abs(), @@ -238,20 +253,25 @@ fn flatten_du_node(n: &DuNode, is_root: bool, build_key: &str, out: &mut HashMap } } -/// Build-independent **ancestral/derived polarity** per SNP **name** from the DecodingUs tree -/// JSON: `name → (ancestral, derived)`. The DecodingUs tree carries true phylogenetic polarity, -/// whereas FTDNA records the GRCh38 *reference* base as "ancestral" — so at sites where the -/// reference carries the derived allele FTDNA's polarity is inverted. This map drives -/// [`normalize_polarity`] to repair an FTDNA tree. Alleles are taken from any one build's -/// coordinate (the mutation's alleles are the same across builds); names are universal. +/// The **ancestral and derived polarity** of each SNP **name**, from the DecodingUs tree JSON, +/// as `name → (ancestral, derived)`. It does not depend on the build. +/// +/// The DecodingUs tree carries the true phylogenetic polarity. FTDNA instead records the GRCh38 +/// *reference* base as the "ancestral" one. At a site where the reference carries the derived +/// allele, the FTDNA polarity is then the wrong way round. [`normalize_polarity`] uses this map +/// to repair an FTDNA tree. +/// +/// The code takes the alleles from the coordinate of any one build, because the alleles of a +/// mutation are the same in every build. The names are universal. pub fn decodingus_polarity_map(data: &str) -> Result, String> { let raw: DuTreeJson = serde_json::from_str(data).map_err(|e| e.to_string())?; let mut out = HashMap::new(); - // The variant-level `link_*` is the authoritative, build-independent polarity — use it directly. - // Only if it is somehow absent do we fall back to a **deterministic** build coordinate (hs1 - // preferred, then sorted keys); iterating `coordinates.values()` (HashMap order) is - // non-deterministic and, where a build records swapped polarity, would pick a different - // orientation per run. + // The `link_*` at the variant level is the authoritative polarity, and it does not depend on + // the build. Use it directly. Fall back to a build coordinate only if `link_*` is absent. + // Choose that coordinate in a **deterministic** way: hs1 first, then the keys in sorted + // order. A walk over `coordinates.values()` follows the HashMap order, which is not + // deterministic. Where a build records a swapped polarity, such a walk would take a different + // orientation on each run. fn pick_polarity(v: &DuVariant) -> Option<(String, String)> { if let Some(p) = v.link_alleles() { return Some(p); @@ -289,12 +309,16 @@ pub fn decodingus_polarity_map(data: &str) -> Result) -> usize { let mut flipped = 0; for node in tree.nodes.values_mut() { @@ -319,13 +343,15 @@ pub fn normalize_polarity(tree: &mut HaploTree, reference: &HashMap) -> Vec { - // |F| — distinct tree sites whose derived allele the sample carries. + // |F|, the count of distinct tree sites whose derived allele the sample carries. let mut carried: HashSet = HashSet::new(); for node in tree.nodes.values() { for locus in &node.loci { @@ -377,37 +403,45 @@ fn complement_base(b: char) -> char { } } -/// Whether a SNP's two alleles are strand-ambiguous (an `A↔T` or `C↔G` transversion): the -/// complement of one allele equals the other, so the strand can't be inferred from the observed -/// base. Complement-matching must be skipped for these. +/// True when the two alleles of a SNP are ambiguous about the strand, which holds for an `A↔T` +/// or a `C↔G` transversion. The complement of one allele equals the other, so the observed base +/// does not tell you the strand. The code must not try a match on the complement for these. fn strand_ambiguous(a: char, d: char) -> bool { let mut pair = [a.to_ascii_uppercase(), d.to_ascii_uppercase()]; pair.sort_unstable(); pair == ['A', 'T'] || pair == ['C', 'G'] } -/// Does the sample carry this locus's derived allele? Accepts the strand-complement of the -/// derived base (some tree variants record alleles on the opposite strand from the reference the -/// alignment was called against — see [`locus_state`]), except for strand-ambiguous SNPs. -/// Whether a locus is a single-base SNP a base-level genotype can evaluate — i.e. NOT an indel / MNP -/// (a multi-character allele). An insertion (`G`→`GAGC…`) or deletion (`GAGC`→`G`) shares its anchor -/// base between the two alleles, so a base-vs-allele compare can't tell them apart and reads every -/// sample as derived. The DecodingUs Y tree carries ~12.7k such indel loci, all with a shared anchor; -/// counting them by base turned indel-heavy nodes into homoplasy magnets that captured the placement. -/// Indel loci are instead genotyped by [`crate::caller::call_indels_at`], which writes a resolved -/// **sentinel** into the genotype map ([`INDEL_DERIVED`] / [`INDEL_ANCESTRAL`]); `locus_state` / -/// `locus_carried` read that sentinel for a non-SNP locus. +/// True when a locus is a single-base SNP that a genotype at the base level can read. That is, +/// it is NOT an indel and NOT an MNP, which carry an allele of more than one character. +/// +/// An insertion (`G`→`GAGC…`) or a deletion (`GAGC`→`G`) holds the same anchor base in both of +/// its alleles. A comparison of a base against an allele can then not separate the two, and it +/// reads every sample as derived. The DecodingUs Y tree carries about 12.7k such indel loci, and +/// all of them share an anchor. To count them by base turned a node with many indels into a +/// collector of homoplasy, and such a node then took the placement. +/// +/// [`crate::caller::call_indels_at`] genotypes the indel loci instead. It writes a resolved +/// **sentinel** into the genotype map, which is [`INDEL_DERIVED`] or [`INDEL_ANCESTRAL`]. +/// `locus_state` and `locus_carried` read that sentinel at a locus that is not a SNP. fn is_snp_locus(locus: &Locus) -> bool { locus.derived.chars().count() == 1 && locus.ancestral.chars().count() <= 1 } -/// Sentinel written at an indel locus's anchor position when the sample **carries** the derived -/// insertion/deletion. Not a nucleotide, so it never collides with a SNP base-call (and a rare -/// indel/SNP position collision degrades both to no-call, never a wrong call). +/// The sentinel that goes at the anchor position of an indel locus when the sample **carries** +/// the derived insertion or deletion. It is not a nucleotide, so it never collides with the base +/// call of a SNP. If an indel and a SNP do share a position, which is rare, both fall to a +/// no-call. Neither becomes a wrong call. pub const INDEL_DERIVED: char = '+'; -/// Sentinel written at an indel locus the sample does **not** carry (reference-spanning / ancestral). +/// The sentinel that goes at an indel locus that the sample does **not** carry. Such a locus +/// covers the reference, and it is ancestral. pub const INDEL_ANCESTRAL: char = '-'; +/// Does the sample carry the derived allele of this locus? The code accepts the strand +/// complement of the derived base, except at a SNP that is ambiguous about the strand. Some tree +/// variants record their alleles on the other strand from the reference that the caller used on +/// the alignment. See [`locus_state`]. +/// fn locus_carried(locus: &Locus, calls: &HashMap) -> bool { if !is_snp_locus(locus) { // Indel locus: carried iff the indel genotyper resolved it to the derived sentinel. @@ -426,21 +460,25 @@ fn locus_carried(locus: &Locus, calls: &HashMap) -> bool { !ambiguous && complement_base(b) == d } -/// The sample's state at one defining SNP: carries the derived allele, carries the ancestral -/// allele, or has no confident call. A locus with no derived allele (indel-only / marker-less) -/// is `NoCall`. +/// The state of the sample at one SNP that defines a branch. The sample carries the derived +/// allele, or it carries the ancestral allele, or it has no confident call. A locus with no +/// derived allele, which holds only an indel or no marker, gives `NoCall`. +/// +/// Some haplotree variants record their ancestral and derived alleles on the **other strand** +/// from the reference that the caller genotyped the alignment against. FTDNA and YBrowse report +/// on the discovery strand of the SNP. A clean read then shows the complement of both tree +/// alleles, and it matches neither the literal ancestral allele nor the literal derived one. The +/// code also accepts a match on the strand complement, which is what the chip reconciliation +/// does. /// -/// Some haplotree variants record their ancestral/derived alleles on the **opposite strand** from -/// the reference the alignment was genotyped against (FTDNA/YBrowse report on the SNP's discovery -/// strand). A clean read then shows the complement of both tree alleles — neither the literal -/// ancestral nor derived — so we also accept a strand-complement match, mirroring the chip -/// reconciliation. The exception is strand-ambiguous (`A↔T` / `C↔G`) SNPs, where the complement of -/// one allele *is* the other and strand can't be inferred; those keep strict literal matching. A -/// base that matches neither strand of either allele is a genuine third allele → `NoCall` (not a -/// branch contradiction). +/// A SNP that is ambiguous about the strand (`A↔T` or `C↔G`) is the exception. There the +/// complement of one allele *is* the other, and the data does not tell you the strand. Those +/// keep a strict literal match. A base that matches neither strand of either allele is a true +/// third allele, and it gives `NoCall`. It does not contradict the branch. fn locus_state(locus: &Locus, calls: &HashMap) -> CallState { - // Indel / MNP loci can't be read as a single base — they are genotyped separately and their - // resolved state arrives as a sentinel ([`INDEL_DERIVED`]/[`INDEL_ANCESTRAL`]) at the anchor. + // The code can not read an indel or an MNP locus as a single base. It genotypes those + // separately, and their resolved state comes in as a sentinel at the anchor, which is + // [`INDEL_DERIVED`] or [`INDEL_ANCESTRAL`]. if !is_snp_locus(locus) { return match calls.get(&locus.position) { Some(&INDEL_DERIVED) => CallState::Derived, @@ -494,9 +532,10 @@ fn dfs( // Add this node's loci to the path (skip positions already seen on the path). let mut added: Vec<(i64, bool)> = Vec::new(); for locus in &node.loci { - // Indel/MNP loci participate only when the indel genotyper resolved them (a sentinel is - // present); an un-called indel must not inflate a node's expected/matched set. SNP loci - // count as before (a no-call SNP still contributes to `expected`). + // An indel or MNP locus counts only when the indel genotyper resolved it, which is when + // a sentinel is present. An indel with no call must not make the expected set or the + // matched set of a node larger. A SNP locus counts as before, and a SNP with no call + // still goes into `expected`. if !is_snp_locus(locus) { match calls.get(&locus.position) { Some(&INDEL_DERIVED) | Some(&INDEL_ANCESTRAL) => {} @@ -547,8 +586,8 @@ fn dfs( lineage.pop(); } -/// Every position that defines some branch → the name of a haplogroup that uses it (for -/// annotating off-path private variants). Recurrent positions keep one name. +/// A map from every position that defines a branch to the name of a haplogroup that uses it. It +/// puts a note on a private variant that is off the path. A recurrent position keeps one name. pub fn tree_positions(tree: &HaploTree) -> HashMap { let mut m = HashMap::new(); for n in tree.nodes.values() { @@ -559,12 +598,17 @@ pub fn tree_positions(tree: &HaploTree) -> HashMap { m } -/// Polarity map for the consensus interpreter: **SNP name → (ancestral, derived)** over every -/// defining locus in the tree. This is the tree-of-record's per-SNP polarity, applied at read time by -/// `navigator_domain::consensus::interpret` so a corrected tree flips states with no re-genotyping. -/// Use for any parsed [`HaploTree`] (mtDNA rCRS, FTDNA) where a JSON polarity map isn't available; -/// for the DecodingUs Y JSON prefer [`decodingus_polarity_map`] (true phylogenetic polarity). Loci -/// without a name or derived allele are skipped; a recurrent name keeps its first-seen polarity. +/// The polarity map for the consensus interpreter: **SNP name → (ancestral, derived)**, over +/// every locus in the tree that defines a branch. This is the polarity of each SNP in the tree of +/// record. `navigator_domain::consensus::interpret` applies it when it reads the data, so a +/// corrected tree changes the states and no sample needs a new genotype. +/// +/// Use this for any parsed [`HaploTree`], such as the mtDNA rCRS tree or an FTDNA tree, where +/// there is no JSON polarity map. For the DecodingUs Y JSON, use [`decodingus_polarity_map`] +/// instead, which gives the true phylogenetic polarity. +/// +/// The code skips a locus that has no name or no derived allele. A name that occurs more than +/// once keeps the polarity of its first occurrence. pub fn polarity_from_tree(tree: &HaploTree) -> std::collections::BTreeMap { let mut m: std::collections::BTreeMap = std::collections::BTreeMap::new(); for n in tree.nodes.values() { @@ -590,7 +634,8 @@ fn build_parent_map(tree: &HaploTree) -> HashMap { parent } -/// The defining-SNP positions on the root→`node_id` path (the placement's backbone). +/// The positions of the SNPs that define a branch, on the path from the root to `node_id`. They +/// are the backbone of the placement. pub fn path_positions(tree: &HaploTree, node_id: i64) -> HashSet { let parent = build_parent_map(tree); let mut positions = HashSet::new(); @@ -607,7 +652,7 @@ pub fn path_positions(tree: &HaploTree, node_id: i64) -> HashSet { positions } -/// The sample's state at a defining SNP. +/// The state of the sample at a SNP that defines a branch. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CallState { /// Carries the branch's derived allele. @@ -618,7 +663,7 @@ pub enum CallState { NoCall, } -/// One defining SNP of a branch, with the sample's state. +/// One SNP that defines a branch, with the state of the sample. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SnpEvidence { pub name: String, @@ -631,21 +676,23 @@ pub struct SnpEvidence { pub base: Option, } -/// A child branch below the reported terminal, with the per-SNP evidence that explains why -/// descent did or didn't continue into it. +/// A child branch below the terminal that the report names, with the evidence at each SNP. That +/// evidence explains why the descent went into that branch, or why it did not. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BranchEvidence { pub name: String, pub snps: Vec, - /// How many of this branch's defining SNPs the sample carries (derived). + /// The count of the SNPs that define this branch, and that the sample carries as derived. pub derived: usize, } -/// For the node `node_id` (typically the reported terminal), evaluate each child branch's -/// defining SNPs against the sample `calls` — `Derived` / `Ancestral` / `NoCall` per SNP. -/// This explains a stop: a child with all-`Ancestral` SNPs is an unsupported split; one -/// with `NoCall` SNPs is unresolved for lack of coverage. Children without defining SNPs -/// are omitted. +/// For the node `node_id`, which is usually the terminal that the report names, examine the SNPs +/// that define each child branch against the sample `calls`. Each SNP gets `Derived`, +/// `Ancestral` or `NoCall`. +/// +/// This explains why the descent stopped. A child whose SNPs are all `Ancestral` is a split that +/// the data does not support. A child with `NoCall` SNPs has too little coverage to resolve. The +/// result leaves out a child that has no SNP to define it. pub fn child_evidence(tree: &HaploTree, calls: &HashMap, node_id: i64) -> Vec { let Some(node) = tree.nodes.get(&node_id) else { return Vec::new(); @@ -683,8 +730,10 @@ pub fn child_evidence(tree: &HaploTree, calls: &HashMap, node_id: i64 out } -/// One node on the root→terminal path with its defining SNPs and the sample's per-SNP state — the -/// grouped form of [`lineage_evidence`], used to draw a YFull-style per-node descent report. +/// One node on the path from the root to the terminal. It holds the SNPs that define that node, +/// and the state of the sample at each of those SNPs. It is the grouped form of +/// [`lineage_evidence`], and it draws a descent report in the YFull style, one row for each +/// node. #[derive(Debug, Clone, PartialEq, Eq)] pub struct NodeEvidence { pub name: String, @@ -693,11 +742,14 @@ pub struct NodeEvidence { pub snps: Vec, } -/// Group the root→`terminal_id` path into per-node defining-SNP evidence (root→terminal order): -/// walk the tree from the terminal up to the root, and for each node attach its loci with the -/// sample's state taken from `state_by_name` (`NoCall` for an equivalent the sample didn't call). -/// Keyed by **SNP name** (build-independent: a name like `M269` is the same across coordinate -/// systems), so a cached variant profile placed under any build can colour an FTDNA-tree path. +/// Group the path from the root to `terminal_id` into the evidence at each node, in root to +/// terminal order. The code walks the tree from the terminal up to the root. At each node it +/// attaches the loci of that node, with the state of the sample from `state_by_name`. An +/// equivalent that the sample did not call gets `NoCall`. +/// +/// The key is the **SNP name**, which does not depend on the build. A name such as `M269` is the +/// same in every coordinate system. A cached variant profile that the code placed under any build +/// can colour a path in an FTDNA tree. pub fn descent_by_node( tree: &HaploTree, terminal_id: i64, @@ -736,9 +788,10 @@ pub fn descent_by_node( .collect() } -/// Per-SNP evidence along the lineage root→`terminal_id`: every defining SNP of every node on -/// the path, with the sample's `Derived`/`Ancestral`/`NoCall` state. Used to compare exactly -/// which defining mutations a sample carries (e.g. GRCh38 vs a lifted CHM13 call). +/// The evidence at each SNP along the lineage from the root to `terminal_id`. It holds every SNP +/// that defines a node on the path, with the `Derived`, `Ancestral` or `NoCall` state of the +/// sample. Use it to compare which of those mutations a sample carries, for example a GRCh38 +/// call against a CHM13 call that came through a liftover. pub fn lineage_evidence(tree: &HaploTree, calls: &HashMap, terminal_id: i64) -> Vec { // child → parent, to walk the terminal back to the root. let parent = build_parent_map(tree); @@ -770,22 +823,29 @@ pub fn lineage_evidence(tree: &HaploTree, calls: &HashMap, terminal_i // ---- path-supported parsimony guard ------------------------------------------ // -// The Kulczynski `score` ranks every node by *proportional* set-similarity, and on real -// data that places the terminal well (validated: GFX0457637 → R-FGC29071). Its one weakness -// is the distal-Y paralog artifact: a deep node reached only by *tunnelling through a branch -// the sample contradicts* can still score highly off a few coincidental matches. +// The Kulczynski `score` ranks every node by *proportional* set similarity. On real data that +// places the terminal well, and a check on GFX0457637 gave R-FGC29071. It has one weakness: the +// paralog artifact on the distal Y. A deep node can still score high on a few matches that are +// only coincidence. That happens when the path reaches the node only by a tunnel *through a +// branch that the sample contradicts*. +// +// Parsimony guards that exact failure. It refuses any candidate whose lineage from the root to +// the node crosses a branch that the sample contradicts. It does not change the proportional +// rank, which gets the clean case right. // -// Parsimony guards exactly that failure — it rejects any candidate whose root→node lineage -// crosses a contradicted branch — without disturbing the proportional ranking that gets the -// clean case right. (A descent-style "follow the most-derived subtree" router was tried and -// derailed onto a bushier wrong fork on the 4× GFX sample: absolute derived count favours -// long/bushy paths, where Kulczynski's proportion does not. The proportional rank + this -// guard is the validated combination. The remaining paralog *false-positive* defence — when -// the wrong branch carries spurious derived calls rather than honest ancestral ones — is the -// haploid allele-balance filter, a separate Phase-1 item.) See PangenomeExpansion.md. - -/// Per-node tally over evaluable defining SNPs (loci with a derived allele): how many the -/// sample calls derived, ancestral (a contradiction), or has no confident base for. +// An earlier try used a router in the descent style, which follows the most derived subtree. It +// went onto a wrong fork with more branches on the 4x GFX sample. An absolute count of derived +// sites prefers a long path with many branches, where the Kulczynski proportion does not. The +// proportional rank plus this guard is the combination that the checks support. +// +// One paralog defence is still open: the *false-positive* case, where the wrong branch carries +// derived calls that are not real, and not honest ancestral ones. The haploid allele-balance +// filter covers that, and it is a separate Phase-1 item. See PangenomeExpansion.md. + +/// The tally at each node. It covers the SNPs that define a branch and that the code can +/// examine, which are the loci with a derived allele. It counts three things: how many the sample +/// calls derived, how many it calls ancestral, and how many it has no confident base for. An +/// ancestral call is a contradiction. fn node_counts(node: &HaploNode, calls: &HashMap) -> (usize, usize, usize) { let (mut d, mut a, mut n) = (0usize, 0usize, 0usize); for l in &node.loci { @@ -801,8 +861,8 @@ fn node_counts(node: &HaploNode, calls: &HashMap) -> (usize, usize, u (d, a, n) } -/// Public view of a node's `(derived, ancestral, no-call)` defining-SNP tally against `calls` — for -/// diagnostics / tracing a placement path. +/// The public view of the `(derived, ancestral, no-call)` tally of a node against `calls`, over +/// the SNPs that define the node. Use it for diagnostics, and to follow a placement path. pub fn node_call_counts(tree: &HaploTree, calls: &HashMap, node_id: i64) -> (usize, usize, usize) { tree.nodes .get(&node_id) @@ -810,15 +870,19 @@ pub fn node_call_counts(tree: &HaploTree, calls: &HashMap, node_id: i .unwrap_or((0, 0, 0)) } -/// Find a node by name for the branch-report tool: matches a **haplogroup name** (e.g. `R-FGC29071`) -/// or any of the node's **defining-marker names** (e.g. `FGC29071`), case-insensitively. Returns the -/// node id. A marker-name match wins over none; among several nodes the first in a stable id order is -/// returned (defining markers are node-unique in practice, so this is deterministic in practice). +/// Find a node by name, for the branch-report tool. It matches a **haplogroup name** such as +/// `R-FGC29071`, or the name of any **marker that defines the node**, such as `FGC29071`. The +/// case does not matter. It returns the node id. +/// +/// A match on a marker name wins over no match at all. When more than one node matches, the +/// function returns the first one in a stable order of the ids. In practice a marker that defines +/// a node belongs to that node alone, so the result is deterministic. pub fn find_node(tree: &HaploTree, query: &str) -> Option { let q = query.trim().to_ascii_uppercase(); let mut ids: Vec = tree.nodes.keys().copied().collect(); ids.sort_unstable(); - // Haplogroup-name match first, then marker-name match — both over the stable id order. + // Try a match on the haplogroup name first, then on the marker name. Both go over the + // stable order of the ids. ids.iter() .find(|&&id| tree.nodes.get(&id).is_some_and(|n| n.name.to_ascii_uppercase() == q)) .or_else(|| { @@ -831,9 +895,10 @@ pub fn find_node(tree: &HaploTree, query: &str) -> Option { .copied() } -/// One row of a branch report: a defining marker of some node within a reported subtree, with the -/// sample's state. `node`/`parent` are haplogroup names; `snp` carries the marker + observed base + -/// derived/ancestral/no-call state (via [`locus_state`]). +/// One row of a branch report. It holds a marker that defines a node in the subtree that the +/// report covers, with the state of the sample. `node` and `parent` are haplogroup names. `snp` +/// carries the marker, the observed base, and the derived, ancestral or no-call state, which +/// [`locus_state`] gives. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BranchReportRow { pub node: String, @@ -841,11 +906,16 @@ pub struct BranchReportRow { pub snp: SnpEvidence, } -/// Every defining marker in the subtree rooted at `root_id`, in **pre-order** (a node's own markers -/// before its descendants'), each scored against `calls`. Children are visited in a stable -/// name-then-id order so the report is deterministic. `max_depth` bounds descent from the root -/// (`None` = unbounded; `Some(0)` = the root node's markers only). This is the descendant-subtree -/// counterpart to [`lineage_evidence`] (which walks root→node ancestors). +/// Every marker that defines a node in the subtree below `root_id`, in **pre-order**. The own +/// markers of a node come before the markers of its descendants. The code scores each marker +/// against `calls`. +/// +/// The walk goes over the children in a stable order, by name and then by id, so the report is +/// deterministic. `max_depth` limits how far the walk goes down from the root. `None` sets no +/// limit, and `Some(0)` gives the markers of the root node alone. +/// +/// This function covers the subtree of the descendants. [`lineage_evidence`] is its counterpart, +/// and it walks the ancestors from the root down to the node. pub fn subtree_report( tree: &HaploTree, calls: &HashMap, @@ -881,7 +951,8 @@ pub fn subtree_report( None => true, }; if descend { - // Push children in reverse stable order so they pop in ascending (pre-order) order. + // Push the children in the reverse of the stable order, so that they come off the + // stack in pre-order, from the lowest up. let mut kids = node.children.clone(); kids.sort_by(|a, b| { let na = tree.nodes.get(a).map(|n| n.name.as_str()).unwrap_or(""); @@ -896,36 +967,47 @@ pub fn subtree_report( out } -/// A node is *contradicted* when the sample carries the ancestral allele at more of the -/// node's defining SNPs than the derived allele — it confidently does **not** belong to this -/// branch. A no-evidence node (all no-call, `d == a == 0`) is *not* contradicted: it is a -/// pass-through, so low coverage never blocks a lineage. A stray ancestral at an otherwise -/// well-supported node (`d >= a`) is tolerated for the same reason. +/// A node is *contradicted* when the sample carries the ancestral allele at more of its SNPs +/// than it carries the derived allele. The sample then clearly does **not** belong to this +/// branch. +/// +/// A node with no evidence, where every SNP is a no-call and `d == a == 0`, is *not* +/// contradicted. The walk passes through it, so low coverage never blocks a lineage. One stray +/// ancestral at a node that the data otherwise supports, where `d >= a`, is acceptable for the +/// same reason. fn is_contradicted(node: &HaploNode, calls: &HashMap) -> bool { let (d, a, _) = node_counts(node, calls); a > d } -/// Derived defining-SNPs that must appear *below* a contradicted ancestor (on the path toward the -/// candidate) before that contradiction stops vetoing the lineage. A lone ancestral at a sparse -/// intermediate node is usually a genotyping artifact — common on targeted-Y data (FTDNA Big Y), -/// whose coverage gaps turn most intermediate SNPs into no-calls — and would otherwise veto an -/// entire deep lineage that the terminal overwhelmingly supports (one stray ancestral at R-Z16250 -/// blocking R-CTS4466 with 10 derived / 0 ancestral). A real off-branch *tunnel* artifact, by -/// contrast, carries only a coincidental hit or two below the contradicted branch-point, so it -/// stays vetoed. The threshold sits above the coincidental noise and well below a genuine clade's -/// derived count. +/// The count of derived branch SNPs that must occur *below* a contradicted ancestor, on the +/// path toward the candidate. Below that count, the contradiction keeps its veto on the +/// lineage. +/// +/// One ancestral call at a thin intermediate node is usually an artifact of the genotype. It is +/// common on targeted-Y data such as FTDNA Big Y, whose coverage gaps turn most intermediate SNPs +/// into no-calls. Without this threshold, such a call would veto a whole deep lineage that the +/// terminal strongly supports. One stray ancestral at R-Z16250 blocked R-CTS4466, which had 10 +/// derived and 0 ancestral. +/// +/// A true off-branch *tunnel* artifact is different. It carries only one or two coincidental hits +/// below the contradicted branch point, so the veto stays. The threshold sits above the noise of +/// coincidence, and well below the derived count of a true clade. const REDEEM_DERIVED: usize = 4; -/// Is the root→`node_id` lineage free of any *unredeemed* contradicted branch? An off-path paralog -/// artifact sits below a branch the sample is ancestral for, so it fails this guard; the genuine -/// lineage (derived or merely no-call along its length) passes. Used to veto otherwise high-scoring -/// tunnel artifacts from the [`score`] ranking. +/// Is the lineage from the root to `node_id` free of every contradicted branch that nothing +/// below it redeems? +/// +/// An off-path paralog artifact lies below a branch for which the sample is ancestral, so it +/// fails this guard. The true lineage clears the guard, because along its length the sample is +/// derived or has only a no-call. [`score`] uses this to veto a tunnel artifact that would +/// otherwise rank high. /// -/// A contradicted ancestor only vetoes when it isn't *redeemed* by derived support further down the -/// path: a single stray ancestral at a sparse intermediate node (a Big Y miscall) is overridden -/// when ≥[`REDEEM_DERIVED`] derived SNPs below it confirm the branch, while a coincidental tunnel — -/// a contradicted branch-point with only a hit or two beneath it — stays vetoed. +/// A contradicted ancestor vetoes only when derived support further down the path does not +/// *redeem* it. Below it, [`REDEEM_DERIVED`] derived SNPs or more confirm the branch. They +/// override one stray ancestral at a thin intermediate node, which is a Big Y miscall. A tunnel +/// of coincidence keeps its veto, because it is a contradicted branch point with only one or two +/// hits below it. pub fn path_admissible(tree: &HaploTree, calls: &HashMap, node_id: i64) -> bool { let parent = build_parent_map(tree); // Root→node path (root first), with each node's derived-call count. @@ -943,13 +1025,18 @@ pub fn path_admissible(tree: &HaploTree, calls: &HashMap, node_id: i6 for (i, id) in path.iter().enumerate() { let Some(node) = tree.nodes.get(id) else { continue }; if is_contradicted(node, calls) { - // A **confident divergence** — the sample carries the ancestral allele here and *none* of - // the node's derived SNPs — is never redeemable: the sample left the lineage above this - // node, so anything below is a different branch (a sibling clade, or a homoplasy / indel - // block that happens to score). Redeeming it is what lets a big spurious-derived block on a - // parallel branch tunnel into an unreachable terminal. Only a *mixed block* (`d > 0`: the - // sample carries some of the node's derived SNPs, and the ancestral ones are an unresolved - // downstream split) can be redeemed by strong derived support below. + // Nothing can redeem a **confident divergence**, where the sample carries the + // ancestral allele here and *none* of the derived SNPs of the node. The sample left + // the lineage above this node, so everything below is a different branch. It is a + // clade beside this one, or a block of homoplasy or indels that happens to score. + // + // To redeem such a node lets a large block of false derived calls tunnel through + // to a terminal that the sample can not reach. That block lies on a parallel + // branch. + // + // Only a *mixed block* can take redemption from strong derived support below. A + // mixed block has `d > 0`. The sample carries some of the derived SNPs of the node, + // and the ancestral ones are a downstream split that the tree has not resolved. if derived[i] == 0 { return false; } @@ -963,22 +1050,25 @@ pub fn path_admissible(tree: &HaploTree, calls: &HashMap, node_id: i6 true } -/// Minimum derived defining-SNPs of a child the sample must carry for [`deepen_terminal`] to -/// descend into it. Two independent shared-derived mutations confirm membership while staying -/// robust to a lone recurrent/artefactual match. +/// The count of derived SNPs of a child that the sample must carry before [`deepen_terminal`] +/// goes down into that child. Two shared derived mutations that are independent confirm that the +/// sample belongs there, and they stay robust against one recurrent or false match. const MIN_DERIVED_TO_DEEPEN: usize = 2; -/// From the guard-selected `start`, descend further into any child the sample has *clearly -/// entered* — carries at least [`MIN_DERIVED_TO_DEEPEN`] of its derived SNPs and is not -/// contradicted (`ancestral ≤ derived`). Routes by derived count (ties → lower id). +/// From the `start` that the guard chose, go further down into any child that the sample has +/// *clearly entered*. Such a child meets two conditions: the sample carries at least +/// [`MIN_DERIVED_TO_DEEPEN`] of its derived SNPs, and nothing contradicts it, which means +/// `ancestral ≤ derived`. The route follows the derived count, and a tie goes to the lower id. +/// +/// This corrects a low call at a **tree node that nobody has split yet**. Published trees hold +/// many of those, and the FTDNA tree most of all. Take a node whose SNP block has no +/// sub-branches yet. A sample on one sub-lineage is then derived for the SNPs of its own line. +/// It is ancestral for the SNPs of the *other* sub-lineages, which the tree has not split. /// -/// This corrects under-calling at **unsplit tree nodes** — a common case in published trees -/// (FTDNA especially): when a node's SNP block has not yet been divided into sub-branches, a -/// sample on one sub-lineage is derived for the SNPs defining its own line and ancestral for -/// the SNPs of the *other* (not-yet-split) sub-lineages. The node then looks "half ancestral", -/// so its proportional [`score`] falls just below its parent's and the guard stops one node -/// too shallow — even though the sample genuinely carries several of the node's mutations. The -/// ancestral SNPs are an unresolved downstream split, not a contradiction. +/// The node then looks half ancestral. Its proportional [`score`] falls just below the score of +/// its parent, and the guard stops one node too high. The sample truly carries some of the +/// mutations of the node. Those ancestral SNPs are a downstream split that nobody has resolved, +/// and they are not a contradiction. pub fn deepen_terminal(tree: &HaploTree, calls: &HashMap, start: i64) -> i64 { let mut current = start; while let Some(node) = tree.nodes.get(¤t) { @@ -1008,29 +1098,31 @@ pub fn deepen_terminal(tree: &HaploTree, calls: &HashMap, start: i64) // ---- induced subtree (the block-tree substrate) ------------------------------ -/// One node of an [`induced_subtree`], carrying the branch's equivalent defining SNPs — a *block* -/// in the FTDNA "block tree" sense: every sample below this node carries all of `loci`, and nothing -/// observed separates them. +/// One node of an [`induced_subtree`]. It holds the equivalent SNPs that define the branch. It +/// is a *block* in the sense of the FTDNA "block tree". Every sample below this node carries all +/// of `loci`, and no observation separates them. #[derive(Debug, Clone, PartialEq, Eq)] pub struct InducedNode { pub id: i64, pub name: String, /// Parent **within the induced subtree** (`None` at an induced root). pub parent: Option, - /// Depth **within the induced subtree**, root = 0. This is the layout coordinate: the induced - /// root is the members' common ancestor, not the tree's, so full-tree depth would waste the - /// whole left margin on nodes nobody is placed under. + /// The depth **inside the induced subtree**, where the root is 0. This is the coordinate for + /// the layout. The induced root is the common ancestor of the members, and not the root of + /// the tree. A depth from the full tree would waste the whole left margin on nodes that hold + /// no sample. pub depth: usize, - /// The equivalent SNPs defining this branch (the node's own loci). + /// The equivalent SNPs that define this branch, which are the own loci of the node. pub loci: Vec, } -/// Haplogroup name → node id, over the whole tree. Callers placing *many* samples should build this -/// once: the per-subject path ([`crate::haplo::descent_by_node`]'s caller) does a linear scan for its -/// one terminal, which is fine for one subject and quadratic for a cohort. +/// A map from a haplogroup name to a node id, over the whole tree. A caller that places *many* +/// samples must build this once. The path for one subject, which is the caller of +/// [`crate::haplo::descent_by_node`], does a linear scan for its one terminal. That is correct +/// for one subject, and quadratic for a cohort. /// -/// Names are assumed unique within a tree. On a duplicate the **lowest id** wins, so the result does -/// not depend on `HashMap` iteration order. +/// The code takes the names to be unique inside a tree. If two are the same, the **lowest id** +/// wins, so the result does not depend on the iteration order of a `HashMap`. pub fn name_index(tree: &HaploTree) -> HashMap<&str, i64> { let mut idx: HashMap<&str, i64> = HashMap::with_capacity(tree.nodes.len()); for n in tree.nodes.values() { @@ -1041,20 +1133,24 @@ pub fn name_index(tree: &HaploTree) -> HashMap<&str, i64> { idx } -/// The **induced subtree** spanning `terminals`: every node lying on a root→terminal path for at -/// least one of them, emitted in pre-order (a parent always precedes its children). +/// The **induced subtree** that covers `terminals`. It holds every node that lies on a path from +/// the root to a terminal, for one terminal or more. It comes out in pre-order, so a parent +/// always comes before its children. /// -/// This is the skeleton of a cohort block tree — the union of the members' descent paths, which is -/// exactly the set of branches that any of them share. Ids absent from `tree` are ignored, so a -/// caller may pass terminals resolved against a different provider/build without pre-filtering. +/// This is the skeleton of a cohort block tree. It is the union of the descent paths of the +/// members, which is exactly the set of branches that any of them share. The code ignores an id +/// that `tree` does not hold. A caller may give terminals that came from a different provider or +/// build, and it does not have to remove them first. /// -/// Sibling order is by `(name, id)` and roots likewise, so the emitted order is deterministic -/// regardless of `HashMap` iteration order — layout and snapshot tests depend on that. +/// The order of the nodes beside each other goes by `(name, id)`, and the order of the roots does +/// the same. The output order is deterministic, and it does not depend on the iteration order of +/// a `HashMap`. The layout tests and the snapshot tests need that. pub fn induced_subtree(tree: &HaploTree, terminals: &[i64]) -> Vec { let parent = build_parent_map(tree); - // Every node on some root→terminal path. `seen` also breaks a malformed cycle: a node already - // kept means the rest of its path is kept too, so we can stop climbing. + // Every node on a path from the root to a terminal. `seen` also breaks a cycle in a tree that + // is not correct. A node that the set already holds means that the code also kept the rest of + // its path, so the walk up can stop there. let mut kept: HashSet = HashSet::new(); for &t in terminals { if !tree.nodes.contains_key(&t) { @@ -1072,9 +1168,10 @@ pub fn induced_subtree(tree: &HaploTree, terminals: &[i64]) -> Vec return Vec::new(); } - // Induced roots: kept nodes whose parent is absent from the kept set. Normally exactly one (the - // members' common ancestor), but a tree with several roots — the DecodingUs document has a - // `roots` array — can yield several, and so can a cohort spanning them. + // The induced roots are the nodes that the code kept, and whose parent it did not keep. + // There is usually exactly one, which is the common ancestor of the members. But a tree with + // more than one root can give more than one, and the DecodingUs document does have a `roots` + // array. A cohort that reaches across those roots can do the same. let sort_key = |id: &i64| tree.nodes.get(id).map(|n| (n.name.clone(), n.id)); let mut roots: Vec = kept .iter() @@ -1085,7 +1182,7 @@ pub fn induced_subtree(tree: &HaploTree, terminals: &[i64]) -> Vec let mut out = Vec::with_capacity(kept.len()); let mut stack: Vec<(i64, Option, usize)> = Vec::new(); - // Reversed, so popping yields the sorted order. + // In the reverse order, so that the nodes come off the stack in the sorted order. stack.extend(roots.iter().rev().map(|&id| (id, None, 0))); while let Some((id, par, depth)) = stack.pop() { let Some(node) = tree.nodes.get(&id) else { continue }; @@ -1125,14 +1222,14 @@ mod tests { pairs.iter().copied().collect() } - /// A branching tree, for the induced-subtree/block cases: + /// A tree with more than one branch, for the induced-subtree and block cases: /// /// ```text /// root ──> R ──> R1 ──> R1a /// └─> R2 └─> R1b /// ``` /// - /// `R1` carries two equivalent SNPs — the block case. + /// `R1` carries two equivalent SNPs, which is the block case. const BRANCHY: &str = r#"{ "allNodes": { "1": {"haplogroupId": 1, "name": "root", "isRoot": true, "variants": [], "children": [2]}, @@ -1182,7 +1279,7 @@ mod tests { assert!(pos(parent_name) < pos(&n.name), "{} preceded its parent", n.name); } } - // Depth is measured from the induced root, not the full tree. + // The depth counts from the induced root, and not from the full tree. assert_eq!(nodes[0].depth, 0); assert_eq!(nodes[0].parent, None); assert_eq!(nodes.iter().find(|n| n.name == "R1a").unwrap().depth, 3); @@ -1193,7 +1290,7 @@ mod tests { let t = parse_ftdna_json(BRANCHY).unwrap(); let nodes = induced_subtree(&t, &[4]); let r1 = nodes.iter().find(|n| n.name == "R1").unwrap(); - // R1's two SNPs are phylogenetically equivalent — that pair *is* the block. + // The two SNPs of R1 are equivalent in the phylogeny. That pair *is* the block. let mut markers: Vec<&str> = r1.loci.iter().map(|l| l.name.as_str()).collect(); markers.sort_unstable(); assert_eq!(markers, vec!["M173", "M306"]); @@ -1202,11 +1299,11 @@ mod tests { #[test] fn induced_subtree_ignores_terminals_absent_from_the_tree() { let t = parse_ftdna_json(BRANCHY).unwrap(); - // 999 doesn't exist (provider/build skew); the real terminal still resolves. + // 999 does not exist (provider/build skew); the real terminal still resolves. let nodes = induced_subtree(&t, &[4, 999]); let names: Vec<&str> = nodes.iter().map(|n| n.name.as_str()).collect(); assert_eq!(names, vec!["root", "R", "R1", "R1a"]); - // ...and a cohort of nothing but unknowns yields nothing, rather than panicking. + // ...and a cohort of unknowns alone gives nothing. It does not panic. assert!(induced_subtree(&t, &[999]).is_empty()); assert!(induced_subtree(&t, &[]).is_empty()); } @@ -1214,7 +1311,8 @@ mod tests { #[test] fn induced_subtree_is_deterministic_across_runs() { let t = parse_ftdna_json(BRANCHY).unwrap(); - // HashMap iteration order varies per process; the emitted order must not. + // The iteration order of a HashMap changes from one process to the next. The output + // order must not. let first = induced_subtree(&t, &[4, 5, 6]); for _ in 0..8 { assert_eq!(induced_subtree(&t, &[6, 5, 4]), first); @@ -1232,7 +1330,8 @@ mod tests { #[test] fn descent_by_node_buckets_path_with_state() { let t = parse_ftdna_json(TREE).unwrap(); - // Sample is derived at H (A146G) and H2 (A263G); H2a's SNP (C750T) was never called. + // The sample carries the derived allele at H (A146G) and at H2 (A263G). Nobody called + // the SNP of H2a, which is C750T. let state: HashMap = [ ("A146G".to_string(), CallState::Derived), ("A263G".to_string(), CallState::Derived), @@ -1241,7 +1340,7 @@ mod tests { .collect(); let grouped = descent_by_node(&t, 4, &state); // terminal H2a - // root → H → H2 → H2a, root carries no defining loci. + // root → H → H2 → H2a. The root carries no locus that defines a branch. let names: Vec<&str> = grouped.iter().map(|n| n.name.as_str()).collect(); assert_eq!(names, vec!["root", "H", "H2", "H2a"]); assert!(grouped[0].snps.is_empty()); // root has no loci @@ -1250,7 +1349,7 @@ mod tests { assert_eq!(grouped[1].snps[0].name, "A146G"); assert_eq!(grouped[1].snps[0].state, CallState::Derived); assert_eq!(grouped[2].snps[0].state, CallState::Derived); - // H2a's defining SNP was never called → NoCall (drawn grey). + // Nobody called the SNP that defines H2a, so it gives NoCall, which the UI draws grey. assert_eq!(grouped[3].snps[0].state, CallState::NoCall); } @@ -1337,10 +1436,12 @@ mod tests { #[test] fn decodingus_link_polarity_overrides_swapped_coordinate() { - // Real DecodingUs quirk: ~1.4% of variants carry a per-build `coordinates.ancestral/derived` - // that is the *swap* of the authoritative variant-level `link_ancestral/link_derived`. Both - // the tree parse and the polarity map must trust `link_*` — otherwise a backbone SNP the - // sample carries (derived) reads as ancestral (the huF98AFD "peppered ancestral" bug). + // This is a real property of the DecodingUs data. About 1.4% of the variants carry a + // `coordinates.ancestral/derived` in a build that is the *exchange* of the authoritative + // `link_ancestral/link_derived` at the variant level. Both the tree parse and the + // polarity map must trust `link_*`. If they do not, a backbone SNP that the sample + // carries as derived reads as ancestral. That was the huF98AFD bug, where the report was + // full of ancestral calls. let json = r#"{ "roots": [ {"id": 1, "name": "A0-T", "variants": [ @@ -1357,7 +1458,8 @@ mod tests { let locus = &t.nodes[&1].loci[0]; assert_eq!(locus.position, 6964116); assert_eq!((locus.ancestral.as_str(), locus.derived.as_str()), ("G", "A")); - // A sample carrying the derived allele A now genotypes as Derived (was Ancestral pre-fix). + // A sample that carries the derived allele A now genotypes as Derived. Before the fix it + // gave Ancestral. let calls: HashMap = [(6964116, 'A')].into_iter().collect(); assert_eq!(locus_state(locus, &calls), CallState::Derived); // The polarity map agrees. @@ -1367,7 +1469,8 @@ mod tests { #[test] fn parse_decodingus_picks_target_build_and_flattens() { - // hs1: both M207 and M173 resolve; the GRCh38-only variant is dropped. + // In hs1, both M207 and M173 resolve. The code drops the variant that exists only in + // GRCh38. let t = parse_decodingus_json(DU_TREE, "hs1").unwrap(); assert_eq!(t.nodes.len(), 2); assert!(t.nodes[&10].is_root && !t.nodes[&11].is_root); @@ -1387,14 +1490,18 @@ mod tests { #[test] fn a_node_whose_every_variant_lacks_the_build_survives_with_no_snps() { - // The shape behind `1087`'s truncated descent. Most DecodingUs-discovered (`DU`-named) SNPs - // exist in CHM13 coordinates alone — only a few hundred were mapped back to the older - // references — so a terminal defined by one of them has *nothing* under GRCh38. Dropping the - // loci does not drop the node: it stays, named and on the path, with an empty `loci`, and - // `descent_by_node` faithfully reports it with no SNPs. A renderer that hides empty blocks - // (correctly — the root is genuinely empty) then shows the lineage stopping one branch - // short, while the terminal *name* remains right. Hence `DECODINGUS_NATIVE_BUILD`: parse in - // hs1 wherever the join is by SNP name. + // This is the shape behind the short descent of `1087`. Most of the SNPs that DecodingUs + // found, which carry a `DU` name, exist in CHM13 coordinates alone. Only a few hundred of + // them went back to the older references. A terminal that one of them defines has + // *nothing* under GRCh38. + // + // To drop the loci does not drop the node. The node stays, with its name and on the path, + // and its `loci` is empty. `descent_by_node` then reports it correctly, with no SNPs. But + // a renderer that hides an empty block, which is correct because the root is truly empty, + // shows the lineage one branch short. The *name* of the terminal stays right. + // + // `DECODINGUS_NATIVE_BUILD` answers this: parse in hs1 wherever the join goes by SNP + // name. let json = r#"{ "roots": [ {"id": 1, "name": "R-BY57568", "variants": [ @@ -1418,7 +1525,8 @@ mod tests { .into_iter() .collect(); - // GRCh38: the terminal is on the path, named, and empty — the bug. + // In GRCh38 the terminal is on the path, it has its name, and it is empty. That is the + // bug. let g38 = parse_decodingus_json(json, "GRCh38").unwrap(); assert!(g38.nodes.contains_key(&2), "the node itself is never dropped"); assert!(g38.nodes[&2].loci.is_empty(), "its only locus has no GRCh38 coordinate"); @@ -1441,7 +1549,7 @@ mod tests { assert_eq!(terminal.snps[0].state, CallState::Derived); } - /// Mirrors `navigator_app::DECODINGUS_NATIVE_BUILD`, which this crate sits below and so cannot + /// Mirrors `navigator_app::DECODINGUS_NATIVE_BUILD`, which this crate sits below and so can not /// import. The test above is the reason that constant exists. const DECODINGUS_NATIVE_BUILD_FOR_TEST: &str = "hs1"; @@ -1469,8 +1577,9 @@ mod tests { #[test] fn child_evidence_explains_an_unsupported_split() { - // H2 has a child H2a (derived T@750). Sample is ancestral (C) at 750 -> the split - // into H2a is not supported, shown per-SNP. + // H2 has a child H2a, whose derived allele is T at 750. The sample is ancestral, C, at + // 750. The data does not support the split into H2a, and the report shows that at each + // SNP. let t = parse_ftdna_json(TREE).unwrap(); let ranked = score(&t, &calls(&[(146, 'G'), (263, 'G'), (750, 'C')])); assert_eq!(ranked[0].name, "H2"); // stops at H2 (750 ancestral) @@ -1508,9 +1617,10 @@ mod tests { #[test] fn reference_polarity_comes_from_the_tree_not_the_reference() { - // The CHM13 trap: at a Y-SNP the tree calls ancestral=A, derived=G, and the analysis - // reference (CHM13 chrY = HG002, haplogroup J) carries the DERIVED base G. Polarity - // must come from comparing the SAMPLE's base to the tree — never from the reference. + // This is the CHM13 trap. At a Y-SNP the tree gives ancestral=A and derived=G. The + // analysis reference carries the DERIVED base G, because CHM13 chrY is HG002, which is a + // haplogroup-J Y. The polarity must come from a comparison of the base of the SAMPLE + // against the tree. It must never come from the reference. let locus = Locus { position: 146, ancestral: "A".into(), @@ -1524,10 +1634,12 @@ mod tests { // Sample carries the DERIVED allele (G, == the reference here): Derived, from the tree. assert_eq!(locus_state(&locus, &calls(&[(146, 'G')])), CallState::Derived); - // End-to-end: a sample ANCESTRAL at the J-derived backbone site 146 does not carry H's - // defining mutation, so it must not be placed into H — even though the CHM13 reference - // base there is the derived G. A REF-as-ancestral assumption would flip 146 and wrongly - // descend; tree-driven, H is contradicted and the call stays at root. + // This is the end-to-end case. A sample that is ANCESTRAL at the J-derived backbone site + // 146 does not carry the mutation that defines H. The code must not place it into H. + // That holds even though the CHM13 reference base there is the derived G. An assumption + // that the REF base is the ancestral one would turn 146 around and descend wrongly. The + // tree controls the polarity, so the data contradicts H and the call stays at the + // root. let t = parse_ftdna_json(TREE).unwrap(); // root→H(146 A→G)→H2(263)→H2a(750) let c = calls(&[(146, 'A'), (263, 'G'), (750, 'T')]); assert!( @@ -1539,10 +1651,11 @@ mod tests { #[test] fn opposite_strand_reads_match_via_the_complement() { - // A non-ambiguous SNP whose tree alleles are recorded on the opposite strand from the - // reference the alignment was genotyped against: ancestral=A, derived=C. A derived sample - // read on the reference strand shows G (complement of C); an ancestral one shows T - // (complement of A). Neither matches a tree allele literally — the strand-complement does. + // A SNP that is not ambiguous. Its tree alleles sit on the other strand from the + // reference that the caller genotyped the alignment against: ancestral=A, derived=C. A + // derived sample, read on the reference strand, shows G, which is the complement of C. An + // ancestral one shows T, the complement of A. Neither one matches a tree allele + // literally, but the strand complement does. let locus = Locus { position: 146, ancestral: "A".into(), @@ -1554,8 +1667,9 @@ mod tests { assert!(locus_carried(&locus, &calls(&[(146, 'G')]))); assert!(!locus_carried(&locus, &calls(&[(146, 'T')]))); - // Strand-ambiguous SNP (C↔G): the complement of derived G is the ancestral C, so strand - // can't be inferred — keep strict literal matching and don't complement-flip. + // A SNP that is ambiguous about the strand (C↔G). The complement of the derived G is the + // ancestral C, so the data does not tell you the strand. Keep a strict literal match, and + // do not turn the base to its complement. let palindrome = Locus { position: 200, ancestral: "C".into(), @@ -1570,8 +1684,9 @@ mod tests { #[test] fn normalize_polarity_flips_ftdna_reference_as_ancestral_inversion() { - // FTDNA records the GRCh38 reference base as "ancestral"; at PF1016 the reference carries - // the derived allele, so FTDNA lists T>C where the true polarity (DecodingUs) is C>T. + // FTDNA records the GRCh38 reference base as the "ancestral" one. At PF1016 the + // reference carries the derived allele. FTDNA lists T>C, where the true polarity, from + // DecodingUs, is C>T. let mut tree = HaploTree { nodes: HashMap::new() }; tree.nodes.insert( 1, @@ -1586,14 +1701,15 @@ mod tests { derived: "C".into(), name: "PF1016".into(), }, - // Already-aligned SNP — must be left untouched. + // This SNP already agrees with the reference map. It must not change. Locus { position: 200, ancestral: "A".into(), derived: "G".into(), name: "M168".into(), }, - // Strand-different alleles (G>A vs C>T) — not a pure swap, left untouched. + // The alleles sit on different strands: G>A against C>T. That is not a + // clean exchange of the two roles, so it must not change. Locus { position: 300, ancestral: "G".into(), @@ -1633,7 +1749,8 @@ mod tests { #[test] fn guard_rejects_a_contradicted_terminal_but_admits_its_parent() { - // Ancestral (C) at 750 -> H2a is contradicted; the report falls back to H2. + // The sample is ancestral, C, at 750. That contradicts H2a, and the report falls back to + // H2. let t = parse_ftdna_json(TREE).unwrap(); let c = calls(&[(146, 'G'), (263, 'G'), (750, 'C')]); assert!(!path_admissible(&t, &c, id_of(&t, "H2a"))); @@ -1643,17 +1760,18 @@ mod tests { #[test] fn no_calls_admit_the_whole_tree() { - // Empty calls: nothing is contradicted, so every lineage is admissible (the guard is - // a veto, not a selector — Kulczynski still picks root for lack of matches). + // With no calls at all, nothing contradicts a node, so every lineage passes the guard. + // The guard is a veto and not a selector. Kulczynski still takes the root, because it + // finds no match. let t = parse_ftdna_json(TREE).unwrap(); let c = calls(&[]); assert!(path_admissible(&t, &c, id_of(&t, "H2a"))); assert_eq!(guarded_terminal(&t, &c), "root"); } - // root -> H(146) -> B(500, contradicted) -> Bdeep(900, coincidental derived). - // Kulczynski is lured to Bdeep (matches 146 + 900); the guard must veto it (tunnels - // through the contradicted B) and fall back to H. + // root -> H(146) -> B(500, contradicted) -> Bdeep(900, derived by coincidence). + // Kulczynski goes to Bdeep, because that node matches 146 and 900. The guard must veto it, + // because the path tunnels through the contradicted B, and fall back to H. const TUNNEL_TREE: &str = r#"{ "allNodes": { "1": {"haplogroupId": 1, "name": "root", "isRoot": true, "variants": [], "children": [2]}, @@ -1671,7 +1789,7 @@ mod tests { let t = parse_ftdna_json(TUNNEL_TREE).unwrap(); // Carries 146 (H) and a coincidental 900 (Bdeep) but is ANCESTRAL (C) at 500. let c = calls(&[(146, 'G'), (500, 'C'), (900, 'A')]); - // Kulczynski alone is lured deeper by the coincidental match... + // Kulczynski alone goes deeper, because of the match by coincidence... assert_eq!(score(&t, &c)[0].name, "Bdeep"); // ...but Bdeep tunnels through the contradicted B, so the guard reports H. assert!(!path_admissible(&t, &c, id_of(&t, "Bdeep"))); @@ -1693,9 +1811,10 @@ mod tests { #[test] fn indel_locus_is_not_evaluable_from_a_raw_base() { let t = parse_ftdna_json(INDEL_TREE).unwrap(); - // Derived at H(146); at the insertion position the sample carries the anchor base G — which a - // naive first-base compare reads as the insertion's derived allele. A *raw base* (not the indel - // sentinel) at an indel position must NOT place onto the indel-defined node. + // The sample carries the derived allele at H (146). At the insertion position it carries + // the anchor base G. A simple comparison of the first base reads that G as the derived + // allele of the insertion. A *raw base* at an indel position, which is not the indel + // sentinel, must NOT place the sample onto the node that the indel defines. let c = calls(&[(146, 'G'), (200, 'G')]); assert!(!locus_carried(&t.nodes[&3].loci[0], &c)); assert_eq!(locus_state(&t.nodes[&3].loci[0], &c), CallState::NoCall); @@ -1715,7 +1834,8 @@ mod tests { assert_eq!(node_call_counts(&t, &derived, 3), (1, 0, 0)); assert_eq!(guarded_terminal(&t, &derived), "Ins"); - // Resolved as ABSENT: the ancestral sentinel keeps the terminal at H (Ins is contradicted). + // The genotyper resolved the indel as ABSENT. The ancestral sentinel keeps the terminal + // at H, and the data contradicts Ins. let ancestral = calls(&[(146, 'G'), (200, INDEL_ANCESTRAL)]); assert!(!locus_carried(indel, &ancestral)); assert_eq!(locus_state(indel, &ancestral), CallState::Ancestral); @@ -1727,10 +1847,12 @@ mod tests { assert_eq!(guarded_terminal(&t, &uncalled), "H"); } - // root -> H(146) -> B(500) -> Bdeep(five derived SNPs). The sample is ANCESTRAL at B (carries NONE - // of B's derived) yet coincidentally matches all five of Bdeep's SNPs — a big homoplasy / indel - // block on a diverged sibling. The redeem clause (>= REDEEM_DERIVED derived below) would tunnel to - // Bdeep; a *confident* divergence (d == 0) must never be redeemed. + // root -> H(146) -> B(500) -> Bdeep(five derived SNPs). The sample is ANCESTRAL at B, so it + // carries NONE of the derived alleles of B. But by coincidence it matches all five SNPs of + // Bdeep. That is a large block of homoplasy or indels on a clade beside this one, which + // diverged earlier. The redeem clause, which needs REDEEM_DERIVED derived alleles below or + // more, would tunnel to Bdeep. Nothing must ever redeem a *confident* divergence, where + // d == 0. const CONFIDENT_DIVERGENCE_TREE: &str = r#"{ "allNodes": { "1": {"haplogroupId": 1, "name": "root", "isRoot": true, "variants": [], "children": [2]}, @@ -1751,8 +1873,9 @@ mod tests { #[test] fn guard_never_redeems_a_confident_divergence() { let t = parse_ftdna_json(CONFIDENT_DIVERGENCE_TREE).unwrap(); - // Derived at H(146); ancestral at B(500) → carries none of B's derived (d == 0); but matches - // all five of Bdeep's SNPs (a homoplasy block → 5 derived below B, past REDEEM_DERIVED). + // Derived at H(146), and ancestral at B(500), so it carries none of the derived alleles + // of B and d == 0. But it matches all five SNPs of Bdeep. That homoplasy block gives 5 + // derived alleles below B, which is past REDEEM_DERIVED. let c = calls(&[ (146, 'G'), (500, 'C'), @@ -1762,10 +1885,11 @@ mod tests { (903, 'A'), (904, 'A'), ]); - // Kulczynski is lured to Bdeep by the five coincidental matches... + // Kulczynski goes to Bdeep, because of the five matches by coincidence... assert_eq!(score(&t, &c)[0].name, "Bdeep"); - // ...but B is a confident divergence (zero derived), never redeemed despite the derived block - // below it. The terminal is H, not Bdeep — the fix that stops a parallel-branch indel tunnel. + // ...but B is a confident divergence, with zero derived alleles, and nothing redeems it. + // The derived block below it does not help. The terminal is H and not Bdeep. That is the + // fix that stops an indel tunnel on a parallel branch. assert!(!path_admissible(&t, &c, id_of(&t, "Bdeep"))); assert_eq!(guarded_terminal(&t, &c), "H"); } @@ -1789,14 +1913,16 @@ mod tests { let full = calls(&[(146, 'G'), (263, 'G')]); assert!(path_admissible(&t, &full, id_of(&t, "D"))); assert_eq!(guarded_terminal(&t, &full), "D"); - // 263 no-call (low coverage): D is *still* admissible (a no-call is not a - // contradiction) — the guard never blocks for lack of coverage. Kulczynski stops at H. + // 263 gives a no-call, because the coverage is low. D is *still* admissible, because a + // no-call is not a contradiction. The guard never blocks a path for lack of coverage. + // Kulczynski stops at H. let sparse = calls(&[(146, 'G')]); assert!(path_admissible(&t, &sparse, id_of(&t, "D"))); assert_eq!(guarded_terminal(&t, &sparse), "H"); } - // root -> H(146) -> D with three defining SNPs, used to exercise the net contradiction rule. + // root -> H(146) -> D, with three SNPs that define D. This covers the rule about the net + // contradiction. const NET_TREE: &str = r#"{ "allNodes": { "1": {"haplogroupId": 1, "name": "root", "isRoot": true, "variants": [], "children": [2]}, @@ -1810,9 +1936,10 @@ mod tests { } }"#; - // root → P (2 derived SNPs) → C, an UNSPLIT node: 3 SNPs that define it + 3 SNPs of a - // not-yet-split sub-branch. A sample on C's trunk is derived for the first 3 and ancestral - // for the other 3 — so C looks half-ancestral and Kulczynski can rank it below P. + // root → P (2 derived SNPs) → C, where C is a node that nobody has SPLIT yet. C holds 3 SNPs + // that define it, and 3 SNPs of a sub-branch that the tree has not split. A sample on the + // trunk of C carries the derived allele at the first 3, and the ancestral allele at the + // other 3. C then looks half ancestral, and Kulczynski can rank it below P. const UNSPLIT_TREE: &str = r#"{ "allNodes": { "1": {"haplogroupId": 1, "name": "root", "isRoot": true, "variants": [], "children": [2]}, @@ -1845,10 +1972,12 @@ mod tests { (700, 'A'), (800, 'A'), ]); - // Deepen enters C from P: it carries 3 derived (≥2) and isn't contradicted (3 anc ≤ 3 der). - // (The "Kulczynski stops at the parent" condition needs a long backbone — validated on - // the real WGS229 short-read sample, where the guard stops at R-FGC29067 and deepen - // recovers R-FGC29071.) + // Deepen goes into C from P. C carries 3 derived alleles, which is 2 or more, and + // nothing contradicts it, because 3 ancestral is not more than 3 derived. + // + // The condition where Kulczynski stops at the parent needs a long backbone. A check on + // the real WGS229 short-read sample showed it: the guard stops at R-FGC29067, and deepen + // recovers R-FGC29071. assert_eq!(deepen_terminal(&t, &c, id_of(&t, "P")), id_of(&t, "C")); } @@ -1867,7 +1996,7 @@ mod tests { (800, 'A'), ]); assert_eq!(deepen_terminal(&t, &lone, id_of(&t, "P")), id_of(&t, "P")); - // 2 derived but 4 ancestral → contradicted (a > d), don't enter even at ≥2 derived. + // 2 derived but 4 ancestral → contradicted (a > d), do not enter even at ≥2 derived. let net_anc = calls(&[ (100, 'G'), (200, 'G'), diff --git a/crates/navigator-analysis/src/heteroplasmy.rs b/crates/navigator-analysis/src/heteroplasmy.rs index 174a857d..a9233b0e 100644 --- a/crates/navigator-analysis/src/heteroplasmy.rs +++ b/crates/navigator-analysis/src/heteroplasmy.rs @@ -1,15 +1,18 @@ //! mtDNA heteroplasmy detection (plan §4b reconciliation, phase 6). //! -//! Unlike the haploid consensus caller — which collapses each position to a single -//! base — heteroplasmy is the *coexistence* of two mitochondrial alleles in one -//! individual. We detect it by scanning every chrM position's A/C/G/T pileup and -//! flagging sites where a second allele is present above a noise floor: a minor-allele -//! fraction in `[min_minor_fraction, 0.5]` backed by at least `min_minor_count` reads. +//! The haploid consensus caller brings each position down to one base. Heteroplasmy is the +//! opposite: two mitochondrial alleles that live together in one individual. //! -//! This is a screening pass, not a clinical caller: it reports observed allele -//! fractions so a curator can judge real heteroplasmy versus sequencing artefacts -//! (NUMT contamination, strand bias, homopolymer noise). chrM is ~16.5 kb, so the -//! whole contig is tallied in a single dense pass via the caller's `tally_region`. +//! The code finds it with a scan over the A/C/G/T pileup at every chrM position. It flags a site +//! where a second allele sits above a noise floor. That means a minor-allele fraction inside +//! `[min_minor_fraction, 0.5]`, with `min_minor_count` reads behind it or more. +//! +//! This pass looks for candidates. It is not a clinical caller. It reports the observed allele +//! fractions, so that a curator can judge real heteroplasmy against an artifact of the sequencing. +//! Contamination from a NUMT, strand bias and homopolymer noise are such artifacts. +//! +//! chrM is about 16.5 kb, so the code tallies the whole contig in one dense pass, through +//! `tally_region` in the caller. use std::path::Path; @@ -18,36 +21,39 @@ use serde::{Deserialize, Serialize}; use crate::caller::{self, HaploidCallerParams}; use crate::error::AnalysisError; -/// A position carrying two alleles above the noise floor. +/// A position that holds two alleles above the noise floor. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct HeteroplasmySite { /// 1-based position on the contig. pub position: i64, - /// Passing depth (reads clearing the quality filters) at this position. + /// The depth at this position that passes, which is the count of reads that clear the quality + /// filters. pub depth: u32, /// The dominant base. pub major_base: char, - /// Reads supporting the major base. + /// The count of reads behind the major base. pub major_count: u32, /// The second-most-common base. pub minor_base: char, - /// Reads supporting the minor base. + /// The count of reads behind the minor base. pub minor_count: u32, - /// `minor_count / depth` — the heteroplasmy level. + /// `minor_count / depth`. It is the level of the heteroplasmy. pub minor_fraction: f64, } -/// Thresholds for calling a site heteroplasmic. Defaults are conservative screening -/// values (gated by the §4c parity harness, like the rest of the caller). +/// The thresholds that a site must meet before the code calls it heteroplasmic. The defaults are +/// careful values, for a search over candidates. The parity harness of §4c gates them, as it gates +/// the rest of the caller. #[derive(Debug, Clone, Copy, PartialEq)] pub struct HeteroplasmyParams { - /// Sites below this passing depth are skipped (too shallow to judge). + /// The code skips a site whose depth, over the reads that pass, is below this. Such a site is + /// too thin to judge. pub min_depth: u32, /// Minor-allele fraction must reach this to flag (the noise floor). pub min_minor_fraction: f64, - /// Minor allele must be backed by at least this many reads. + /// The minor allele needs this many reads behind it, or more. pub min_minor_count: u32, - /// Reads below this MAPQ are dropped. + /// The code drops a read below this MAPQ. pub min_mapping_quality: u8, /// Bases below this quality are not counted. pub min_base_quality: u8, @@ -55,8 +61,9 @@ pub struct HeteroplasmyParams { impl Default for HeteroplasmyParams { fn default() -> Self { - // 3% noise floor with ≥3 supporting reads is a common screening default for - // mtDNA heteroplasmy on short-read data; depth 20 keeps fractions meaningful. + // A noise floor of 3%, with 3 reads behind the allele or more, is a common default for a + // search over mtDNA heteroplasmy candidates on short-read data. A depth of 20 keeps the + // fractions meaningful. HeteroplasmyParams { min_depth: 20, min_minor_fraction: 0.03, @@ -84,8 +91,8 @@ fn top_two(counts: &[u32; 4]) -> ((usize, u32), (usize, u32)) { (first, second) } -/// Scan every position on `contig` and return the heteroplasmic sites, ascending by -/// position. Tallies the full contig in one pass (fine for chrM-sized contigs). +/// Scan every position on `contig`, and return the heteroplasmic sites, from the lowest position +/// up. It tallies the whole contig in one pass, which is correct for a contig the size of chrM. pub fn detect_heteroplasmy( bam_path: &Path, contig: &str, @@ -93,8 +100,9 @@ pub fn detect_heteroplasmy( reference: Option<&Path>, ) -> Result, AnalysisError> { let length = caller::read_contig_length(bam_path, contig, reference)?; - // Reuse the caller's pileup with matching quality gates; allele-fraction/min-depth - // gating here is heteroplasmy-specific, so the caller params stay permissive. + // Use the pileup of the caller again, with the same quality gates. The gates here, on the + // allele fraction and the minimum depth, belong to heteroplasmy alone. So the parameters of + // the caller stay open. let caller_params = HaploidCallerParams { min_depth: 1, min_mapping_quality: params.min_mapping_quality, diff --git a/crates/navigator-analysis/src/ibd.rs b/crates/navigator-analysis/src/ibd.rs index 53beba67..75b7ccaa 100644 --- a/crates/navigator-analysis/src/ibd.rs +++ b/crates/navigator-analysis/src/ibd.rs @@ -1,12 +1,17 @@ -//! IBD (identity-by-descent) detection + relationship estimation — port of the Scala -//! `ibd.engine` (PairwiseIbdDetector / GeneticMap / RelationshipEstimator). Pure math: -//! the network matching layer (crypto/protocol/relay) is out of scope here. +//! IBD detection, which is identity by descent, and the estimate of a relationship. This is the +//! port of the Scala `ibd.engine`, which held PairwiseIbdDetector, GeneticMap and +//! RelationshipEstimator. It is pure arithmetic. The network layer that matches two people, with +//! its crypto, protocol and relay, is outside this module. //! -//! Input is per-chromosome diploid dosage genotypes (0/1/2, -1 no-call) — exactly what -//! [`crate::caller::genotype_sites`] produces. The detector classifies IBS at shared -//! sites, finds high-IBS runs with a sliding window + error tolerance, converts spans to -//! centiMorgans via a [`GeneticMap`], filters by length, and merges nearby segments. -//! Total shared cM maps to a relationship category (Shared cM Project / ISOGG values). +//! The input is diploid dosage genotypes, one set for each chromosome, at 0, 1 or 2, with -1 for a +//! no-call. That is exactly what [`crate::caller::genotype_sites`] gives. +//! +//! The detector classifies the IBS at the shared sites. It finds the runs of high IBS with a window +//! that slides, and it accepts some errors. It turns each span into centiMorgans, through a +//! [`GeneticMap`]. It then filters by length, and it merges the segments that lie near each other. +//! +//! The total shared cM maps to a class of relationship, with the values of the Shared cM Project +//! and of ISOGG. use std::collections::{BTreeSet, HashMap}; @@ -110,8 +115,9 @@ impl MatchSummary { } } -/// Diploid dosage genotypes for one chromosome. `dosages`: 0 hom-ref, 1 het, 2 hom-alt, -/// -1 no-call. `positions` must be sorted ascending and the same length as `dosages`. +/// The diploid dosage genotypes of one chromosome. In `dosages`, 0 is hom-ref, 1 is het, 2 is +/// hom-alt, and -1 is a no-call. `positions` must come in order, from the lowest up, and it must +/// hold as many entries as `dosages`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ChromosomeGenotypes { pub chromosome: String, @@ -214,7 +220,7 @@ impl GeneticMap { bincode::serialize(self).map_err(|e| AnalysisError::Message(format!("genetic map encode: {e}"))) } - /// Build from per-chromosome `(name, positions, cm)` marker arrays. + /// Build from the `(name, positions, cm)` marker arrays of each chromosome. pub fn from_markers(markers: impl IntoIterator, Vec)>) -> Self { let maps = markers .into_iter() @@ -322,7 +328,8 @@ impl PairwiseIbdDetector { out } - /// Sliding-window candidate segments: `(start_idx, end_idx, snp_count, ibs2_count)`. + /// The candidate segments, from a window that slides. Each one is + /// `(start_idx, end_idx, snp_count, ibs2_count)`. fn find_candidate_segments(&self, positions: &[i32], ibs: &[i8]) -> Vec<(usize, usize, usize, usize)> { let n = positions.len(); if n < self.config.window_size { @@ -499,8 +506,8 @@ mod tests { #[test] fn detects_a_long_identical_segment_but_not_a_discordant_region() { - // 300 SNPs at 50 kb spacing (15 Mb -> 15 cM at 1 cM/Mb). First 250 identical - // (IBS-2), last 50 opposite homozygotes (IBS-0). + // 300 SNPs, with 50 kb between them, which is 15 Mb, and 15 cM at 1 cM/Mb. The first 250 + // are identical, at IBS-2. The last 50 are opposite homozygotes, at IBS-0. let n = 300; let s1 = chrom(n, 50_000, |i| (i % 3) as i8); // 0,1,2 cycling let s2 = chrom(n, 50_000, |i| if i < 250 { (i % 3) as i8 } else { 2 - (i % 3) as i8 }); diff --git a/crates/navigator-analysis/src/ibd_attest.rs b/crates/navigator-analysis/src/ibd_attest.rs index a2c35e64..629a1509 100644 --- a/crates/navigator-analysis/src/ibd_attest.rs +++ b/crates/navigator-analysis/src/ibd_attest.rs @@ -1,22 +1,29 @@ //! IBD segment-exchange payload + match attestation (gap §4 application layer). //! -//! IBD detection needs **both** peers' genotypes, so over the encrypted edge channel each peer sends -//! its dosages at the canonical IBD-panel sites ([`IbdSite`], an [`IbdExchangeMsg::Dosages`]); each -//! then runs the symmetric [`crate::ibd::PairwiseIbdDetector`] locally → an identical -//! [`crate::ibd::MatchSummary`]. Each peer signs an [`IbdAttestation`] over its computed summary and -//! exchanges it; agreement = both signed attestations carry the same `summary_hash` (proof both -//! computed the same result). Only panel dosages cross the wire — encrypted, never seen by the broker. +//! IBD detection needs the genotypes of **both** peers. So over the encrypted edge channel each +//! peer sends its dosages at the canonical sites of the IBD panel. Those are [`IbdSite`] values, in +//! an [`IbdExchangeMsg::Dosages`]. //! -//! This module is pure: it defines the wire types, the canonical signing string, and the summary -//! hash. Signing (the device key) and verification (`du_atproto::verify_did_key`) happen in the app. +//! Each peer then runs the symmetric [`crate::ibd::PairwiseIbdDetector`] on its own machine. Both +//! get the same [`crate::ibd::MatchSummary`]. Each peer signs an [`IbdAttestation`] over the +//! summary that it computed, and the two exchange those. They agree when both signed attestations +//! carry the same `summary_hash`, and that is proof that both computed the same result. +//! +//! The panel dosages are the only thing that crosses the wire. They go encrypted, and the broker +//! never sees them. +//! +//! This module is pure. It gives the wire types, the canonical string that a peer signs, and the +//! hash of a summary. The app does the signature, with the device key, and the verification, +//! through `du_atproto::verify_did_key`. use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::ibd::MatchSummary; -/// One panel-site dosage on the wire — the minimal input the IBD detector consumes (the heavy -/// [`crate::caller::SiteGenotype`] fields aren't sent). `dosage` is 0/1/2, or -1 for no-call. +/// The dosage at one panel site, on the wire. It is the smallest input that the IBD detector +/// reads. The heavy fields of a [`crate::caller::SiteGenotype`] do not go out. `dosage` is 0, 1 or +/// 2, or -1 for a no-call. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct IbdSite { pub contig: String, @@ -36,7 +43,8 @@ pub struct IbdAttestation { pub session_id: String, /// The attester's account DID. pub attesting_did: String, - /// Opaque references to each side's biosample (never an identifier — a guid/at-uri). + /// A reference to the biosample of each side, and it says nothing about that biosample. It is + /// never an identifier: it is a guid or an at-uri. pub attesting_sample_ref: Option, pub partner_sample_ref: Option, pub total_shared_cm: f64, @@ -44,7 +52,8 @@ pub struct IbdAttestation { pub longest_segment_cm: f64, /// The relationship band (`RelationshipEstimate` debug name). pub relationship: String, - /// SHA-256 (base64) of the attester's match summary — the agreement fingerprint. + /// The SHA-256 of the match summary of the attester, in base64. It is the fingerprint that + /// shows whether the two peers agree. pub summary_hash: String, /// RFC3339 timestamp. pub attested_at: String, @@ -84,8 +93,9 @@ impl IbdAttestation { } } - /// The canonical `\n`-joined message that gets signed/verified — every field except the signature - /// and the signing key (which carries the signature). Both peers build it byte-identically. + /// The canonical message that a peer signs, and that the other verifies. A `\n` joins its + /// parts. It holds every field except the signature, and except the key that signs, which + /// carries that signature. Both peers build it to the same bytes. pub fn canonical(&self) -> String { [ self.match_request_uri.as_str(), @@ -104,8 +114,9 @@ impl IbdAttestation { } } -/// SHA-256 (lowercase hex) of a match summary's salient fields — the cross-peer agreement -/// fingerprint. Deterministic so both peers, computing the same segments, get the same hash. +/// The SHA-256 of the fields of a match summary that matter, in lower-case hex. It is the +/// fingerprint that shows whether the two peers agree. It is deterministic, so two peers that +/// compute the same segments get the same hash. pub fn summary_hash(s: &MatchSummary) -> String { let canon = format!( "{:.4}|{}|{:.4}|{:?}", @@ -123,13 +134,15 @@ pub fn summary_hash(s: &MatchSummary) -> String { pub enum IbdExchangeMsg { /// The sender's panel dosages (`build` = the panel's reference build, e.g. `hs1`). Dosages { build: String, sites: Vec }, - /// The sender's signed match attestation (boxed — much larger than the other variant). + /// The signed match attestation of the sender. It sits in a `Box`, because it is much larger + /// than the other variant. Attest(Box), } impl IbdExchangeMsg { - /// Gzipped JSON bytes for the channel. The dosage payload is large (a panel of sites), and the - /// relay caps an envelope at 1 MiB, so it's compressed (the dosage vector compresses well). + /// The JSON bytes for the channel, through gzip. The dosage payload is large, because it + /// holds a whole panel of sites, and the relay limits one envelope to 1 MiB. So the code + /// compresses it, and a vector of dosages compresses well. pub fn to_bytes(&self) -> Result, String> { use flate2::{write::GzEncoder, Compression}; use std::io::Write; diff --git a/crates/navigator-analysis/src/ibd_panel.rs b/crates/navigator-analysis/src/ibd_panel.rs index 20ba3c42..6e8c6556 100644 --- a/crates/navigator-analysis/src/ibd_panel.rs +++ b/crates/navigator-analysis/src/ibd_panel.rs @@ -1,17 +1,24 @@ -//! Multi-build, chip-compatible IBD reference panel (ancestry-ibd-asset-wiring B2/B2c). +//! The IBD reference panel. It covers more than one build, and a chip can use it. See the +//! ancestry-ibd asset design, B2 and B2c. //! -//! IBD matching needs a neutral, dense SNP set that's also **assayed by consumer arrays** — chip -//! kits outnumber WGS by orders of magnitude, so the panel must be where chip and WGS overlap. -//! Each site carries its `(contig, pos, REF, ALT)` on **CHM13, GRCh37, and GRCh38** (built once via -//! allele-aware GATK liftover, offline), so a chip genotype on *any* build resolves to the canonical -//! CHM13 site + orientation with **no runtime liftover** — the panel pre-computes it. +//! IBD matching needs a neutral, dense set of SNPs that a **consumer array also assays**. Chip kits +//! outnumber WGS runs by orders of magnitude, so the panel must sit where a chip and a WGS run +//! overlap. //! -//! Two correctness rules: -//! - The per-build loci carry the **same biological alleles** (GATK reverse-complements / swaps -//! REF↔ALT on inverted chain blocks), so "count of that build's ALT" == "count of the CHM13 ALT". -//! The dosage is therefore build-agnostic. -//! - **Strand-ambiguous palindromes (A/T, C/G) are excluded** ([`is_palindromic`]) — `rc(A)=T` is -//! also a valid allele, so a chip's strand can't be disambiguated by allele comparison. +//! Each site carries its `(contig, pos, REF, ALT)` on **CHM13, GRCh37 and GRCh38**. An offline +//! GATK liftover that knows about alleles builds those once. So a chip genotype on *any* build +//! resolves to the canonical CHM13 site, with its orientation. The app needs **no liftover at run +//! time**, because the panel already holds the answer. +//! +//! Two rules keep this correct: +//! +//! - The locus of each build carries the **same biological alleles**. On an inverted chain block, +//! GATK takes the reverse complement, and it exchanges REF with ALT. The count of the ALT of +//! that build then equals the count of the CHM13 ALT. The dosage does not depend on the +//! build. +//! - The panel **leaves out a palindrome that is ambiguous about the strand**, which is A/T or +//! C/G. See [`is_palindromic`]. There `rc(A)=T` is also a correct allele, so a comparison of the +//! alleles can not tell you the strand of a chip. use std::collections::HashMap; @@ -76,11 +83,16 @@ impl IbdPanel { bincode::serialize(self).map_err(|e| AnalysisError::Message(format!("ibd panel encode: {e}"))) } - /// Build from sites, **retaining** strand-ambiguous palindromes (A/T, C/G). The panel is a probe - /// superset: WGS + ancestry genotype palindromic sites fine (a read gives the reference-strand - /// base), and only the CHIP path can't orient them — so [`resolve_chip`] skips palindromes at - /// resolve time rather than excluding them from the panel here. Returns `(panel, n_palindromic)` - /// (the retained palindrome count, for the build log). + /// Build from a set of sites, and **keep** a palindrome that is ambiguous about the strand, + /// which is A/T or C/G. + /// + /// The panel is a superset of probes. A WGS run and the ancestry path genotype a palindromic + /// site without trouble, because a read gives the base on the reference strand. The CHIP path + /// alone can not orient one. So [`resolve_chip`] skips a palindrome when it resolves, and this + /// function does not remove one from the panel. + /// + /// Returns `(panel, n_palindromic)`, where the second value is the count of palindromes that + /// stayed, for the build log. pub fn from_sites(build: impl Into, sites: Vec) -> (Self, usize) { let palindromic = sites .iter() @@ -95,18 +107,25 @@ impl IbdPanel { ) } - /// Resolve chip calls (on `build`, as `(contig, pos, a1, a2)`) to canonical CHM13 dosages. - /// Indexes the panel by the build's `(contig, position)`, then counts copies of the **canonical - /// CHM13 ALT** directly from each observed pair — direct or reverse-complement - /// ([`dosage_from_alleles`]) — and emits it as a [`SiteGenotype`] at the CHM13 locus. No runtime - /// liftover; no alignment. Unmatched / no-call / non-reconciling calls are dropped. + /// Resolve the chip calls, which are on `build` and come as `(contig, pos, a1, a2)`, to + /// canonical CHM13 dosages. /// - /// We deliberately score against the **CHM13** `(REF, ALT)`, not the build locus's: chip allele - /// letters are absolute, and the asset's per-build `(REF, ALT)` labels are *not* reliably oriented - /// to the CHM13 ALT (a large fraction are ref/alt-swapped relative to CHM13 — the GRCh37 reference - /// allele is often the CHM13 ALT), so scoring against the build ALT flips the dosage 0↔2 at those - /// sites. Comparing the chip alleles to the CHM13 alleles (with the rc retry for strand) is - /// orientation-bug-proof; the build locus is used only to look the site up by position. + /// It indexes the panel by the `(contig, position)` of that build. It then counts the copies of + /// the **canonical CHM13 ALT** straight from each observed pair, either directly or through the + /// reverse complement. See [`dosage_from_alleles`]. It emits the result as a [`SiteGenotype`] + /// at the CHM13 locus. There is no liftover at run time, and no alignment. It drops a call that + /// does not match a site, a no-call, and a call that does not reconcile. + /// + /// The score goes against the **CHM13** `(REF, ALT)`, and not against the locus of the build. + /// That is deliberate. The allele letters of a chip are absolute. But the `(REF, ALT)` labels + /// of each build, in the asset, do not reliably point to the CHM13 ALT. A large share of them + /// have REF and ALT the other way round, and the reference allele of GRCh37 is often the CHM13 + /// ALT. At those sites, a score against the ALT of the build turns the dosage from 0 to 2, and + /// from 2 to 0. + /// + /// A comparison of the chip alleles against the CHM13 alleles, with the reverse-complement + /// retry for the strand, can not go wrong that way. The locus of the build has one use: to look + /// the site up by position. pub fn resolve_chip(&self, build: &str, calls: &[(String, i64, char, char)]) -> Vec { let mut index: HashMap<(&str, i64), &IbdPanelSite> = HashMap::new(); for s in &self.sites { @@ -119,9 +138,10 @@ impl IbdPanel { let Some(site) = index.get(&(contig.as_str(), *pos)) else { continue; }; - // Strand-ambiguous palindromes (A/T, C/G) can't be oriented from a chip's reported - // alleles — skip them for the chip path (WGS/ancestry still use them via direct base - // calls). The probe panel retains them; this is where the chip-only exclusion lives. + // Take a palindrome that is ambiguous about the strand, which is A/T or C/G. The + // reported alleles of a chip give it no orientation. So skip it on the chip path. A + // WGS run and the ancestry path still use it, from a direct base call. The probe panel + // keeps it, and this line is where the chip path alone leaves it out. if is_palindromic(site.chm13.reference, site.chm13.alternate) { continue; } @@ -133,18 +153,26 @@ impl IbdPanel { out } - /// Resolve a **whole-genome, variant-only** source (a WGS VCF or CompleteGenomics masterVar) to - /// canonical CHM13 dosages over the panel. Unlike a chip — which reports a genotype at every - /// array site — such a source lists *only* the non-reference sites, so every panel site the - /// source could have called but didn't is taken as **homozygous reference** (dosage 0). That - /// assumption is valid **only** for a source that genotyped the whole genome (absent ⇒ hom-ref, - /// not no-call); never pass a targeted panel (Big Y / Sanger) here. + /// Resolve a source that covers the **whole genome and lists variants alone** to canonical + /// CHM13 dosages over the panel. A WGS VCF and a CompleteGenomics masterVar are such + /// sources. + /// + /// A chip reports a genotype at every array site. Such a source instead lists *only* the sites + /// that are not reference. So every panel site that the source could have called, and did not, + /// counts as **homozygous reference**, at dosage 0. /// - /// `variant_calls` are the source's variant sites on `build` as `(contig, pos, a1, a2)` - /// reference-forward allele pairs. Contigs match `chr`-insensitively (a source's `chr1` lines up - /// with a panel `grch37` locus stored as `1`). A variant whose alleles don't reconcile to the - /// site (multiallelic mismatch) is dropped, not mis-called hom-ref. Palindromic (A/T, C/G) sites - /// are skipped — strand-ambiguous across builds, exactly as [`resolve_chip`]. + /// That assumption holds **only** for a source that genotyped the whole genome, where an + /// absent site means hom-ref and not a no-call. Never give this function a targeted panel, + /// such as a Big Y or a Sanger run. + /// + /// `variant_calls` holds the variant sites of the source on `build`, as `(contig, pos, a1, a2)` + /// allele pairs, forward on the reference. A contig matches whatever `chr` prefix it carries, + /// so a `chr1` from the source lines up with a `grch37` panel locus stored as `1`. + /// + /// A variant whose alleles do not reconcile to the site, which is a mismatch at a site with + /// more than two alleles, goes away. The code does not call it hom-ref by mistake. It also + /// skips a palindromic site, A/T or C/G, because that site is ambiguous about the strand across + /// builds, exactly as in [`resolve_chip`]. pub fn resolve_whole_genome(&self, build: &str, variant_calls: &[(String, i64, char, char)]) -> Vec { let norm = crate::contig::bare_upper; let variants: HashMap<(String, i64), (char, char)> = variant_calls @@ -172,16 +200,24 @@ impl IbdPanel { out } - /// Re-key genotypes taken at **this build's** loci back to canonical CHM13 dosages. The caller - /// genotypes a non-CHM13 alignment's BAM at each site's `locus(build)` (that build's - /// contig/pos/REF/ALT); the resulting `dosage` counts the *build* ALT. Because the per-build - /// REF/ALT are often swapped relative to CHM13 (see [`resolve_chip`]), we reconstruct the observed - /// alleles from that dosage and re-score them against the **CHM13** REF/ALT (direct or - /// reverse-complement), emitting each at its CHM13 locus with the alignment's depth/GQ preserved. - /// Palindromes (A/T, C/G) are skipped — strand-ambiguous across builds, exactly as the chip and - /// whole-genome resolvers. This is the alignment analogue of [`resolve_chip`], and it lets a - /// GRCh37/GRCh38 WGS reach the CHM13-coordinate ancestry panel without a runtime liftover (the - /// panel already carries every build's coordinates). + /// Take genotypes that the caller made at the loci of **this build**, and key them back to + /// canonical CHM13 dosages. + /// + /// The caller genotypes the BAM of an alignment that is not on CHM13. It works at the + /// `locus(build)` of each site, which holds the contig, the position, the REF and the ALT of + /// that build. The `dosage` that comes back counts the ALT of the *build*. + /// + /// The REF and ALT of a build are often the other way round from CHM13. See [`resolve_chip`]. + /// So this function builds the observed alleles again from that dosage. It scores them against + /// the **CHM13** REF and ALT, either directly or through the reverse complement. It emits each + /// one at its CHM13 locus, with the depth and the GQ of the alignment. + /// + /// It skips a palindrome, A/T or C/G. That site is ambiguous about the strand across builds, + /// exactly as in the chip resolver and the whole-genome one. + /// + /// This is the counterpart of [`resolve_chip`] for an alignment. It lets a GRCh37 or GRCh38 WGS + /// run reach the ancestry panel, whose coordinates are CHM13, with no liftover at run time. The + /// panel already carries the coordinates of every build. pub fn resolve_alignment(&self, build: &str, genotypes: &[SiteGenotype]) -> Vec { let norm = crate::contig::bare_upper; let mut index: HashMap<(String, i64), &IbdPanelSite> = HashMap::new(); @@ -201,8 +237,9 @@ impl IbdPanel { if is_palindromic(site.chm13.reference, site.chm13.alternate) { continue; } - // The genotype was called against the build REF/ALT (g.reference_allele/g.alternate_allele). - // Reconstruct the observed diploid alleles from the dosage, then re-score vs the CHM13 alleles. + // The caller made this genotype against the REF and ALT of the build, which are + // g.reference_allele and g.alternate_allele. Build the observed diploid alleles again + // from the dosage, and then score them against the CHM13 alleles. let br = g.reference_allele.chars().next().unwrap_or('N'); let ba = g.alternate_allele.chars().next().unwrap_or('N'); let (a1, a2) = match g.dosage { @@ -233,8 +270,8 @@ impl IbdPanel { out } - /// The canonical CHM13 `(contig, position)` sites — the targets a WGS caller genotypes so its - /// dosages line up with the chip path. + /// The canonical CHM13 `(contig, position)` sites. A WGS caller genotypes these targets, and + /// its dosages then line up with those of the chip path. pub fn chm13_sites(&self) -> Vec<(&str, i64)> { self.sites .iter() @@ -243,8 +280,9 @@ impl IbdPanel { } } -/// Build a diploid [`SiteGenotype`] at a panel site's canonical CHM13 locus with the given alt -/// dosage — the shared emit for the chip and whole-genome resolvers (no depth/quality; dosage only). +/// Build a diploid [`SiteGenotype`] at the canonical CHM13 locus of a panel site, with the given +/// alt dosage. The chip resolver and the whole-genome one share this. It carries no depth and no +/// quality, and the dosage alone. fn panel_site_genotype(site: &IbdPanelSite, dosage: i32) -> SiteGenotype { SiteGenotype { name: site.rsid.clone(), @@ -264,8 +302,9 @@ fn panel_site_genotype(site: &IbdPanelSite, dosage: i32) -> SiteGenotype { } } -/// Whether `(a, b)` is a strand-ambiguous palindrome (A/T or C/G) — excluded from a chip-compatible -/// panel because reverse-complement can't disambiguate the array's strand. +/// True when `(a, b)` is a palindrome that is ambiguous about the strand, which is A/T or C/G. A +/// panel that a chip can use leaves those out, because the reverse complement can not tell you the +/// strand of the array. pub fn is_palindromic(a: char, b: char) -> bool { matches!( (a.to_ascii_uppercase(), b.to_ascii_uppercase()), @@ -324,8 +363,9 @@ mod tests { #[test] fn resolve_chip_same_and_opposite_strand() { - // rs1: GRCh37 1:500 A/G (same alleles as CHM13 chr1:100 A/G). - // rs2: GRCh37 1:600 T/C — CHM13 chr1:200 A/G (a strand flip: GRCh37 alleles are rc). + // rs1: GRCh37 1:500 A/G, which holds the same alleles as CHM13 chr1:100 A/G. + // rs2: GRCh37 1:600 T/C, against CHM13 chr1:200 A/G. That is a strand flip, and the GRCh37 + // alleles are the reverse complement. let (panel, _) = IbdPanel::from_sites( "chm13v2.0", vec![ @@ -333,7 +373,8 @@ mod tests { site("rs2", (200, 'A', 'G'), Some((600, 'T', 'C'))), ], ); - // Chip on GRCh37: rs1 het AG → dosage 1; rs2 het TC → reconciles via rc → dosage 1. + // A chip on GRCh37. rs1 is het AG, so the dosage is 1. rs2 is het TC. That reconciles + // through the reverse complement, so its dosage is 1 too. let calls = vec![ ("1".to_string(), 500, 'A', 'G'), ("1".to_string(), 600, 'T', 'C'), @@ -350,9 +391,11 @@ mod tests { #[test] fn resolve_chip_ref_alt_swapped_against_chm13() { - // The asset's GRCh37 locus is ref/alt-SWAPPED vs CHM13: chm13 chr1:100 G/T (ALT=T) but - // grch37 1:500 T/G (ALT=G). A chip hom for G is hom-CHM13-REF → dosage 0. Scoring against the - // build ALT (G) would wrongly give 2; scoring against the CHM13 ALT (T) gives the correct 0. + // The GRCh37 locus of the asset has REF and ALT the OTHER WAY ROUND from CHM13. CHM13 + // chr1:100 is G/T, with ALT=T. GRCh37 1:500 is T/G, with ALT=G. A chip that is hom for G is + // then hom for the CHM13 REF, and the dosage is 0. A score against the ALT of the build, + // which is G, would wrongly give 2. A score against the CHM13 ALT, which is T, gives the + // correct 0. let (panel, _) = IbdPanel::from_sites( "chm13v2.0", vec![site("rs_swap", (100, 'G', 'T'), Some((500, 'T', 'G')))], @@ -391,8 +434,9 @@ mod tests { site("rs3", (300, 'A', 'T'), Some((700, 'A', 'T'))), // palindrome — always skipped ], ); - // A whole-genome source that lists ONLY rs1 as a het (GRCh37 forward alleles T/C, contig - // "chr1" to prove chr-insensitive matching). rs2 is unlisted ⇒ hom-ref; rs3 skipped. + // A whole-genome source that lists ONLY rs1, as a het. Its GRCh37 forward alleles are + // T/C. Its contig is "chr1", which shows that the match ignores the `chr` prefix. rs2 is + // not in the list, so it is hom-ref. The code skips rs3. let calls = vec![("chr1".to_string(), 500, 'T', 'C')]; let g = panel.resolve_whole_genome("GRCh37", &calls); let by_pos: std::collections::HashMap = g.iter().map(|s| (s.position, s.dosage)).collect(); @@ -411,9 +455,13 @@ mod tests { site("rs2", (200, 'C', 'T'), Some((600, 'C', 'T'))), ], ); - // rs1 hom-alt (G/G → dosage 2); rs2 listed but the alleles are internally inconsistent with - // the biallelic site: C matches ref directly, A only matches alt(T) under rc — neither a - // pure direct nor a pure rc pair, so it doesn't reconcile → dropped, NOT called hom-ref. + // rs1 is hom-alt, G/G, so its dosage is 2. + // + // The rs2 record is in the list, but its alleles do not agree with the site, which has two + // alleles. C matches the ref directly. A matches the + // alt, T, only under the reverse complement. The pair is then neither a pure direct match + // nor a pure reverse-complement one. It does not reconcile. The code drops it, and it does + // NOT call it hom-ref. let calls = vec![("1".to_string(), 500, 'G', 'G'), ("1".to_string(), 600, 'C', 'A')]; let g = panel.resolve_whole_genome("GRCh37", &calls); let by_pos: std::collections::HashMap = g.iter().map(|s| (s.position, s.dosage)).collect(); @@ -471,8 +519,10 @@ mod tests { site_b("rs_pal", (300, 'A', 'T'), ("chr1", 700, 'A', 'T'), "38"), ]; let (panel, _) = IbdPanel::from_sites("chm13v2.0", sites); - // Genotypes at the grch38 loci (build-oriented dosage): - // rs1 het → chm13 dosage 1; rs_swap hom grch38-ALT (G/G) — G is the CHM13 REF → chm13 dosage 0. + // The genotypes at the grch38 loci, whose dosage points at the build. + // + // rs1 is het, so the CHM13 dosage is 1. rs_swap is hom for the grch38 ALT, which is G/G. G + // is the CHM13 REF, so the CHM13 dosage is 0. let raw = vec![ geno("rs1", "chr1", 500, "A", "G", 1), geno("rs_swap", "chr1", 600, "T", "G", 2), @@ -504,7 +554,7 @@ mod tests { (out[0].position, out[0].dosage, out[0].contig.as_str()), (100, 2, "chr1") ); - // A no-call (dosage < 0) is dropped. + // The code drops a no-call, which is a dosage below 0. assert!(panel .resolve_alignment("GRCh37", &[geno("rs1", "1", 500, "A", "G", -1)]) .is_empty()); diff --git a/crates/navigator-analysis/src/index.rs b/crates/navigator-analysis/src/index.rs index 1ee42f8f..73252a45 100644 --- a/crates/navigator-analysis/src/index.rs +++ b/crates/navigator-analysis/src/index.rs @@ -1,17 +1,20 @@ -//! Building the coordinate index a BAM/CRAM needs for **region queries**. +//! Build the coordinate index that a BAM or CRAM needs for a **region query**. //! -//! The per-contig walker ([`crate::unified`]), the callable-interval scan -//! ([`crate::coverage::callable_intervals`]), and the de-novo / STR callers all seek to a -//! `contig:start-end` via `noodles`' indexed readers, which autoload a sibling `.bai` (BAM) or -//! `.crai` (CRAM). Aligned files usually ship with one, but a re-exported or single-imported file -//! often does not — without it those paths either error outright or fall back to a whole-file linear -//! scan. This module builds the missing index once, up front, so every query path is fast. +//! Three paths seek to a `contig:start-end`: the walker over the contigs ([`crate::unified`]), the +//! scan for callable intervals ([`crate::coverage::callable_intervals`]), and the de-novo and STR +//! callers. They all go through the indexed readers of `noodles`, which load a `.bai` beside a +//! BAM, or a `.crai` beside a CRAM. //! -//! Index construction is a single sequential pass over the file (the same cost as one analysis -//! read), so it is worth surfacing to the user with progress. The BAM path reports a true byte -//! fraction (the compressed offset of the bgzf stream); the CRAM path delegates to `noodles`' -//! container walk, which exposes no offset hook, so it reports *indeterminate* progress -//! (`total = None`) — the UI shows a spinner for it. +//! An aligned file usually comes with one. But a file that somebody exported again, or imported on +//! its own, often does not. Without an index, those paths either fail outright, or fall back to a +//! linear scan of the whole file. This module builds the missing index once, at the start, so that +//! every query path is fast. +//! +//! To build an index is one sequential pass over the file, which costs the same as one analysis +//! read. So it is worth a progress report to the user. The BAM path gives a true fraction of the +//! bytes, from the compressed offset of the bgzf stream. The CRAM path hands the work to the +//! container walk of `noodles`, which offers no hook for an offset. So that path reports progress +//! with no end, at `total = None`, and the UI shows a spinner for it. use std::fs::File; use std::path::{Path, PathBuf}; @@ -29,13 +32,15 @@ use noodles::sam::{ use crate::error::AnalysisError; use crate::reader::{detect_format, has_region_index, Format}; -/// Progress sink for index construction: `(done_bytes, total_bytes)`. `total` is `Some` for BAM -/// (compressed file length) and `None` for CRAM (no offset hook — indeterminate). +/// The sink for the progress of an index build, as `(done_bytes, total_bytes)`. `total` is `Some` +/// for a BAM, where it holds the compressed length of the file. It is `None` for a CRAM, which +/// offers no hook for an offset, so the progress there has no end. pub type ProgressFn<'a> = &'a mut dyn FnMut(u64, Option); -/// The sibling index path this module writes for `path`: `foo.bam` → `foo.bam.bai`, -/// `foo.cram` → `foo.cram.crai`. (This is the `.bam.bai` / `.cram.crai` spelling; the query readers -/// also accept the `.bai` / `.crai` spelling, but we always write the dotted form `samtools` does.) +/// The index path that this module writes beside `path`: `foo.bam` gives `foo.bam.bai`, and +/// `foo.cram` gives `foo.cram.crai`. That is the `.bam.bai` and `.cram.crai` form. The query +/// readers also accept the `.bai` and `.crai` form, but this module always writes the dotted form +/// that `samtools` writes. pub fn index_path_for(path: &Path) -> PathBuf { let ext = match detect_format(path) { Format::Bam => "bai", @@ -47,14 +52,19 @@ pub fn index_path_for(path: &Path) -> PathBuf { path.with_file_name(file_name) } -/// Build the coordinate index for `path` **if one is not already present**, returning the path of -/// the index that was written (`Ok(None)` when a `.bai`/`.crai` already existed — nothing to do). +/// Build the coordinate index of `path`, **if one is not there already**. Returns the path of the +/// index that it wrote. It returns `Ok(None)` when a `.bai` or `.crai` already existed, and there +/// was nothing to do. +/// +/// A BAM input must be in coordinate order, with `SO:coordinate` in its header. A file that is not +/// sorted gives a clear error, and not a corrupt index. /// -/// The BAM input must be coordinate-sorted (`SO:coordinate` in the header); an unsorted file yields -/// a clear error rather than a corrupt index. `reference` is unused: the BAM path never needs it, -/// and the CRAM path *would* need it for multi-reference slices but has no way to supply it — see -/// [`build_crai`]. It is kept in the signature to stay drop-in with the reader/decode helpers, and -/// because threading it through is what a fixed CRAM indexer would want. +/// Nothing uses `reference`. The BAM path never needs it. The CRAM path *would* need it for a +/// slice that holds more than one reference, and it has no way to get it there. See +/// [`build_crai`]. +/// +/// The argument stays in the signature for two reasons. It keeps this function interchangeable +/// with the reader and decode helpers, and a CRAM indexer that somebody fixes would want it. pub fn ensure_index( path: &Path, _reference: Option<&Path>, @@ -71,9 +81,11 @@ pub fn ensure_index( Ok(Some(dst)) } -/// Index a coordinate-sorted BAM, reporting a byte fraction from the bgzf compressed offset. This -/// mirrors `noodles`' `bam::fs::index`, but drives the record loop ourselves so we can emit -/// progress against the on-disk (compressed) file length. +/// Index a BAM that is in coordinate order. It reports a fraction of the bytes, from the +/// compressed offset of the bgzf stream. +/// +/// It has the same shape as `bam::fs::index` in `noodles`. But it drives the record loop itself, +/// so that it can emit progress against the compressed length of the file on disk. fn build_bai(path: &Path, dst: &Path, progress: ProgressFn) -> Result<(), AnalysisError> { let total = File::open(path) .and_then(|f| f.metadata()) @@ -120,7 +132,7 @@ fn build_bai(path: &Path, dst: &Path, progress: ProgressFn) -> Result<(), Analys .add_record(alignment_context, chunk) .map_err(|e| AnalysisError::io(path, e))?; - // Report on ~32 MB of compressed progress so a multi-GB BAM doesn't flood the channel. + // Report on ~32 MB of compressed progress so a multi-GB BAM does not flood the channel. let done = end_position.compressed(); if done.saturating_sub(last_reported) >= 32_000_000 { last_reported = done; @@ -135,17 +147,22 @@ fn build_bai(path: &Path, dst: &Path, progress: ProgressFn) -> Result<(), Analys Ok(()) } -/// Index a CRAM by delegating to `noodles`' container walk. It exposes no incremental offset, so -/// progress is reported as indeterminate: one `(0, None)` heartbeat at the start, then completion. +/// Index a CRAM. It hands the work to the container walk of `noodles`. That walk offers no offset +/// as it goes, so the progress has no end: one `(0, None)` beat at the start, and then the +/// finish. /// -/// **This does not work on every CRAM.** A *single*-reference slice is cheap to index — its span -/// comes straight from the slice header — but a *multi*-reference slice has no one span, so noodles -/// decodes its records to derive one. Reconstructing a mapped record's sequence needs the reference -/// bases, and `cram::fs::index` hands the decoder an empty `fasta::Repository` (its own `// TODO`, -/// still open as of noodles-cram 0.95), so it panics there instead of erroring. Aligners write -/// their unmapped/decoy tail as multi-reference slices, so most real whole-genome CRAMs hit this — -/// and only at the very end of the file, after the walk has already done nearly all the work. -/// [`multi_reference_panic`] turns that panic into an actionable error rather than a crash. +/// **This does not work on every CRAM.** A slice with *one* reference costs little to index, +/// because its span comes straight from the slice header. A slice with *more than one* reference +/// has no single span, so noodles decodes its records to get one. +/// +/// To build the sequence of a mapped record again needs the reference bases. `cram::fs::index` +/// gives the decoder an empty `fasta::Repository`, which carries its own `// TODO`, still open at +/// noodles-cram 0.95. So it panics there, and it does not return an error. +/// +/// An aligner writes its unmapped and decoy tail as slices with more than one reference. So most +/// real whole-genome CRAMs reach this. They reach it at the very end of the file, after the walk +/// has done almost all of the work. [`multi_reference_panic`] turns that panic into an error that +/// a user can act on, and not a crash. fn build_crai(path: &Path, dst: &Path, progress: ProgressFn) -> Result<(), AnalysisError> { progress(0, None); let index = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cram::fs::index(path))) { @@ -157,15 +174,19 @@ fn build_crai(path: &Path, dst: &Path, progress: ProgressFn) -> Result<(), Analy Ok(()) } -/// The panic text noodles emits when a multi-reference slice's record decode asks the (empty) -/// repository for reference bases. Distinct from the single-reference slice's "invalid **slice** -/// reference sequence name", which would mean something genuinely different — a reference that -/// really is missing a contig — so match on the record-level wording only. +/// The panic text that noodles gives when the record decode of a slice with more than one +/// reference asks the empty repository for reference bases. +/// +/// It is different from "invalid **slice** reference sequence name", which comes from a slice with +/// one reference. That message means something else: a reference that truly does not hold a +/// contig. So match on the text at the record level alone. const MULTI_REFERENCE_PANIC: &str = "invalid reference sequence name"; -/// Explain a panic escaping the CRAM index walk. The multi-reference case is known and has a -/// concrete workaround, so name it and give the command; anything else reports its own text -/// instead of a guess. +/// Explain a panic that comes out of the CRAM index walk. +/// +/// Somebody has already found the case of a slice with more than one reference, and there is a +/// concrete way around it. So this function names that case and gives the command. Any other +/// panic reports its own text, and this code does not guess. fn index_panic_error(path: &Path, payload: &(dyn std::any::Any + Send)) -> AnalysisError { let text = crate::error::panic_text(payload).unwrap_or("no further detail"); if multi_reference_panic(text) { @@ -208,9 +229,10 @@ fn alignment_context(record: &bam::Record) -> std::io::Result<(Option, Op mod tests { use super::*; - /// The two noodles panics differ by one word, and they mean opposite things: the record-level - /// one is the indexer's own limitation, the slice-level one means the reference really is - /// missing a contig. Telling a user to run `samtools index` for the latter would be wrong. + /// The two panics of noodles differ by one word, and they mean opposite things. The one at + /// the record level is a limit of the indexer itself. The one at the slice level means that + /// the reference truly does not hold a contig. To tell a user to run `samtools index` for that + /// second one would be wrong. #[test] fn distinguishes_the_indexer_limitation_from_a_genuinely_missing_contig() { assert!(multi_reference_panic("invalid reference sequence name")); @@ -230,7 +252,8 @@ mod tests { "gives the command: {known}" ); - // An unclassified panic reports its own text rather than borrowing the known diagnosis. + // A panic that this code does not know reports its own text. It does not take the + // diagnosis of the known case. let other = index_panic_error(path, &String::from("not yet implemented")).to_string(); assert!(other.contains("not yet implemented"), "quotes the panic: {other}"); assert!(!other.contains("samtools"), "no bogus workaround: {other}"); diff --git a/crates/navigator-analysis/src/lai.rs b/crates/navigator-analysis/src/lai.rs index 42a581a0..daa32385 100644 --- a/crates/navigator-analysis/src/lai.rs +++ b/crates/navigator-analysis/src/lai.rs @@ -2,19 +2,26 @@ //! reference-haplotype panel ([`HaplotypeReference`]). //! //! The allele-frequency painter ([`crate::ancestry::paint_local_ancestry_phased`]) scores each site -//! against per-population *allele frequencies*, which throws away the haplotype/linkage structure -//! that separates closely-related populations (e.g. British vs continental NW-European) — so it -//! collapses fine structure to the super-population. This module instead models each phased side as a -//! **mosaic of reference haplotypes**: a Li & Stephens copying HMM whose hidden state is *which -//! reference haplotype we are copying*, with the copied haplotype's population label read off as the -//! local ancestry. Long shared haplotypes (IBD-like tracts) pull the posterior toward the right -//! sub-population, recovering the structure frequency emission can't see. +//! against the *allele frequency* of each population. That throws away the haplotype and linkage +//! structure. Two populations that are near each other, such as the British and the continental +//! north-west European, need that structure to separate. So the painter collapses the fine +//! structure down to the super-population. //! -//! Per phased side and contig: forward–backward over the `K` reference haplotypes (O(N·K) thanks to -//! the rank-1 "stay or uniform-jump" transition), aggregate the per-site copy posterior by population -//! label, then a Viterbi over labels smooths the noisy per-site calls into coherent segments. Output -//! is the same [`AncestrySegment`] type the frequency painter produces, with `fine_population_code` -//! set to the resolved sub-population and `population_code` its super-population roll-up. +//! This module instead models each phased side as a **mosaic of reference haplotypes**. It is a Li +//! & Stephens copying HMM. Its hidden state is *which reference haplotype the model copies*, and +//! the population label of that haplotype is the local ancestry. A long shared haplotype, which is +//! a tract like an IBD segment, pulls the posterior toward the correct sub-population. That +//! recovers the structure that a frequency emission can not see. +//! +//! For each phased side and each contig, the code does three things. First it runs +//! forward-backward over the `K` reference haplotypes. That costs O(N·K), because the transition +//! has rank 1: it is a "stay, or jump to a uniform choice". Then it adds up the copy posterior at +//! each site, by population label. Last, a Viterbi over the labels turns the noisy call at each +//! site into coherent segments. +//! +//! The output is the same [`AncestrySegment`] type that the frequency painter makes. +//! `fine_population_code` holds the resolved sub-population, and `population_code` holds the +//! super-population that it rolls up to. use std::collections::{BTreeMap, HashMap}; @@ -24,86 +31,121 @@ use crate::ancestry::HaplotypeReference; use crate::ibd::GeneticMap; use crate::phasing::{PhasedGenotypes, PhasedSite}; -/// Tuning for [`paint_copying_lai`]. +/// The controls of [`paint_copying_lai`]. #[derive(Debug, Clone)] pub struct CopyingLaiParams { - /// Copy mismatch rate μ: probability the copied reference allele is observed flipped (mutation / - /// divergence since the shared ancestor). Higher tolerates more mismatch before switching copy. + /// The copy mismatch rate μ. It is the probability that the copied reference allele reads the + /// other way round, from a mutation or from divergence since the shared ancestor. A higher + /// value accepts more mismatch before the model changes the haplotype that it copies. pub mismatch: f64, - /// Reference-haplotype switch intensity per centiMorgan (the copying model's recombination): how - /// readily the mosaic jumps to a different reference haplotype. Too high and the mosaic re-picks - /// the local allele match almost per-site, discarding the long-haplotype signal that separates - /// populations; too low and it commits to one reference haplotype for a whole ~10 cM tract, which - /// at this panel's density (~0.5 markers/Mb) rests each call on a handful of sites — the regime - /// where drifted isolates (Finnish/Sardinian/Basque) win on chance matches. + /// The switch intensity of the reference haplotype in one centiMorgan. It is the + /// recombination of the copying model, and it says how easily the mosaic jumps to a different + /// reference haplotype. + /// + /// Set it too high, and the mosaic takes the local allele match again at almost every site. + /// That throws away the long-haplotype signal which separates the populations. Set it too low, + /// and the mosaic holds one reference haplotype for a whole tract of about 10 cM. At the + /// density of this panel, which is about 0.5 markers/Mb, each call then stands on a few sites. + /// That is where a drifted isolate, such as the Finnish, Sardinian or Basque, wins on a match + /// by chance. /// - /// Calibrated on held-out reference individuals (`navigator-panelbuild validate-lai`): over - /// 0.1 → 1.0 accuracy rises and isolate over-call falls monotonically, flattening around 0.5. + /// The calibration ran on held-out reference individuals, with + /// `navigator-panelbuild validate-lai`. From 0.1 to 1.0, the accuracy rises and the over-call + /// of an isolate falls, both without a reverse, and the curve goes flat near 0.5. pub recomb_per_cm: f64, - /// Ancestry (population-label) switch intensity per cM for the smoothing Viterbi. Lower → longer, - /// more confident ancestry segments (needs sustained evidence to switch population). + /// The switch intensity of the ancestry, which is the population label, in one cM, for the + /// Viterbi that smooths the path. A lower value gives longer ancestry segments with more + /// confidence, because the model then needs evidence over a longer run to change + /// population. pub switch_per_cm: f64, /// A reference population with fewer than this many haplotypes is not a callable label; its /// haplotypes fold into their super-population (suppresses tiny stray-labelled reference groups). pub min_ref_haps: usize, - /// Balance the reference by capping each population at this many haplotypes. Without any cap the - /// largest 1000G samples (Iberian/Tuscan, ~214 haps) out-vote by count and paint a NW-European - /// as southern; a per-size *division* over-swings to the tiny HGDP isolates. Capping puts the - /// populations on a common footing so they compete on match quality, not sample count. + /// Balance the reference: hold each population to this many haplotypes at most. + /// + /// With no limit, the largest 1000G samples win by count alone, and they paint a north-west + /// European as southern. Those are the Iberian and the Tuscan, at about 214 haplotypes. A + /// *division* by the size instead swings too far, to the very small HGDP isolates. A limit + /// puts the populations on common ground. They then compete on the quality of the match, and + /// not on the size of the sample. + /// + /// The limit must stay **well above** the small reference populations: HGDP Orcadian has 30, + /// Adygei 32, and Basque 46. Set near them, it throws away most of the haplotypes of the big + /// populations. A population that the code thins copies worse, so the isolates then win by + /// default. /// - /// The cap must stay **well above** the small reference populations (HGDP Orcadian 30, Adygei 32, - /// Basque 46): set near them it throws away most of the big populations' haplotypes, and a - /// thinned population copies worse, so the isolates win by default. `validate-lai` measures - /// exactly that — at 50 a held-out individual scores 12.9% fine with 27.2% of its genome called - /// into a drifted isolate; at 200, 24.6% and 11.8%. Above ~300 accuracy falls again (the big - /// populations stop being balanced at all). + /// `validate-lai` measures exactly that. At 50, a held-out individual scores 12.9% fine, and + /// 27.2% of its genome goes to a drifted isolate. At 200, those become 24.6% and 11.8%. Above + /// about 300 the accuracy falls again, because the big populations are no longer in balance at + /// all. /// - /// Capping is not the only size correction — see [`Self::size_normalize`], which divides by the - /// haplotype count and, on a dense panel, does the work capping cannot. + /// This limit is not the only correction for size. See [`Self::size_normalize`], which divides + /// by the haplotype count. On a dense panel that division does the work that this limit can + /// not. pub max_ref_haps: usize, - /// Runs shorter than this many **centiMorgans** merge into the neighbouring segment. + /// A run shorter than this many **centiMorgans** merges into the segment beside it. /// - /// Expressed in genetic distance, not in sites, because the same site count means different - /// things on different panels: the shipped 15.6k-site panel carries one marker per ~200 kb while - /// the dense one carries one per ~19 kb, and `validate-lai` finds the same *physical* optimum on - /// both (~4 cM — 20 sites on the sparse panel, 200 on the dense). A site-count threshold would - /// silently mean a 40 Mb minimum segment if the panel got 10x denser and the number stayed put. + /// This is a genetic distance, and not a count of sites, because the same count of sites means + /// different things on different panels. The 15.6k-site panel that shipped carries one marker + /// in about 200 kb, and the dense one carries one in about 19 kb. `validate-lai` finds the same + /// *physical* best value on both, at about 4 cM. That is 20 sites on the sparse panel and 200 + /// on the dense one. + /// + /// Take a threshold in sites, and make the panel 10x denser, and leave the number where it + /// was. That threshold then means a smallest segment of 40 Mb, and nobody would see it + /// happen. pub min_segment_cm: f64, - /// Correct the copy posterior for how many haplotypes each population contributes: divide its - /// aggregated copy mass by `count^size_normalize` (`0.0` = off, `1.0` = a full per-haplotype - /// average). Capping ([`Self::max_ref_haps`]) can only bring populations *down* to a common - /// size, and this panel still spans 30 (HGDP Orcadian) to 200 haplotypes; dividing lets a small - /// population keep all its haplotypes and still compete per-haplotype. + /// Correct the copy posterior for the count of haplotypes that each population gives. The code + /// divides the copy mass of a population by `count^size_normalize`. `0.0` turns the correction + /// off, and `1.0` gives a full mean over the haplotypes. + /// + /// The limit in [`Self::max_ref_haps`] can bring a population *down* to a common size and + /// nothing more. This panel still runs from 30 haplotypes (HGDP Orcadian) to 200. The division + /// instead lets a small population keep all of its haplotypes, and still compete on a + /// haplotype-for-haplotype basis. + /// + /// **The correct value depends on the marker density, and it went the other way when the panel + /// got denser.** On the old 15.6k-site panel, every setting made the result worse, and this + /// control stayed off. + /// + /// On the 165k-site panel that ships, `validate-lai` measures the opposite. The small HGDP + /// populations come back. At 0.25: Sardinian 10→13% / 33→36%, Basque 33→37% / 37→39%, + /// Orcadian 6→10%, Russian 10→12% / 4→9%. GBR and TSI also get better, and CEU does not + /// move. /// - /// **This knob's answer depends on marker density, and it reversed when the panel got denser.** - /// On the old 15.6k-site panel every setting made things worse and it was rejected. On the - /// shipped 165k-site panel `validate-lai` measures the opposite — the small HGDP populations come - /// back (at 0.25: Sardinian 10→13% / 33→36%, Basque 33→37% / 37→39%, Orcadian 6→10%, Russian - /// 10→12% / 4→9%) while GBR and TSI also improve and CEU stays put. + /// The metric that the UI claims sets this value. The accuracy at each site does not set it. /// - /// The value is set by the metric the UI actually claims, not by per-site accuracy. Per-site - /// accuracy keeps rising to 1.0 (32.0→36.4%), but the *largest called label* stops being the - /// right one for NW-European subjects above 0.25 (top-1 50%→42%), because boosted small - /// populations displace it — and drifted-isolate over-call climbs 9.6→11→15→18%. 0.25 is the - /// knee: two thirds of the rescue, top-1 intact, +1.3 pts of isolate noise. + /// That accuracy continues to rise up to 1.0, from 32.0% to 36.4%. But above 0.25 the *largest + /// called label* stops being the correct one for a north-west European subject. Top-1 falls + /// from 50% to 42%, because the small populations that the correction lifts displace it. The + /// over-call of a drifted isolate also climbs, 9.6→11→15→18%. + /// + /// 0.25 is the knee of the curve. It gives two thirds of the rescue, it leaves top-1 intact, + /// and it costs 1.3 points of isolate noise. pub size_normalize: f64, - /// Global-composition gate: reference haplotypes whose super-population is below this fraction of - /// the genome-wide `prior` are dropped from the copying set entirely (the dominant super-pop is - /// always kept). Without it the fine-grained reference invites spurious short copies from every - /// continent — a 99%-European is painted with scattered East-Asian/African/American specks. `0.0` - /// disables the gate. + /// The gate on the global composition. The code drops a reference haplotype from the copying + /// set when its super-population is below this fraction of the genome-wide `prior`. It always + /// keeps the dominant super-population. + /// + /// Without the gate, the fine-grained reference invites short false copies from every + /// continent. A subject who is 99% European then gets small East-Asian, African and American + /// marks over the whole genome. `0.0` turns the gate off. pub min_ancestry: f64, } impl Default for CopyingLaiParams { - /// Calibrated against known truth by `navigator-panelbuild validate-lai` (held-out reference - /// individuals + simulated admixture over the shipped `ancestry_haps` panel) rather than by - /// inspecting one kit's painting. On the dense (165k-site) panel, 41 cases spanning 1000G and - /// HGDP populations: **32.0% fine-population accuracy against an 11.4% chance level, 68.1% at - /// the regional level, 98.5% super-population, and 9.6%** of a non-isolate genome called into a - /// drifted isolate. `recomb_per_cm`, `switch_per_cm` and `mismatch` are flat within - /// case-to-case noise at this density; `min_segment_cm` is the knob that matters (4 cM beats - /// 2 cM by 6 points of accuracy and 8 of regional accuracy). + /// `navigator-panelbuild validate-lai` calibrated these defaults against known truth. It used + /// held-out reference individuals, and simulated admixture over the `ancestry_haps` panel that + /// ships. Nobody looked at the painting of one kit and chose a value. + /// + /// The run covered the dense 165k-site panel, over 41 cases across the 1000G and HGDP + /// populations. The result was **32.0% accuracy at the fine population, against a chance level + /// of 11.4%. At the regional level it was 68.1%, and at the super-population 98.5%.** And + /// **9.6%** of a genome that is not an isolate went to a drifted isolate. + /// + /// At this density, `recomb_per_cm`, `switch_per_cm` and `mismatch` are flat inside the noise + /// between one case and the next. `min_segment_cm` is the control that matters: 4 cM beats + /// 2 cM by 6 points of accuracy, and by 8 points of regional accuracy. fn default() -> Self { Self { mismatch: 0.02, @@ -142,12 +184,16 @@ fn kept_super_pops(prior: &[(String, f64)], min_ancestry: f64) -> Option = Vec::new(); @@ -184,9 +231,10 @@ pub fn paint_copying_lai( return Vec::new(); } - // Callable label per KEPT haplotype (aligned to `kept_haps`): its fine population when that - // population has enough kept haplotypes, else the super-population it rolls up to (folding tiny - // stray-labelled groups). + // The label of each haplotype that the code KEPT, in line with `kept_haps`. It is the fine + // population of that haplotype, when the code kept enough haplotypes of that population. + // Otherwise it is the super-population that the fine one rolls up to. That puts a very small + // group with a stray label back into a larger one. let mut pop_counts = vec![0usize; reference.populations.len()]; for &h in &kept_haps { pop_counts[reference.hap_pop[h] as usize] += 1; @@ -208,7 +256,8 @@ pub fn paint_copying_lai( }); } let n_labels = labels.len(); - // Per-label divisor for the size correction: (kept haplotypes carrying the label)^size_normalize. + // The divisor of each label, for the size correction. It is the count of kept haplotypes with + // that label, raised to the power size_normalize. let mut label_weight = vec![1.0f64; n_labels]; if params.size_normalize != 0.0 { let mut counts = vec![0usize; n_labels]; @@ -280,9 +329,9 @@ pub fn paint_copying_lai( segments } -/// Forward–backward over the reference haplotypes for one side of one contig. Returns the per-site -/// posterior aggregated by population label: `post[site][label]`. Uses per-site normalization for -/// numerical stability (which cancels in the posterior). +/// Forward-backward over the reference haplotypes, for one side of one contig. Returns the +/// posterior at each site, added up by population label, as `post[site][label]`. It normalizes at +/// each site to keep the arithmetic stable, and that normalization cancels in the posterior. #[allow(clippy::too_many_arguments)] fn copying_posteriors( cols: &[usize], @@ -345,7 +394,8 @@ fn copying_posteriors( } } - // Backward, accumulating the label-aggregated posterior as we go (no stored β trellis). + // The backward pass. It adds up the posterior over the labels as it goes, and it stores no β + // trellis. let mut post = vec![vec![0.0f64; n_labels]; n]; let mut beta = vec![1.0f64 / k as f64; k]; accumulate_post(&alpha[n - 1], &beta, hap_label, label_weight, &mut post[n - 1]); @@ -376,9 +426,9 @@ fn copying_posteriors( post } -/// `post_out[label] = Σ_{hap: label(hap)=label} normalized(α·β) · label_weight[label]`, renormalized -/// to sum to 1 so the size correction rescales labels against each other without changing the scale -/// the smoothing Viterbi sees. +/// `post_out[label] = Σ_{hap: label(hap)=label} normalized(α·β) · label_weight[label]`. The code +/// then normalizes the result to sum to 1. The size correction then scales the labels against each +/// other, and the scale that the Viterbi sees does not move. fn accumulate_post(alpha_i: &[f64], beta_i: &[f64], hap_label: &[usize], label_weight: &[f64], post_out: &mut [f64]) { let mut gs = 0.0; for (a, b) in alpha_i.iter().zip(beta_i) { @@ -400,8 +450,9 @@ fn accumulate_post(alpha_i: &[f64], beta_i: &[f64], hap_label: &[usize], label_w } } -/// Viterbi over population labels with the per-site posterior as (log) emission and a -/// distance-scaled stay/switch transition — smooths noisy per-site calls into coherent segments. +/// Viterbi over the population labels. The emission is the log of the posterior at each site, and +/// the transition, which is a stay or a switch, scales with distance. It turns the noisy call at +/// each site into coherent segments. fn smooth_viterbi( post: &[Vec], positions: &[i64], @@ -453,10 +504,10 @@ fn smooth_viterbi( path } -/// Collapse a per-site label path into segments (merging runs shorter than `min_cm` of genetic -/// distance into the previous run), tagging each with the side `copy`, its super-population, and — -/// when the label is a fine population — the fine code. `posterior` is the mean per-site posterior -/// of the label. +/// Collapse a path that holds a label at each site into segments. A run shorter than `min_cm` of +/// genetic distance merges into the run before it. Each segment carries the side `copy`, the +/// super-population, and, when the label is a fine population, the fine code. `posterior` is the +/// mean posterior of the label over the sites. #[allow(clippy::too_many_arguments)] fn collapse_labels( contig: &str, @@ -476,8 +527,9 @@ fn collapse_labels( _ => runs.push((l, i, i)), } } - // A run's genetic span; the map is the same one the copying model used, so a missing interval - // (contig absent from the map) falls back to 0 and the run merges — the conservative direction. + // The genetic span of a run. The map is the same one that the copying model used. A missing + // interval, where the map does not hold the contig, falls back to 0, and the run merges. That + // is the careful direction. let span_cm = |lo: usize, hi: usize| { map.interval_cm(contig, positions[lo] as i32, positions[hi] as i32) .unwrap_or(0.0) @@ -523,8 +575,8 @@ mod tests { use super::*; use crate::ancestry::HapSite; - /// Deterministic xorshift64* — these are numeric gates, so the same run must give the same - /// numbers twice. + /// A deterministic xorshift64*. These are numeric gates, so two runs must give the same + /// numbers. struct Rng(u64); impl Rng { @@ -541,8 +593,9 @@ mod tests { let (u1, u2) = (self.next_f64().max(1e-12), self.next_f64()); (-2.0 * u1.ln()).sqrt() * (std::f64::consts::TAU * u2).cos() } - /// Allele frequency after drift of magnitude `fst` from `p` (normal approximation to the - /// Balding–Nichols draw — good enough to give populations distinguishable frequencies). + /// The allele frequency after drift of size `fst` away from `p`. It is the normal + /// approximation to the draw of Balding and Nichols (1995). That is good enough to give + /// the populations frequencies that a test can separate. fn drift(&mut self, p: f64, fst: f64) -> f64 { (p + self.normal() * (fst * p * (1.0 - p)).sqrt()).clamp(0.01, 0.99) } @@ -578,9 +631,10 @@ mod tests { } } - /// Two distinguishable reference populations (GBR→EUR carrying pattern G, YRI→AFR carrying the - /// opposite pattern). A side that matches GBR haplotypes on the first half and YRI on the second - /// must paint GBR (EUR) then YRI (AFR) with the fine codes resolved. + /// Two reference populations that a test can separate. GBR, which rolls up to EUR, holds + /// pattern G. YRI, which rolls up to AFR, holds the opposite pattern. Take a side that matches + /// the GBR haplotypes on the first half and the YRI ones on the second. The painter must give + /// GBR (EUR) and then YRI (AFR), with both fine codes resolved. #[test] fn copies_the_matching_population_and_resolves_fine() { let n = 60usize; @@ -626,8 +680,9 @@ mod tests { assert_eq!(side0[1].fine_population_code.as_deref(), Some("YRI")); assert_eq!(side0[1].population_code, "AFR"); - // Global-composition gate: an EUR-only prior drops the AFR (YRI) reference haplotypes, so even - // the YRI-matching second half can no longer be painted AFR — the whole side stays European. + // The gate on the global composition. A prior that holds EUR alone drops the AFR (YRI) + // reference haplotypes. Even the second half, which matches YRI, can then no longer get + // an AFR paint. The whole side stays European. let segs_gated = paint_copying_lai(&phased, &reference, &map, &[("EUR".to_string(), 1.0)], ¶ms); assert!( segs_gated.iter().all(|s| s.population_code == "EUR"), @@ -635,8 +690,9 @@ mod tests { ); } - /// A tiny reference population (below `min_ref_haps`) folds into its super-pop, so it's never a - /// callable fine label: a 1-hap TSI group is never painted "TSI" (it folds to EUR). + /// A very small reference population, below `min_ref_haps`, goes into its super-population. + /// It is then never a fine label that the painter can call. A TSI group of one haplotype never + /// gets the paint "TSI", because it goes to EUR. #[test] fn tiny_populations_fold_into_super_pop() { let n = 30usize; @@ -659,7 +715,8 @@ mod tests { min_segment_cm: 5.0, ..CopyingLaiParams::default() }; - // Empty prior → gate disabled (keep all haplotypes), so the folding path is exercised. + // An empty prior turns the gate off, and every haplotype stays. This test then covers the + // path that puts a small group into its super-population. let segs = paint_copying_lai(&phased, &reference, &map, &[], ¶ms); // The folded tiny pop must never surface as a fine call. for s in segs.iter().filter(|s| s.copy == 0) { @@ -667,20 +724,24 @@ mod tests { } } - /// A synthetic three-population reference with realistic *drift* structure: two close European - /// populations (GBR, TSI) split off a common European branch, plus FIN — a small, heavily - /// drifted isolate whose low internal diversity is what made it over-attract the copy in the - /// field. Haplotypes are drawn from each population's own frequencies. + /// A synthetic reference of three populations, with a realistic *drift* structure. Two close + /// European populations, GBR and TSI, come off a common European branch. FIN is the third, and + /// it is a small isolate with much drift. Its low internal diversity is what made it attract + /// too much of the copy in the field. The code draws the haplotypes from the frequencies of + /// each population. /// /// Returns `(reference, sides)` where `sides` are two haplotypes drawn from GBR's frequencies /// and **not** in the reference (so nothing self-copies). fn drifted_reference(n_sites: usize) -> (HaplotypeReference, [Vec; 2]) { let mut rng = Rng(0x5EED_1234_5678_9ABC); - // Ancestral → European branch → {GBR, TSI}; FIN drifts harder off the same branch. AFR - // (YRI) splits at the root. The sibling separation here (Fst 0.02) is deliberately wider - // than real intra-European Fst (~0.005) and the sites are independent: this gate tests the - // *model*, given information sufficient to separate the populations. Whether the shipped - // panel carries that much information is the separate question `validate-lai` answers on + // The ancestral node goes to a European branch, and that branch goes to {GBR, TSI}. FIN + // drifts further off the same branch. AFR (YRI) splits at the root. + // + // The separation between the two close populations here is an Fst of 0.02. That is wider + // than the real Fst inside Europe, which is about 0.005, and the sites here are + // independent. Both choices are deliberate. This gate tests the *model*, and it gives the + // model enough information to separate the populations. Whether the panel that ships + // carries that much information is a separate question, and `validate-lai` answers it on // the real reference. let mut freqs: Vec<[f64; 4]> = Vec::with_capacity(n_sites); // GBR, TSI, FIN, YRI for _ in 0..n_sites { @@ -693,8 +754,8 @@ mod tests { rng.drift(anc, 0.150), ]); } - // Panel sizes deliberately unequal (the balance the capping knob exists to fix): a big GBR - // and TSI, a small FIN, a big YRI. + // The panel sizes are not equal, and that is deliberate. It is the balance that the limit + // on each population exists to correct: a big GBR and TSI, a small FIN, and a big YRI. let counts = [60usize, 60, 24, 60]; let mut rows: Vec> = Vec::new(); let mut hap_pop: Vec = Vec::new(); @@ -721,7 +782,7 @@ mod tests { (reference, sides) } - /// Per-site share of each called fine label across both sides, keyed by label. + /// The share of each called fine label, at each site, over both sides. The key is the label. fn call_shares(segs: &[AncestrySegment], positions: &[i64]) -> BTreeMap { let mut shares: BTreeMap = BTreeMap::new(); let mut total = 0.0; @@ -740,12 +801,16 @@ mod tests { shares } - /// **Numeric gate** (the property the calibration commits were chasing, as an assertion rather - /// than a look at the painted chromosomes): at the shipped defaults a GBR individual painted - /// against a reference holding a small, heavily drifted FIN must be called GBR — over the - /// isolate, over its sibling population, above chance, and never outside its continent. The - /// end-to-end equivalent on the real panel is `navigator-panelbuild validate-lai`, which scores - /// held-out reference individuals against the same properties. + /// **A numeric gate.** It holds the property that the calibration commits went after, as an + /// assertion, and not as a look at the painted chromosomes. + /// + /// At the defaults that ship, take a GBR individual, and paint it against a reference that + /// holds a small FIN with much drift. The painter must call it GBR. GBR must beat the isolate, + /// the population beside it on the tree, and chance. And the painter must never go outside the + /// continent. + /// + /// `navigator-panelbuild validate-lai` is the end-to-end equivalent on the real panel. It + /// scores held-out reference individuals against the same properties. #[test] fn drifted_isolate_does_not_out_call_the_true_population() { let n = 600usize; @@ -770,7 +835,8 @@ mod tests { // 1. The drifted isolate barely features at all. assert!(share("FIN") < 0.05, "drifted FIN over-attracted the copy: {shares:?}"); - // 2. The true population beats its sibling clearly, not by a nose. + // 2. The true population clearly beats the population beside it on the tree. The margin + // is not small. assert!( share("GBR") > share("TSI") * 1.5, "sibling TSI not separated from the true GBR: {shares:?}" @@ -784,12 +850,17 @@ mod tests { ); } - /// **Calibration regression gate.** The previous defaults (`recomb_per_cm` 0.1, `max_ref_haps` - /// 50) were tuned by eye and are measurably worse: committing the mosaic to whole ~10 cM tracts - /// rests each call on a handful of markers, and capping populations down near the size of the - /// small HGDP isolates thins the large ones until the isolates win by default. `validate-lai` - /// measures the same gradient on the real panel (fine accuracy 12.9% → 24.6%, isolate over-call - /// 27.2% → 11.8%). This pins the direction, so reverting the calibration fails here. + /// **A gate against a return of the old calibration.** Somebody chose the defaults before + /// these, which were `recomb_per_cm` 0.1 and `max_ref_haps` 50, by eye. A measurement shows + /// that they are worse. + /// + /// There are two reasons. To hold the mosaic on a whole tract of about 10 cM leaves each call + /// on a few markers alone. And a limit near the size of the small HGDP isolates thins the + /// large populations until the isolates win by default. + /// + /// `validate-lai` measures the same gradient on the real panel: the fine accuracy goes from + /// 12.9% to 24.6%, and the over-call of an isolate from 27.2% to 11.8%. This test holds the + /// direction, so a change back to the old calibration fails here. #[test] fn shipped_calibration_beats_the_previous_one() { let n = 600usize; diff --git a/crates/navigator-analysis/src/lib.rs b/crates/navigator-analysis/src/lib.rs index 6bf8c7d7..5f08968b 100644 --- a/crates/navigator-analysis/src/lib.rs +++ b/crates/navigator-analysis/src/lib.rs @@ -1,13 +1,18 @@ -//! Navigator analysis — the htsjdk/GATK replacement, Navigator-side. +//! The analysis crate of Navigator. It replaces htsjdk and GATK, on the Navigator side. //! -//! Owns the `noodles` BAM/CRAM/FASTA/BGZF/index I/O layer (kept out of shared -//! `du-bio`, which stays IO-light coordinate math + text parsing), the ported GATK -//! walkers (`coverage`, `read_metrics`, `sv`, `sex`), and the purpose-built haploid -//! variant caller: force-call genotyping at known sites plus de-novo Y/mtDNA discovery -//! for private-variant matching and branch creation. +//! It owns the I/O layer over `noodles`, for BAM, CRAM, FASTA, BGZF and the index files. That layer +//! stays out of the shared `du-bio` crate. `du-bio` holds coordinate arithmetic, and it reads text, +//! and it does little I/O. //! -//! Built on `du-bio` for liftover/callable/coordinate primitives. A GATK-vs-Rust -//! golden-truth parity harness gates cutover. Implemented in roadmap phases 2–3. +//! It also owns the GATK walkers that this project ported: `coverage`, `read_metrics`, `sv` and +//! `sex`. And it owns the haploid variant caller that somebody built for this purpose. +//! +//! That caller does two things. It force-calls a genotype at a known site. And it discovers +//! de-novo on the Y and the mtDNA, for private-variant matching and for a new branch. +//! +//! It stands on `du-bio` for the primitives of liftover, callability and coordinates. A parity +//! harness, of GATK against Rust, over a golden truth, gates the cutover. Phases 2 and 3 of the +//! roadmap built it. pub mod ancestry; pub mod archaic; @@ -33,9 +38,10 @@ pub mod library_stats; pub mod manifest; pub mod mask; pub mod mastervar; -/// mtDNA variant derivation + CHM13 `chrM`↔rCRS liftover. Moved to the shared `du-bio` crate -/// so the AppView and Navigator share one implementation; re-exported here under the original -/// path so existing `navigator_analysis::mtvariants::…` call sites are unchanged. +/// The derivation of the mtDNA variants, and the liftover between the CHM13 `chrM` and the rCRS. +/// This code moved to the shared `du-bio` crate, so that the AppView and Navigator share one +/// implementation. This module exports it again, under its original path, so that a call site that +/// says `navigator_analysis::mtvariants::…` does not change. pub use du_bio::mt as mtvariants; pub mod parity; pub mod phasing; diff --git a/crates/navigator-analysis/src/library_stats.rs b/crates/navigator-analysis/src/library_stats.rs index 7fcc1bb1..cf647b24 100644 --- a/crates/navigator-analysis/src/library_stats.rs +++ b/crates/navigator-analysis/src/library_stats.rs @@ -1,12 +1,16 @@ -//! Library / instrument inference from an alignment — a Rust port of the Scala -//! `LibraryStatsProcessor`. The header probe ([`crate::probe`]) only reads `@RG PL/PM`, which -//! many vendor BAMs (FGC/YSEQ/Dante…) leave sparse; the instrument serial that identifies the -//! physical sequencer (and, via the crowd-sourced AppView map, the lab) lives in the **read -//! names**. This scans a bounded prefix of records, classifies each read's platform from its -//! qname, extracts the instrument + flowcell, and reports the most-frequent of each plus the -//! `@RG SM/LB/PU` tags. +//! Infer the library and the instrument from an alignment. This is the Rust port of the Scala +//! `LibraryStatsProcessor`. //! -//! The `instrument_id` is the key datum for resolving the sequencing facility (roadmap D8). +//! The header probe ([`crate::probe`]) reads `@RG PL` and `@RG PM` alone, and many vendor BAMs +//! leave those thin. FGC, YSEQ and Dante are examples. The serial of the instrument names the +//! physical sequencer, and, through the AppView map that the community builds, the lab. That +//! serial lives in the **read names**. +//! +//! This code scans a bounded prefix of the records. At each read it takes the platform from the +//! qname, and it takes out the instrument and the flowcell. It then reports the most common of +//! each, and the `@RG SM`, `LB` and `PU` tags. +//! +//! The `instrument_id` is the key datum that resolves the sequencing facility. See roadmap D8. use std::collections::HashMap; use std::path::Path; @@ -16,8 +20,8 @@ use noodles::sam::header::record::value::map::read_group; use crate::error::AnalysisError; use crate::reader::open_seq; -/// Default cap on records scanned — enough to settle the most-frequent instrument without -/// reading a whole multi-GB file. +/// The default limit on how many records the scan reads. It is enough to settle which instrument +/// is the most common, and it does not read a whole file of some GB. pub const DEFAULT_MAX_READS: usize = 10_000; /// What the read-name scan + `@RG` tags inferred. Every field is best-effort (`None`/`Unknown`). @@ -39,8 +43,8 @@ pub struct LibraryStats { pub instrument_model: Option, /// Most-frequent flowcell id from read names. pub flowcell_id: Option, - /// `PAIRED` if the scanned reads carry the SAM segmented (0x1) flag, else `SINGLE`; `None` - /// when no primary reads were scanned. + /// `PAIRED` when the scanned reads carry the SAM segmented flag, which is 0x1. Else `SINGLE`. + /// It is `None` when the scan read no primary record. pub library_layout: Option, /// Most-frequent read chemistry/mode inferred from the read names: `SHORT` / `HIFI` / `CLR` / /// `ONT_SIMPLEX` / `ONT_DUPLEX`. The only edge signal that tells HiFi from CLR (both PacBio); @@ -123,7 +127,8 @@ pub fn scan_library_stats( _ => None, }; - // Majority vote: PAIRED if most scanned reads are segmented (handles a few stray flags). + // A majority vote. It gives PAIRED when most of the scanned reads carry the segmented flag. + // It thereby survives a few stray flags. let library_layout = (read_count > 0).then(|| { if segmented_count * 2 >= read_count { "PAIRED" @@ -147,10 +152,17 @@ pub fn scan_library_stats( }) } -/// Classify a read's chemistry/mode from its name + already-detected platform. PacBio splits on the -/// movie-name suffix — a `/ccs` segment **is** HiFi (CCS), anything else is CLR (continuous/subread). -/// Nanopore is simplex unless the name is a dorado duplex pair (two read ids joined by `;`). Short- -/// read platforms are `SHORT`. `None` when the platform is unknown (contributes no vote). +/// Take the chemistry and mode of a read, from its name and from the platform that the code +/// already found. +/// +/// PacBio splits on the suffix of the movie name. A `/ccs` segment **is** HiFi, which is CCS. +/// Anything else is CLR, which is continuous, or a subread. +/// +/// Nanopore is simplex, unless the name holds a duplex pair from dorado, which is two read ids with +/// a `;` between them. +/// +/// A short-read platform gives `SHORT`. The result is `None` when the code does not know the +/// platform, and that adds no vote. fn detect_read_type_from_qname(qname: &str, platform: &str) -> Option<&'static str> { match platform { "PacBio" => Some(if qname.rsplit('/').next() == Some("ccs") { @@ -180,8 +192,9 @@ fn val>(v: &T) -> String { /// Classify a read's platform from its name. Mirrors the Scala heuristics (MGI prefixes / /// colon-shaped Illumina / UUID Nanopore / `m#####` PacBio); `Unknown` when nothing matches. fn detect_platform_from_qname(qname: &str) -> &'static str { - // MGI: a V300/E100/CL100/G400/G99 instrument prefix, or a colon-delimited name whose first - // field starts V/E/CL/G with a flowcell field starting "L". + // MGI. The name starts with a V300, E100, CL100, G400 or G99 instrument prefix. Or it has + // colons between its fields, its first field starts with V, E, CL or G, and its flowcell field + // starts with "L". if qname.len() > 15 { let prefix = qname.get(0..5).unwrap_or("").to_ascii_uppercase(); if ["V300", "E100", "CL100", "G400", "G99"] @@ -229,7 +242,7 @@ fn is_illumina_qname(q: &str) -> bool { .any(|w| num(w[0]) && alnum(w[1]) && w[2].len() == 1 && num(w[2]) && num(w[3]) && num(w[4]) && num(w[5])) } -/// A leading UUID (`8-4-4-4-12` hex) — Oxford Nanopore. +/// A UUID at the start of the name, in the `8-4-4-4-12` hex form. That marks Oxford Nanopore. fn is_nanopore_uuid(q: &str) -> bool { let b = q.as_bytes(); if b.len() < 36 { diff --git a/crates/navigator-analysis/src/manifest.rs b/crates/navigator-analysis/src/manifest.rs index 734cb195..6aec1ac5 100644 --- a/crates/navigator-analysis/src/manifest.rs +++ b/crates/navigator-analysis/src/manifest.rs @@ -1,6 +1,9 @@ -//! Asset integrity manifest (ancestry-ibd-asset-wiring, cross-cutting). `navigator-panelbuild` -//! writes `ancestry_manifest_.json` listing each built `.bin`'s SHA-256; the app verifies a -//! loaded asset against it and refuses a mismatch — a cheap integrity guard for CDN-delivered assets. +//! The manifest that holds the integrity of each asset. See the ancestry-ibd asset design, which +//! this crosses. +//! +//! `navigator-panelbuild` writes `ancestry_manifest_.json`. That file lists the SHA-256 of +//! each `.bin` that the build made. The app checks a loaded asset against it, and it refuses one +//! that does not match. That guard costs little, and it covers an asset that came over a CDN. use std::collections::BTreeMap; @@ -8,8 +11,9 @@ use serde::{Deserialize, Serialize}; use crate::error::AnalysisError; -/// Lowercase-hex SHA-256 of `bytes`. Re-exported from the shared `du-bio` helper so existing -/// `manifest::sha256_hex` callers (e.g. `navigator-panelbuild`) keep working. +/// The SHA-256 of `bytes`, in lower-case hex. It comes from the shared helper in `du-bio`, and +/// this module exports it again, so that a caller of `manifest::sha256_hex` still compiles. +/// `navigator-panelbuild` is such a caller. pub use du_bio::hash::sha256_hex; /// One asset's integrity record. @@ -19,7 +23,8 @@ pub struct AssetEntry { pub bytes: u64, } -/// Per-build asset manifest: asset filename (on-disk name) → integrity record. +/// The asset manifest of one build. It maps the file name of an asset, as it sits on disk, to its +/// integrity record. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct AssetManifest { pub build: String, @@ -49,8 +54,9 @@ impl AssetManifest { ); } - /// Verify `bytes` for `filename`. `Ok` when the manifest has no entry for the file (advisory — - /// unlisted assets aren't gated) or the digest matches; `Err(expected, got)` on a mismatch. + /// Check `bytes` for `filename`. It gives `Ok` in two cases. The manifest holds no entry for + /// that file: this check is advisory, and it gates nothing that the manifest does not list. Or + /// the digest matches. It gives `Err(expected, got)` when the two differ. pub fn verify(&self, filename: &str, bytes: &[u8]) -> Result<(), (String, String)> { if let Some(e) = self.assets.get(filename) { let got = sha256_hex(bytes); @@ -75,7 +81,8 @@ mod tests { }; m.insert("ancestry_panel_chm13v2.0.bin", b"hello"); assert_eq!(m.assets["ancestry_panel_chm13v2.0.bin"].bytes, 5); - // Matching bytes verify; tampered bytes are rejected; unlisted files pass (advisory). + // Bytes that match pass the check. Bytes that somebody changed do not. A file that the + // manifest does not list passes, because the check is advisory. assert!(m.verify("ancestry_panel_chm13v2.0.bin", b"hello").is_ok()); assert!(m.verify("ancestry_panel_chm13v2.0.bin", b"hELLo").is_err()); assert!(m.verify("not_listed.bin", b"anything").is_ok()); diff --git a/crates/navigator-analysis/src/mask.rs b/crates/navigator-analysis/src/mask.rs index 0686fc85..86736488 100644 --- a/crates/navigator-analysis/src/mask.rs +++ b/crates/navigator-analysis/src/mask.rs @@ -1,10 +1,11 @@ -//! Callable-region mask from a BED file — restricts variant calls to reliable regions -//! (e.g. the Poznik/1KG callable-Y mask, `b38_sites.bed`). Without it, a whole-chrY -//! de-novo sweep is dominated by palindrome/heterochromatin/repeat artifacts. +//! The mask of the callable regions, from a BED file. It holds the variant calls to the regions +//! that the code can trust, such as the callable-Y mask of Poznik and 1KG, in `b38_sites.bed`. +//! Without it, artifacts from a palindrome, from heterochromatin and from a repeat control a +//! de-novo sweep over the whole chrY. //! -//! BED is 0-based, half-open `[start, end)`; our positions are 1-based. Intervals for the -//! requested contig are loaded, sorted, and coalesced so [`RegionMask::contains`] is a -//! binary search. +//! BED is 0-based and half-open, as `[start, end)`. The positions in this project are 1-based. +//! The code loads the intervals of the contig that the caller asked for, sorts them, and joins the +//! ones that touch. [`RegionMask::contains`] is then a binary search. use std::io::BufRead; use std::path::Path; @@ -20,9 +21,10 @@ pub struct RegionMask { } impl RegionMask { - /// Load the intervals for `contig` from a BED file (other contigs ignored). A gzip- or - /// BGZF-compressed BED is transparently decompressed (detected by content), so large bundled - /// masks can ship compressed — including multi-block bgzipped files. + /// Load the intervals of `contig` from a BED file. It ignores every other contig. It also + /// decompresses a BED that came through gzip or BGZF, and the caller sees no difference. It + /// finds the compression from the content of the file. A large mask that ships with the app + /// can then stay compressed, and that includes a bgzipped file of more than one block. pub fn from_bed(path: &Path, contig: &str) -> Result { let reader = crate::gzio::open_maybe_gz(path).map_err(|e| AnalysisError::io(path, e))?; let mut intervals = Vec::new(); @@ -61,8 +63,9 @@ impl RegionMask { RegionMask { intervals: merged } } - /// The coalesced `[start, end)` intervals, for callers that need to transform them (lifting a - /// mask to another build, say) rather than only query membership. + /// The `[start, end)` intervals, after the code joined the ones that touch. Use this when a + /// caller must change them, for example to lift a mask to another build. Membership is not the + /// only question you can ask. pub fn intervals(&self) -> &[(i64, i64)] { &self.intervals } @@ -95,26 +98,33 @@ impl RegionMask { } } -/// The structural class of a chrY region — for *down-weighting* (not dropping) Y calls by how -/// reliably short reads map there. Each class carries a **quality modifier** in `(0, 1]` (a port of -/// the Scala `YRegionAnnotator` ladder): unique / X-degenerate sequence is full weight (no class, -/// modifier 1.0); paralog-prone and repeat zones get progressively lower weight. A position not in -/// any class is treated as unique (modifier 1.0). +/// The structural class of a chrY region. It gives a Y call *less weight*, and it does not drop +/// that call. The weight follows how well a short read maps there. +/// +/// Each class carries a **quality modifier** in `(0, 1]`. That ladder is the port of the Scala +/// `YRegionAnnotator`. Unique sequence, and X-degenerate sequence, carry the full weight: they have +/// no class, and their modifier is 1.0. A zone that is prone to a paralog carries less weight, and +/// so does a repeat zone. The more difficult the zone, the less it carries. A position in no class +/// counts as unique, at a modifier of 1.0. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum YRegionClass { - /// Pseudoautosomal region (recombines with X) — modifier 0.5. + /// A pseudoautosomal region, which recombines with X. The modifier is 0.5. Par, - /// Palindrome / inverted repeat (gene-conversion + mapping risk) — modifier 0.4. + /// A palindrome or an inverted repeat. Gene conversion and a wrong mapping are both a risk. + /// The modifier is 0.4. Palindrome, - /// X-transposed region (~99% X-identical, contamination risk) — modifier 0.3. + /// An X-transposed region. It is about 99% the same as X, so contamination is a risk. The + /// modifier is 0.3. Xtr, - /// Ampliconic block — near-identical repeat copies, high paralog risk — modifier 0.3. + /// An ampliconic block. Its repeat copies are almost the same, so a paralog is a large risk. + /// The modifier is 0.3. Amplicon, - /// Short-tandem-repeat region (recLOH / stutter risk) — modifier 0.25. + /// A short-tandem-repeat region. recLOH and stutter are both a risk. The modifier is 0.25. Str, - /// Centromeric region (nearly unmappable) — modifier 0.1. + /// A centromeric region. Almost nothing maps there. The modifier is 0.1. Centromere, - /// Yq12 heterochromatin / AZF-DYZ satellite (unmappable) — modifier 0.1. (Was `AzfDyz`.) + /// Yq12 heterochromatin, and the AZF-DYZ satellite. Nothing maps there. The modifier is 0.1. + /// The old name of this was `AzfDyz`. #[serde(alias = "AzfDyz")] Heterochromatin, } @@ -132,8 +142,9 @@ impl YRegionClass { } } - /// Quality modifier in `(0, 1]`: how much a call here counts in haplogroup concordance scoring - /// (lower = more paralog-/mapping-suspect). Unique sequence (no class) is 1.0. + /// The quality modifier, in `(0, 1]`. It says how much a call here counts when the code scores + /// the concordance of a haplogroup. A lower value means more doubt about a paralog or a + /// mapping. Unique sequence, which has no class, is 1.0. pub fn modifier(self) -> f64 { match self { YRegionClass::Par => 0.5, @@ -145,26 +156,30 @@ impl YRegionClass { } } -/// CHM13v2.0 chrY PAR1 (`chrY:1–2,458,320`), as a 0-based half-open interval. The pseudoautosomal -/// regions recombine with X, so Y-SNP placement never lives here — flagged for QC, not dropped. +/// PAR1 of chrY in CHM13v2.0, at `chrY:1–2,458,320`, as a 0-based half-open interval. A +/// pseudoautosomal region recombines with X, so a Y-SNP placement never sits here. The code flags +/// such a position for QC, and it does not drop it. const CHM13_PAR1: (i64, i64) = (0, 2_458_320); /// CHM13v2.0 chrY PAR2 (`chrY:62,122,809–62,460,029`), 0-based half-open. const CHM13_PAR2: (i64, i64) = (62_122_808, 62_460_029); -/// CHM13v2.0 chrY Yq12 heterochromatin bound (`chrY:26,637,971–62,122,809`), 0-based half-open — -/// the validated constant carried over from the Scala port (mostly satellite, unmappable). +/// The bound of the Yq12 heterochromatin on chrY in CHM13v2.0, at `chrY:26,637,971–62,122,809`, +/// 0-based and half-open. This constant came over from the Scala port, and a check confirmed it. +/// The region is mostly satellite, and nothing maps there. const CHM13_YQ12_HET: (i64, i64) = (26_637_970, 62_122_809); -/// Per-build chrY PAR + heterochromatin bounds, 0-based half-open. +/// The chrY PAR and heterochromatin bounds of each build, 0-based and half-open. +/// +/// The code does **not lift** these. It takes them from each build natively, for two reasons. Each +/// assembly documents them exactly. And a chain is least reliable in exactly these places: chrX +/// shares a pseudoautosomal region, and Yq12 is satellite. A lift through either one is as likely +/// to be wrong as it is to be absent. /// -/// These are **not lifted** — they are taken natively per build, because they are precisely -/// documented for each assembly and because a chain is least trustworthy exactly here: the -/// pseudoautosomal regions are shared with chrX and Yq12 is satellite, so a lift through either is -/// as likely to be wrong as absent. The palindromes and amplicons *are* lifted, since they sit in -/// male-specific euchromatin where the chain is reliable. +/// The palindromes and the amplicons *do* go through a lift. They sit in the euchromatin that only +/// males have, and the chain is reliable there. /// -/// GRCh38 (GCA_000001405.15): PAR1 `chrY:10,001–2,781,479`, PAR2 `chrY:56,887,903–57,217,415`; -/// the male-specific euchromatin ends and the heterochromatic arm begins at ~26.6 Mb, running to -/// the end of the assembly. +/// GRCh38 (GCA_000001405.15) has PAR1 at `chrY:10,001–2,781,479`, and PAR2 at +/// `chrY:56,887,903–57,217,415`. The euchromatin that only males have ends at about 26.6 Mb. The +/// heterochromatic arm begins there, and it goes to the end of the assembly. const GRCH38_PAR1: (i64, i64) = (10_000, 2_781_479); const GRCH38_PAR2: (i64, i64) = (56_887_902, 57_217_415); const GRCH38_YQ12_HET: (i64, i64) = (26_600_000, 57_227_415); @@ -207,10 +222,12 @@ pub fn y_landmarks(build: &str) -> Option { } } -/// Curated CHM13 chrY structural regions with quality modifiers, for down-weighting calls in -/// paralog-prone / unmappable zones. [`classify`](Self::classify) returns the **most-impactful** -/// (lowest-modifier) class containing a position; [`quality_modifier`](Self::quality_modifier) -/// returns its modifier (1.0 for unique sequence). +/// The curated structural regions of chrY on CHM13, with a quality modifier at each one. They give +/// less weight to a call in a zone that is prone to a paralog, or where nothing maps. +/// +/// [`classify`](Self::classify) returns the class that matters most at a position, which is the +/// class with the lowest modifier. [`quality_modifier`](Self::quality_modifier) returns the +/// modifier of that class, and 1.0 for unique sequence. #[derive(Debug, Clone)] pub struct YStructuralRegions { par: RegionMask, @@ -221,9 +238,10 @@ pub struct YStructuralRegions { } impl YStructuralRegions { - /// Load from the three CHM13 chrY BEDs (amplicons, inverted-repeats/palindromes, AZF/DYZ), - /// adding the hardcoded CHM13 PAR1/PAR2 and Yq12-heterochromatin constants (the AZF/DYZ BED - /// covers the satellite arrays; the constant fills the broader heterochromatic q-arm). + /// Load from the three chrY BEDs of CHM13: the amplicons, the inverted repeats and + /// palindromes, and AZF/DYZ. It then adds the CHM13 PAR1, PAR2 and Yq12-heterochromatin + /// constants that this file holds. The AZF/DYZ BED covers the satellite arrays, and the + /// constant fills the wider heterochromatic q-arm. pub fn from_beds(amplicon: &Path, palindrome: &Path, azf_dyz: &Path) -> Result { Ok(Self::from_masks( RegionMask::from_intervals(vec![CHM13_PAR1, CHM13_PAR2]), @@ -233,19 +251,21 @@ impl YStructuralRegions { )) } - /// The palindrome and amplicon masks, for lifting to another build. PAR and heterochromatin are - /// deliberately not exposed for that purpose — see [`y_landmarks`]. + /// The palindrome mask and the amplicon mask, for a lift to another build. This does not give + /// out the PAR mask or the heterochromatin mask for that purpose, and that is deliberate. See + /// [`y_landmarks`]. pub fn structural_masks(&self) -> (&RegionMask, &RegionMask) { (&self.palindrome, &self.amplicon) } - /// The AZF/DYZ + heterochromatin mask, for lifting the satellite-array intervals. + /// The AZF/DYZ and heterochromatin mask, for a lift of the satellite-array intervals. pub fn heterochromatin_mask(&self) -> &RegionMask { &self.heterochromatin } - /// Build from explicit masks (the seam the BED loader + unit tests share). XTR/STR/centromere - /// masks aren't sourced yet — those tiers exist in [`YRegionClass`] for when their data lands. + /// Build from masks that the caller gives. The BED loader and the unit tests share this seam. + /// Nobody has found a source for the XTR, STR and centromere masks yet. Those tiers exist in + /// [`YRegionClass`], ready for the day that their data arrives. pub fn from_masks( par: RegionMask, palindrome: RegionMask, @@ -260,11 +280,14 @@ impl YStructuralRegions { } } - /// The most-impactful (lowest-modifier) structural class containing the 1-based `position`, or - /// `None` if it is in unique (reliably-mappable / X-degenerate) sequence. + /// The structural class that matters most at the 1-based `position`, which is the class with + /// the lowest modifier. It is `None` when that position sits in unique sequence, which is + /// sequence that maps reliably, or that is X-degenerate. pub fn classify(&self, position: i64) -> Option { - // Checked in ascending-modifier (most-impactful-first) order so overlaps resolve to the - // strongest down-weight (e.g. an amplicon inside the heterochromatic arm → Heterochromatin). + // The checks run from the lowest modifier up, so the class that matters most comes first. + // Where two classes overlap, the result is then the one that takes the most weight + // away. + // An amplicon inside the heterochromatic arm gives Heterochromatin. if self.heterochromatin.contains(position) { Some(YRegionClass::Heterochromatin) } else if self.amplicon.contains(position) { @@ -289,8 +312,9 @@ impl YStructuralRegions { mod y_landmark_tests { use super::*; - /// PAR and heterochromatin are per-build constants rather than lifted, so a wrong one silently - /// mis-masks a whole assembly. Pin the documented coordinates. + /// The PAR and heterochromatin bounds are a constant of each build, and no lift makes them. A + /// wrong one then masks a whole assembly wrongly, and nobody sees it. This test holds the + /// documented coordinates. #[test] fn each_build_carries_its_own_chry_geometry() { let l = y_landmarks("hs1").expect("CHM13"); @@ -363,7 +387,8 @@ mod tests { let path = dir.join("m.bed.gz"); let mut enc = flate2::write::GzEncoder::new(std::fs::File::create(&path).unwrap(), flate2::Compression::default()); - // chrX ignored; two chrY intervals, one of them coalescing. + // The loader ignores chrX. There are two chrY intervals, and the code joins one of them + // to its neighbour. enc.write_all(b"chrY\t100\t200\nchrX\t0\t50\nchrY\t150\t260\n").unwrap(); enc.finish().unwrap(); let m = RegionMask::from_bed(&path, "chrY").unwrap(); @@ -374,9 +399,10 @@ mod tests { #[test] fn reads_multiblock_bgzf_bed_without_truncation() { - // A bgzipped BED is a concatenation of independent gzip members. The old GzDecoder - // path stopped after the first member and silently dropped every later interval; - // this guards that all members are read (via MultiGzDecoder in gzio::open_maybe_gz). + // A bgzipped BED is a chain of independent gzip members. The GzDecoder path before this + // one stopped after the first member, and it dropped every later interval where nobody saw + // it. This test holds that the code reads every member, through MultiGzDecoder in + // gzio::open_maybe_gz. use std::io::Write; let dir = std::env::temp_dir().join(format!("dun-maskbgzf-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -415,8 +441,9 @@ mod tests { #[test] fn y_structural_classifies_most_impactful_first() { - // Disjoint synthetic regions (the BED loader keys on chrY; a chrX line is ignored). The - // production constructor also bakes in CHM13 PAR1/PAR2 + the Yq12 heterochromatin bound. + // Synthetic regions that do not overlap. The BED loader keys on chrY, and it ignores a + // chrX line. The constructor that production uses also holds the CHM13 PAR1 and PAR2, and + // the Yq12 heterochromatin bound. let dir = std::env::temp_dir().join(format!("dun-ymask-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let write = |name: &str, body: &str| { diff --git a/crates/navigator-analysis/src/mastervar.rs b/crates/navigator-analysis/src/mastervar.rs index 1f987c0a..5b569378 100644 --- a/crates/navigator-analysis/src/mastervar.rs +++ b/crates/navigator-analysis/src/mastervar.rs @@ -1,19 +1,25 @@ -//! CompleteGenomics **masterVar** reader — the whole-genome variant table `cgatools` emitted -//! for the old CG sequencing service (`var--ASM.tsv[.bz2]`, `FORMAT_VERSION 2.0`). +//! The reader of a CompleteGenomics **masterVar** file. That is the whole-genome variant table +//! that `cgatools` wrote for the old CG sequencing service. Its name is +//! `var--ASM.tsv[.bz2]`, and its `FORMAT_VERSION` is 2.0. //! -//! The file is a tab-separated per-allele call over the *entire* genome: `ref` / `no-call` -//! blocks interleaved with the actual `snp` / `ins` / `del` / `sub` calls. Each locus is one -//! or more consecutive rows sharing the leading `locus` id — a diploid heterozygous SNP is two -//! rows (allele `1` and allele `2`), a homozygous or haploid call one row (allele `all` / `1`). -//! Coordinates are **0-based half-open** (`begin`/`end`); the 1-based SNP position is `begin + 1`. +//! The file holds one call for each allele, over the *whole* genome, with tabs between the +//! fields. Blocks of `ref` and `no-call` sit between the real `snp`, `ins`, `del` and `sub` +//! calls. //! -//! We extract SNPs only (matching the VCF/CSV variant importer, which is SNP-only): each locus -//! becomes at most one biallelic [`VariantCall`] with a reconstructed genotype (`1`, `0/1`, -//! `1/1`, `1/2`, or `1/.` when the partner allele is a no-call). Indels/substitutions and the -//! `ref`/`no-call` spans are skipped. NCBI build 37 ⇒ the calls are on **GRCh37** (chrM = rCRS). +//! One locus is one row, or more than one row in a line, and they share the `locus` id at the +//! front. A diploid heterozygous SNP is two rows, for allele `1` and allele `2`. A homozygous or +//! haploid call is one row, for allele `all` or `1`. The coordinates are **0-based and +//! half-open**, in `begin` and `end`, and the 1-based position of a SNP is `begin + 1`. //! -//! The whole file is streamed (a genome masterVar is multi-GB uncompressed), transparently -//! decoding `.bz2` / `.gz` via [`crate::gzio::open_maybe_compressed`]. +//! This reader takes SNPs alone. That matches the variant importer for a VCF or a CSV, which also +//! takes SNPs alone. Each locus becomes at most one biallelic [`VariantCall`], with a genotype +//! that the code builds: `1`, `0/1`, `1/1`, `1/2`, or `1/.` when the partner allele has no call. +//! It skips an indel, a substitution, and the `ref` and `no-call` spans. NCBI build 37 means that +//! the calls are on **GRCh37**, where chrM is the rCRS. +//! +//! The code streams the whole file, because a genome masterVar is some GB when nothing compresses +//! it. It decodes a `.bz2` or a `.gz` on the way, through +//! [`crate::gzio::open_maybe_compressed`]. use std::io::BufRead; use std::path::Path; @@ -30,16 +36,18 @@ pub enum MasterVarError { Format(String), } -/// The result of reading a masterVar file: the SNP calls plus the sample id / reference build -/// parsed from the `#`-comment header, and a couple of tallies for the import summary. +/// The result of a read of a masterVar file. It holds the SNP calls. It also holds the sample id +/// and the reference build, which the code read out of the `#` comment header. And it holds two +/// tallies for the import summary. #[derive(Debug, Default, Clone)] pub struct MasterVarImport { /// The `#SAMPLE` header value, if present (e.g. `GS00253-DNA_A01`). pub sample_id: Option, - /// Reference build the calls are on — `"GRCh37"` for `NCBI build 37` (the only build the CG - /// service shipped), else a best-effort echo of the `#GENOME_REFERENCE` header. + /// The reference build that the calls sit on. It is `"GRCh37"` for `NCBI build 37`, which is + /// the only build that the CG service ever sent out. Otherwise it repeats the + /// `#GENOME_REFERENCE` header as best it can. pub reference_build: String, - /// The extracted biallelic SNP calls (one per SNP locus). + /// The biallelic SNP calls that the code took out, one at each SNP locus. pub calls: Vec, /// Total loci examined (distinct `locus` ids). pub loci_seen: u64, @@ -62,7 +70,7 @@ struct Columns { impl Columns { /// Map the masterVar column header (the `>`-prefixed line) to field indices by name. Returns - /// `None` if a required column is missing (so a look-alike table isn't parsed as masterVar). + /// `None` if a required column is missing (so a look-alike table is not parsed as masterVar). fn from_header(line: &str) -> Option { let header = line.strip_prefix('>').unwrap_or(line); let names: Vec<&str> = header.split('\t').map(str::trim).collect(); @@ -96,8 +104,9 @@ enum VarType { Other, } -/// One parsed data row (only the fields we need). Borrows nothing — a locus group is buffered -/// before it is resolved, so the owning line has already been consumed. +/// One data row that the code parsed, with the fields that it needs and no others. It borrows +/// nothing. A group of rows for one locus goes into a buffer before the code resolves it, so the +/// line that held them is already gone. struct Row { locus: u64, ploidy: u8, @@ -170,7 +179,7 @@ enum Hap { } /// Resolve one locus's rows into at most one biallelic SNP [`VariantCall`]. Returns `None` for -/// a locus with no `snp` row (a `ref`/`no-call`/indel span) or one whose SNP alleles aren't a +/// a locus with no `snp` row (a `ref`/`no-call`/indel span) or one whose SNP alleles are not a /// clean single-base substitution. fn locus_call(rows: &[Row]) -> Option { // Site anchor: the first snp row carries the reference base + coordinates. @@ -182,8 +191,8 @@ fn locus_call(rows: &[Row]) -> Option { // A dbSNP rsID from any of the locus's rows. let rs_id = rows.iter().find_map(|r| r.rs_id.clone()); - // Haploid contigs (chrY, chrM, male non-PAR chrX) come through as ploidy 1 with a single - // called allele — emit the alt with a hemizygous genotype. + // A haploid contig comes through at ploidy 1, with one called allele. chrY, chrM and the male + // non-PAR part of chrX are haploid. Emit the alt with a hemizygous genotype. if snp.ploidy == 1 { return snp_call(contig, position, reference, &snp.allele_seq, rs_id, Some("1".into())); } @@ -196,9 +205,10 @@ fn locus_call(rows: &[Row]) -> Option { return snp_call(contig, position, reference, &all.allele_seq, rs_id, Some("1/1".into())); } let hap = |which: Allele| -> Hap { - // A compound locus can list several rows for one allele (e.g. a `ref` segment beside the - // `snp` segment). Prefer the SNP call for that allele; fall back to ref, else missing — - // so a snp isn't hidden behind a same-allele ref/no-call row and the locus lost. + // A compound locus can list more than one row for one allele, such as a `ref` segment + // beside the `snp` segment. Take the SNP call of that allele first. Fall back to the ref + // call, and then to a missing call. A `ref` or `no-call` row on the same allele can then + // not hide a snp, and the locus does not go away. let mut result = Hap::Missing; for r in rows.iter().filter(|r| r.allele == which) { match r.var_type { @@ -216,7 +226,8 @@ fn locus_call(rows: &[Row]) -> Option { (Hap::Alt(a), Hap::Alt(_)) => (a.clone(), "1/2"), // tri-allelic het; keep allele 1's alt (Hap::Alt(a), Hap::Ref) | (Hap::Ref, Hap::Alt(a)) => (a.clone(), "0/1"), (Hap::Alt(a), Hap::Missing) | (Hap::Missing, Hap::Alt(a)) => (a.clone(), "1/."), - // No alt on either haplotype — not a variant (shouldn't occur given the snp row above). + // There is no alt on either haplotype, so this is not a variant. The snp row above means + // that this must not happen. _ => return None, }; snp_call(contig, position, reference, &alt, rs_id, Some(genotype.into())) @@ -230,12 +241,14 @@ fn build_for_reference(genome_reference: &str) -> String { } else if g.contains("build 38") || g.contains("grch38") || g.contains("hg38") { "GRCh38".to_string() } else { - // Unknown/blank header: CG service data is build 37, so default there rather than to None. + // The header is absent, or the code does not know it. The data of the CG service is + // build 37, so default to that, and not to None. "GRCh37".to_string() } } -/// Read and parse a masterVar file (transparently decompressing `.bz2` / `.gz`). +/// Read a masterVar file, and parse it. It decodes a `.bz2` or a `.gz` on the way, and the caller +/// sees no difference. pub fn parse_file(path: &Path) -> Result { let reader = gzio::open_maybe_compressed(path)?; parse_reader(reader) @@ -397,9 +410,9 @@ mod tests { #[test] fn compound_locus_snp_hidden_behind_same_allele_ref() { - // A single locus can list several segments for one allele: here allele 1 has a `ref` - // segment *before* its `snp` segment. The snp must still win (not be masked by the ref), - // so the locus yields a 0/1 call rather than being dropped. + // One locus can list more than one segment for one allele. Here allele 1 has a `ref` + // segment *before* its `snp` segment. The snp must still win, and the ref must not hide + // it. The locus then gives a 0/1 call, and the code does not drop it. let out = parse( "500\t2\t1\tchr1\t9000\t9005\tref\tACGTA\tACGTA\t\t\t\t\t\n\ 500\t2\t1\tchr1\t9005\t9006\tsnp\tT\tC\t80\t80\tVQHIGH\t\tdbsnp.1:rs99\n\ diff --git a/crates/navigator-analysis/src/parity.rs b/crates/navigator-analysis/src/parity.rs index dbf96ce8..5f65e3f9 100644 --- a/crates/navigator-analysis/src/parity.rs +++ b/crates/navigator-analysis/src/parity.rs @@ -1,11 +1,14 @@ -//! §4c golden-truth parity harness — compares the Rust haploid caller's de-novo SNP -//! calls against a GATK truth VCF and reports concordance (precision/recall). +//! The parity harness of §4c, against a golden truth. It compares the de-novo SNP calls of the +//! Rust haploid caller against a truth VCF from GATK. It then reports the concordance, as a +//! precision and a recall. //! -//! The v1 caller is SNP-only, so GATK indel/MNP alleles are counted separately and -//! excluded from the SNP concordance rather than scored as misses — an honest fair -//! comparison until local realignment lands (plan §4b). This is the cutover gate and -//! the regression guard for the analysis layer; the comparison logic is pure and -//! unit-tested, and an ignored test drives it against real GATK output. +//! The v1 caller handles a SNP alone. So an indel allele, and an MNP allele, from GATK go into +//! their own count, and they stay out of the SNP concordance. The harness does not score them as +//! misses. That is an honest comparison, until the local realignment of plan §4b arrives. +//! +//! This is the gate of the cutover, and the guard against a regression in the analysis layer. The +//! comparison logic is pure, and unit tests cover it. A test that carries `#[ignore]` runs it +//! against real GATK output. use std::collections::BTreeSet; use std::io::BufReader; @@ -16,8 +19,8 @@ pub use du_bio::vcf::VcfVariant; use crate::caller::VariantCall; use crate::error::AnalysisError; -/// Parse a (plain-text) truth VCF — e.g. GATK output decompressed with `bgzip -d`. -/// Reuses the shared `du-bio` variant-column parser. +/// Parse a truth VCF in plain text, such as GATK output that `bgzip -d` decompressed. It uses the +/// shared parser of the variant columns, from `du-bio`. pub fn parse_truth_vcf(path: &Path) -> Result, AnalysisError> { let file = std::fs::File::open(path).map_err(|e| AnalysisError::io(path, e))?; du_bio::vcf::parse(BufReader::new(file)) diff --git a/crates/navigator-analysis/src/phasing.rs b/crates/navigator-analysis/src/phasing.rs index 0df80f1a..3aa90e1a 100644 --- a/crates/navigator-analysis/src/phasing.rs +++ b/crates/navigator-analysis/src/phasing.rs @@ -1,20 +1,24 @@ -//! Statistical haplotype phasing — the "which allele came from which parent" step the chromosome -//! painter needs to split ancestry into two internally-consistent parental sides. +//! Statistical phasing of the haplotypes. It answers "which allele came from which parent". The +//! chromosome painter needs that answer, to split the ancestry into two parental sides that agree +//! with themselves. //! -//! Navigator analyses **one subject at a time**, so there is no cohort to phase against; we phase -//! against a bundled panel of **phased reference haplotypes** ([`HaplotypeReference`]) using the -//! Li & Stephens copying model — the same reference-based mode EAGLE2/Beagle use. Each of the -//! sample's two haplotypes is modelled as a mosaic of reference haplotypes; the ordered pair of -//! copied reference haplotypes at each site implies the phase. +//! Navigator analyses **one subject at a time**, so there is no cohort to phase against. It phases +//! against a bundled panel of **phased reference haplotypes**, which is a +//! [`HaplotypeReference`], with the Li & Stephens copying model. That is the same mode against a +//! reference that EAGLE2 and Beagle use. The model takes each of the two haplotypes of the sample +//! as a mosaic of reference haplotypes. The ordered pair of copied reference haplotypes at each +//! site then gives the phase. //! -//! The exact diploid HMM has `K²` states (K = number of reference haplotypes, ~5000), which is -//! infeasible. [`ReferencePhaser`] uses a **beam search** over ordered pair-states, with switch -//! targets restricted to a per-site candidate set of the reference haplotypes sharing the longest -//! IBS run with the sample (a cheap, PBWT-like heuristic). That makes it `O(N · B · M)` in the -//! number of sites `N`, beam width `B`, and candidate count `M`. +//! The exact diploid HMM has `K²` states, where K is the count of reference haplotypes, at about +//! 5000. Nobody can run that. [`ReferencePhaser`] uses a **beam search** over the ordered pair +//! states instead. It limits the targets of a switch to a candidate set at each site: the +//! reference haplotypes with the longest IBS run against the sample. That heuristic costs little, +//! and it works like a PBWT. The cost is then `O(N · B · M)`, over the count of sites `N`, the +//! beam width `B`, and the candidate count `M`. //! -//! The [`Phaser`] trait keeps the seam stable so a Mendelian [`TrioPhaser`] (used when a parent -//! sample is in the workspace) or a full PBWT phaser can drop in without touching callers. +//! The [`Phaser`] trait holds the seam steady. Two other phasers can then go in, and no caller +//! changes. One is a Mendelian [`TrioPhaser`], for a workspace that holds a parent sample. The +//! other is a full PBWT phaser. use std::collections::HashMap; @@ -22,10 +26,13 @@ use crate::ancestry::HaplotypeReference; use crate::caller::SiteGenotype; use crate::ibd::GeneticMap; -/// One phased site: the coordinate plus the allele placed on each of the two sides (`0` = ref -/// allele, `1` = alt). `side0`/`side1` are consistent across the whole chromosome (a genuine -/// parental split), not sorted per site. `confidence` is the phase confidence (1.0 at homozygous -/// sites, which are unambiguous; lower at heterozygous sites the model is unsure about). +/// One phased site. It holds the coordinate, and the allele on each of the two sides, where `0` is +/// the ref allele and `1` is the alt. +/// +/// `side0` and `side1` keep the same sense across the whole chromosome, so they are a true parental +/// split. The code does not sort them at each site. `confidence` is the confidence of the phase. It +/// is 1.0 at a homozygous site, which is not ambiguous, and lower at a heterozygous site that the +/// model is unsure about. #[derive(Debug, Clone, PartialEq)] pub struct PhasedSite { pub contig: String, @@ -49,18 +56,22 @@ pub trait Phaser { fn phase(&self, genotypes: &[SiteGenotype]) -> PhasedGenotypes; } -/// Tuning knobs for [`ReferencePhaser`]. +/// The controls of [`ReferencePhaser`]. #[derive(Debug, Clone)] pub struct PhaseParams { - /// Beam width `B`: ordered pair-states kept per site. Larger → more accurate, slower. + /// The beam width `B`. It is the count of ordered pair states that the code keeps at each + /// site. A larger value is more accurate, and slower. pub beam: usize, - /// Per-site switch-target candidates `M`: the reference haplotypes with the longest current - /// IBS run with the sample that a side may recombine onto. + /// The count `M` of switch-target candidates at each site. Those are the reference haplotypes + /// with the longest current IBS run against the sample, and a side may recombine onto one of + /// them. pub candidates: usize, - /// Copying mutation/mismatch rate μ: probability a copied reference allele is observed flipped. + /// The mutation and mismatch rate μ of the copy. It is the probability that a copied reference + /// allele reads the other way round. pub mutation: f64, - /// Recombination intensity (expected copy switches per centiMorgan). Sets the distance-scaled - /// switch probability `1 - exp(-d_cM · rate)`. + /// The recombination intensity. It is the count of copy switches that the model expects in one + /// centiMorgan. It sets the switch probability, which scales with distance, as + /// `1 - exp(-d_cM · rate)`. pub recomb_per_cm: f64, } @@ -95,8 +106,9 @@ impl<'a> ReferencePhaser<'a> { Self { reference, map, params } } - /// ln P(observe genotype `g` | the two copied reference alleles `c0`, `c1`) under independent - /// per-copy mutation at rate μ. `c*`/`o*` are 0/1 alleles; `g` is the unordered dosage. + /// ln P(the code observes genotype `g` | the two copied reference alleles are `c0` and `c1`), + /// under independent mutation on each copy, at the rate μ. `c*` and `o*` are alleles, 0 or 1. + /// `g` is the dosage, and it holds no order. fn emit_ln(&self, g: u8, c0: u8, c1: u8) -> f64 { let mu = self.params.mutation; // P(observe o | copied c): faithful with prob 1-μ, flipped with μ. @@ -110,9 +122,11 @@ impl<'a> ReferencePhaser<'a> { prob.max(1e-300).ln() } - /// Given the MAP copied alleles `(c0, c1)` at a heterozygous site, the ordered `(side0, side1)` - /// alleles and a confidence. When the copies disagree (one 0, one 1) the phase is determined; - /// when they agree the het is explained by a mutation and phase is ambiguous (low confidence). + /// From the MAP copied alleles `(c0, c1)` at a heterozygous site, this gives the ordered + /// alleles `(side0, side1)`, and a confidence. + /// + /// When the two copies disagree, at one 0 and one 1, the phase follows. When they agree, a + /// mutation explains the het, and the phase is ambiguous, so the confidence is low. fn resolve_het(c0: u8, c1: u8) -> (u8, u8, f32) { match (c0, c1) { (0, 1) => (0, 1, 1.0), @@ -122,7 +136,8 @@ impl<'a> ReferencePhaser<'a> { } } - /// Phase one contig's usable sites. Returns `(side0, side1, confidence)` per site. + /// Phase the sites of one contig that the code can use. It returns a + /// `(side0, side1, confidence)` at each site. fn phase_contig(&self, contig: &str, sites: &[UsableSite]) -> Vec<(u8, u8, f32)> { let n = sites.len(); let k = self.reference.n_haplotypes; @@ -131,10 +146,13 @@ impl<'a> ReferencePhaser<'a> { } let allele = |col: usize, hap: usize| self.reference.allele(hap, col); - // Running IBS match length per reference haplotype: consecutive recent sites where the - // haplotype's allele is consistent with the observed genotype (homozygous sites only - // discriminate; heterozygous sites are consistent with every haplotype). Used to pick the - // per-site switch candidates — the reference haplotypes sharing the longest tract. + // The IBS match length of each reference haplotype, as the walk goes on. It counts the + // recent sites in a row where the allele of that haplotype agrees with the observed + // genotype. A homozygous site separates the haplotypes, and a heterozygous one agrees with + // every haplotype. + // + // The code uses this to take the switch candidates at each site: the reference haplotypes + // with the longest shared tract. let mut match_len = vec![0u32; k]; // Beam state: (side0 copied hap, side1 copied hap, ln prob, backpointer into prev beam). @@ -177,7 +195,8 @@ impl<'a> ReferencePhaser<'a> { } }; - // Initialise the beam at site 0 from the leading candidate set (uniform prior over pairs). + // Set up the beam at site 0, from the first candidate set. The prior over the pairs is + // uniform. update_match(&mut match_len, sites[0].ref_col, sites[0].dosage); let cand0 = candidates_at(&match_len, self.params.candidates); let mut beam: Vec = Vec::new(); @@ -206,8 +225,8 @@ impl<'a> ReferencePhaser<'a> { let sw = 1.0 - (-d_cm * self.params.recomb_per_cm).exp(); let sw = sw.clamp(1e-6, 0.999); let stay_ln = (1.0 - sw).ln(); - // A recombination lands on a specific candidate with prob sw/K; we only enumerate the - // strong candidates but charge the per-target sw/K mass. + // A recombination lands on one candidate with a probability of sw/K. This code walks + // the strong candidates alone, and it still charges the sw/K mass of each target. let jump_ln = (sw / k as f64).max(1e-300).ln(); update_match(&mut match_len, sites[i].ref_col, sites[i].dosage); @@ -217,7 +236,8 @@ impl<'a> ReferencePhaser<'a> { let col = sites[i].ref_col; let g = sites[i].dosage; - // Collect candidate successor states, keyed by (x,y), keeping the best incoming lp. + // Collect the candidate successor states, keyed by (x,y). Keep the best lp that comes + // in. let mut next: HashMap<(u32, u32), (f64, u32)> = HashMap::new(); let consider = |x: u32, y: u32, base_lp: f64, trans_ln: f64, bp: u32, next: &mut HashMap<(u32, u32), (f64, u32)>| { @@ -412,8 +432,9 @@ mod tests { let phased = phaser.phase(&genos); assert_eq!(phased.sites.len(), n); - // Count het sites resolved to the correct ordering (allowing a global side-swap, since - // side labels are arbitrary until anchored). + // Count the het sites that came out in the correct order. A swap of the two sides across + // the whole chromosome is acceptable, because the side labels mean nothing until an anchor + // fixes them. let mut agree_direct = 0; let mut agree_swapped = 0; let mut hets = 0; @@ -434,7 +455,7 @@ mod tests { } assert!(hets > 0); let best = agree_direct.max(agree_swapped); - // Allow a couple of switch errors but demand the phase is overwhelmingly recovered. + // A few switch errors are acceptable. But the phase must come back almost complete. assert!( best as f64 >= 0.9 * hets as f64, "recovered {best}/{hets} het sites (direct {agree_direct}, swapped {agree_swapped})" diff --git a/crates/navigator-analysis/src/postprocess/bamio.rs b/crates/navigator-analysis/src/postprocess/bamio.rs index cef6c1b9..71d4049d 100644 --- a/crates/navigator-analysis/src/postprocess/bamio.rs +++ b/crates/navigator-analysis/src/postprocess/bamio.rs @@ -1,21 +1,24 @@ //! Multithreaded BAM I/O for the post-processing stages. //! -//! Stage C moves the whole alignment through BGZF four times over — the sort reads the mapped BAM, -//! writes its spilled runs, reads them back to merge, and writes the sorted output — and duplicate -//! marking and CRAM emission each read it once more. On a 30x WGS that is hundreds of GB of -//! inflate and deflate, and every pass of it was running on one thread: the first WGS-scale run -//! measured the sort at 4 h 44 m, the most expensive stage in the pipeline, ahead of the mapping -//! it feeds. +//! Stage C moves the whole alignment through BGZF four times. The sort reads the mapped BAM, +//! writes its spilled runs, reads them back to merge, and writes the sorted output. The duplicate +//! mark and the CRAM output then each read it once more. //! -//! BGZF is a *block*-gzip stream, so compression and decompression parallelize across blocks while -//! the record stream stays sequential and byte-identical. That is the same reasoning -//! [`crate::reader::open_seq`] already applies to reading vendor BAMs; these stages simply never -//! got it, because they were written against the plain constructors. +//! On a 30x WGS that is hundreds of GB of inflate and deflate, and every pass of it ran on one +//! thread. The first run at WGS scale measured the sort at 4 h 44 m. That made it the stage in the +//! pipeline that cost the most, ahead of the mapping that it feeds. //! -//! Compression *level* is deliberately unchanged. These are all intermediates consumed once, so a -//! faster level is tempting, but it would inflate the scratch footprint that -//! `navigator-app::realign_job`'s preflight is calibrated against — a separate change, with a -//! recalibration attached. +//! BGZF is a stream of gzip *blocks*. So the compression and the decompression run in parallel +//! across those blocks, while the stream of records stays sequential, and its bytes do not +//! change. +//! That is the same reasoning that [`crate::reader::open_seq`] already applies to a read of a +//! vendor BAM. These stages never got it, because somebody wrote them against the plain +//! constructors. +//! +//! The compression *level* does not change, and that is deliberate. These are all intermediates +//! that one step reads once, so a faster level looks attractive. But a faster level makes the +//! scratch space larger, and the preflight of `navigator-app::realign_job` holds a calibration +//! against that space. That is a separate change, and it carries a new calibration with it. use std::fs::File; use std::io::{BufReader, BufWriter, Write}; @@ -40,10 +43,12 @@ pub(crate) type BamWriter = bam::io::Writer bool { use std::io::{Read, Seek, SeekFrom}; @@ -86,19 +94,22 @@ pub(crate) fn open(path: &Path) -> Result { Ok(bam::io::Reader::from(inner)) } -/// Open `path` as one of many streams that will be read concurrently. +/// Open `path` as one of many streams that the code reads at the same time. +/// +/// [`open`] is correct for a stream that stands alone. There, threads on its inflation are the +/// difference between one core and six, on a file that the stage reads from end to end. It is very +/// wrong for a stream that is one of hundreds. /// -/// [`open`] is right for a stream that is alone: threading its inflation is the difference between -/// one core and six on a file the stage reads end to end. It is badly wrong for a stream that is -/// one of hundreds. The sort's merge opens every spilled run at once — 688 of them on a 30x WGS at -/// the default budget — and giving each its own worker pool spawned **4,843 threads** in a measured -/// run, each independently reading ahead. The machine did 15,000 IOPS and 6.6 GB/s of disk reads to -/// produce 5 MB/s of merged output, WindowServer could not get scheduled, and the run was killed by -/// the watchdog that noticed. +/// The merge of the sort opens every spilled run at one time. That is 688 of them on a 30x WGS, at +/// the default budget. A worker pool for each one started **4,843 threads** in a measured run, and +/// each of those read ahead on its own. The machine did 15,000 IOPS, and 6.6 GB/s of reads from +/// the disk, to make 5 MB/s of merged output. WindowServer could not get onto a core, and its +/// watchdog saw that and killed the run. /// -/// There is nothing for a worker pool to do here anyway: the merge is already parallel across runs, -/// and it consumes one record at a time from each. So this inflates on the calling thread, behind a -/// modest buffer — 688 of these cost 688 buffers and no threads at all. +/// A worker pool has nothing to do here in any case. The merge already runs in parallel across the +/// runs, and it takes one record at a time from each. So this function inflates on the thread that +/// calls it, behind a buffer of modest size. There, 688 of these cost 688 buffers, and no +/// threads. pub(crate) fn open_many(path: &Path) -> Result { let file = File::open(path).map_err(|e| AnalysisError::io(path, e))?; let inner = bgzf::io::Reader::new(BufReader::with_capacity(READ_BUFFER, file)); @@ -117,13 +128,13 @@ pub(crate) fn create(path: &Path) -> Result { /// Finish the stream, including the BGZF end-of-file block. /// -/// A BGZF file without its EOF block is indistinguishable from a truncated one, and the plain -/// writer's `try_finish` does not exist on this type — the equivalent is draining the workers. +/// Nothing can separate a BGZF file with no EOF block from one that stops early. This type has no +/// `try_finish`, which the plain writer has. Here the equivalent is to run the workers dry. /// -/// The finished file is then synced, which matters more here than it looks: that EOF block is -/// exactly what [`navigator_app::realign_job`]'s resume uses to tell a stage output it can trust -/// from one a killed job left half-written. A marker still sitting in the page cache would be a -/// promise the disk has not made. +/// The code then syncs the finished file, and that matters more than it looks. The resume in +/// [`navigator_app::realign_job`] uses that EOF block. It separates a stage output that it can +/// trust from one that a killed job left half written. A marker that still sits in the page cache +/// is a promise that the disk has not made. pub(crate) fn finish(mut writer: BamWriter, path: &Path) -> Result<(), AnalysisError> { let mut buffered = writer.get_mut().finish().map_err(|e| AnalysisError::io(path, e))?; buffered.flush().map_err(|e| AnalysisError::io(path, e))?; diff --git a/crates/navigator-analysis/src/postprocess/cram.rs b/crates/navigator-analysis/src/postprocess/cram.rs index dbf6e269..6a2d5dc6 100644 --- a/crates/navigator-analysis/src/postprocess/cram.rs +++ b/crates/navigator-analysis/src/postprocess/cram.rs @@ -1,30 +1,33 @@ -//! Compress a sorted, marked alignment to CRAM and index it — the last step of stage C. +//! Compress a sorted alignment that carries its duplicate marks to a CRAM, and make its index. +//! This is the last step of stage C. //! -//! CRAM stores each read as its *difference* from the reference rather than as bases, which is why -//! it is dramatically smaller than BAM and why it needs the reference to read back. Navigator -//! already reads CRAM this way (`reader::open_seq` takes a reference for exactly this), so a -//! realigned alignment lands in the same shape as a vendor one. +//! A CRAM stores each read as its *difference* from the reference, and not as bases. That is why a +//! CRAM is much smaller than a BAM, and why a reader needs the reference to read it back. +//! Navigator already reads a CRAM this way, and `reader::open_seq` takes a reference for exactly +//! that. So a realigned alignment comes out in the same shape as a vendor one. //! //! ## The reference is part of the file //! -//! A CRAM is unreadable without the reference it was written against — not "degraded", unreadable. -//! That makes the reference argument here a correctness matter rather than a tuning knob, and it is -//! why the realigned alignment's row records `reference_path` alongside `bam_path` in stage D. -//! Compressing against the wrong reference produces a file that decodes to wrong bases rather than -//! failing, so the caller must pass the reference the reads were actually mapped to. +//! Nothing can read a CRAM without the reference that it went out against. The file is not +//! "worse": no reader can read it at all. So the reference argument here is a matter of +//! correctness, and not a control that somebody tunes. It is also why the row of a realigned +//! alignment records `reference_path` beside `bam_path`, in stage D. //! -//! ## Order matters, and it is checked +//! A compression against the wrong reference makes a file that decodes to the wrong bases. It does +//! not fail. So the caller must give the reference that the mapper mapped the reads to. //! -//! CRAM's compression assumes coordinate-sorted, reference-adjacent records; feeding it read-order -//! input produces a file that is both slow to write and larger than the BAM it replaced. Rather -//! than trust the caller to have sorted first, [`write_cram`] reads the `@HD SO` the sort stamped -//! and refuses input that does not claim coordinate order. +//! ## The order matters, and the code checks it +//! +//! The compression of a CRAM needs records in coordinate order, and near to the reference. Read +//! order instead makes a file that is slow to write, and larger than the BAM that it replaced. +//! [`write_cram`] does not trust the caller to have sorted first. It reads the `@HD SO` that the +//! sort wrote, and it refuses input that does not say coordinate order. //! //! ## The index //! -//! A `.crai` is what turns a region query from a full-file scan into a seek. It is built by reading -//! the finished CRAM back and recording where each container starts, so it necessarily happens -//! after the file is complete rather than during the write. +//! A `.crai` turns a region query from a scan of the whole file into a seek. The code builds it +//! from a read of the finished CRAM, and it notes where each container starts. That must happen +//! after the file is complete, and not during the write. use std::path::{Path, PathBuf}; @@ -48,21 +51,25 @@ pub struct CramOutput { pub index: PathBuf, /// Records written. pub records: u64, - /// Non-primary records dropped for carrying no `SEQ` — see [`is_unencodable`]. + /// The count of records that are not primary, and that the code dropped because they hold no + /// `SEQ`. See [`is_unencodable`]. pub sequenceless_dropped: u64, } -/// Can this record not be written as differences from the reference? +/// Can the code not write this record as a difference from the reference? +/// +/// A CRAM stores a read as its difference against the reference. To encode one means a walk over +/// its CIGAR, and a comparison of the bases. A record with an aligned CIGAR, and a `SEQ` of `*`, +/// has no base to compare. There, noodles indexes the empty sequence, and it does not check first. +/// The result is a panic from inside the writer, ten hours into a WGS job: +/// `range end index N out of range for slice of length 0`. /// -/// CRAM stores a read as its diff against the reference, so encoding one means walking its CIGAR -/// and comparing bases. A record with an aligned CIGAR but `SEQ: *` has no bases to compare, and -/// noodles indexes the empty sequence rather than checking — a panic (`range end index N out of -/// range for slice of length 0`) from inside the writer, ten hours into a WGS job. +/// This input is not malformed. SAM lets a secondary alignment carry a `SEQ` of `*`, and minimap2 +/// uses that. The primary alignment alone holds the bases. A secondary one points at another locus +/// that the same read could have come from. /// -/// This is not malformed input. SAM permits `SEQ: *` on a secondary alignment, and minimap2 uses -/// that permission: only the primary carries the bases, and the secondaries point at other loci -/// the same read could have come from. So the reads are not lost by dropping these — the primary -/// holds the sequence — but the records cannot be represented and must not reach the writer. +/// So to drop these loses no read, because the primary holds the sequence. But a CRAM can not +/// represent these records, and they must not reach the writer. fn is_unencodable(record: &noodles::sam::alignment::RecordBuf) -> bool { !record.cigar().as_ref().is_empty() && record.sequence().as_ref().is_empty() } @@ -84,9 +91,9 @@ pub fn write_cram( require_coordinate_sorted(&header, input)?; let repository = fasta_repository(reference)?; - // The final CRAM is tens of GB and was the last writer in the pipeline still handing its output - // straight to the page cache: `build_from_path` opens the file itself, which is how an encoder - // ends up holding a raw `File` that nothing paces and nothing counts. + // The final CRAM is tens of GB. It was the last writer in the pipeline that still gave its + // output straight to the page cache. `build_from_path` opens the file itself, and that is how + // an encoder comes to hold a raw `File` that nothing paces and nothing counts. let file = std::fs::File::create(output).map_err(|e| AnalysisError::io(output, e))?; let mut writer = cram::io::writer::Builder::default() .set_reference_sequence_repository(repository) @@ -107,10 +114,12 @@ pub fn write_cram( if is_unencodable(&record) { let flags = record.flags(); - // A secondary or supplementary record is a second opinion about where a read could go; - // the read itself survives on its primary, so dropping it costs no sequence. A *primary* - // with no SEQ would be an actual read going missing, which is not something to absorb - // quietly after the hours it took to get here. + // A secondary record, and a supplementary one, each give a second opinion about where + // a read could go. The read itself lives on its primary record, so to drop one of + // those costs no sequence. + // + // A *primary* record with no SEQ is a real read that goes missing. Nothing may take + // that in without a word, after the hours that this run took to reach here. if !flags.is_secondary() && !flags.is_supplementary() { return Err(AnalysisError::Message(format!( "{}: primary record {} has an aligned CIGAR but no SEQ, so it cannot be encoded \ @@ -132,15 +141,17 @@ pub fn write_cram( records += 1; } - // CRAM buffers records into containers and only writes the last one — and the end-of-file - // marker — on shutdown. A dropped writer leaves a file that looks complete and is not. + // A CRAM collects records into containers. It writes the last container, and the end-of-file + // marker, at shutdown alone. A writer that drops leaves a file that looks complete and is + // not. writer.try_finish(&header).map_err(|e| AnalysisError::io(output, e))?; { use std::io::Write as _; let buffered = writer.get_mut(); buffered.flush().map_err(|e| AnalysisError::io(output, e))?; - // Synced before it is indexed and handed to the workspace: `index_cram` reads the file back - // immediately, and everything downstream treats this path as the finished alignment. + // Sync it before the code makes its index and gives it to the workspace. `index_cram` + // reads the file back at once, and every later step takes this path as the finished + // alignment. buffered.get_ref().sync().map_err(|e| AnalysisError::io(output, e))?; } progress(records); @@ -154,10 +165,10 @@ pub fn write_cram( }) } -/// Build the `.crai` for a finished CRAM, returning its path. +/// Build the `.crai` of a finished CRAM. Returns its path. /// -/// Separate from [`write_cram`] because indexing reads the completed file back, so it is also what -/// a "reindex this alignment" repair would call. +/// It sits apart from [`write_cram`] because it reads the completed file back. So it is also the +/// function that a repair which makes the index again would call. pub fn index_cram(cram_path: &Path) -> Result { let index = cram::fs::index(cram_path).map_err(|e| AnalysisError::io(cram_path, e))?; let index_path = crai_path(cram_path); @@ -174,9 +185,9 @@ pub fn crai_path(cram_path: &Path) -> PathBuf { /// Refuse input that does not declare coordinate order. /// -/// The sort stamps `@HD SO:coordinate`; anything else here means the caller skipped the sort, and -/// the resulting CRAM would be slow to write, larger than the BAM, and unusable for region queries. -/// Failing now beats discovering that after hours of compression. +/// The sort writes `@HD SO:coordinate`. Anything else here means that the caller left the sort +/// out. The CRAM would then be slow to write, larger than the BAM, and of no use for a region +/// query. To fail now is better than to find that out after hours of compression. fn require_coordinate_sorted(header: &sam::Header, input: &Path) -> Result<(), AnalysisError> { use noodles::sam::header::record::value::map::header::tag; diff --git a/crates/navigator-analysis/src/postprocess/finalize.rs b/crates/navigator-analysis/src/postprocess/finalize.rs index 42f5691a..9cdf44ae 100644 --- a/crates/navigator-analysis/src/postprocess/finalize.rs +++ b/crates/navigator-analysis/src/postprocess/finalize.rs @@ -1,28 +1,29 @@ //! The last step of stage C: put the marked alignment where it belongs and index it. //! -//! ## Why this emits BAM rather than CRAM +//! ## Why this gives a BAM, and not a CRAM //! -//! CRAM is the better container on paper — reference-based compression makes a 30x WGS roughly -//! 17 GB against 60–80 GB of BAM — and stage C was built to produce it. Two defects in -//! `noodles-cram` 0.94 make that unreachable today, and both surfaced only at real scale: +//! On paper a CRAM is the better container. Its compression against a reference makes a 30x WGS +//! about 17 GB, against 60 to 80 GB for a BAM. Stage C exists to make one. Two defects in +//! `noodles-cram` 0.94 put that out of reach today, and both appeared only at real scale: //! -//! - **Writing** panics on a secondary alignment with `SEQ: *`, which is legal SAM and what -//! minimap2 emits, because CRAM encodes a read as its difference from the reference and there -//! are no bases to difference. [`super::cram`] handles it by dropping those records. -//! - **Indexing** then panics on any *multi-reference* slice: `cram::fs::index` decodes records -//! with `fasta::Repository::default()` — an empty one, still carrying its `// TODO` upstream — -//! and the reader `expect`s a name that cannot be there. With 25 contigs, every slice that -//! straddles a contig boundary is multi-reference, so a whole-genome CRAM cannot be indexed at -//! all. Only the coordinates are actually needed, which is why the decode is a bug rather than -//! a requirement. +//! - The **write** panics on a secondary alignment with a `SEQ` of `*`. That is legal SAM, and it +//! is what minimap2 gives. A CRAM encodes a read as its difference from the reference, and there +//! is no base to take a difference of. [`super::cram`] handles it, and drops those records. +//! - The **index** then panics on any slice that holds *more than one reference*. +//! `cram::fs::index` decodes the records with a `fasta::Repository::default()`, which is empty +//! and still carries its `// TODO` upstream. The reader then calls `expect` on a name that can +//! not be there. With 25 contigs, every slice that crosses a contig boundary holds more than one +//! reference, so nothing can index a whole-genome CRAM at all. The index needs the coordinates +//! alone, and that is why the decode is a bug, and not a requirement. //! -//! Emitting BAM avoids both, and costs less than it looks: duplicate marking has already written -//! exactly the bytes that belong in the output, so finalising is a rename and an index instead of -//! a whole re-compression pass over the alignment. The trade is disk, and for the handful of -//! whole genomes a desktop user holds that is the cheaper side. +//! A BAM avoids both, and it costs less than it looks. The duplicate mark already wrote exactly +//! the bytes that belong in the output. So this last step is a rename and an index, and not a +//! second compression pass over the whole alignment. The cost is disk space. For the few whole +//! genomes that a desktop user holds, that is the cheaper side. //! -//! The CRAM path is kept and still tested — the defects are upstream and fixable, and the choice -//! should be revisitable without rebuilding the stage. +//! The CRAM path stays, and the tests still cover it. The defects are upstream, and somebody can +//! fix them. The choice must then be open again, and nobody should have to build the stage +//! anew. use std::path::{Path, PathBuf}; @@ -30,7 +31,7 @@ use noodles::bam; use crate::error::AnalysisError; -/// What finalising produced. +/// What this last step made. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FinalizedAlignment { pub bam: PathBuf, @@ -39,8 +40,9 @@ pub struct FinalizedAlignment { /// Move `input` to `output` and write its `.bai`. /// -/// A rename when both sit on one filesystem, which is the normal case — the scratch directory -/// lives beside the output precisely so this costs nothing. Falls back to a copy across devices. +/// This is a rename when both paths sit on one filesystem, and that is the usual case. The scratch +/// directory lives beside the output for exactly that reason, so the move costs nothing. Across +/// two devices it falls back to a copy. pub fn finalize_bam(input: &Path, output: &Path) -> Result { if let Some(parent) = output.parent() { std::fs::create_dir_all(parent).map_err(|e| AnalysisError::io(parent, e))?; @@ -58,10 +60,10 @@ pub fn finalize_bam(input: &Path, output: &Path) -> Result Result { let index = bam::fs::index(bam_path).map_err(|e| AnalysisError::io(bam_path, e))?; let index_path = bai_path(bam_path); diff --git a/crates/navigator-analysis/src/postprocess/markdup.rs b/crates/navigator-analysis/src/postprocess/markdup.rs index 8aa342b3..97bd7b19 100644 --- a/crates/navigator-analysis/src/postprocess/markdup.rs +++ b/crates/navigator-analysis/src/postprocess/markdup.rs @@ -1,36 +1,41 @@ //! Mark PCR/optical duplicates on a coordinate-sorted alignment. //! -//! Two fragments that start and end at the same place, on the same strands, are almost certainly -//! copies of one original molecule rather than two independent observations. Counting both inflates -//! coverage and, worse, makes a sequencing error present as a confidently-supported variant. So one -//! member of each such group keeps its flag clear and the rest get `0x400`. +//! Take two fragments that start and end at the same place, on the same strands. They are almost +//! surely copies of one original molecule, and not two independent observations. To count both +//! makes the coverage too high. Worse, it makes a sequencing error look like a variant with strong +//! support. So one member of each such group keeps its flag clear, and the rest get `0x400`. //! -//! Nothing is removed. `0x400` is advice, and every consumer decides for itself — the coverage walk -//! honours it, a structural-variant caller may not. Deleting reads would take that choice away. +//! The code removes nothing. `0x400` is advice, and each consumer decides for itself. The coverage +//! walk follows it, and a structural-variant caller may not. To delete a read would take that +//! choice away. //! -//! ## Short reads only +//! ## Short reads alone //! -//! Long-read libraries (HiFi, ONT) are typically PCR-free, and long reads genuinely share endpoints -//! far less often, so the inference "same endpoints therefore same molecule" does not hold. Marking -//! them would discard real coverage, which is why [`MarkDupParams::enabled`] exists and why stage C -//! turns this off for the long-read presets. +//! A long-read library, from HiFi or ONT, usually has no PCR step, and two long reads share their +//! endpoints far less often. So the inference from the same endpoints to the same molecule does +//! not hold. A mark on them would throw away real coverage. That is why +//! [`MarkDupParams::enabled`] exists, and why stage C turns this off for the long-read presets. //! -//! ## Unclipped positions +//! ## The position before the clip //! -//! Grouping uses each end's **unclipped** 5′ position, not its alignment start. Two copies of one -//! molecule can be soft-clipped differently — a mismatch near one copy's end is enough — which -//! moves the alignment start without moving the fragment. Adding the clipped bases back recovers -//! where the molecule actually began, which is the thing being compared. +//! The groups use the 5′ position of each end **before its clip**, and not its alignment start. +//! Two copies of one molecule can carry different soft clips, and a mismatch near the end of one +//! copy is enough to cause that. The alignment start then moves, and the fragment does not. To add +//! the clipped bases back gives the place where the molecule began, and that is what the code must +//! compare. //! //! ## Both ends of a template must agree //! -//! A template with one end marked and the other not is a real corruption: consumers that filter on -//! `0x400` would see half a pair. This marks each end independently, which is safe only because a -//! template's signature is symmetric — it combines this end's 5′ position and strand with its -//! mate's, so both ends of two duplicate templates land in groups with identical membership, and -//! first-seen-in-coordinate-order picks the same template at both. The sort upstream is -//! deterministic precisely so that "first" means the same thing on every run. There is a test that -//! holds this property directly rather than trusting the argument. +//! A template with a mark on one end, and none on the other, is real damage. A consumer that +//! filters on `0x400` would then see half of a pair. +//! +//! This code marks each end on its own. That is safe for one reason only: the signature of a +//! template is symmetric. It puts the 5′ position and strand of this end together with those of +//! its mate. Both ends of two duplicate templates land in groups with the same membership. The +//! rule "the first one in coordinate order" then takes the same template at both ends. +//! +//! The sort before this is deterministic for exactly that reason: "first" must mean the same thing +//! on every run. A test holds this property directly, and it does not depend on the argument. use std::collections::{HashMap, VecDeque}; use std::path::Path; @@ -47,14 +52,15 @@ use crate::error::AnalysisError; const CANCEL_CHECK_INTERVAL: u64 = 4096; -/// How far back a candidate duplicate can sit and still be compared, in bases. +/// How far back a candidate duplicate can sit, in bases, and still enter a comparison. /// -/// Only clipping separates two copies' alignment starts, so this needs to exceed the largest -/// plausible clip — a read length, comfortably. It also bounds memory: only signatures within the -/// window are held. +/// A clip is the only thing that separates the alignment starts of two copies. So this value must +/// be more than the largest clip that can occur, and one read length covers that with room to +/// spare. It also bounds the memory, because the code holds a signature only while it is inside +/// the window. const DEFAULT_WINDOW: usize = 1_000; -/// Tuning for [`mark_duplicates`]. +/// The controls of [`mark_duplicates`]. #[derive(Debug, Clone)] pub struct MarkDupParams { /// Off for long-read data. See the module docs. @@ -72,23 +78,24 @@ impl Default for MarkDupParams { } } -/// What the marking did. +/// What this pass did. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct MarkDupStats { - /// Records read, and written — marking never drops one. + /// The count of records that the code read, and wrote. This pass never drops one. pub records: u64, /// Records flagged `0x400`. pub duplicates: u64, /// Records not eligible: unmapped, secondary, or supplementary. pub ineligible: u64, - /// True when [`MarkDupParams::enabled`] was false and records were copied through unmarked. + /// True when [`MarkDupParams::enabled`] was false, and the code copied every record through + /// with no mark. pub skipped: bool, } /// Mark duplicates in `input`, writing to `output`. /// -/// `input` must be coordinate-sorted — grouping relies on copies of a molecule being near each -/// other in the file, which is only true after the sort. +/// `input` must be in coordinate order. The groups depend on the copies of a molecule that lie +/// near each other in the file, and that holds only after the sort. pub fn mark_duplicates( input: &Path, output: &Path, @@ -123,8 +130,9 @@ pub fn mark_duplicates( if params.enabled { match signature(&record) { Some(sig) => { - // Recompute from scratch: an input that was marked before (a re-run, or a - // vendor file) must not inherit a verdict this pass did not reach. + // Compute this again from nothing. Take an input that already carries a mark, + // from a second run or from a vendor. It must not keep an answer that this + // pass did not reach. set_duplicate(&mut record, false); if seen.is_duplicate(sig) { set_duplicate(&mut record, true); @@ -149,8 +157,9 @@ pub fn mark_duplicates( /// What makes two records copies of one molecule. /// -/// Symmetric across a pair by construction: it carries this end's 5′ position and strand *and* the -/// mate's, so both ends of two duplicate templates produce groups with the same membership. +/// It is symmetric across a pair by construction. It carries the 5′ position and strand of this +/// end, *and* those of the mate. Both ends of two duplicate templates then make groups with the +/// same membership. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] struct Signature { reference: usize, @@ -161,11 +170,12 @@ struct Signature { mate_reverse: bool, } -/// The signature of an eligible record, or `None` when the record cannot be marked. +/// The signature of a record that the code may mark, or `None` when it may not mark that +/// record. /// -/// Unmapped records have no position to compare. Secondary and supplementary records describe an -/// alignment of a read whose primary is elsewhere; marking them would double-count a template that -/// the primary already represents. +/// A record with no mapping has no position to compare. A secondary record, and a supplementary +/// one, each describe an alignment of a read whose primary record sits elsewhere. To mark one of +/// those would count a template twice, because its primary record already stands for it. fn signature(record: &RecordBuf) -> Option { let flags = record.flags(); if flags.is_unmapped() || flags.is_secondary() || flags.is_supplementary() { @@ -176,8 +186,8 @@ fn signature(record: &RecordBuf) -> Option { let (leading, trailing, span) = clip_and_span(record); let reverse = flags.is_reverse_complemented(); - // The 5′ end of the molecule: the alignment start for a forward read, the alignment end for a - // reverse one — in both cases with the clipped bases added back. + // The 5′ end of the molecule. For a forward read that is the alignment start. For a reverse + // read it is the alignment end. In both cases the code adds the clipped bases back. let five_prime = if reverse { start + span - 1 + trailing } else { @@ -205,10 +215,10 @@ fn signature(record: &RecordBuf) -> Option { }) } -/// Leading clip, trailing clip, and reference span, from the CIGAR. +/// The clip at the start, the clip at the end, and the reference span, from the CIGAR. /// -/// Both soft (`S`) and hard (`H`) clips count: a hard clip means bases were removed from the -/// record, but the molecule still started that far back. +/// Both a soft clip (`S`) and a hard clip (`H`) count. A hard clip means that somebody removed +/// bases from the record. But the molecule still began that far back. fn clip_and_span(record: &RecordBuf) -> (i64, i64, i64) { use noodles::sam::alignment::record::Cigar as _; @@ -250,11 +260,11 @@ fn set_duplicate(record: &mut RecordBuf, duplicate: bool) { *record.flags_mut() = flags; } -// ---- the sliding window --------------------------------------------------- +// ---- the window that slides ----------------------------------------------- /// Signatures seen recently, evicted once the file has moved past them. /// -/// Bounded by the window rather than the file, so a WGS costs the same as an exome. +/// The window bounds this, and the file does not, so a WGS costs the same as an exome. struct SeenSignatures { window: i64, live: HashMap, @@ -270,8 +280,8 @@ impl SeenSignatures { } } - /// Whether this signature has been seen inside the window. The first record of a group is the - /// one kept; everything matching it afterwards is a duplicate. + /// True when this signature already occurred inside the window. The code keeps the first + /// record of a group. Everything after it that matches is a duplicate. fn is_duplicate(&mut self, sig: Signature) -> bool { self.evict(sig.five_prime); if self.live.contains_key(&sig) { @@ -282,7 +292,7 @@ impl SeenSignatures { false } - /// Drop signatures the file has moved past. Anything further back than the window cannot be a + /// Drop signatures the file has moved past. Anything further back than the window can not be a /// duplicate of the current record, because only clipping separates two copies' positions. fn evict(&mut self, position: i64) { while let Some((pos, sig)) = self.order.front().copied() { @@ -295,8 +305,9 @@ impl SeenSignatures { } } -/// Copy `header` and every record through unchanged. Used when marking is disabled, so callers get -/// the same output path either way rather than branching on whether stage C ran. +/// Copy `header`, and every record, through unchanged. The code calls this when the mark is off. A +/// caller then gets the same output path either way, and it does not have to ask whether stage C +/// ran. pub fn copy_through(input: &Path, output: &Path) -> Result { let mut reader = bamio::open(input)?; let header: sam::Header = reader.read_header().map_err(|e| AnalysisError::io(input, e))?; diff --git a/crates/navigator-analysis/src/postprocess/mod.rs b/crates/navigator-analysis/src/postprocess/mod.rs index 0cc28d7c..e5f4e706 100644 --- a/crates/navigator-analysis/src/postprocess/mod.rs +++ b/crates/navigator-analysis/src/postprocess/mod.rs @@ -1,9 +1,9 @@ -//! Stage C of the realignment pipeline — turning the mapper's output into an alignment the rest -//! of Navigator can use (`documents/design/realignment-module.md`). +//! Stage C of the realignment pipeline. It turns the output of the mapper into an alignment that +//! the rest of Navigator can use. See `documents/design/realignment-module.md`. //! -//! The mapper emits records in the order the reads arrived, which is useless to every consumer: -//! coverage walks, region queries, and the variant caller all assume coordinate order, and CRAM's -//! compression assumes it too. So stage C is: +//! The mapper gives its records in the order that the reads arrived, and no consumer can use that +//! order. A coverage walk, a region query and the variant caller all need coordinate order, and +//! the compression of a CRAM needs it too. So stage C does this: //! //! ```text //! mapped BAM (read order) @@ -15,12 +15,13 @@ //! CRAM + .crai <- next //! ``` //! -//! Duplicate marking is deliberately short-read-only. It identifies reads that start and end at -//! the same place with the same orientation, on the reasoning that two such fragments are almost -//! certainly PCR copies of one original. For HiFi and ONT that reasoning does not hold — long -//! reads genuinely share endpoints far less often, and the libraries are usually PCR-free — so -//! marking them would discard real coverage. This matches standard practice and the design's -//! Stage C note. +//! The mark on a duplicate goes onto a short read alone, and that is deliberate. The step finds +//! two reads that start and end at the same place, with the same orientation. Two such fragments +//! are almost surely PCR copies of one original. +//! +//! That reasoning does not hold for HiFi and ONT. Two long reads share their endpoints far less +//! often, and such a library usually has no PCR step. A mark on them would then throw away real +//! coverage. This agrees with standard practice, and with the Stage C note of the design. pub mod bamio; mod cram; diff --git a/crates/navigator-analysis/src/postprocess/sort.rs b/crates/navigator-analysis/src/postprocess/sort.rs index 506409df..cd2cdcdc 100644 --- a/crates/navigator-analysis/src/postprocess/sort.rs +++ b/crates/navigator-analysis/src/postprocess/sort.rs @@ -1,29 +1,34 @@ //! Coordinate-sort a BAM, on disk. //! -//! Same shape as [`crate::revert::collate`] and for the same reason: a WGS alignment does not fit -//! in memory, so this fills a budget, sorts it, spills a run, and k-way merges the runs at the end. -//! Peak memory is the budget plus one buffered block per run, independent of input size. +//! This has the same shape as [`crate::revert::collate`], and for the same reason. A WGS alignment +//! does not fit in memory. So this code fills a budget, sorts it, spills a run, and then merges +//! the runs k at a time at the end. The peak memory is the budget, plus one buffered block for +//! each run, and it does not grow with the input. //! -//! The budget comes from the machine ([`navigator_resource::spill_budget`]), not from a constant. -//! It was 512 MB for everyone, which on a 30x WGS spilled **688 runs** that the merge then opened -//! at once — bounded memory by design, and a great deal of fan-in to buy on a machine with 128 GB -//! sitting idle. +//! The budget comes from the machine. See [`navigator_resource::spill_budget`]. It is not a +//! constant. It was 512 MB for everybody, and on a 30x WGS that spilled **688 runs**, which the +//! merge then opened at one time. The memory had a bound by design. But that is a great deal of +//! fan-in to buy on a machine with 128 GB that nothing uses. //! -//! ## Runs are ordinary BAM files +//! ## A run is an ordinary BAM file //! -//! The revert stage spills a bespoke binary encoding because its records are four small fields. -//! An alignment record is not — it has a CIGAR, a tag dictionary, and per-base sequence and -//! quality — so inventing a serialization for it would mean re-deriving BAM's encoding badly. -//! Each run is written as a real BAM instead, using the same noodles encoder as the final output. -//! It costs a header per run (a few kB against runs of hundreds of MB) and buys a format that is -//! already correct, already tested, and inspectable with any tool when something looks wrong. +//! The revert stage spills a binary encoding of its own, because its records hold four small +//! fields. An alignment record does not. It holds a CIGAR, a tag dictionary, and a sequence and a +//! quality at each base. To invent a serialization for that would mean a second, worse version of +//! the BAM encoding. //! -//! ## Order +//! So each run goes out as a real BAM, through the same noodles encoder as the final output. That +//! costs one header for each run, which is a few kB against runs of hundreds of MB. It buys a +//! format that is already correct, already tested, and that any tool can open when something looks +//! wrong. //! -//! SAM's coordinate order is by reference sequence, then by alignment start, with **unplaced reads -//! last**. Those unplaced reads are not an afterthought here: recovering reads the old reference -//! could not place is a large part of why realignment exists, so they have to survive the sort and -//! land somewhere a reader will find them, rather than being dropped for having no coordinate. +//! ## The order +//! +//! The coordinate order of SAM goes by reference sequence, then by alignment start, and it puts +//! **the reads with no place last**. Those reads matter here. To recover a read that the old +//! reference could not place is a large part of why realignment exists. So they must survive the +//! sort, and land where a reader will find them. The sort must not drop a read because it has no +//! coordinate. use std::cmp::Reverse; use std::collections::BinaryHeap; @@ -37,20 +42,23 @@ use super::bamio; use crate::cancel::CancelToken; use crate::error::AnalysisError; -/// How often the record loop asks whether it has been cancelled — same cadence as the walkers. +/// How often the record loop asks whether somebody cancelled it. This is the same rate as in the +/// walkers. const CANCEL_CHECK_INTERVAL: u64 = 4096; -/// Tuning for [`sort_alignment`]. +/// The controls of [`sort_alignment`]. #[derive(Debug, Clone)] pub struct SortParams { - /// Approximate bytes of records held before a sorted run is spilled to scratch. + /// About how many bytes of records the code holds before it sorts them and spills a run to + /// the scratch space. pub buffer_bytes: usize, } impl Default for SortParams { - /// Sized from the machine, not from a constant — see [`navigator_resource::spill_budget`], which - /// also documents `NAVIGATOR_SORT_MB`. The constant this replaced was 512 MB, which spilled 688 - /// runs on a 30x WGS regardless of whether the machine had 8 GB or 128 GB to work with. + /// The size comes from the machine, and not from a constant. See + /// [`navigator_resource::spill_budget`], which also documents `NAVIGATOR_SORT_MB`. The constant + /// before it was 512 MB. That spilled 688 runs on a 30x WGS, whether the machine had 8 GB or + /// 128 GB to work with. fn default() -> Self { Self { buffer_bytes: navigator_resource::spill_budget("NAVIGATOR_SORT_MB") as usize, @@ -61,7 +69,7 @@ impl Default for SortParams { /// What the sort did. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct SortStats { - /// Records read, and written — the sort never drops one. + /// The count of records that the code read, and wrote. The sort never drops one. pub records: u64, /// Records with no coordinate, which sort to the end. pub unplaced: u64, @@ -69,10 +77,10 @@ pub struct SortStats { pub runs: usize, } -/// Coordinate-sort `input` into `output`, using `scratch` for spilled runs. +/// Sort `input` into `output`, in coordinate order. It uses `scratch` for the runs that it spills. /// -/// The output header is the input's, with `@HD SO:coordinate` set — the claim readers rely on to -/// decide whether they may binary-search or must scan. +/// The output header is the header of the input, with `@HD SO:coordinate` set. A reader depends on +/// that claim to decide whether it may do a binary search, or must scan. pub fn sort_alignment( input: &Path, output: &Path, @@ -116,7 +124,7 @@ pub fn sort_alignment( // ---- spilling ------------------------------------------------------------- -/// Accumulates records, spilling a sorted BAM run whenever the budget is reached. +/// It collects records. Whenever they reach the budget, it sorts them and spills a BAM run. struct Spiller { dir: PathBuf, header: sam::Header, @@ -151,10 +159,10 @@ impl Spiller { if self.buffered.is_empty() { return Ok(()); } - // `sort_by_key` (stable) rather than `sort_unstable_by_key`: records that share a - // coordinate keep their input order, so the whole sort is deterministic. A duplicate - // marker downstream picks a representative from a group of identical positions, and it - // should pick the same one on every run. + // This uses `sort_by_key`, which is stable, and not `sort_unstable_by_key`. Records that + // share a coordinate then keep their input order, so the whole sort is deterministic. A + // later step marks the duplicates. It takes one record from a group at the same position, + // and it must take the same one on every run. self.buffered.sort_by_key(sort_key); let path = self.dir.join(format!("sort-run-{:05}.bam", self.paths.len())); @@ -181,8 +189,9 @@ impl Spiller { } } -/// The spilled runs, deleted on drop so a failed or cancelled sort leaves no scratch behind — at -/// WGS scale these are the size of the alignment itself. +/// The runs that the code spilled. They go away when this drops, so a sort that failed, or that +/// somebody cancelled, leaves no scratch space behind. At WGS scale they are as large as the +/// alignment itself. struct Runs { paths: Vec, } @@ -212,8 +221,9 @@ impl PartialEq for Entry { impl Eq for Entry {} impl Ord for Entry { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // Ties break on run index so the merge is deterministic; combined with the stable sort - // within each run, equal-coordinate records come out in a repeatable order. + // A tie breaks on the run index, so the merge is deterministic. With the stable sort + // inside each run, two records at the same coordinate always come out in the same + // order. self.key.cmp(&other.key).then(self.run.cmp(&other.run)) } } @@ -234,7 +244,7 @@ fn merge( let mut writer = bamio::create(output)?; writer.write_header(header).map_err(|e| AnalysisError::io(output, e))?; - // Each run is read through its own reader; only one record per run is resident at a time. + // Each run has its own reader. Only one record of each run sits in memory at a time. let mut readers = Vec::with_capacity(runs.paths.len()); for path in &runs.paths { let mut reader = bamio::open_many(path)?; @@ -279,8 +289,8 @@ fn merge( bamio::finish(writer, output)?; - // A sort that loses records is the failure this whole stage must not have, and it would be - // invisible downstream — coverage would simply read low. + // A sort that loses records is the one failure that this stage must not have. No later step + // would see it, because the coverage would only read low. if written != stats.records { return Err(AnalysisError::Message(format!( "sort lost records: read {}, wrote {written}", @@ -293,8 +303,9 @@ fn merge( struct RunReader { path: PathBuf, header: sam::Header, - /// Single-threaded by design — see [`bamio::open_many`]. Every run in the merge is open at - /// once, so a worker pool per run is thousands of threads for work that is already parallel. + /// It runs on one thread, and that is deliberate. See [`bamio::open_many`]. Every run of the + /// merge is open at one time. A worker pool for each run is then thousands of threads, for + /// work that already runs in parallel. reader: bamio::PlainBamReader, } @@ -324,20 +335,25 @@ fn sort_key(record: &RecordBuf) -> (u32, u64) { /// What one record costs the buffer. /// -/// This used to be the variable-length parts plus a flat 256, which read as a reasonable stand-in -/// for "fixed fields and allocator overhead" and was not one. It left out the tag dictionary -/// entirely, and a mapped record carries a dozen tags — `NM`, `MD`, `AS`, `ms`, `nn`, `tp`, `cm`, -/// `s1`, `s2`, `de`, `rl` from minimap2 alone — each an entry in a `Vec<(Tag, Value)>`. The buffer -/// therefore held well over its stated budget, which mattered little against a constant picked with -/// an unwritten margin and matters a great deal now that the budget is a fraction of the machine -/// (see [`navigator_resource::spill_budget`]). +/// This was the parts of variable length, plus a flat 256. That 256 read as a reasonable stand-in +/// for the fixed fields and the allocator overhead, and it was not one. It left the tag dictionary +/// out completely. +/// +/// A mapped record carries a dozen tags. minimap2 alone gives `NM`, `MD`, `AS`, `ms`, `nn`, `tp`, +/// `cm`, `s1`, `s2`, `de` and `rl`. Each one is an entry in a `Vec<(Tag, Value)>`. So the buffer +/// held well over the budget that it stated. +/// +/// That mattered little against a constant that somebody chose with a margin nobody wrote down. It +/// matters a great deal now that the budget is a fraction of the machine. See +/// [`navigator_resource::spill_budget`]. /// -/// Still an estimate: it does not chase a `Value`'s own heap (a string tag's bytes) or a `Vec`'s -/// spare capacity. It is close enough to size a buffer by, and it no longer omits a whole field. +/// This is still an estimate. It does not follow the own heap of a `Value`, which holds the bytes +/// of a string tag. It also does not count the spare capacity of a `Vec`. It is close enough to +/// size a buffer by, and it no longer leaves a whole field out. pub(super) fn heap_bytes(record: &RecordBuf) -> usize { use noodles::sam::alignment::record::Cigar as _; - // The record itself sits inline in the buffer's `Vec`, so its size is part of what a record - // costs — not something to approximate around. + // The record itself sits inside the `Vec` of the buffer. Its size is part of what a record + // costs, and not something to work around. std::mem::size_of::() + record.name().map(|n| n.len()).unwrap_or(0) + record.sequence().len() @@ -350,15 +366,16 @@ pub(super) fn heap_bytes(record: &RecordBuf) -> usize { /// Bytes one tag occupies in a record's `Vec<(Tag, Value)>`. /// -/// Pinned by `a_tag_entry_is_not_larger_than_the_estimate_assumes`, so a noodles upgrade that grows -/// `Value` fails a test here rather than quietly halving the buffer's honesty. +/// `a_tag_entry_is_not_larger_than_the_estimate_assumes` holds this value. So an upgrade of +/// noodles that makes `Value` larger fails a test here. Without that test, the buffer would hold +/// twice what it says, and nobody would see it. pub(super) const TAG_ENTRY_BYTES: usize = 48; /// Stamp `@HD SO:coordinate` on the header. /// -/// Not cosmetic: an index is only valid for a coordinate-sorted file, and readers decide whether -/// they may query a region by looking at this. A correctly sorted file that fails to say so is -/// treated as unsorted and re-scanned. +/// This is not for appearance. An index is correct only for a file in coordinate order, and a +/// reader reads this to decide whether it may query a region. A file that the code sorted +/// correctly, and that does not say so, reads as unsorted, and every reader scans it again. fn with_coordinate_sort_order(mut header: sam::Header) -> sam::Header { use noodles::sam::header::record::value::map::header::tag; use noodles::sam::header::record::value::{map, Map}; diff --git a/crates/navigator-analysis/src/postprocess/tests.rs b/crates/navigator-analysis/src/postprocess/tests.rs index 5763dedf..e3f7b899 100644 --- a/crates/navigator-analysis/src/postprocess/tests.rs +++ b/crates/navigator-analysis/src/postprocess/tests.rs @@ -1,9 +1,11 @@ //! Tests for the coordinate sort. //! -//! The property that matters most is that the sort is *lossless* — a sort that quietly drops -//! records would show up downstream only as coverage reading a bit low, which is close to -//! undetectable. The spill/no-spill equivalence test is the second: the result must not depend on -//! how much of the input happened to fit in memory. +//! The property that matters most is that the sort loses nothing. A sort that drops records, where +//! nobody sees it, would show later as a coverage that reads a little low. Almost nobody would +//! find that. +//! +//! The second property is the test that the spill path and the no-spill path agree. The result +//! must not depend on how much of the input fit in memory. use std::path::{Path, PathBuf}; @@ -118,9 +120,10 @@ fn unsorted_fixture(dir: &Path) -> (PathBuf, sam::Header, usize) { // ---- the buffer estimate --------------------------------------------------- -/// The buffer is now sized as a fraction of the machine rather than a hand-picked constant, so what -/// a record is charged against it has to be roughly true. It was not: the tag dictionary was free, -/// and a mapped record carries a dozen tags. +/// The size of the buffer is now a fraction of the machine, and not a constant that somebody +/// chose. So the cost that the code charges a record against that buffer must be about correct. It +/// was not correct before: the tag dictionary cost nothing, and a mapped record carries a dozen +/// tags. #[test] fn the_buffer_estimate_counts_the_tag_dictionary() { use noodles::sam::alignment::record::data::field::Tag; @@ -140,8 +143,8 @@ fn the_buffer_estimate_counts_the_tag_dictionary() { assert_eq!(charged, 3 * TAG_ENTRY_BYTES, "three tags should cost three entries"); } -/// A noodles upgrade that grows `Value` should fail here, rather than quietly making every buffer -/// hold more than its budget says. +/// An upgrade of noodles that makes `Value` larger must fail here. Without this test, every buffer +/// would hold more than its budget says, and nobody would see it. #[test] fn a_tag_entry_is_not_larger_than_the_estimate_assumes() { use noodles::sam::alignment::record::data::field::Tag; @@ -154,7 +157,8 @@ fn a_tag_entry_is_not_larger_than_the_estimate_assumes() { ); } -/// The record's own size is part of what it costs — it lives inline in the buffer's `Vec`. +/// The own size of the record is part of what it costs, because it lives inside the `Vec` of the +/// buffer. #[test] fn the_buffer_estimate_covers_the_record_itself() { let empty = RecordBuf::default(); @@ -191,9 +195,9 @@ fn records_come_out_in_coordinate_order_with_unplaced_reads_last() { } } -/// A sort that loses records would show up downstream only as slightly low coverage — which is -/// close to undetectable. Nothing may be dropped, including the unplaced reads that realignment -/// exists to recover. +/// A sort that loses records would show later as a coverage that is a little low, and almost +/// nobody would find that. The sort must drop nothing, and that includes the reads with no place, +/// which realignment exists to recover. #[test] fn the_sort_is_lossless() { let dir = scratch("lossless"); @@ -212,8 +216,9 @@ fn the_sort_is_lossless() { assert_eq!(names, expected, "every input record must appear exactly once"); } -/// The result must not depend on how much of the input fit in memory — the same claim the revert -/// stage's external sort makes, and the reason the disk path is safe to rely on. +/// The result must not depend on how much of the input fit in memory. The external sort of the +/// revert stage makes the same claim. That is the reason you can rely on the path that uses the +/// disk. #[test] fn spilling_produces_the_same_order_as_sorting_in_memory() { let dir = scratch("spill"); @@ -238,8 +243,9 @@ fn spilling_produces_the_same_order_as_sorting_in_memory() { assert_eq!(read_back(&a).1, read_back(&b).1); } -/// An index is only valid for a coordinate-sorted file, and readers decide whether they may query -/// a region by reading this. Sorting correctly but failing to say so means every reader rescans. +/// An index is correct only for a file in coordinate order, and a reader reads this to decide +/// whether it may query a region. A sort that works, and that does not say so, makes every reader +/// scan the file again. #[test] fn the_output_header_declares_coordinate_order() { let dir = scratch("header"); @@ -289,8 +295,9 @@ fn cancellation_stops_the_sort() { assert!(matches!(err, AnalysisError::Cancelled)); } -/// An already-sorted file is a normal input (a re-run, or a mapper that happened to emit in -/// order), and must come out unchanged rather than subtly reordered. +/// A file that is already sorted is a usual input. It comes from a second run, or from a mapper +/// that gave its output in order. It must come out unchanged, and the sort must not move a record +/// where nobody looks. #[test] fn an_already_sorted_file_is_unchanged() { let dir = scratch("idempotent"); @@ -304,7 +311,7 @@ fn an_already_sorted_file_is_unchanged() { assert_eq!(read_back(&once).1, read_back(&twice).1); } -// ---- duplicate marking ---------------------------------------------------- +// ---- the mark on a duplicate ---------------------------------------------- use super::markdup::{mark_duplicates, MarkDupParams}; @@ -336,8 +343,9 @@ fn pair_record( .set_cigar(parse_cigar(cigar)) .set_mate_reference_sequence_id(mate_ref) .set_mate_alignment_start(noodles::core::Position::new(mate_pos).unwrap()) - // SEQ must be exactly as long as the CIGAR's query consumption — soft clips count, hard - // clips do not. noodles rejects the record otherwise, which is how this was caught. + // SEQ must have exactly the length that the CIGAR takes from the query. A soft clip + // counts, and a hard clip does not. noodles refuses the record if the two differ, and that + // is how somebody found this. .set_sequence(Sequence::from(vec![b'A'; query_len(cigar)])) .set_quality_scores(QualityScores::from(vec![30; query_len(cigar)])) .build() @@ -361,7 +369,7 @@ fn query_len(spec: &str) -> usize { total } -/// Minimal CIGAR parser for the fixtures, e.g. "3S10M". +/// A small CIGAR parser for the fixtures, for example "3S10M". fn parse_cigar(spec: &str) -> noodles::sam::alignment::record_buf::Cigar { use noodles::sam::alignment::record::cigar::op::{Kind, Op}; let mut ops = Vec::new(); @@ -410,8 +418,9 @@ fn mark(input: &Path, dir: &Path, params: MarkDupParams) -> (PathBuf, super::Mar (output, stats) } -/// Two templates from the same molecule: same endpoints, same strands. One survives unmarked, the -/// other is flagged — and an independent template at another position is untouched. +/// Two templates from the same molecule, with the same endpoints and the same strands. One of them +/// stays without a mark, and the code flags the other. A separate template at another position does +/// not change. #[test] fn identical_fragments_are_marked_and_one_representative_is_kept() { let dir = scratch("dupes"); @@ -462,14 +471,15 @@ fn both_ends_of_a_duplicate_template_are_marked_alike() { assert_eq!(verdicts("b"), vec![true, true], "both ends of 'b' agree"); } -/// Copies of one molecule can be clipped differently — a mismatch near an end is enough. Grouping -/// on the alignment start would miss them; grouping on the unclipped 5' position finds them. +/// Two copies of one molecule can carry different clips, and one mismatch near an end is enough to +/// cause that. A group on the alignment start would miss them. A group on the 5' position before +/// the clip finds them. #[test] fn differently_clipped_copies_of_one_fragment_are_still_duplicates() { let dir = scratch("clipping"); let hdr = header(&[("chr1", 100_000)]); - // Both molecules begin at 100: one aligns from 100 with no clip, the other is clipped by 3 and - // so *starts* at 103 while describing the same fragment. + // Both molecules begin at 100. One aligns from 100 with no clip. The other carries a clip of + // 3, so it *starts* at 103, and it covers the same fragment. let records = vec![ pair_record("plain", 0, 100, false, true, 0, 500, true, "10M"), pair_record("clipped", 0, 103, false, true, 0, 500, true, "3S10M"), @@ -482,8 +492,9 @@ fn differently_clipped_copies_of_one_fragment_are_still_duplicates() { assert!(duplicate_flags(&output)[1].1, "the clipped copy is the duplicate"); } -/// Same start, different mate — two independent molecules that happen to share one endpoint. A -/// signature that ignored the mate would collapse them and delete real coverage. +/// The same start, and a different mate. These are two separate molecules that share one endpoint +/// by chance. A signature that left out the mate would put them together, and that would delete +/// real coverage. #[test] fn fragments_sharing_one_end_but_not_the_other_are_not_duplicates() { let dir = scratch("mate"); @@ -515,8 +526,9 @@ fn opposite_strands_are_not_duplicates() { assert_eq!(stats.duplicates, 0); } -/// Long-read libraries are usually PCR-free and long reads rarely share endpoints by chance, so -/// marking them would throw away real coverage. Disabling must actually disable. +/// A long-read library usually has no PCR step, and two long reads rarely share an endpoint by +/// chance. A mark on them would then throw away real coverage. The option that turns the mark off +/// must turn it off. #[test] fn marking_can_be_turned_off_for_long_reads() { let dir = scratch("disabled"); @@ -541,8 +553,9 @@ fn marking_can_be_turned_off_for_long_reads() { assert!(duplicate_flags(&output).iter().all(|(_, d)| !d)); } -/// Unmapped, secondary, and supplementary records have no molecule of their own to represent — -/// their primary already does. They pass through unmarked and are counted as ineligible. +/// An unmapped record, a secondary record and a supplementary record represent no molecule of +/// their own, because the primary record already does. They go through with no mark, and the count +/// puts them outside the set that the pass can mark. #[test] fn ineligible_records_pass_through_unmarked() { let dir = scratch("ineligible"); @@ -567,8 +580,8 @@ fn ineligible_records_pass_through_unmarked() { assert!(duplicate_flags(&output).iter().all(|(_, d)| !d)); } -/// A re-run, or an input a vendor already marked, must get this pass's verdict rather than -/// inheriting one it did not reach. +/// A second run, and an input that a vendor already marked, must both take the answer of this +/// pass. Neither may keep an answer that this pass did not reach. #[test] fn pre_existing_duplicate_flags_are_recomputed() { let dir = scratch("recompute"); @@ -587,7 +600,7 @@ fn pre_existing_duplicate_flags_are_recomputed() { ); } -/// Marking never drops a record — it only changes a flag. +/// The mark never drops a record. It changes a flag and nothing else. #[test] fn marking_is_lossless() { let dir = scratch("mdlossless"); @@ -605,9 +618,10 @@ fn marking_is_lossless() { use super::cram::{crai_path, index_cram, write_cram}; -/// CRAM stores reads as differences from the reference, so a test needs a real one — bases the -/// records actually match, plus the `.fai` the repository reads. A stub would either fail to build -/// or silently encode mismatches for every base. +/// A CRAM stores a read as the difference from the reference, so a test needs a real reference. +/// That means bases that the records match, and the `.fai` that the repository reads. A stub +/// reference would either fail to build, or encode a mismatch at every base where nobody sees +/// it. fn write_reference_fasta(dir: &Path, name: &str, len: usize) -> PathBuf { let path = dir.join("ref.fa"); let bases: Vec = (0..len).map(|i| b"ACGT"[(i * 7 + 3) % 4]).collect(); @@ -621,7 +635,8 @@ fn write_reference_fasta(dir: &Path, name: &str, len: usize) -> PathBuf { } std::fs::write(&path, &text).unwrap(); - // A minimal `.fai`: name, length, offset of the first base, bases per line, bytes per line. + // A small `.fai`. Its five fields are the name, the length, the offset of the first base, the + // bases in a line, and the bytes in a line. let offset = name.len() + 2; // ">name\n" std::fs::write( dir.join("ref.fa.fai"), @@ -635,9 +650,9 @@ fn reference_bases_at(len: usize) -> Vec { (0..len).map(|i| b"ACGT"[(i * 7 + 3) % 4]).collect() } -/// A record whose sequence matches the reference at `pos`, so CRAM has nothing to store but the -/// position — which is the case worth testing, since a mismatch-heavy fixture would not exercise -/// reference-based compression at all. +/// A record whose sequence matches the reference at `pos`. A CRAM then has nothing to store but +/// the position. That is the case that this test needs, because a fixture with many mismatches +/// would not cover the compression against a reference at all. fn matching_record(name: &str, pos: usize, len: usize, reference: &[u8]) -> RecordBuf { RecordBuf::builder() .set_name(name) @@ -650,7 +665,7 @@ fn matching_record(name: &str, pos: usize, len: usize, reference: &[u8]) -> Reco .build() } -/// A sorted BAM plus the reference it was aligned to. +/// A sorted BAM, and the reference that the mapper aligned it to. fn cram_fixture(dir: &Path, count: usize) -> (PathBuf, PathBuf, usize) { let contig_len = 10_000; let reference = write_reference_fasta(dir, "chr1", contig_len); @@ -660,8 +675,8 @@ fn cram_fixture(dir: &Path, count: usize) -> (PathBuf, PathBuf, usize) { let mut records: Vec = (0..count) .map(|i| matching_record(&format!("r{i:03}"), 1 + i * 50, 50, &bases)) .collect(); - // Coordinate order is the precondition; the fixture is already in it, but stamp the header the - // way the sort would so the check under test sees what it expects. + // Coordinate order is the condition. The fixture is already in that order. But write the + // header the way that the sort would, so that the check under test sees what it expects. records.sort_by_key(|r| r.alignment_start().map(|p| p.get()).unwrap_or(0)); let bam = dir.join("sorted.bam"); @@ -669,7 +684,7 @@ fn cram_fixture(dir: &Path, count: usize) -> (PathBuf, PathBuf, usize) { (bam, reference, count) } -/// Like [`write_bam`], but stamping `@HD SO:coordinate` as the sort does. +/// The same as [`write_bam`], and it also writes `@HD SO:coordinate`, as the sort does. fn write_bam_sorted(path: &Path, header: &sam::Header, records: &[RecordBuf]) { use noodles::sam::header::record::value::map::header::tag; use noodles::sam::header::record::value::{map, Map}; @@ -682,8 +697,8 @@ fn write_bam_sorted(path: &Path, header: &sam::Header, records: &[RecordBuf]) { write_bam(path, &header, records); } -/// The headline: a sorted BAM becomes a CRAM that reads back with the same records, and gets an -/// index beside it. +/// The main property. A sorted BAM becomes a CRAM that reads back with the same records, and an +/// index sits beside it. #[test] fn cram_round_trips_every_record_and_writes_an_index() { let dir = scratch("cram"); @@ -702,8 +717,8 @@ fn cram_round_trips_every_record_and_writes_an_index() { "an empty index would leave every query scanning the whole file" ); - // Read back through the same path Navigator uses for vendor CRAMs — if the realigned output - // is not readable that way, it is not usable by any existing analysis. + // Read it back through the same path that Navigator uses for a vendor CRAM. If no reader can + // take the realigned output that way, then no analysis in the app can use it. let (header, mut reader) = crate::reader::open_seq(&out, Some(&reference)).unwrap(); let names: Vec = reader .records(&header) @@ -713,8 +728,9 @@ fn cram_round_trips_every_record_and_writes_an_index() { assert_eq!(names[0], "r000"); } -/// CRAM reconstructs bases from the reference. If that round trip were wrong the sequences would -/// come back altered rather than the read failing, so the bases are compared explicitly. +/// A CRAM builds the bases again from the reference. If that round trip were wrong, the sequences +/// would come back changed, and the read itself would not fail. So this test compares the bases +/// one by one. #[test] fn sequences_survive_reference_based_compression() { let dir = scratch("crambases"); @@ -736,8 +752,8 @@ fn sequences_survive_reference_based_compression() { } } -/// Compressing read-order input produces a file that is slow to write, larger than the BAM, and -/// useless for region queries. Refusing up front beats discovering that after hours. +/// Input in read order makes a file that is slow to write, larger than the BAM, and of no use for +/// a region query. To refuse it at the start is better than to find that out after hours. #[test] fn unsorted_input_is_refused_before_compressing() { let dir = scratch("cramunsorted"); @@ -802,9 +818,10 @@ fn cancellation_stops_cram_emission() { assert!(matches!(err, AnalysisError::Cancelled)); } -/// A secondary alignment with `SEQ: *` — legal SAM, and what minimap2 emits, since only the -/// primary carries the bases. It cannot be encoded as differences from the reference, so it is -/// dropped and counted rather than panicking the writer from inside noodles. +/// A secondary alignment with `SEQ: *`. That is legal SAM, and it is what minimap2 gives, because +/// the primary alignment alone carries the bases. The code can not encode it as a difference from +/// the reference. So it drops that record and counts it. Without that, the writer would panic from +/// inside noodles. #[test] fn cram_drops_a_secondary_record_that_carries_no_sequence() { let dir = scratch("cram-secondary"); @@ -832,8 +849,9 @@ fn cram_drops_a_secondary_record_that_carries_no_sequence() { assert_eq!(result.sequenceless_dropped, 1, "the secondary is dropped, and counted"); } -/// The same shape on a *primary* is a read going missing, not a redundant record, so it fails -/// loudly instead of being absorbed into a drop count. +/// The same shape on a *primary* alignment means that a read goes missing. It is not a record that +/// says nothing new. So the code fails loudly there, and it does not put that record into a count +/// of drops. #[test] fn cram_refuses_a_primary_record_that_carries_no_sequence() { let dir = scratch("cram-primary-noseq"); @@ -866,9 +884,10 @@ fn cram_refuses_a_primary_record_that_carries_no_sequence() { ); } -/// Finalising moves the marked BAM into place and indexes it — across *several* contigs, which is -/// the shape that cannot be indexed as CRAM: every slice straddling a contig boundary is -/// multi-reference, and `cram::fs::index` decodes those against an empty reference repository. +/// The last step moves the marked BAM into place and makes its index. This test covers *more than +/// one* contig, which is the shape that the code can not index as a CRAM. Every slice that crosses +/// a contig boundary holds more than one reference, and `cram::fs::index` decodes those against an +/// empty reference repository. #[test] fn finalizing_moves_the_bam_into_place_and_indexes_it() { use super::finalize::{bai_path, finalize_bam}; @@ -908,11 +927,12 @@ fn finalizing_moves_the_bam_into_place_and_indexes_it() { assert_eq!(read_back.len(), 60, "every record survives finalising"); } -// ---- completeness --------------------------------------------------------- +// ---- is the file complete ------------------------------------------------- // -// `is_complete_bam` is what `navigator-app`'s realignment resume trusts when it decides whether a -// killed attempt's 60 GB intermediate can be picked up or has to be re-derived over several hours. -// Getting it wrong in either direction is expensive, so both directions are pinned here. +// The realignment resume in `navigator-app` trusts `is_complete_bam`. That answer decides one +// thing. Can the app take up the 60 GB intermediate of a run that somebody killed, or must it +// derive that file again over some hours? A wrong answer in either direction costs much, so this +// section holds both directions. #[test] fn a_finished_bam_is_complete() { @@ -924,8 +944,9 @@ fn a_finished_bam_is_complete() { assert!(crate::postprocess::is_complete_bam(&path)); } -/// The case this exists for: a writer that was killed mid-stream. The file is large, plausible, -/// and readable up to the cut — only the end-of-file marker distinguishes it from a good one. +/// This is the case that the check exists for: somebody killed a writer in the middle of its +/// stream. The file is large, it looks correct, and a reader can read it up to the cut. Only the +/// end-of-file marker separates it from a good file. #[test] fn a_truncated_bam_is_not_complete() { let dir = scratch("complete-truncated"); @@ -950,8 +971,9 @@ fn a_missing_or_empty_file_is_not_complete() { assert!(!crate::postprocess::is_complete_bam(&empty)); } -/// The writers the pipeline actually uses go through `bamio`, which now paces its flushes and -/// syncs on the way out. Whatever that path does, what it leaves behind has to read as complete. +/// The writers that the pipeline uses go through `bamio`, which now paces its flushes and its +/// syncs on the way out. Whatever that path does, the file that it leaves must read as +/// complete. #[test] fn what_the_pipeline_writes_reads_as_complete() { let dir = scratch("complete-pipeline"); diff --git a/crates/navigator-analysis/src/preflight.rs b/crates/navigator-analysis/src/preflight.rs index e2f77135..67ad00cd 100644 --- a/crates/navigator-analysis/src/preflight.rs +++ b/crates/navigator-analysis/src/preflight.rs @@ -1,33 +1,37 @@ -//! Answering *why* an alignment can't be read, instead of guessing. +//! This module says *why* the code can not read an alignment. It does not guess. //! -//! The reader helpers all funnel their failures through [`AnalysisError::io`], which formats the -//! path **the caller passed**, not the file that actually failed. That is fine for a hot loop but -//! actively misleading at the edge: `open_indexed` hands it the CRAM, yet the open it performs -//! also autoloads the sibling `.crai`, resolves the reference FASTA and that FASTA's `.fai`. So an -//! unreadable index reports `io error on sample.cram`, and whoever reads that message goes looking -//! at the CRAM. Worse, [`crate::reader::has_region_index`] is built on `Path::exists`, which -//! answers `false` for *both* "no index here" and "the OS refused to tell me" — the two cases with -//! completely different fixes. +//! Every reader helper sends its failure through [`AnalysisError::io`], which prints the path +//! **that the caller gave**, and not the file that failed. That is correct in a hot loop, but at +//! the edge it points the user the wrong way. `open_indexed` gives it the CRAM. But the open that +//! it does also loads the `.crai` beside that CRAM, resolves the reference FASTA, and resolves +//! the `.fai` of that FASTA. So an index that the code can not read reports +//! `io error on sample.cram`, and the person who reads that message looks at the CRAM. //! -//! This module takes the opposite approach: probe each participating file **separately**, name it -//! explicitly, and keep the raw `errno` rather than collapsing everything to a `bool`. The errno is -//! the whole diagnosis on macOS, where the three failures look identical in a status bar but mean -//! unrelated things: +//! [`crate::reader::has_region_index`] is worse. It stands on `Path::exists`, which answers +//! `false` for *both* "there is no index here" and "the OS refused to tell me". Those two cases +//! need completely different fixes. //! -//! | errno | name | meaning | fix | +//! This module does the opposite. It probes each file that takes part **on its own**, it names +//! that file, and it keeps the raw `errno`. It does not collapse everything into a `bool`. On +//! macOS the errno is the whole diagnosis. The three failures look the same in a status bar, and +//! they have nothing to do with each other: +//! +//! | errno | name | what it says | the fix | //! |---|---|---|---| -//! | 2 | `ENOENT` | the file is not there | create/fetch it | -//! | 13 | `EACCES` | Unix mode bits deny it | `chmod` / `chown` | -//! | 1 | `EPERM` | **macOS privacy (TCC) denied it** | grant Full Disk Access, or move the file | +//! | 2 | `ENOENT` | the file is not there | create it, or fetch it | +//! | 13 | `EACCES` | the Unix mode bits deny it | `chmod` or `chown` | +//! | 1 | `EPERM` | **macOS privacy (TCC) denied it** | grant Full Disk Access | +//! +//! `EPERM` is the reason that this module exists. It is not a Unix permission failure, because +//! those give `EACCES`. It is macOS that refuses the process, whatever the mode bits say. That is +//! why a file at `chmod 777` in `~/Desktop` still fails. //! -//! `EPERM` is the one that motivated this module. It is not a Unix permission failure — those are -//! `EACCES` — it is macOS refusing the process regardless of mode bits, which is why a `chmod 777` -//! file in `~/Desktop` still fails. Reading a directory listing is enough to distinguish the cases, -//! so [`diagnose`] does that too: an index that `stat` denies but that shows up in the parent -//! directory is a privacy denial, full stop. +//! A read of a directory listing is enough to separate the cases, and [`diagnose`] does that too. +//! Take an index that `stat` refuses, and that the listing of the parent directory shows. That is +//! a privacy denial, and nothing else. //! -//! Nothing here mutates, downloads, or decodes more than a header and one region query, so it is -//! always safe to run — including on a file that is already failing. +//! Nothing here changes a file, downloads a file, or decodes more than a header and one region +//! query. It is always safe to run, and that includes a file that already fails. use std::fmt; use std::fs::File; @@ -37,9 +41,9 @@ use noodles::core::Region; use crate::reader::{self, detect_format, Format}; -/// How a single check came out. `Warn` is for a condition that degrades behaviour but has a -/// working fallback (a missing index still reads sequentially); `Fail` is for one that stops the -/// operation outright. +/// The result of one check. `Warn` marks a condition that makes the behaviour worse, but that has +/// a fallback which works. A missing index is one: the code still reads the file sequentially. +/// `Fail` marks a condition that stops the operation completely. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] #[serde(rename_all = "lowercase")] pub enum Status { @@ -58,8 +62,9 @@ impl Status { } } -/// Which check this is. Callers branch on the identity, not the display string — a batch deciding -/// whether to skip a sample must not depend on prose that can be reworded or translated. +/// Which check this is. A caller branches on the identity, and not on the string that the UI +/// shows. A batch that decides whether to skip a sample must not depend on prose. Somebody can +/// change that prose, or translate it. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] #[serde(rename_all = "snake_case")] pub enum CheckId { @@ -74,7 +79,8 @@ pub enum CheckId { } impl CheckId { - /// The human label. Single source of truth, so a check's name and its identity cannot drift. + /// The human label. It is the single source of truth, so the name of a check and its + /// identity always agree. pub fn label(self) -> &'static str { match self { CheckId::Format => "format", @@ -88,26 +94,31 @@ impl CheckId { } } - /// Whether failing this check makes the file unreadable *entirely*, sequential passes included. + /// True when a failure of this check makes the file unreadable *completely*, and that includes + /// a sequential pass. + /// + /// This is what decides whether a caller may skip a sample. A broken index, and anything that + /// stands on it, blocks a region query and nothing else. Read metrics, coverage and sex fall + /// back to a sequential walk, and they still succeed. To read that as "nobody can analyze this + /// sample" would throw away results that do work. + /// + /// The list is narrow, and that is deliberate. Only the open of the file, and the read of its + /// header, count. Those are the minimum that every sequential path does. /// - /// The distinction drives what a caller may skip. A broken index (or anything built on it) - /// blocks only region queries — read metrics, coverage and sex fall back to a sequential walk - /// and still succeed — so treating that as "this sample is unanalyzable" would throw away - /// results that do work. + /// A problem with the reference is *not* on the list, although some steps need one. How much + /// it matters depends on the format and on the step. And a read of the header of a CRAM + /// already needs the reference, so a reference that is truly unusable fails + /// [`CheckId::ReadHeader`] in any case. /// - /// Deliberately narrow: only opening the file and reading its header qualify, because they are - /// the minimum every sequential path performs. A reference problem is *not* listed even though - /// several steps need one — how much it matters depends on the format and the step, and since - /// reading a CRAM's header already requires the reference, a genuinely unusable reference fails - /// [`CheckId::ReadHeader`] anyway. Since skipping discards work that might have succeeded, it - /// should follow only from a failure that leaves nothing to try. + /// To skip a sample throws away work that could have succeeded. So it must follow only from a + /// failure that leaves nothing to try. pub fn blocks_sequential_reads(self) -> bool { matches!(self, CheckId::AlignmentFile | CheckId::ReadHeader) } } -/// One named check against one named file. `path` is the file *this* check actually touched — the -/// point of the whole module — so a failure is never attributed to a file that was merely nearby. +/// One named check against one named file. `path` is the file that *this* check touched, which is +/// the point of the whole module. A failure never goes to a file that only sat nearby. #[derive(Debug, Clone, serde::Serialize)] pub struct Check { pub id: CheckId, @@ -137,7 +148,7 @@ impl Check { } } -/// The outcome of diagnosing one alignment. +/// The result of a diagnosis of one alignment. #[derive(Debug, Clone, serde::Serialize)] pub struct Report { pub alignment: PathBuf, @@ -146,23 +157,25 @@ pub struct Report { } impl Report { - /// Whether any check failed outright (warnings don't count — they have fallbacks). + /// True when a check failed completely. A warning does not count, because a warning has a + /// fallback. pub fn failed(&self) -> bool { self.checks.iter().any(|c| c.status == Status::Fail) } - /// The first failing check — the one whose fix unblocks the rest, since later checks depend on - /// earlier ones succeeding. + /// The first check that failed. A fix to that one clears the rest, because a later check needs + /// the earlier checks to pass. pub fn first_failure(&self) -> Option<&Check> { self.checks.iter().find(|c| c.status == Status::Fail) } - /// Whether the file cannot be read *at all* — not even by a sequential pass. + /// True when nothing can read the file, and that includes a sequential pass. /// - /// This is the question a batch has to answer before deciding to skip a sample. A failure that - /// only blocks region queries (a missing or unreadable index) must not skip it: read metrics, - /// coverage and sex still complete via the sequential fallback, and discarding those because - /// the Y step can't run would lose results the user would otherwise get. + /// A batch must answer this question before it skips a sample. Take a failure that blocks a + /// region query alone, such as an index that is missing, or one that the code can not read. + /// That failure must not skip the sample. Read metrics, coverage and sex still finish through + /// the sequential fallback. To throw those away because the Y step can not run would lose + /// results that the user would otherwise get. pub fn blocks_sequential_reads(&self) -> bool { self.checks .iter() @@ -175,8 +188,8 @@ impl Report { } impl fmt::Display for Report { - /// A pasteable plain-text report. This is the format a user drops into a bug report, so it - /// leads with the failing check rather than making the reader scan for it. + /// A plain-text report that a user can paste. This is the form that goes into a bug report, + /// so the check that failed comes first. The reader does not have to look for it. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!(f, "alignment: {}", self.alignment.display())?; match &self.reference { @@ -206,15 +219,17 @@ impl fmt::Display for Report { } } -/// Explain an I/O error in terms of what the user has to *do*, keyed on the raw errno. The -/// distinction that matters is `EPERM` vs `EACCES`: they render almost identically in a status bar -/// ("Operation not permitted" vs "Permission denied") and have nothing to do with each other. +/// Explain an I/O error by what the user must *do*. The key is the raw errno. The difference that +/// matters is `EPERM` against `EACCES`. A status bar shows the two almost the same way, +/// "Operation not permitted" against "Permission denied", and they have nothing to do with each +/// other. fn explain(path: &Path, e: &std::io::Error) -> (Status, String, Option) { let errno = e.raw_os_error(); - // "Not found" is keyed on the portable `ErrorKind`, not a raw errno: Unix returns ENOENT (2), - // but Windows returns ERROR_FILE_NOT_FOUND (2) *or* ERROR_PATH_NOT_FOUND (3) depending on which - // component of the path is absent — both of which map to `NotFound`. The remaining branches stay - // errno-keyed because they draw a distinction (EPERM vs EACCES) that `ErrorKind` collapses. + // The key of "not found" is the portable `ErrorKind`, and not a raw errno. Unix returns + // ENOENT (2). Windows returns ERROR_FILE_NOT_FOUND (2) *or* ERROR_PATH_NOT_FOUND (3), and + // which one depends on the component of the path that is absent. Both of those map to + // `NotFound`. The other branches keep the errno as their key, because they draw a difference, + // EPERM against EACCES, that `ErrorKind` collapses. if e.kind() == std::io::ErrorKind::NotFound { return (Status::Fail, format!("not found: {}", path.display()), errno); } @@ -236,11 +251,11 @@ fn explain(path: &Path, e: &std::io::Error) -> (Status, String, Option) { (Status::Fail, detail, errno) } -/// What a single file looks like to this process: does it exist, and can we actually open it? +/// What one file looks like to this process. Does it exist, and can the process open it? /// -/// Both halves are necessary. `metadata` alone answers a different question than `open` on macOS — -/// a privacy denial can let `stat` through and refuse the `open`, or refuse both — so the check -/// that matters is the one the reader will actually perform, which is opening it. +/// Both halves are necessary. On macOS, `metadata` answers a different question from `open`. A +/// privacy denial can let `stat` through and refuse the `open`, or it can refuse both. So the +/// check that matters is the one that the reader itself does, which is the open. fn probe_file(id: CheckId, path: &Path) -> Check { match File::open(path) { Ok(_) => { @@ -249,9 +264,9 @@ fn probe_file(id: CheckId, path: &Path) -> Check { } Err(e) => { let (status, mut detail, errno) = explain(path, &e); - // A file the OS won't open but that is visible in its own directory listing is being - // withheld, not absent — worth saying, because "not found" would send the user looking - // for a file that is sitting right there. + // Take a file that the OS will not open, and that its own directory listing shows. + // The OS holds that file back, and the file is not absent. Say so, because "not + // found" would send the user to look for a file that is right there. if e.kind() == std::io::ErrorKind::NotFound && directory_lists(path) { detail = format!( "{detail}\n (the parent directory lists this name, so it exists but \ @@ -281,8 +296,9 @@ fn directory_lists(path: &Path) -> bool { entries.flatten().any(|e| e.file_name() == file) } -/// Every path that could serve as the coordinate index for `path`, in the order the readers accept -/// them: the dotted `foo.cram.crai` spelling `samtools` writes, then the replaced `foo.crai`. +/// Every path that can be the coordinate index of `path`, in the order that the readers accept +/// them. First comes the `foo.cram.crai` form that `samtools` writes, and then `foo.crai`, where +/// the extension replaces the old one. pub fn index_candidates(path: &Path) -> Vec { match detect_format(path) { Format::Bam => vec![path.with_extension("bam.bai"), path.with_extension("bai")], @@ -290,13 +306,13 @@ pub fn index_candidates(path: &Path) -> Vec { } } -/// Diagnose an alignment, in dependency order: the file itself, then its index, then the reference -/// and the reference's own index, then the operations built on all of them (header read, indexed -/// open, one region query). +/// Diagnose an alignment, in the order of the dependencies. First the file itself, then its index, +/// then the reference and the index of that reference. Last come the operations that stand on all +/// of those: the read of the header, the indexed open, and one region query. /// -/// The order is the point — each check presupposes the previous one, so [`Report::first_failure`] -/// names the thing to fix rather than the last thing to fall over. Reads at most one header and one -/// region's records; never writes, never downloads. +/// The order is the point. Each check needs the check before it, so [`Report::first_failure`] +/// names the thing to fix, and not the last thing to fall over. This function reads one header at +/// most, and the records of one region. It never writes, and it never downloads. pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { let mut report = Report { alignment: alignment.to_path_buf(), @@ -320,10 +336,12 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { return report; } - // The index. Its absence is a warning, not a failure: sequential walks (read metrics, coverage, - // sex) fall back and succeed, which is exactly why an alignment can look healthy in the UI - // right up until something needs a region query. An index that exists but won't open is a - // failure, and is the case `has_region_index` silently reports as "no index". + // The index. Its absence is a warning and not a failure. The three sequential walks, which + // are the read metrics, the coverage and the sex, fall back and succeed. That is exactly why an alignment + // can look healthy in the UI until something needs a region query. + // + // An index that exists but that will not open is a failure. That is the case where + // `has_region_index` reports "no index", and nobody sees the real cause. let candidates = index_candidates(alignment); let found = candidates.iter().find(|p| directory_lists(p)); let has_index = found.is_some(); @@ -349,7 +367,8 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { let index_ok = probe.status == Status::Ok; report.push(probe); if !index_ok { - // No point attempting the indexed open — it would fail and, worse, blame the CRAM. + // Do not try the indexed open. It would fail, and it would put the blame on the + // CRAM. return report; } } @@ -376,8 +395,9 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { if !reference_ok { return report; } - // CRAM decode goes through an *indexed* FASTA reader, so a missing `.fai` fails the open - // just as hard as a missing FASTA — and reports the FASTA's path when it does. + // A CRAM decode goes through an FASTA reader that uses an *index*. So a missing `.fai` + // fails the open as hard as a missing FASTA does, and it reports the path of the FASTA + // when it fails. let fai = PathBuf::from(format!("{}.fai", r.display())); let probe = probe_file(CheckId::ReferenceIndex, &fai); let fai_ok = probe.status == Status::Ok; @@ -387,7 +407,8 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { } } - // Now the composite operations, in the order the analysis paths perform them. + // Now the operations that put those parts together, in the order that the analysis paths do + // them. match reader::read_header(alignment, reference) { Ok(h) => report.push(Check::ok( CheckId::ReadHeader, @@ -415,10 +436,11 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { v } Err(e) => { - // The whole point of this module: don't repeat the upstream message's mistake of - // blaming the alignment. If we already established there is no index, *that* is the - // finding — an `ENOENT` naming the CRAM here means the reader could not autoload a - // sibling index, not that the CRAM went missing between two reads of it. + // This is the whole point of the module. Do not repeat the mistake of the message + // above, which puts the blame on the alignment. If the code already showed that there + // is no index, *that* is the answer. An `ENOENT` that names the CRAM here means that + // the reader could not load an index beside it. It does not mean that the CRAM went + // away between two reads of it. let check = if has_index { Check::new( CheckId::OpenIndexed, @@ -447,8 +469,9 @@ pub fn diagnose(alignment: &Path, reference: Option<&Path>) -> Report { } }; - // One real region query. Everything above can pass on a file whose index is stale or truncated; - // this is the check that actually exercises a seek, which is what the Y/mtDNA/SV paths do. + // One real region query. Everything above can pass on a file whose index is stale or cut + // short. This is the check that does a seek, and a seek is what the Y, mtDNA and SV paths + // do. let Some(contig) = header .reference_sequences() .keys() @@ -510,9 +533,9 @@ mod tests { assert_eq!(bam[1], PathBuf::from("/d/s.bai")); } - /// EPERM and EACCES are the two that a status bar makes look alike and that have unrelated - /// fixes, so the explanation must separate them — chmod advice on a TCC denial sends the user - /// down a dead end. + /// A status bar makes EPERM and EACCES look alike, and their fixes have nothing to do with + /// each other. So the explanation must separate them. Advice to run chmod, on a TCC denial, + /// sends the user nowhere. #[test] fn explains_tcc_denial_separately_from_unix_permissions() { let p = Path::new("/d/s.cram"); @@ -531,9 +554,10 @@ mod tests { } } - /// A readable file with no index must warn about the *index* and name both accepted spellings, - /// and must not let that warning masquerade as a problem with the alignment — the confusion - /// that made the original bug report unreadable. + /// Take a file that the code can read, and that has no index. The report must warn about the + /// *index*, and it must name both of the forms that the readers accept. It must not let that + /// warning look like a problem with the alignment. That confusion is what made the original + /// bug report impossible to read. #[test] fn missing_index_is_reported_against_the_index_not_the_alignment() { let dir = std::env::temp_dir().join("navigator-preflight-noindex"); @@ -562,10 +586,11 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } - /// The invariant the batch depends on. A file with no index still analyzes fine sequentially - /// (coverage, read metrics, sex), so the missing index — and the `open indexed` failure that - /// follows from it — must not read as "this sample is unanalyzable". Getting this backwards - /// would silently drop results for every un-indexed CRAM in a project. + /// The invariant that the batch depends on. A file with no index still analyzes correctly in + /// a sequential pass, for coverage, read metrics and sex. The missing index, and the + /// `open indexed` failure that follows it, must not read as "nobody can analyze this sample". + /// The wrong answer here would drop the results of every CRAM without an index in a project, + /// and nobody would see it happen. #[test] fn a_missing_index_does_not_block_sequential_reads() { let dir = std::env::temp_dir().join("navigator-preflight-blocking"); @@ -589,8 +614,9 @@ mod tests { assert!(!CheckId::CoordinateIndex.blocks_sequential_reads()); assert!(!CheckId::OpenIndexed.blocks_sequential_reads()); assert!(!CheckId::RegionQuery.blocks_sequential_reads()); - // A reference problem does not skip the sample on its own — a BAM reads without one, and a - // CRAM that truly cannot use it fails the header read, which does. + // A problem with the reference does not skip the sample on its own. A BAM reads without a + // reference. And a CRAM that truly can not use one fails the header read, and that check + // does skip the sample. assert!(!CheckId::ReferenceFasta.blocks_sequential_reads()); assert!(!CheckId::ReferenceIndex.blocks_sequential_reads()); assert!(CheckId::AlignmentFile.blocks_sequential_reads()); @@ -609,10 +635,11 @@ mod tests { let report = diagnose(Path::new("/nonexistent/sample.cram"), None); let first = report.first_failure().expect("missing file must fail"); assert_eq!(first.id, CheckId::AlignmentFile); - // Assert not-found portably: Unix reports ENOENT (2); Windows reports 2 or 3 depending on - // which path component is absent, so key on the message rather than a Unix errno. + // Assert "not found" in a portable way. Unix reports ENOENT (2). Windows reports 2 or 3, + // and which one depends on the component of the path that is absent. So key on the + // message, and not on a Unix errno. assert!(first.detail.starts_with("not found"), "{}", first.detail); - // The reference check must not run — the report stops at the first real blocker. + // The check on the reference must not run. The report stops at the first real block. assert!( !report.checks.iter().any(|c| c.id == CheckId::ReferenceFasta), "{report}" diff --git a/crates/navigator-analysis/src/probe.rs b/crates/navigator-analysis/src/probe.rs index 0fbbfbd2..8ea841ec 100644 --- a/crates/navigator-analysis/src/probe.rs +++ b/crates/navigator-analysis/src/probe.rs @@ -1,7 +1,9 @@ -//! Probe a BAM/CRAM **header** for the metadata a user would otherwise type in: the reference -//! build (from `@SQ`), the aligner (from `@PG`), and the sequencing platform / instrument / -//! test type (from `@RG`). Only the SAM header is read — no records, and (for CRAM) no -//! reference FASTA — so it is cheap and runs before the reference is resolved. +//! Read the **header** of a BAM or CRAM, for the metadata that a user would else type in. That is +//! the reference build, from `@SQ`; the aligner, from `@PG`; and the sequencing platform, the +//! instrument and the test type, from `@RG`. +//! +//! It reads the SAM header and nothing else. It reads no record, and, for a CRAM, no reference +//! FASTA. So it costs little, and it can run before the code resolves the reference. use std::path::Path; @@ -24,13 +26,19 @@ pub struct AlignmentProbe { pub instrument_model: Option, /// Best-guess test type code from the catalog, e.g. `"WGS_HIFI"`, `"WGS"`. pub test_type: Option, - /// Vendor hint scraped from the header (`@RG CN` center / `@RG LB` library / `@PG` / `@CO`), - /// e.g. `"FamilyTreeDNA"`, `"Full Genomes"`, `"YSEQ"` — refines a coverage-detected targeted-Y - /// test into the specific vendor product. `None` when no recognizable vendor token appears. + /// A hint about the vendor, which the code took out of the header. It comes from the `@RG CN` + /// center, the `@RG LB` library, a `@PG` line or a `@CO` line. Examples are + /// `"FamilyTreeDNA"`, `"Full Genomes"` and `"YSEQ"`. + /// + /// It turns a targeted-Y test that the coverage shape found into the specific product of a + /// vendor. It is `None` when no vendor token that the code knows appears. pub vendor_hint: Option, - /// FTDNA Big Y generation code (`"BIG_Y_700"` / `"BIG_Y_500"`) when the recent FTDNA pipeline - /// stamped it into the `@RG LB` library label (e.g. `unknown-library-Big Y-700`). Authoritative - /// — overrides the coverage-shape guess. `None` on older headers that omit the generation. + /// The generation code of an FTDNA Big Y, which is `"BIG_Y_700"` or `"BIG_Y_500"`. It is here + /// when the recent FTDNA pipeline wrote it into the `@RG LB` library label, as in + /// `unknown-library-Big Y-700`. + /// + /// It is authoritative, and it wins over the guess from the coverage shape. It is `None` on an + /// older header that leaves the generation out. pub big_y_code: Option, } @@ -54,9 +62,10 @@ pub fn probe_alignment(path: &Path) -> Result { }) } -/// FTDNA stamps the Big Y generation into the `@RG LB` (library) label on recent pipeline output — -/// `unknown-library-Big Y-700` / `…-Big Y-500`. Map that to the catalog code. `None` when no such -/// label is present (older 2019-era headers just carry `unknown-library`). +/// FTDNA writes the Big Y generation into the `@RG LB` library label, on recent output of its +/// pipeline. That label reads `unknown-library-Big Y-700`, or `…-Big Y-500`. This maps it to the +/// catalog code. It gives `None` when there is no such label. An older header, from about 2019, +/// carries `unknown-library` alone. fn detect_big_y_code(header: &sam::Header) -> Option<&'static str> { for map in header.read_groups().values() { if let Some(lb) = map.other_fields().get(&read_group::tag::LIBRARY) { @@ -94,8 +103,9 @@ fn detect_vendor_hint(header: &sam::Header) -> Option { hay.push_str(&s(cn)); hay.push(' '); } - // The library label carries the product on recent FTDNA output (`…-Big Y-700`), where the - // `@PG` path no longer mentions FTDNA — so scan it too (the "big y" token marks FTDNA). + // On recent FTDNA output the library label carries the product, as `…-Big Y-700`. There + // the `@PG` path no longer names FTDNA. So scan the label too. The "big y" token marks + // FTDNA. if let Some(lb) = map.other_fields().get(&read_group::tag::LIBRARY) { hay.push_str(&s(lb)); hay.push(' '); @@ -124,7 +134,7 @@ fn detect_vendor_hint(header: &sam::Header) -> Option { .map(|(_, canon)| (*canon).to_string()) } -/// Read just the SAM header from a BAM or CRAM (CRAM's header doesn't need the reference). +/// Read just the SAM header from a BAM or CRAM (CRAM's header does not need the reference). fn read_header_only(path: &Path) -> Result { match detect_format(path) { Format::Bam => { @@ -142,16 +152,18 @@ fn read_header_only(path: &Path) -> Result { } } -/// Lossy UTF-8 of a header field value (a `bstr::BString` or sequence name — any `AsRef<[u8]>`). +/// The lossy UTF-8 of the value of a header field. That value is a `bstr::BString`, or a sequence +/// name, or any other `AsRef<[u8]>`. fn s>(v: &T) -> String { String::from_utf8_lossy(v.as_ref()).into_owned() } /// Reference build from `@SQ`: prefer the assembly (`AS`) tag, else the chr1 length signature. fn detect_build(header: &sam::Header) -> Option { - // The Y-PAR-masked + rCRS CHM13 analysis set is indistinguishable from plain chm13v2.0 - // by `@SQ` (same contig names/lengths), so check the reference filename the aligner - // recorded (`@PG CL` / `@SQ UR`) first. Plain chm13 won't match this signature. + // The CHM13 analysis set with a masked Y-PAR and the rCRS looks the same as a plain + // chm13v2.0 through `@SQ`. The contig names and lengths match. So check first the name of the + // reference file that the aligner recorded, in `@PG CL` or `@SQ UR`. A plain chm13 does not + // match this signature. if header_mentions_masked_rcrs(header) { return Some("chm13v2.0_maskedY_rCRS".into()); } @@ -227,8 +239,9 @@ const ALIGNERS: &[(&str, &str)] = &[ ("bwa", "bwa"), ]; -/// Aligner from `@PG`: match the program id / name (`PN`) / command line (`CL`) against the -/// known-aligner list, so non-aligner programs (samtools, gatk, …) are ignored. +/// The aligner, from `@PG`. It matches the program id, the name in `PN`, and the command line in +/// `CL`, against the list of aligners that the code knows. A program that is not an aligner, such +/// as samtools or gatk, then does not match. fn detect_aligner(header: &sam::Header) -> Option { for (id, map) in header.programs().as_ref().iter() { let pn = map.other_fields().get(&program::tag::NAME).map(s).unwrap_or_default(); @@ -247,9 +260,11 @@ fn detect_aligner(header: &sam::Header) -> Option { None } -/// A recognized SAM `@RG PL` value, upper-cased — or `None` for a missing / non-standard string -/// (e.g. Dante/DRAGEN's `PL0` placeholder), so the caller falls back to read-name inference rather -/// than recording a bogus platform. SAM spec values + the common `NANOPORE` alias for `ONT`. +/// An `@RG PL` value from SAM that the code knows, in upper case. It is `None` when the value is +/// absent, or when it is not in the standard. The `PL0` placeholder of Dante and DRAGEN is one such +/// value. The caller then falls back to an inference from the read names, and it does not record a +/// platform that means nothing. The list holds the values of the SAM specification, plus the common +/// `NANOPORE` alias for `ONT`. fn normalize_platform(raw: &str) -> Option { let u = raw.trim().to_uppercase(); matches!( @@ -272,9 +287,10 @@ fn normalize_platform(raw: &str) -> Option { .then_some(u) } -/// Platform (`PL`, validated against the SAM platform vocabulary) + instrument model (`PM`) from the -/// first informative `@RG`. A non-standard `PL` is dropped so read-name inference can supply the -/// real platform. +/// The platform, from `PL`, and the instrument model, from `PM`, out of the first `@RG` line that +/// carries them. The code checks the `PL` against the platform vocabulary of SAM. It drops a `PL` +/// that is not in that vocabulary, so that an inference from the read names can give the real +/// platform. fn detect_platform(header: &sam::Header) -> (Option, Option) { for map in header.read_groups().values() { let pl = map @@ -341,7 +357,7 @@ mod tests { // @RG CN (sequencing center). let rg = header_from_sam("@HD\tVN:1.6\n@SQ\tSN:chrY\tLN:57227415\n@RG\tID:r1\tCN:FamilyTreeDNA\tPL:ILLUMINA\n"); assert_eq!(detect_vendor_hint(&rg).as_deref(), Some("FamilyTreeDNA")); - // @CO free-text comment naming the product. + // A free-text `@CO` comment that names the product. let co = header_from_sam("@HD\tVN:1.6\n@SQ\tSN:chrY\tLN:57227415\n@CO\tFull Genomes Y Elite v2\n"); assert_eq!(detect_vendor_hint(&co).as_deref(), Some("Full Genomes")); // No vendor token. @@ -355,7 +371,8 @@ mod tests { let by700 = header_from_sam("@HD\tVN:1.4\n@SQ\tSN:chrY\tLN:57227415\n@RG\tID:r\tLB:unknown-library-Big Y-700\tSM:s\n"); assert_eq!(detect_big_y_code(&by700), Some("BIG_Y_700")); - // The library label also marks the vendor (via the "big y" token) when @PG/@CN don't. + // The library label also marks the vendor, through the "big y" token, when `@PG` and + // `@CN` do not. assert_eq!(detect_vendor_hint(&by700).as_deref(), Some("FamilyTreeDNA")); let by500 = @@ -369,8 +386,9 @@ mod tests { #[test] fn nonstandard_platform_is_dropped() { - // Dante/DRAGEN writes PL:PL0 (not a SAM platform); the probe drops it so read-name - // inference can recover the real platform instead of recording "PL0". + // Dante and DRAGEN write `PL:PL0`, and that is not a SAM platform. The probe drops it, so + // that an inference from the read names can recover the real platform. Without that, the + // record would hold "PL0". let h = header_from_sam("@HD\tVN:1.6\n@SQ\tSN:chr1\tLN:248956422\n@RG\tID:1\tPL:PL0\n"); let (pl, _) = detect_platform(&h); assert_eq!(pl, None); diff --git a/crates/navigator-analysis/src/read_metrics.rs b/crates/navigator-analysis/src/read_metrics.rs index 316028a5..5deeb8ed 100644 --- a/crates/navigator-analysis/src/read_metrics.rs +++ b/crates/navigator-analysis/src/read_metrics.rs @@ -1,10 +1,14 @@ -//! Read-level metrics walker — Rust port of the Scala `UnifiedMetricsWalker` -//! (replaces GATK `CollectAlignmentSummaryMetrics` + `CollectInsertSizeMetrics`, -//! no R dependency). Single pass over the BAM collects alignment-summary counts, -//! read-length and insert-size distributions, pair orientation, and mean MAPQ. +//! The walker over the metrics at the read level. It is the Rust port of the Scala +//! `UnifiedMetricsWalker`, and it replaces GATK `CollectAlignmentSummaryMetrics` and +//! `CollectInsertSizeMetrics`. It needs no R. //! -//! Parity target is the Scala walker. Primary metrics exclude secondary/supplementary -//! records; insert size is taken from first-of-pair proper pairs only (no double count). +//! One pass over the BAM collects four things. The counts of the alignment summary. The +//! distributions of the read length and the insert size. The orientation of a pair. And the mean +//! MAPQ. +//! +//! The parity target is the Scala walker. The primary metrics leave out a secondary record and a +//! supplementary one. The insert size comes from a correct pair, and from its first read alone, so +//! nothing counts twice. use std::collections::BTreeMap; use std::path::Path; @@ -70,11 +74,13 @@ pub struct ReadMetrics { } impl ReadMetrics { - /// Exact total sequenced yield in base pairs — `Σ length × count` over the read-length - /// histogram. This is the DTC "Gbases" figure (`/ 1e9`) for the standardized test label. Falls - /// back to `total_reads × mean_read_length` when the histogram is empty (e.g. a flagstat sidecar - /// carries no distribution), which is exact only for fixed-length short reads. `None` when - /// neither is available. + /// The exact total yield of the sequencing run, in base pairs. It is `Σ length × count` over + /// the read-length histogram. Divided by 1e9, it is the "Gbases" figure that a DTC label + /// carries. + /// + /// It falls back to `total_reads × mean_read_length` when the histogram is empty, which + /// happens when a flagstat sidecar carries no distribution. That fallback is exact only for a + /// short read of fixed length. It is `None` when neither value is there. pub fn total_bases(&self) -> Option { if !self.read_length_histogram.is_empty() { let sum: u128 = self @@ -116,9 +122,10 @@ impl DistAccum { self.count += 1; } - /// Fold another accumulator in (for the parallel per-contig merge). All fields are - /// commutative sums / set-min-max / histogram unions, so the merged distribution is - /// independent of how records were partitioned across contigs. + /// Fold another accumulator into this one, for the merge of the parallel walk over the + /// contigs. Every field is a sum that commutes, a minimum or maximum, or a union of two + /// histograms. So the merged distribution does not depend on how the code split the records + /// across the contigs. fn merge(&mut self, other: DistAccum) { for (value, count) in other.hist { *self.hist.entry(value).or_insert(0) += count; @@ -167,7 +174,7 @@ fn median_from_hist(hist: &BTreeMap, total: u64) -> f64 { last } -/// Single pass over the BAM collecting read-level metrics. +/// One pass over the BAM. It collects the metrics at the read level. pub fn collect_read_metrics(bam_path: &Path, reference: Option<&Path>) -> Result { let (header, mut reader) = reader::open_seq(bam_path, reference)?; let mut state = ReadMetricsState::default(); @@ -177,9 +184,12 @@ pub fn collect_read_metrics(bam_path: &Path, reference: Option<&Path>) -> Result Ok(state.finish()) } -/// Read-level metrics accumulator shared by the standalone walker and the fused -/// [`crate::unified`] walker (one source of truth → byte-identical numbers). Feed every -/// record via [`ReadMetricsState::accept`], then call [`ReadMetricsState::finish`]. +/// The accumulator of the metrics at the read level. The separate walker and the fused +/// [`crate::unified`] walker share it. There is one source of truth, and the numbers of the two +/// match to the last digit. +/// +/// Give every record to [`ReadMetricsState::accept`], and then call +/// [`ReadMetricsState::finish`]. #[derive(Default)] pub(crate) struct ReadMetricsState { total_reads: u64, @@ -198,9 +208,10 @@ pub(crate) struct ReadMetricsState { } impl ReadMetricsState { - /// Fold another state in (parallel per-contig + unmapped-sweep merge). Every field is a - /// commutative count / histogram union, so the result equals a single sequential pass - /// regardless of how records were split across contigs. + /// Fold another state into this one. The merge of the parallel walk over the contigs, and of + /// the sweep over the unmapped reads, both use it. Every field is a count that commutes, or a + /// union of two histograms. So the result equals that of one sequential pass, at any split of + /// the records across the contigs. pub(crate) fn merge(&mut self, other: ReadMetricsState) { self.total_reads += other.total_reads; self.pf_reads += other.pf_reads; @@ -317,7 +328,8 @@ fn ratio(num: u64, den: u64) -> f64 { } } -/// Pair orientation from a first-of-pair proper pair, mirroring the Scala logic. +/// The orientation of a pair, from the first read of a correct pair. It follows the same logic as +/// the Scala code. fn detect_orientation(record: &impl AlnRead) -> PairOrientation { let read_neg = record.flags().is_reverse_complemented(); let mate_neg = record.flags().is_mate_reverse_complemented(); diff --git a/crates/navigator-analysis/src/reader.rs b/crates/navigator-analysis/src/reader.rs index 4bc98a2c..98f77c13 100644 --- a/crates/navigator-analysis/src/reader.rs +++ b/crates/navigator-analysis/src/reader.rs @@ -1,12 +1,14 @@ -//! Format-agnostic alignment reading. The walkers (coverage, caller, read-metrics) need -//! to read records the same way whether the file is BAM or CRAM, but `noodles` exposes -//! two different reader families: BAM yields borrowed `bam::Record`s, CRAM yields owned -//! `sam::alignment::RecordBuf`s and needs the reference FASTA to decode. This module -//! normalizes both to `RecordBuf` (one owned allocation per record — the same order CRAM -//! pays anyway) so the hot per-base loops stay format-blind and allocation-free. +//! A read of an alignment that does not depend on the format. The walkers, which are coverage, +//! the caller and read-metrics, must read a record the same way from a BAM and from a CRAM. //! -//! noodles is intentionally confined to this crate (see lib.rs); this is the single place -//! that knows about CRAM's reference-sequence repository. +//! But `noodles` gives two different families of reader. A BAM gives a borrowed `bam::Record`. A +//! CRAM gives an owned `sam::alignment::RecordBuf`, and it needs the reference FASTA to decode. +//! This module brings both to a `RecordBuf`. That is one owned allocation at each record, which is +//! the same order of cost that a CRAM pays in any case. So the hot loops over the bases do not +//! know the format, and they allocate nothing. +//! +//! noodles stays inside this crate, and that is deliberate. See lib.rs. This is the one place that +//! knows about the repository of reference sequences of a CRAM. use std::fs::File; use std::num::NonZeroUsize; @@ -17,12 +19,15 @@ use noodles::core::{Position, Region}; use noodles::sam::alignment::RecordBuf; use noodles::{bam, bgzf, cram, fasta, sam}; -/// Worker threads for multithreaded bgzf decompression of BAM sequential reads. bgzf is a -/// block-gzip stream, so block inflation parallelizes while record parsing stays sequential -/// (output is byte-identical — only decompression is threaded). Defaults to the available -/// parallelism minus one (the record-parsing consumer), capped at 6 — beyond a handful of -/// inflate workers the single consumer thread is the limit. Override with -/// `NAVIGATOR_BGZF_THREADS` (clamped to >= 1; set to 1 to disable threading). +/// The count of worker threads that decompress bgzf, for a sequential read of a BAM. +/// +/// bgzf is a stream of gzip blocks. So the inflation of those blocks runs in parallel, while the +/// parse of the records stays sequential. The output does not change, to the last byte, because +/// the threads only decompress. +/// +/// The default is the available parallelism, less one for the consumer that parses the records, +/// and up to 6. Above a few inflate workers, the one consumer thread is the limit. +/// `NAVIGATOR_BGZF_THREADS` overrides it, clamped to 1 or more. Set it to 1 to use one thread. pub(crate) fn bgzf_worker_count() -> NonZeroUsize { if let Some(n) = std::env::var("NAVIGATOR_BGZF_THREADS") .ok() @@ -38,13 +43,17 @@ use crate::cancel::CancelToken; use crate::error::AnalysisError; use crate::readview::{AlnRead, SeqRecord}; -/// Per-thread stack size (bytes) for any thread that decodes a BAM/**CRAM** record. noodles' CRAM -/// decoder recurses proportionally to the data — notably the CRAM **3.1** codecs (range/arithmetic -/// coder, fqzcomp, name tokenizer), which older 3.0 files never exercise — and can recurse deep -/// enough to blow a default thread stack (2 MiB) or even rayon's pools. A stack overflow **aborts -/// the process** (it is not a catchable panic), so a single deeply-encoded file would otherwise take -/// down the whole app/batch. Give decode threads a generous stack. Override with -/// `NAVIGATOR_DECODE_STACK_MB` (whole MiB; clamped to >= 8). +/// The stack size of each thread, in bytes, for any thread that decodes a BAM or **CRAM** record. +/// +/// The CRAM decoder of noodles recurses in proportion to the data. The CRAM **3.1** codecs are the +/// ones that matter: the range and arithmetic coder, fqzcomp, and the name tokenizer. An older 3.0 +/// file never uses those. The decoder can recurse deep enough to overflow a default thread stack +/// of 2 MiB, and even the pools of rayon. +/// +/// A stack overflow **aborts the process**. It is not a panic that the code can catch. One file +/// with a deep encoding would else take down the whole app or batch. So give a decode thread a +/// large stack. `NAVIGATOR_DECODE_STACK_MB` overrides the size, in whole MiB, clamped to 8 or +/// more. pub fn decode_stack_size() -> usize { let mb = std::env::var("NAVIGATOR_DECODE_STACK_MB") .ok() @@ -54,9 +63,10 @@ pub fn decode_stack_size() -> usize { mb * 1024 * 1024 } -/// Build a rayon pool whose worker threads have a decode-safe stack ([`decode_stack_size`]). -/// Use this for any parallel work that decodes CRAM/BAM records — the rayon default (2 MiB) and -/// even a modest fixed bump are not enough for deeply-encoded CRAM 3.1 files. +/// Build a rayon pool whose worker threads have a stack that is safe for a decode. See +/// [`decode_stack_size`]. Use it for any parallel work that decodes a CRAM or BAM record. The +/// rayon default of 2 MiB is not enough for a CRAM 3.1 file with a deep encoding, and neither is a +/// small fixed increase. pub fn decode_pool(threads: usize) -> Result { rayon::ThreadPoolBuilder::new() .num_threads(threads) @@ -91,16 +101,16 @@ pub fn build_repository(reference: &Path) -> Result(path: &Path, reference: Option<&'a Path>) -> Result<&'a Path, AnalysisError> { reference.ok_or_else(|| AnalysisError::Message(format!("CRAM {} requires a reference FASTA", path.display()))) } // ---- sequential (whole-file) reading -------------------------------------- -/// A whole-file reader over BAM or CRAM. Hold it and call [`SeqReader::records`]. The BAM -/// path uses a multithreaded bgzf reader so block decompression runs on a worker pool while -/// records are parsed sequentially (see [`bgzf_worker_count`]). +/// A reader over a whole BAM or CRAM file. Hold it, and call [`SeqReader::records`]. The BAM path +/// uses a bgzf reader with more than one thread. So the block decompression runs on a worker pool, +/// while the parse of the records stays sequential. See [`bgzf_worker_count`]. pub enum SeqReader { Bam { inner: bam::io::Reader>, @@ -112,8 +122,8 @@ pub enum SeqReader { }, } -/// Open `path` for a sequential pass, returning the header and reader. `reference` is -/// required for CRAM (ignored for BAM). +/// Open `path` for a sequential pass. Returns the header and the reader. A CRAM needs +/// `reference`, and a BAM ignores it. pub fn open_seq(path: &Path, reference: Option<&Path>) -> Result<(sam::Header, SeqReader), AnalysisError> { match detect_format(path) { Format::Bam => { @@ -173,10 +183,13 @@ impl SeqReader { } } - /// Iterate every record as a [`SeqRecord`] — the **lazy** counterpart to [`SeqReader::records`]. - /// The BAM path yields the zero-copy `bam::Record` (no owned `RecordBuf` decode/tag-parse, the - /// hot-path win); the CRAM path yields the decoded `RecordBuf` (no cheaper form). The walkers - /// consume `&impl AlnRead`, so `SeqRecord` drives them with no allocation on the BAM path. + /// Walk every record as a [`SeqRecord`]. This is the **lazy** counterpart to + /// [`SeqReader::records`]. + /// + /// The BAM path gives the zero-copy `bam::Record`. It does no decode into an owned + /// `RecordBuf`, and it parses no tag, and that is the gain on the hot path. The CRAM path gives + /// the decoded `RecordBuf`, because there is no cheaper form. The walkers take an + /// `&impl AlnRead`, so a `SeqRecord` drives them with no allocation on the BAM path. pub fn records_lazy<'a>( &'a mut self, header: &'a sam::Header, @@ -217,20 +230,23 @@ pub enum IdxReader { }, } -/// File offsets of the `.crai` containers that can hold records overlapping `interval` on `ref_id`. +/// The file offsets of the `.crai` containers that can hold a record inside `interval` on +/// `ref_id`. +/// +/// **This is the whole reason that a CRAM region query is usable.** A CRAM container is the unit +/// of a decode, and you can not decode part of one. To limit *which containers the code decodes* +/// is the only place where a region query can save work. /// -/// **This is the whole reason CRAM region queries are usable.** A CRAM container is the unit of -/// decode — you cannot decode part of one — so restricting *which containers get decoded* is the -/// only place a region query can save work. noodles' own `Query` (and, before this, our `for_each`) -/// selected containers by reference sequence alone and then discarded non-overlapping records -/// *after* decoding them, which made every query cost a whole chromosome no matter how small the -/// region: measured at 20.9 s for a 1 bp query on chr21 and 116 s on chr1, against 4–6 ms for the -/// same query on a BAM. chr21 of a 30x WGS holds 1,140 containers and a point query needs exactly -/// one of them. +/// The `Query` of noodles, and our own `for_each` before this code, chose the containers by +/// reference sequence alone. They then threw away the records outside the region, *after* they +/// decoded those records. So every query cost a whole chromosome, at any size of region. +/// A measurement gave 20.9 s for a 1 bp query on chr21, and 116 s on chr1. The same query on a BAM +/// takes 4 to 6 ms. chr21 of a 30x WGS holds 1,140 containers, and a point query needs exactly one +/// of them. /// -/// A container whose `alignment_start` is absent is **kept**: that is a container this index cannot -/// place, and dropping it would silently lose records. Skipping is only ever done on positive -/// evidence that the container lies outside the interval. +/// The code **keeps** a container whose `alignment_start` is absent. This index can not place such +/// a container, and to drop it would lose records where nobody sees it happen. The code skips a +/// container only on positive evidence that the container lies outside the interval. fn cram_container_offsets(index: &cram::crai::Index, ref_id: usize, interval: Interval) -> Vec { index .iter() @@ -248,8 +264,8 @@ fn cram_container_offsets(index: &cram::crai::Index, ref_id: usize, interval: In .collect() } -/// Open `path` for indexed region queries (autoloads the `.bai`/`.crai`). `reference` is -/// required for CRAM. +/// Open `path` for a region query over its index. It loads the `.bai` or `.crai` itself. A CRAM +/// needs `reference`. pub fn open_indexed(path: &Path, reference: Option<&Path>) -> Result<(sam::Header, IdxReader), AnalysisError> { match detect_format(path) { Format::Bam => { @@ -285,7 +301,7 @@ pub fn open_indexed(path: &Path, reference: Option<&Path>) -> Result<(sam::Heade } impl IdxReader { - /// Iterate the records overlapping `region` as `RecordBuf`s. + /// Walk the records inside `region` as `RecordBuf` values. pub fn query<'a>( &'a mut self, header: &'a sam::Header, @@ -300,11 +316,14 @@ impl IdxReader { RecordBuf::try_from_alignment_record(header, &rec).map_err(|e| AnalysisError::io(&path, e)) }))) } - // Hand-rolled rather than `inner.query(...)`: noodles' `Query` decodes every container - // of the contig and filters records afterwards, so a 1 bp query costs a whole - // chromosome (see [`cram_container_offsets`]). This decodes only the containers that - // can overlap, lazily — one container at a time, so a caller that stops early (a - // `.take(n)` probe, a cancelled walk) does not pay for the rest. + // This code does the work itself, and it does not call `inner.query(...)`. The + // `Query` of noodles decodes every container of the contig, and it filters the records + // after that. A 1 bp query then costs a whole chromosome. See + // [`cram_container_offsets`]. + // + // This code decodes only the containers that can hold a record in the region, and it + // does that lazily, one container at a time. So a caller that stops early, such as a + // `.take(n)` probe or a walk that somebody cancelled, does not pay for the rest. IdxReader::Cram { inner, repo, path } => { use std::io::{Seek, SeekFrom}; @@ -365,8 +384,9 @@ impl IdxReader { Err(e) => return Some(Err(e)), }; for rec in &records { - // Same per-record overlap test noodles applies post-decode — the - // container filter is a coarse prefilter, not a replacement for it. + // The same test at each record that noodles applies after its + // decode. The container filter is a coarse first pass, and it does + // not replace this test. if let (Some(Ok(start)), Some(Ok(end))) = (rec.alignment_start(), rec.alignment_end()) { if !interval.intersects((start..=end).into()) { continue; @@ -387,9 +407,10 @@ impl IdxReader { } } - /// Iterate the unplaced unmapped records (the BAM tail) as `RecordBuf`s. BAM only — - /// CRAM's `.crai` exposes no unmapped query, so it returns an error (callers needing the - /// unmapped tail for CRAM should take a sequential pass instead). + /// Walk the unmapped records that have no place, which are the tail of a BAM, as `RecordBuf` + /// values. This works for a BAM alone. The `.crai` of a CRAM gives no query for the unmapped + /// records, so this returns an error there. A caller that needs the unmapped tail of a CRAM + /// must take a sequential pass. pub fn query_unmapped<'a>( &'a mut self, header: &'a sam::Header, @@ -411,28 +432,32 @@ impl IdxReader { } } -/// A per-record consumer the indexed reader drives over a region. The `accept` method is generic -/// over [`AlnRead`], so it monomorphizes for each record type: the BAM path hands it the **lazy, -/// zero-copy** `bam::Record` (no per-read owned `RecordBuf` allocation — the hot-path win) and the -/// CRAM path hands it the decoded `RecordBuf`. A single sink serves both. +/// A consumer of one record, which the indexed reader drives over a region. +/// +/// The `accept` method is generic over [`AlnRead`], so the compiler makes one copy for each record +/// type. The BAM path gives it the **lazy, zero-copy** `bam::Record`. There is no owned +/// `RecordBuf` allocation at each read, and that is the gain on the hot path. The CRAM path gives +/// it the decoded `RecordBuf`. One sink serves both. pub trait RecordSink { fn accept(&mut self, record: &impl AlnRead); } -/// How often the record loops poll the cancel token. Tuned to be invisible in a profile while -/// keeping the worst-case delay between a click and a stop well under a frame: at ~1M records/s -/// this is a check every few milliseconds. The check itself is one relaxed atomic load. +/// How often a record loop polls the cancel token. The value keeps the check out of a profile, +/// and it keeps the worst delay between a click and a stop well below one frame. At about 1M +/// records/s that is a check every few milliseconds. The check itself is one relaxed atomic +/// load. const CANCEL_CHECK_RECORDS: u32 = 4096; impl IdxReader { - /// Drive `sink` over every record overlapping `region` (BAM: lazy record; CRAM: `RecordBuf`). - /// A record that fails to read aborts with an error. The allocation-free counterpart to - /// [`IdxReader::query`] (which copies each record into an owned `RecordBuf`). + /// Drive `sink` over every record inside `region`. A BAM gives a lazy record, and a CRAM gives + /// a `RecordBuf`. A record that the code can not read stops the walk with an error. This is the + /// counterpart of [`IdxReader::query`] that allocates nothing, where that one copies each + /// record into an owned `RecordBuf`. /// - /// `cancel` is polled every [`CANCEL_CHECK_RECORDS`] records, so a cancelled walk stops - /// mid-contig instead of at the next contig boundary — on chr1 that is the difference between - /// stopping in milliseconds and stopping in minutes. Pass [`CancelToken::none`] when there is - /// nothing to cancel. + /// The loop polls `cancel` every [`CANCEL_CHECK_RECORDS`] records. So a walk that somebody + /// cancelled stops in the middle of a contig, and not at the next contig boundary. On + /// chr1 that is the difference between a stop in milliseconds and a stop in minutes. Pass + /// [`CancelToken::none`] when there is nothing to cancel. pub fn for_each( &mut self, header: &sam::Header, @@ -455,11 +480,14 @@ impl IdxReader { Ok(()) } IdxReader::Cram { inner, repo, path } => { - // Decode the region's CRAM containers down to borrowed `cram::Record`s and drive the - // sink off them directly — skipping the per-read `RecordBuf` copy the high-level - // `query` iterator pays (~1.74× the per-read decode on a 30× WGS CRAM). This mirrors - // noodles' own `Query`: seek each `.crai` container whose reference matches, decode - // its slices, and keep the records overlapping the query interval. + // Decode the CRAM containers of the region down to borrowed `cram::Record` + // values, and drive the sink from those directly. That leaves out the `RecordBuf` + // copy at each read, which the high-level `query` iterator pays. On a 30x WGS CRAM + // that copy costs about 1.74 times the decode of one read. + // + // This has the same shape as the `Query` of noodles. Seek each `.crai` container + // whose reference matches, decode the slices of that container, and keep the + // records inside the query interval. use std::io::{Seek, SeekFrom}; use noodles::sam::alignment::Record as _; // alignment_start/_end on cram::Record @@ -483,17 +511,20 @@ impl IdxReader { })?; let interval = region.interval(); - // Collect the file offsets of the containers that can overlap the query before - // borrowing `inner` mutably to seek/read (the `.crai` index borrow can't overlap the - // read borrow). Selecting on the interval — not just the contig — is what keeps this - // proportional to the region instead of the chromosome; see + // Collect the file offsets of the containers that can hold a record in the query. + // Do that before the code takes a mutable borrow of `inner` to seek and read. The + // borrow of the `.crai` index can not overlap the borrow for the read. + // + // The choice goes on the interval, and not on the contig alone. That is what keeps + // the cost proportional to the region, and not to the chromosome. See // [`cram_container_offsets`]. let offsets = cram_container_offsets(inner.index(), ref_id, interval); let mut container = cram::io::reader::Container::default(); for offset in offsets { - // Per container rather than per record: a CRAM container is decoded as a unit, - // so this is the finest granularity at which stopping actually saves work. + // The check goes at each container, and not at each record. The code decodes a + // CRAM container as a unit, so this is the smallest step at which a stop saves + // work. cancel.check()?; inner.get_mut().seek(SeekFrom::Start(offset)).map_err(io_err)?; if inner.read_container(&mut container).map_err(io_err)? == 0 { @@ -550,9 +581,10 @@ impl IdxReader { } } -/// Whether a sibling BAM index (`.bai`, as `foo.bam.bai` or `foo.bai`) exists for `path`. -/// The per-contig parallel walker needs one for region queries; callers fall back to a -/// sequential pass when this is false. CRAM is excluded (its `.crai` has no unmapped query). +/// True when a BAM index sits beside `path`, as `foo.bam.bai` or as `foo.bai`. The parallel walker +/// over the contigs needs one for its region queries. A caller falls back to a sequential pass +/// when this is false. A CRAM is out, because its `.crai` gives no query for the unmapped +/// records. pub fn has_bai_index(path: &Path) -> bool { if detect_format(path) != Format::Bam { return false; @@ -570,24 +602,27 @@ pub fn has_crai_index(path: &Path) -> bool { path.with_extension("cram.crai").exists() || path.with_extension("crai").exists() } -/// Whether the file has a coordinate index supporting **per-contig region queries** — a BAM `.bai` -/// or a CRAM `.crai`. The prerequisite for the parallel per-contig walker (CRAM additionally can't -/// region-query the unmapped tail; callers handle that separately). +/// True when the file has a coordinate index that supports a **region query on one contig**. That +/// is a `.bai` for a BAM, or a `.crai` for a CRAM. The parallel walker over the contigs needs this. +/// A CRAM also has no region query for its unmapped tail, and a caller handles that on its own. pub fn has_region_index(path: &Path) -> bool { has_bai_index(path) || has_crai_index(path) } // ---- header-only ---------------------------------------------------------- -/// Read just the SAM header (e.g. to resolve a contig length). `reference` is required -/// for CRAM. +/// Read the SAM header alone, for example to find the length of a contig. A CRAM needs +/// `reference`. pub fn read_header(path: &Path, reference: Option<&Path>) -> Result { open_seq(path, reference).map(|(header, _)| header) } -/// The alignment's reference-sequence (contig) names, in header order. `reference` is required for -/// a CRAM. Used to reconcile a panel/site contig against the file's naming convention — a GRCh37 -/// alignment may use bare `1` where a panel locus stores `chr1` (or vice versa). +/// The names of the reference sequences of the alignment, which are the contigs, in header order. +/// A CRAM needs `reference`. +/// +/// Use it to reconcile the contig of a panel or a site against the names that the file uses. A +/// GRCh37 alignment can use a bare `1` where a panel locus holds `chr1`, and the two can also be +/// the other way round. pub fn contig_names(path: &Path, reference: Option<&Path>) -> Result, AnalysisError> { let header = read_header(path, reference)?; Ok(header @@ -653,9 +688,10 @@ mod tests { } } - /// The new slice-level CRAM `for_each` path (borrowed `cram::Record`) must yield records - /// field-identical to the high-level `query` path (owned `RecordBuf`) — guards the noodles - /// internal-API replication (crai seek + slice decode) against version drift. + /// The new CRAM `for_each` path works at the level of a slice, with a borrowed `cram::Record`. + /// It must give records whose fields all match those from the high-level `query` path, which + /// gives an owned `RecordBuf`. This test guards our copy of the internal noodles API, which is + /// the crai seek and the slice decode, against a change of version. #[test] fn cram_for_each_matches_query_recordbuf() { struct CollectSink(Vec); @@ -688,15 +724,15 @@ mod tests { assert_eq!(sink.0, via_query, "cram::Record path must match RecordBuf path"); } - /// [`cram_container_offsets`] decides which containers are decoded at all, so its boundary - /// behaviour *is* the correctness of every CRAM region query: a container wrongly skipped is - /// reads silently missing from a variant call, which no downstream test would attribute to the - /// reader. Checked at the edges, where an off-by-one actually lives. + /// [`cram_container_offsets`] decides which containers the code decodes at all. Its behaviour + /// at a boundary *is* the correctness of every CRAM region query. A container that it skips + /// wrongly means reads that a variant call never sees, and no later test would trace that back + /// to the reader. This test looks at the edges, where an off-by-one error lives. #[test] fn container_offsets_select_only_overlapping_containers() { let p = |n: usize| Position::new(n).unwrap(); - // ref 0 containers spanning [1000,1099], [2000,2099], [3000,3099]; one on ref 1; and one - // the index cannot place. + // Three containers on ref 0, which cover [1000,1099], [2000,2099] and [3000,3099]. One + // container on ref 1. And one that the index can not place. let idx: cram::crai::Index = vec![ cram::crai::Record::new(Some(0), Some(p(1000)), 100, 10, 0, 0), cram::crai::Record::new(Some(0), Some(p(2000)), 100, 20, 0, 0), @@ -706,15 +742,17 @@ mod tests { ]; let sel = |a: usize, b: usize| cram_container_offsets(&idx, 0, (p(a)..=p(b)).into()); - // A point inside one container decodes that container — not the contig. This single + // A point inside one container decodes that container, and not the contig. This one // assertion is the difference between 8 ms and 21 s on a real chr21. assert_eq!(sel(2050, 2050), vec![20, 50]); - // Boundaries: touching the first/last base of a container counts as overlap. + // The boundaries. A query that reaches the first base or the last base of a container + // counts as an overlap. assert_eq!(sel(2099, 2099), vec![20, 50], "last base of a container overlaps"); assert_eq!(sel(2000, 2000), vec![20, 50], "first base of a container overlaps"); assert_eq!(sel(2100, 2100), vec![50], "one past the end does not"); assert_eq!(sel(1999, 1999), vec![50], "one before the start does not"); - // A span crossing several containers takes exactly those it crosses. + // A span that crosses more than one container takes exactly the containers that it + // crosses. assert_eq!(sel(1050, 2050), vec![10, 20, 50]); // The other reference is never selected, even at identical coordinates. assert_eq!(cram_container_offsets(&idx, 1, (p(2050)..=p(2050)).into()), vec![40]); @@ -725,9 +763,9 @@ mod tests { ); } - /// Our hand-rolled `query` must return exactly what noodles' own `Query` returns. We replaced - /// it for speed, and a reimplementation that quietly drops records would look like a faster - /// caller rather than a broken one — the failure mode worth a test. + /// The `query` in this module must return exactly what the `Query` of noodles returns. We + /// wrote our own for speed. A version of it that drops records where nobody looks would seem + /// to be a faster caller, and not a broken one. That failure needs a test. #[test] fn cram_query_matches_noodles_query() { let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); diff --git a/crates/navigator-analysis/src/readview.rs b/crates/navigator-analysis/src/readview.rs index 0bb047db..c5233e21 100644 --- a/crates/navigator-analysis/src/readview.rs +++ b/crates/navigator-analysis/src/readview.rs @@ -1,14 +1,19 @@ -//! `AlnRead` — the minimal record view the quality-metrics walkers need, abstracted over the -//! **lazy** `bam::Record` (zero-copy, the hot path) and the owned `RecordBuf` (CRAM). The walkers -//! used to consume `RecordBuf`, which forced a per-read owned copy of the sequence, qualities, -//! CIGAR, name, and *every* optional tag — measured at ~half the per-read CPU on a WGS BAM. The -//! walkers touch only a handful of scalar fields plus the CIGAR ops and per-base qualities, all of -//! which `bam::Record` exposes as borrowed views, so this trait lets the same accumulator code run -//! over either record type with no allocation on the BAM path. +//! `AlnRead` is the smallest view of a record that the quality-metrics walkers need. It covers two +//! types: the **lazy** `bam::Record`, which is zero-copy and the hot path, and the owned +//! `RecordBuf`, which a CRAM gives. //! -//! Implementations map noodles' lazy `io::Result` accessors to plain `Option`/values, treating a -//! decode error as "absent" (skips that field) rather than aborting the walk — strictly more robust -//! than the old `RecordBuf` conversion, which would have errored the whole pass on a bad record. +//! The walkers once took a `RecordBuf`. That forced an owned copy, at each read, of the sequence, +//! the qualities, the CIGAR, the name, and *every* optional tag. A measurement on a WGS BAM put +//! that at about half of the CPU at each read. +//! +//! The walkers touch a few scalar fields, plus the CIGAR operations and the quality of each base. +//! `bam::Record` gives all of those as borrowed views. So this trait lets the same accumulator +//! code run over either record type, and it allocates nothing on the BAM path. +//! +//! An implementation maps the lazy `io::Result` accessors of noodles to a plain `Option` or a +//! value. It reads a decode error as "absent", and it skips that field. It does not stop the walk. +//! That is more robust than the old conversion to a `RecordBuf`, which would have failed the whole +//! pass on one bad record. use noodles::sam::alignment::record::cigar::op::Kind; use noodles::sam::alignment::record::data::field::Tag; @@ -28,22 +33,31 @@ pub trait AlnRead { fn mapping_quality(&self) -> Option; fn template_length(&self) -> i32; fn sequence_len(&self) -> usize; - /// Read (template) name as raw bytes, or `None` when unset. Borrowed — the caller decides - /// whether to pay for a `String`. + /// The name of the read, which is its template name, as raw bytes. It is `None` when the + /// record has none. This borrows the bytes, so the caller decides whether to pay for a + /// `String`. fn name(&self) -> Option<&[u8]>; - /// A string-valued auxiliary tag (e.g. `SA`), or `None` when absent, undecodable, or of another - /// type. Owned because the three record types spell their `Data` view differently; the tags this - /// serves (`SA`) appear on a small minority of reads, so the allocation is not on the hot path. + /// An auxiliary tag whose value is a string, such as `SA`. It is `None` when the tag is + /// absent, when the code can not decode it, or when it holds another type. + /// + /// This owns the value, because the three record types each give their `Data` view a + /// different shape. The one tag that this serves is `SA`, and it occurs on a small minority of + /// the reads. So the allocation does not sit on the hot path. fn string_tag(&self, tag: Tag) -> Option; - /// Run `f` with an iterator of CIGAR `(kind, len)` ops. The callback form keeps the lazy - /// `bam::Record`'s borrowed view alive for the duration; undecodable ops are skipped. Prefer - /// this over [`AlnRead::pileup_with`] when only the CIGAR is needed — the CRAM impl of - /// `pileup_with` materializes the quality scores, which this skips. + /// Run `f` with an iterator over the CIGAR operations, as `(kind, len)`. The callback form + /// keeps the borrowed view of the lazy `bam::Record` alive while `f` runs. It skips an + /// operation that the code can not decode. + /// + /// Use this, and not [`AlnRead::pileup_with`], when you need the CIGAR alone. The CRAM version + /// of `pileup_with` builds the quality scores, and this function does not. fn cigar_with(&self, f: impl FnOnce(&mut dyn Iterator) -> T) -> T; - /// Run `f` with the per-base phred qualities (raw, no +33; indexable by query offset) and an - /// iterator of CIGAR `(kind, len)` ops. Via a callback so the lazy `bam::Record` views (which - /// borrow the record's buffer through a temporary wrapper) stay alive for the duration — no - /// per-read allocation. Undecodable CIGAR ops are skipped. + /// Run `f` with two things: the phred quality of each base, and an iterator over the CIGAR + /// operations as `(kind, len)`. The qualities are raw, with no +33, and you index them by the + /// offset into the query. + /// + /// It uses a callback so that the views of the lazy `bam::Record` stay alive while `f` runs. + /// Those views borrow the buffer of the record through a temporary wrapper. There is then no + /// allocation at each read. It skips a CIGAR operation that the code can not decode. fn pileup_with(&self, f: impl FnOnce(&[u8], &mut dyn Iterator) -> T) -> T; } @@ -145,14 +159,20 @@ impl AlnRead for noodles::bam::Record { } } -/// A borrowed view over a decoded **CRAM** record (`noodles::cram::Record`) paired with the header, -/// implementing [`AlnRead`] by delegating to the `sam::alignment::Record` trait. CRAM stores the -/// sequence as deltas against the reference, so a `cram::Record` already holds the per-read data in -/// borrowed/lightweight form — driving the walkers off it directly skips the per-read -/// `RecordBuf::try_from_alignment_record` copy (sequence + quals + cigar + name + *every* tag into -/// owned form), measured at ~1.74× the per-read decode cost on a 30× short-read WGS CRAM. The -/// header is only needed by the trait's `reference_sequence_id` accessors (which, for CRAM, ignore -/// it and return the record's stored id — but the signature requires one). +/// A borrowed view over a decoded **CRAM** record, which is a `noodles::cram::Record`, together +/// with the header. It implements [`AlnRead`], and it hands the work to the +/// `sam::alignment::Record` trait. +/// +/// A CRAM stores the sequence as deltas against the reference. So a `cram::Record` already holds +/// the data of each read in a borrowed, light form. To drive the walkers from it directly +/// leaves out the `RecordBuf::try_from_alignment_record` copy at each read. +/// +/// That copy takes the sequence, the qualities, the CIGAR, the name and *every* tag into owned +/// form. A measurement on a 30x short-read WGS CRAM put it at about 1.74 times the cost of the +/// decode of one read. +/// +/// The header serves the `reference_sequence_id` accessors of the trait alone. For a CRAM those +/// ignore it, and they return the id that the record stores. But the signature needs one. pub struct CramRead<'a, 'c> { pub rec: &'a noodles::cram::Record<'c>, pub header: &'a noodles::sam::Header, @@ -211,10 +231,11 @@ impl AlnRead for CramRead<'_, '_> { } fn pileup_with(&self, f: impl FnOnce(&[u8], &mut dyn Iterator) -> T) -> T { use noodles::sam::alignment::Record as _; - // CRAM exposes qualities only via an iterator (not a contiguous slice), so collect them - // once per read — a small (~read-length) allocation, still far cheaper than the full - // `RecordBuf` materialization the high-level reader would do. The cigar is a lazy view over - // the record's features, iterated directly with no allocation. + // A CRAM gives the qualities through an iterator alone, and not as one slice. So the code + // collects them once at each read. That allocation is small, at about one read length, and + // it still costs far less than the full `RecordBuf` that the high-level reader would + // build. The cigar is a lazy view over the features of the record, and the code walks it + // directly, with no allocation. let quals: Vec = self.rec.quality_scores().iter().map(|r| r.unwrap_or(0)).collect(); let cigar = self.rec.cigar(); let mut ops = cigar.iter().filter_map(|op| op.ok().map(|o| (o.kind(), o.len()))); @@ -222,12 +243,17 @@ impl AlnRead for CramRead<'_, '_> { } } -/// A record yielded by a **sequential** (whole-file, no index) walk over either format: the -/// **lazy, zero-copy** `bam::Record` on the BAM path (no owned `RecordBuf` decode/tag-parse — the -/// hot-path win) and the decoded `RecordBuf` on the CRAM path (CRAM has no cheaper lazy form). It -/// implements [`AlnRead`] by delegating to the per-type impls above, so the same accumulator code -/// (`CoverageState`/`ReadMetricsState`/`SexState`) drives both with no allocation on the BAM path — -/// the sequential counterpart to the indexed [`crate::reader::RecordSink`] fan-out. +/// A record that a **sequential** walk gives, over either format. Such a walk covers the whole +/// file and uses no index. +/// +/// On the BAM path it is the **lazy, zero-copy** `bam::Record`. There is then no decode into an +/// owned `RecordBuf`, and no parse of a tag, and that is the gain on the hot path. On the CRAM +/// path it is the decoded `RecordBuf`, because a CRAM has no cheaper lazy form. +/// +/// It implements [`AlnRead`] by a call into the implementation of each type above. The same +/// accumulator code, which is `CoverageState`, `ReadMetricsState` and `SexState`, then drives +/// both, and it allocates nothing on the BAM path. This is the sequential counterpart of the +/// indexed [`crate::reader::RecordSink`] fan-out. pub enum SeqRecord { Bam(noodles::bam::Record), Cram(RecordBuf), diff --git a/crates/navigator-analysis/src/realign.rs b/crates/navigator-analysis/src/realign.rs index 2fde70e0..96650318 100644 --- a/crates/navigator-analysis/src/realign.rs +++ b/crates/navigator-analysis/src/realign.rs @@ -1,18 +1,22 @@ //! Light local realignment around candidate indels (plan §4b mitigation). //! -//! Ambiguous indels in homopolymers/repeats make BWA place the same insertion -//! differently across reads, smearing bases onto neighbouring positions — e.g. on -//! HG002 chrM a +1C in the 16295–16301 C-run makes ~47 reads put a spurious C on the -//! reference T at 16302, a false T>C SNP. GATK avoids this by local reassembly; here we -//! re-fit each read's bases over an active window back onto the reference with a -//! consistent gap model, so the homopolymer bases land in one place and the spurious -//! substitution disappears. +//! An indel in a homopolymer or a repeat is ambiguous. So BWA puts the same insertion at a +//! different place in different reads. That spreads bases onto the positions beside it. //! -//! The aligner is a **fitting alignment**: the read substring is fully consumed, with -//! free end gaps on the reference window (so reads starting/ending inside the window -//! still align). This module is pure and unit-tested; [`crate::caller`] drives it. +//! Here is a real case. On HG002 chrM, a +1C in the C-run at 16295 to 16301 makes about 47 reads +//! put a false C onto the reference T at 16302. That reads as a T>C SNP, and it is not one. +//! +//! GATK avoids this with a local reassembly. This module instead puts the bases of each read, over +//! an active window, back onto the reference, with one consistent gap model. The homopolymer bases +//! then land in one place, and the false substitution goes away. +//! +//! The aligner does a **fit of the read into the window**. It consumes the whole part of the read +//! that it looks at, and the end gaps on the reference window are free. So a read that starts or +//! ends inside the window still aligns. This module is pure, and unit tests cover it. +//! [`crate::caller`] drives it. -/// One aligned column between a read substring (query) and a reference window (target). +/// One aligned column, between a part of a read, which is the query, and a reference window, +/// which is the target. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Op { /// Query base aligned to a reference base (match or mismatch). @@ -23,8 +27,9 @@ pub enum Op { Deletion, } -// Linear-gap scoring. Single-base homopolymer indels dominate, so affine gaps add no -// resolving power here; a clear mismatch penalty drives the insertion choice. +// The score uses a linear gap. An indel of one base in a homopolymer is the case that dominates +// here, so an affine gap separates nothing more. A clear mismatch penalty is what drives the +// choice of where the insertion goes. const MATCH: i32 = 2; const MISMATCH: i32 = -4; const GAP: i32 = -3; @@ -48,12 +53,14 @@ pub fn fitting_align(query: &[u8], target: &[u8]) -> (usize, Vec) { let mut score = vec![vec![0i32; m + 1]; n + 1]; let mut tb = vec![vec![Move::Stop; m + 1]; n + 1]; - // Query must be fully consumed: leading query bases against empty target cost gaps. + // The alignment must consume the whole query. So a query base at the start, against an empty + // target, costs a gap. for i in 1..=n { score[i][0] = GAP * i as i32; tb[i][0] = Move::Up; } - // Row 0: free leading target gap (query can start anywhere in target) -> stays 0. + // Row 0. A target gap at the start is free, so the query can start anywhere in the target. + // The row stays at 0. for i in 1..=n { for j in 1..=m { @@ -80,7 +87,7 @@ pub fn fitting_align(query: &[u8], target: &[u8]) -> (usize, Vec) { } } - // Free trailing target gap: best score across the last query row. + // A target gap at the end is free, so take the best score across the last query row. let mut end_j = 0; let mut best = i32::MIN; for (j, &s) in score[n].iter().enumerate() { @@ -90,7 +97,7 @@ pub fn fitting_align(query: &[u8], target: &[u8]) -> (usize, Vec) { } } - // Traceback to row 0 (free leading target gap). + // Trace back to row 0, because a target gap at the start is free. let mut ops = Vec::new(); let (mut i, mut j) = (n, end_j); while i > 0 { @@ -167,7 +174,8 @@ mod tests { #[test] fn query_fits_into_a_substring_of_target() { - // query aligns to target[2..6]; free leading/trailing target gaps. + // The query aligns to target[2..6]. The target gaps at the start and at the end are + // free. assert_eq!(aligned_string(b"CGTA", b"AACGTACG"), (2, "MMMM".into())); } @@ -177,7 +185,7 @@ mod tests { // insertion so the read's T aligns to the ref T (not a C smeared onto T). let (_start, ops) = fitting_align(b"CCCCCCCCT", b"CCCCCCCT"); assert_eq!(ops.iter().filter(|o| **o == Op::Insertion).count(), 1); - // last column aligns the trailing T. + // The last column aligns the T at the end. assert_eq!(*ops.last().unwrap(), Op::Aligned); // Projection puts the T on the last reference position, never a C. @@ -190,8 +198,8 @@ mod tests { #[test] fn read_ending_in_homopolymer_does_not_reach_the_trailing_base() { - // read ends in the C-run (no T); after realignment it should not place any base - // on the reference T position — the spurious-SNP fix. + // The read ends inside the C-run, and it holds no T. After the realignment it must put no + // base at all on the reference T position. That is the fix for the false SNP. let ref_window = b"CCCCCCCT"; // ref C-run + T let read = b"CCCCCCCC"; // 8 C's, no T (read ended in the homopolymer) let (start, ops) = fitting_align(read, ref_window); diff --git a/crates/navigator-analysis/src/reassembly.rs b/crates/navigator-analysis/src/reassembly.rs index 525b559a..0baca9b9 100644 --- a/crates/navigator-analysis/src/reassembly.rs +++ b/crates/navigator-analysis/src/reassembly.rs @@ -1,24 +1,35 @@ -//! Haploid **local reassembly** resolver (private-Y Option B, phase 1) — pure Rust, no external -//! tools, Windows/MSVC-clean. +//! The haploid **local reassembly** resolver, which is Option B of the private-Y work, phase 1. It +//! is pure Rust, it needs no external tool, and it is clean on Windows and MSVC. //! -//! Design: `documents/design/haploid-reassembly-caller.md`. This module owns **Stages B–E** over a single -//! active window: read selection (mapping-quality gate + fragment dedup), candidate haplotypes -//! (per-SNV in v1: reference vs reference-with-one-substitution), read↔haplotype likelihood via a -//! **base-quality-aware PairHMM** (`bio::stats::pairhmm`), and haploid genotyping by the aggregate -//! log-odds. `caller.rs` owns Stage A (active-region detection — it already tallies the per-position -//! counts) and Stage F (turning [`ReassemblyCall`]s into `VariantCall`s); this module is deliberately -//! **I/O-free** so it is unit-testable on synthetic windows. +//! The design is in `documents/design/haploid-reassembly-caller.md`. This module owns **Stages B to +//! E** over one active window. Those four stages are: //! -//! Why it exists: the pileup caller (`caller.rs`) rejects a position whose pileup is ~50/50 as a -//! suspected paralog artifact (`is_paralogous`). At Y segmental-duplication / ampliconic loci that -//! throws away *true* derived SNVs, because reads from a paralogous region mismap and carry the -//! reference base onto the site. GATK resolves these by local reassembly + a base-quality PairHMM; -//! this is the haploid-only equivalent. Proven on WGS229 (POC `examples/reassembly_probe.rs`): the -//! base-quality PairHMM recovers the misaligned-ref sites the crude match/mismatch pileup ties. +//! - the selection of the reads, with a gate on the mapping quality and a dedup of the fragments; +//! - the candidate haplotypes, which in v1 is one for each SNV: the reference, against the +//! reference with one substitution; +//! - the likelihood of a read against a haplotype, from a **PairHMM that knows the base +//! qualities** (`bio::stats::pairhmm`); +//! - the haploid genotype, from the log-odds over all of the reads. //! -//! v1 is **per-candidate-SNV** (one alternate haplotype per candidate position). Linked variants and -//! short indels via POA multi-haplotype assembly are the v2 extension (see the design doc); POA still -//! serves here as an optional cross-check for the caller. +//! `caller.rs` owns Stage A and Stage F. Stage A finds the active region, and that code already +//! tallies the counts at each position. Stage F turns a [`ReassemblyCall`] into a `VariantCall`. +//! This module does **no I/O**, and that is deliberate: a unit test can then run it on a synthetic +//! window. +//! +//! Here is why it exists. The pileup caller in `caller.rs` refuses a position whose pileup is near +//! 50/50, because it suspects a paralog artifact. See `is_paralogous`. At a Y locus with a +//! segmental duplication, or an ampliconic one, that throws away a *true* derived SNV. Reads from +//! a paralogous region map to the wrong place and bring the reference base onto the site. +//! +//! GATK resolves those by local reassembly and a PairHMM over the base qualities. This module is +//! the haploid-only equivalent. The POC in `examples/reassembly_probe.rs` showed it on WGS229: the +//! PairHMM over base qualities recovers the misaligned-reference sites where the crude +//! match-against-mismatch pileup gives a tie. +//! +//! v1 works on **one candidate SNV at a time**, with one alternate haplotype for each candidate +//! position. Linked variants and short indels, through a POA assembly of more than one haplotype, +//! are the v2 extension. See the design document. POA still serves here as an optional +//! cross-check for the caller. use std::collections::HashMap; @@ -30,28 +41,38 @@ use bio::stats::{LogProb, Prob}; /// Natural-log → Phred scale factor (`10 / ln 10`); `LogProb` is base-*e*. const PHRED_PER_NAT: f64 = 4.342_944_819_032_518; -/// Tuning for the reassembly resolver. Defaults are the POC-validated starting points; the design -/// doc's §Open-questions flags τ / window size for calibration on the full truth set. +/// The controls of the reassembly resolver. The defaults are the start points that the POC +/// checked. The §Open-questions of the design document marks τ and the window size for a +/// calibration against the full truth set. #[derive(Debug, Clone, Copy, PartialEq)] pub struct ReassemblyParams { - /// Reads below this mapping quality are excluded (GATK default) — this is what drops the - /// ambiguously-placed paralog reads that masquerade as high-base-quality reference support. + /// The code drops a read below this mapping quality, which is the GATK default. That is what + /// removes a paralog read whose place is in doubt, and which otherwise looks like reference + /// support at a high base quality. pub min_mapping_quality: u8, /// Minimum aggregate log-odds (nats) for a haploid DERIVED call; symmetric for ANCESTRAL. pub min_log_odds: f64, - /// A DERIVED call needs at least this many alt-supporting fragments (post-dedup). + /// A DERIVED call needs this many fragments that support the alt allele, or more, after the + /// dedup. pub min_alt_fragments: u32, - /// v2: assemble the alternate haplotype from the alt-supporting reads (majority consensus over - /// the reference frame — [`assemble_alt_haplotype`]) so linked variants the true reads carry - /// don't penalise them against reference. **Default off**: it helps the synthetic linked-variant - /// case but on real WGS229 it perturbs marginal ~50/50 sites (regressed `chrY:4284195`), and - /// there is no real linked-variant truth site yet to validate the benefit. The mechanism is - /// unit-tested and opt-in (this flag / `NAVIGATOR_REASSEMBLY_ASSEMBLE=1`) pending that validation; - /// the read-likelihood floor below is the default-on v2 win. See `haploid-reassembly-caller.md`. + /// A v2 option: build the alternate haplotype from the reads that support the alt allele. It + /// is a majority consensus over the reference frame. See [`assemble_alt_haplotype`]. A linked + /// variant that the true reads carry then does not count against them, in the comparison with + /// the reference. + /// + /// **The default is off.** It helps the synthetic case with a linked variant. But on the real + /// WGS229 data it moves a site that sits near 50/50, and it broke `chrY:4284195`. There is also + /// no real truth site with a linked variant yet, so nobody can check the gain. + /// + /// Unit tests cover the mechanism, and you turn it on with this flag or with + /// `NAVIGATOR_REASSEMBLY_ASSEMBLE=1`, until somebody checks it. The floor on the read + /// likelihood below is the v2 gain that is on by default. See + /// `haploid-reassembly-caller.md`. pub assemble_alt: bool, - /// v2: drop a read whose best (ref-or-alt) haplotype log-likelihood is below this — it matches - /// *neither* local haplotype, i.e. paralog/junk from another locus. Roughly `-9 nats` per - /// mismatch, so `-90` tolerates real divergence (~9–10 mismatches) before excluding a read. + /// A v2 rule: drop a read whose best log-likelihood, over the reference haplotype and the alt + /// one, is below this. Such a read matches *neither* local haplotype. It is a paralog, or junk + /// from another locus. One mismatch costs about `-9 nats`, so `-90` accepts real divergence, at + /// about 9 or 10 mismatches, before the code drops a read. pub min_read_loglik: f64, } @@ -75,16 +96,19 @@ pub struct SiteObs { pub qual: u8, } -/// A read projected onto the active window's reference frame. Construction (the CIGAR walk that -/// yields the window-frame sequence, per-base qualities, and per-candidate [`SiteObs`]) is the -/// caller's job; this module consumes the projection so it stays I/O-free and testable. +/// A read projected onto the reference frame of the active window. +/// +/// The caller builds it. That build is the CIGAR walk that gives the sequence in the window frame, +/// the quality of each base, and a [`SiteObs`] at each candidate. This module reads the +/// projection, so it stays free of I/O and a test can cover it. #[derive(Debug, Clone)] pub struct WindowRead { - /// Fragment identity (query name) — same name for a read and its mate, used for dedup. + /// The identity of the fragment, which is the query name. A read and its mate share the name, + /// and the dedup uses it. pub name: Vec, /// Window-frame bases (uppercase), for the whole-read PairHMM realignment. pub seq: Vec, - /// Per-base Phred qualities, parallel to `seq`. + /// The Phred quality of each base, in line with `seq`. pub quals: Vec, /// Mapping quality of the source record. pub mapq: u8, @@ -108,25 +132,28 @@ pub enum Zygosity { Derived, /// The reference haplotype explains the reads (drop it). Ancestral, - /// Neither wins by `min_log_odds` — genuinely undecided (do not call). + /// Neither side wins by `min_log_odds`. The data does not decide this site, so do not call + /// it. Ambiguous, } -/// A genotyped candidate. The caller keeps [`Zygosity::Derived`] calls and turns them into -/// `VariantCall`s (Stage F); the others are returned so tests and diagnostics can see the decision. +/// A candidate with a genotype. The caller keeps a [`Zygosity::Derived`] call and turns it into a +/// `VariantCall`, in Stage F. The others come back too, so that a test and a diagnostic can see +/// the decision. #[derive(Debug, Clone, Copy, PartialEq)] pub struct ReassemblyCall { pub position: i64, pub ref_base: u8, pub alt_base: u8, - /// Fragments spanning the site after the MAPQ gate + mate dedup. + /// The count of fragments that cover the site, after the MAPQ gate and the dedup of the + /// mates. pub depth: u32, - /// Spanning fragments whose site base is the alternate allele. + /// The count of those fragments whose base at the site is the alternate allele. pub alt_depth: u32, pub allele_fraction: f64, /// Aggregate `Σ ln P(read|alt) − ln P(read|ref)` (nats); >0 favours the alt haplotype. pub log_odds: f64, - /// Phred-scaled confidence of the winning genotype (GQ-like). + /// The confidence of the genotype that won, on the Phred scale. It is like a GQ. pub quality: f64, pub genotype: Zygosity, } @@ -150,8 +177,9 @@ pub fn genotype_window( .collect() } -/// Resolve a single candidate: build the alt haplotype, select + dedup spanning reads, score each -/// against ref vs alt with the PairHMM, and genotype by the aggregate log-odds. +/// Resolve one candidate. It builds the alt haplotype. It takes the reads that cover the site, and +/// dedups them. It scores each of those against the reference and the alt with the PairHMM. It +/// then gives a genotype from the log-odds over all of them. fn genotype_candidate( hmm: &mut PairHMM, ref_window: &[u8], @@ -163,13 +191,16 @@ fn genotype_candidate( ) -> ReassemblyCall { let off = (cand.position - window_start) as usize; - // Stage B — select reads that clear the MAPQ gate and span this candidate, then collapse - // overlapping mate pairs to one fragment (keep the record whose site base has higher quality). + // Stage B. Take the reads that clear the MAPQ gate and that cover this candidate. Then put a + // mate pair that overlaps into one fragment, and keep the record whose base at the site has + // the higher quality. let kept = dedup_spanning_fragments(reads, ci, params); - // Stage C — alternate haplotype. v2: POA-assemble the alt-supporting reads so linked variants - // they carry don't penalise them against reference; fall back to reference-plus-one-substitution - // when assembly is degenerate. v1 behaviour is the fallback, so simple sites are unchanged. + // Stage C, the alternate haplotype. In v2, a POA builds it from the reads that support the + // alt allele. A linked variant that those reads carry then does not count against them in the + // comparison with the reference. When the assembly is degenerate, the code falls back to the + // reference plus one substitution. That fallback is the v1 behaviour, so a simple site does + // not change. let mut single_snv = ref_window.to_vec(); if off < single_snv.len() { single_snv[off] = cand.alt_base; @@ -180,8 +211,9 @@ fn genotype_candidate( single_snv }; - // Stages D/E — per-fragment likelihood ratio and site-base vote, with the absolute-likelihood - // floor excluding reads that match neither haplotype (paralog/junk from another locus). + // Stages D and E. Take the likelihood ratio of each fragment, and the vote of its base at the + // site. The floor on the absolute likelihood drops a read that matches neither haplotype, + // which is a paralog, or junk from another locus. let mut log_odds = 0.0f64; let mut depth = 0u32; let mut alt_depth = 0u32; @@ -225,9 +257,12 @@ fn genotype_candidate( } } -/// Reads clearing the MAPQ gate and spanning candidate `ci`, with overlapping mate pairs collapsed -/// to one fragment (keep the record whose site base has higher quality). Returns read indices sorted -/// by fragment name so downstream assembly is deterministic (`HashMap` order is not). +/// The reads that clear the MAPQ gate and that cover the candidate `ci`. A mate pair that overlaps +/// goes into one fragment, and the code keeps the record whose base at the site has the higher +/// quality. +/// +/// It returns the read indices in the order of the fragment names. The assembly that follows is +/// then deterministic, where the order of a `HashMap` is not. fn dedup_spanning_fragments(reads: &[WindowRead], ci: usize, params: &ReassemblyParams) -> Vec { let mut by_fragment: HashMap<&[u8], usize> = HashMap::new(); for (ri, read) in reads.iter().enumerate() { @@ -252,17 +287,23 @@ fn dedup_spanning_fragments(reads: &[WindowRead], ci: usize, params: &Reassembly kept } -/// Build the alternate haplotype from the alt-supporting fragments (site base == `alt_base`) by -/// **majority consensus over the reference frame**: reference, plus every position where a strict -/// majority of the covering alt reads concordantly carry the same non-reference base, plus the -/// candidate substitution at `site_off`. Returns `None` (→ caller falls back to reference+SNV) when -/// there are fewer than two alt reads. +/// Build the alternate haplotype from the fragments that support the alt allele, which are the +/// ones whose base at the site is `alt_base`. The method is a **majority consensus over the +/// reference frame**. +/// +/// It takes three things. The reference. Every position where a strict majority of the alt reads +/// that cover it agree on the same non-reference base. And the candidate substitution at +/// `site_off`. It returns `None` with fewer than two alt reads, and the caller then falls back to +/// the reference plus the SNV. +/// +/// This is *not* a raw POA, and that is deliberate. A real read has ragged ends, and it covers the +/// window only in part. A POA over such reads gives a noisy consensus, and that consensus scores a +/// site near 50/50 wrongly. In a test it broke `chrY:4284195`. /// -/// This is deliberately *not* raw POA. POA over ragged, partially-spanning real reads produces a -/// noisy consensus that mis-scores marginal 50/50 sites (it regressed `chrY:4284195` in testing). -/// The majority rule reduces to reference+SNV when the alt reads carry no concordant linked variant -/// — so it never hurts a site without linked context — while still adding real linked variants so -/// the true reads match cleanly. (Short indels are v2b, via POA over the confirmed alt reads.) +/// The majority rule comes down to the reference plus the SNV when the alt reads carry no linked +/// variant that they agree on. So it never hurts a site with no linked context. And it still adds +/// a real linked variant, so that the true reads match cleanly. A short indel is v2b, through a +/// POA over the alt reads that the code confirmed. fn assemble_alt_haplotype( reads: &[WindowRead], kept: &[usize], @@ -280,7 +321,8 @@ fn assemble_alt_haplotype( return None; } - // Tally each alt read's bases per reference position (via pairwise projection onto the window). + // Tally the bases of each alt read at each reference position, through a pairwise projection + // onto the window. let mut counts = vec![[0u32; 4]; ref_window.len()]; let mut cover = vec![0u32; ref_window.len()]; for r in &alt_reads { @@ -298,7 +340,8 @@ fn assemble_alt_haplotype( hap[pos] = BASES[bi]; } } - // The candidate substitution is why we're here — force it (its column may be exactly 50/50). + // The candidate substitution is the reason for this call, so force it in. Its column can sit + // at exactly 50/50. if site_off < hap.len() { hap[site_off] = alt_base; } @@ -328,8 +371,9 @@ fn argmax4(counts: &[u32; 4]) -> (usize, u32) { (bi, counts[bi]) } -/// Add `seq`'s bases to the per-reference-position `counts`/`cover` tallies by semiglobally aligning -/// it to `ref_window` (only aligned match/mismatch columns contribute; insertions/deletions don't). +/// Add the bases of `seq` to the `counts` and `cover` tallies at each reference position. It aligns +/// `seq` to `ref_window` in a semiglobal way. Only a column that aligns as a match or a mismatch +/// counts. An insertion and a deletion do not. fn project_read_onto_ref(seq: &[u8], ref_window: &[u8], counts: &mut [[u32; 4]], cover: &mut [u32]) { let score = |a: u8, b: u8| if a == b { 1i32 } else { -4i32 }; let mut aligner = PwAligner::new(-5, -1, score); @@ -356,20 +400,23 @@ fn project_read_onto_ref(seq: &[u8], ref_window: &[u8], counts: &mut [[u32; 4]], } } -/// Log-probability that `read` (with `quals`) was produced by `hap`, marginalised over alignments. +/// The log-probability that `hap` gave `read`, which carries `quals`. It marginalises over the +/// alignments. fn hap_likelihood(hmm: &mut PairHMM, read: &[u8], quals: &[u8], hap: &[u8]) -> LogProb { hmm.prob_related(&ReadHapEmission { read, quals, hap }, &Semiglobal, None) } // ---- base-quality-aware PairHMM emission model (POC-validated) -------------------------------- -/// Phred score → error probability, clamped to Q2–Q60 (never a certain match/mismatch). +/// The error probability that a Phred score gives, clamped to Q2 and Q60. A match or a mismatch is +/// then never sure. fn phred_err(q: u8) -> f64 { let q = q.clamp(2, 60) as f64; 10f64.powf(-q / 10.0) } -/// Emission: `x` = read (carries per-base quality), `y` = candidate haplotype. +/// The emission. `x` is the read, which carries a quality at each base. `y` is the candidate +/// haplotype. struct ReadHapEmission<'a> { read: &'a [u8], quals: &'a [u8], @@ -416,7 +463,8 @@ impl GapParameters for GapParams { } } -/// Semiglobal in the read: free leading/trailing offset so window-edge trimming isn't penalised. +/// Semiglobal in the read. The offset at the start and at the end is free, so a cut at the edge of +/// the window costs nothing. struct Semiglobal; impl StartEndGapParameters for Semiglobal { fn free_start_gap_x(&self) -> bool { @@ -449,9 +497,10 @@ mod tests { read_muts(name, site_base, &[], qual, mapq) } - /// Like [`read`] but also applies `muts` (offset → base) to the window sequence — for building - /// reads that carry linked variants (or, with many muts, paralog junk). `site_obs` reflects only - /// the candidate site base, as the caller's CIGAR-walk extraction would produce it. + /// The same as [`read`], and it also applies `muts`, which maps an offset to a base, to the + /// sequence of the window. Use it to make reads that carry a linked variant, or, with many + /// muts, paralog junk. `site_obs` holds the base at the candidate site alone, as the CIGAR + /// walk of the caller would give it. fn read_muts(name: &str, site_base: u8, muts: &[(usize, u8)], qual: u8, mapq: u8) -> WindowRead { let mut seq = REF.to_vec(); seq[CAND_OFF] = site_base; @@ -479,7 +528,7 @@ mod tests { #[test] fn clean_derived_site_is_called() { - // Twelve fragments all carrying the alt allele → strongly DERIVED. + // Twelve fragments, and all of them carry the alt allele. The call is strongly DERIVED. let reads: Vec<_> = (0..12).map(|i| read(&format!("r{i}"), b'T', 35, 60)).collect(); let c = call(&reads); assert_eq!(c.genotype, Zygosity::Derived); @@ -491,9 +540,10 @@ mod tests { #[test] fn low_mapq_paralog_reference_reads_are_dropped_recovering_the_site() { - // The misaligned-ref case: 8 clean alt fragments (MAPQ 60) + 6 paralog reference fragments - // that carry the ref base but are ambiguously placed (MAPQ 5). The MAPQ gate excludes the - // paralogs, so the site is recovered as DERIVED instead of rejected as ~50/50. + // The case where the reference alignment is wrong. There are 8 clean alt fragments at + // MAPQ 60, and 6 paralog fragments that carry the ref base but whose place is in doubt, at + // MAPQ 5. The MAPQ gate drops the paralogs, so the site comes back as DERIVED. Without the + // gate it would go out as a 50/50 rejection. let mut reads: Vec<_> = (0..8).map(|i| read(&format!("alt{i}"), b'T', 35, 60)).collect(); reads.extend((0..6).map(|i| read(&format!("par{i}"), b'A', 35, 5))); let c = call(&reads); @@ -504,8 +554,8 @@ mod tests { #[test] fn genuinely_balanced_high_quality_site_is_not_called() { - // Specificity: an even split of high-quality, well-placed ref and alt fragments is truly - // undecided — reassembly must NOT invent a call. + // A test of the specificity. An even split of ref and alt fragments, all of high quality + // and all placed well, decides nothing. The reassembly must NOT invent a call. let mut reads: Vec<_> = (0..6).map(|i| read(&format!("alt{i}"), b'T', 35, 60)).collect(); reads.extend((0..6).map(|i| read(&format!("ref{i}"), b'A', 35, 60))); let c = call(&reads); @@ -514,8 +564,9 @@ mod tests { #[test] fn overlapping_mates_are_counted_once() { - // Four distinct alt fragments plus a read and its mate (same name) both covering the site. - // Fragment dedup must collapse the mate pair so depth is 5, not 6. + // Four separate alt fragments, plus a read and its mate, which share a name, and which + // both cover the site. The dedup must put the mate pair into one fragment, so that the + // depth reads 5 and not 6. let mut reads: Vec<_> = (0..4).map(|i| read(&format!("f{i}"), b'T', 35, 60)).collect(); reads.push(read("pair", b'T', 20, 60)); // read reads.push(read("pair", b'T', 35, 60)); // its mate (higher qual → the kept one) @@ -541,10 +592,12 @@ mod tests { #[test] fn assembled_alt_haplotype_lifts_confidence_on_linked_variant_site() { - // True reads (majority) carry the derived allele PLUS two linked variants; reference reads - // are clean. Against a reference+single-SNV alt haplotype (v1) the linked variants penalise - // the true reads; the POA-assembled haplotype (v2) lets them match cleanly, so the call is - // both DERIVED and more confident than v1. + // The true reads are the majority, and they carry the derived allele PLUS two linked + // variants. The reference reads are clean. + // + // Against a v1 alt haplotype, which is the reference plus one SNV, those linked variants + // count against the true reads. The v2 haplotype, which a POA assembles, lets them match + // cleanly. The call is then DERIVED, and it carries more confidence than in v1. let mut reads: Vec<_> = (0..10) .map(|i| read_muts(&format!("alt{i}"), b'T', LINKED, 35, 60)) .collect(); @@ -570,11 +623,13 @@ mod tests { #[test] fn paralog_junk_read_matching_neither_haplotype_is_filtered() { - // Five clean reference reads + one "read" carrying the alt base but riddled with mismatches - // *throughout* the window (a paralog fragment from another locus). Spread matters: the - // semiglobal PairHMM clips clean prefixes/suffixes, so only mismatches distributed across the - // read make it match neither haplotype. The likelihood floor must exclude it, so it neither - // inflates depth nor tilts the call away from ANCESTRAL. + // Five clean reference reads, plus one "read" that carries the alt base and holds + // mismatches *across the whole* window. That is a paralog fragment from another locus. + // + // The spread of those mismatches matters. The semiglobal PairHMM cuts a clean start and a + // clean end off a read. So only mismatches that lie across the whole read make it match + // neither haplotype. The floor on the likelihood must drop it. It must not raise the + // depth, and it must not move the call away from ANCESTRAL. let junk_muts: Vec<(usize, u8)> = (0..REF.len()) .step_by(2) .filter(|&k| k != CAND_OFF) @@ -591,8 +646,9 @@ mod tests { #[test] fn assembly_falls_back_to_single_snv_when_alt_reads_are_too_few() { - // One lone alt read (< 2) can't seed an assembly → fall back to reference+SNV; with only one - // alt fragment against ten reference reads the site stays ANCESTRAL (no spurious call). + // One alt read, which is fewer than 2, can not start an assembly. So the code falls back + // to the reference plus the SNV. With one alt fragment against ten reference reads, + // the site stays ANCESTRAL, and there is no false call. let mut reads: Vec<_> = (0..10).map(|i| read(&format!("ref{i}"), b'A', 35, 60)).collect(); reads.push(read("lone", b'T', 35, 60)); let c = call(&reads); diff --git a/crates/navigator-analysis/src/revert/collate.rs b/crates/navigator-analysis/src/revert/collate.rs index 38640bee..faef6734 100644 --- a/crates/navigator-analysis/src/revert/collate.rs +++ b/crates/navigator-analysis/src/revert/collate.rs @@ -1,20 +1,24 @@ //! Collate reverted reads by name, on disk. //! -//! A coordinate-sorted WGS BAM holds ~10⁹ records and its mates are scattered, so pairing needs the -//! records in name order. Two things rule out the obvious approaches: a name→record hash map does -//! not fit in memory at that scale, and a second indexed pass costs a full extra decode of the -//! file. So this is a textbook **external merge sort** — fill a fixed memory budget, sort it, spill -//! a run to scratch, repeat; then merge the runs with a k-way heap. +//! A WGS BAM in coordinate order holds about 10⁹ records, and its mates lie far apart. To put two +//! mates together, the code needs the records in name order. //! -//! The property that matters is that peak memory is the budget plus one buffered block per run, -//! *independent of input size*. That is what lets the same code path revert a 5 GB exome and a -//! 200 GB WGS on the same laptop, and it is why this is disk-backed rather than clever. The budget -//! itself is sized from the machine — see [`navigator_resource::spill_budget`]. +//! Two things rule out the two direct methods. A hash map from a name to a record does not fit in +//! memory at that scale. And a second pass over the index costs a full extra decode of the file. //! -//! Runs use a plain length-prefixed binary encoding rather than a serialization framework: the -//! format is written and read in this one file, it is a hot path measured in billions of records, -//! and an explicit encoder is easier to reason about than a derived one when the question is "how -//! many bytes did that read cost". +//! So this is a standard **external merge sort**. Fill a fixed memory budget, sort it, spill a run +//! to the scratch space, and repeat. Then merge the runs with a heap, k at a time. +//! +//! The property that matters is the peak memory: the budget, plus one buffered block for each run, +//! and *nothing that grows with the input*. That is what lets one code path revert a 5 GB exome +//! and a 200 GB WGS on the same laptop. It is also why this code writes to disk, instead of a +//! method that looks smarter. The budget itself comes from the machine. See +//! [`navigator_resource::spill_budget`]. +//! +//! A run holds a plain binary encoding, where a length comes before each field. It uses no +//! serialization framework. There are three reasons. This one file writes the format and reads it. +//! It is a hot path, over billions of records. And an encoder that you can read answers the +//! question "how many bytes did that read cost", where one that a macro derived does not. use std::cmp::Reverse; use std::collections::BinaryHeap; @@ -25,12 +29,13 @@ use std::path::{Path, PathBuf}; use super::transform::{Mate, RevertedRead}; use crate::error::AnalysisError; -/// Buffer size for run spill/read-back. Large enough that the merge's per-run reads stay -/// sequential, small enough that the runs a WGS produces don't add up to real memory when the merge -/// holds all of them open at once. +/// The buffer size for a spill of a run, and for the read back. It is large enough that the reads +/// of the merge, on each run, stay sequential. It is also small enough that the runs of a WGS do +/// not add up to real memory. The merge holds all of them open at one time. const RUN_IO_BUFFER: usize = 256 * 1024; -/// Accumulates reverted reads, spilling sorted runs to scratch when the budget is reached. +/// It collects the reverted reads. When they reach the budget, it sorts them and spills a run to +/// the scratch space. pub struct Collator { dir: PathBuf, buffer: Vec, @@ -69,9 +74,10 @@ impl Collator { let path = self.dir.join(format!("revert-run-{:05}.bin", self.runs.len())); let file = File::create(&path).map_err(|e| AnalysisError::io(&path, e))?; - // Paced: the spill runs are the biggest thing this pipeline writes — the scratch peak is - // here, not in the post-processing stages — so they are exactly what must not be allowed to - // pile up dirty in the page cache. See `navigator_resource::PacedFile`. + // The code paces this write. The spill runs are the largest thing that this pipeline + // writes, and the peak of the scratch space is here, not in a later stage. So these are + // exactly the writes that must not collect in the page cache with nothing behind them. + // See `navigator_resource::PacedFile`. let mut w = BufWriter::with_capacity(RUN_IO_BUFFER, navigator_resource::PacedFile::new(file)); for read in &self.buffer { write_read(&mut w, read).map_err(|e| AnalysisError::io(&path, e))?; @@ -84,17 +90,17 @@ impl Collator { Ok(()) } - /// Spill whatever is left and open the merge over all runs. + /// Spill the remaining reads, and then open the merge over all of the runs. pub fn finish(mut self) -> Result { self.spill()?; Merged::open(self.runs) } } -/// A k-way merge over the spilled runs, yielding reads in `(name, mate)` order. +/// A merge over the spilled runs, k at a time. It gives the reads back in `(name, mate)` order. /// -/// Owns its run files and deletes them on drop, so a cancelled or failed revert does not leave -/// tens of gigabytes of scratch behind. +/// It owns its run files, and it deletes them when it drops. So a revert that somebody cancelled, +/// or that failed, leaves no scratch space behind. That space is tens of gigabytes. pub struct Merged { runs: Vec, readers: Vec>, @@ -140,7 +146,8 @@ impl Merged { Ok(Self { runs, readers, heap }) } - /// How many runs were spilled. One means the input fit in the memory budget. + /// The count of runs that the code spilled. A value of one means that the input fit in the + /// memory budget. pub fn run_count(&self) -> usize { self.runs.len() } @@ -156,11 +163,11 @@ impl Merged { Ok(Some(read)) } - /// The next group of reads sharing a name — i.e. one template. + /// The next group of reads that share a name. That group is one template. /// - /// Reads arrive name-ordered, so a group is just the run of equal names at the front. This is - /// where pairing becomes possible: a template's records are finally adjacent, which is the - /// thing coordinate order took away. + /// The reads arrive in name order, so a group is the run of equal names at the front, and + /// nothing more. This is where two mates can come together at last. The records of a template + /// finally sit beside each other, and coordinate order had taken that away. pub fn next_group(&mut self, group: &mut Vec) -> Result { group.clear(); let Some(first) = self.next_read()? else { @@ -168,7 +175,8 @@ impl Merged { }; group.push(first); - // Peek: the heap's top is the next read overall, so compare names without consuming. + // Look at the top of the heap, which is the next read of all. Compare the names, and do + // not take it off. while let Some(Reverse(entry)) = self.heap.peek() { if entry.read.name != group[0].name { break; @@ -182,8 +190,8 @@ impl Merged { impl Drop for Merged { fn drop(&mut self) { - // Close the handles before unlinking; best-effort, since a failed cleanup must not mask - // whatever error is already unwinding. + // Close the handles before the code removes the files. This is best effort. A cleanup + // that fails must not hide an error that is already on its way out. self.readers.clear(); for path in &self.runs { let _ = std::fs::remove_file(path); @@ -211,9 +219,9 @@ fn mate_from_byte(b: u8) -> Mate { /// `name_len:u32 | name | mate:u8 | seq_len:u32 | sequence | qualities` /// -/// Qualities are not length-prefixed: [`super::transform::revert_record`] guarantees they match the -/// sequence length, and re-encoding a number we already know would cost 4 bytes per read across -/// billions of reads for nothing. +/// The qualities carry no length before them. [`super::transform::revert_record`] makes sure that +/// their length matches the sequence. To write a number that the code already knows would cost 4 +/// bytes at each read, over billions of reads, and it would buy nothing. fn write_read(w: &mut W, read: &RevertedRead) -> std::io::Result<()> { w.write_all(&(read.name.len() as u32).to_le_bytes())?; w.write_all(&read.name)?; @@ -250,8 +258,8 @@ fn read_read(r: &mut R) -> std::io::Result> { })) } -/// A clean end-of-run is the only place a short read is expected; anywhere else it is corruption -/// and `read_exact` will say so. +/// A clean end of a run is the one place where a short read is correct. Anywhere else, the data +/// carries damage, and `read_exact` says so. fn read_u32_or_eof(r: &mut R) -> std::io::Result> { let mut buf = [0u8; 4]; let mut filled = 0; diff --git a/crates/navigator-analysis/src/revert/mod.rs b/crates/navigator-analysis/src/revert/mod.rs index 607be822..f17ce16b 100644 --- a/crates/navigator-analysis/src/revert/mod.rs +++ b/crates/navigator-analysis/src/revert/mod.rs @@ -1,39 +1,44 @@ -//! Revert an aligned BAM/CRAM back to the unaligned reads it was built from — stage A of the -//! realignment pipeline (`documents/design/realignment-module.md`). +//! Turn an aligned BAM or CRAM back into the unaligned reads that made it. This is stage A of the +//! realignment pipeline. See `documents/design/realignment-module.md`. //! -//! This is the GATK `RevertSam` + `SamToFastq` / `samtools collate | fastq` job, in Rust on -//! noodles. It is deliberately **backend-agnostic**: nothing here knows which mapper the reads are -//! headed for, so it is worth having even if the aligner decision changes underneath it. +//! It is the job of GATK `RevertSam` plus `SamToFastq`, or of `samtools collate | fastq`, in Rust +//! and on noodles. It does not know about any backend, and that is deliberate. Nothing here knows +//! which mapper the reads go to, so it holds its value even if somebody changes the aligner under +//! it. //! //! ## Why this is the hard part //! -//! Recovering the original reads from a coordinate-sorted alignment is not a filter, it is a -//! regrouping. Four things make it awkward: +//! To recover the original reads from an alignment in coordinate order is not a filter. It is a +//! regroup. Four things make it awkward: //! -//! 1. **Mates are far apart.** In coordinate order a read and its mate can sit gigabases away, so -//! pairing requires grouping by name. A read-name→record hash map is not an option at WGS scale -//! (~10⁹ records), so [`collate`] does a disk-backed external merge sort instead: fill a memory -//! budget, sort, spill a run, then k-way merge the runs back. Memory stays flat and bounded -//! regardless of input size, which is the whole point. -//! 2. **Aligners rewrite the read.** A reverse-strand alignment stores SEQ and QUAL -//! reverse-complemented relative to the sequencer's output, so both must be restored. -//! 3. **Only primaries carry the full read.** Secondary and supplementary records are dropped; -//! supplementaries are typically hard-clipped, meaning sequence has already been discarded. -//! 4. **Unmapped reads must survive.** They are not an edge case to tolerate — reads that failed to -//! map on GRCh38 are exactly the ones that may land in CHM13-resolved sequence, so they are the -//! realignment payoff and have to reach the FASTQ. +//! 1. **A read and its mate are far apart.** In coordinate order they can sit gigabases from each +//! other, so the code can rebuild the pair only from a group by name. At WGS scale, which is +//! about 10⁹ records, a hash map from a read name to a record is not possible. So [`collate`] +//! does an external merge sort on disk. It fills a memory budget, sorts, spills a run, and +//! merges the runs back k at a time. The memory then stays flat at any input size. +//! 2. **An aligner rewrites the read.** A reverse-strand alignment stores SEQ and QUAL as the +//! reverse complement of what the sequencer gave, so the code must restore both. +//! 3. **Only a primary record carries the full read.** The code drops a secondary record and a +//! supplementary one. A supplementary record usually carries a hard clip, which means that the +//! aligner already threw sequence away. +//! 4. **An unmapped read must survive.** It is not an edge case to accept. A read that did not map +//! on GRCh38 is exactly the read that may land in sequence that CHM13 resolves. Those reads are +//! the gain of the whole realignment, and they must reach the FASTQ. //! //! ## What comes out //! -//! Paired FASTQ (`_1.fastq` / `_2.fastq`, kept in lockstep) plus a singletons file for anything -//! that did not pair — an unpaired library, a mate whose partner was dropped, or a read whose -//! flags disagree with themselves. Read names are written bare, without `/1` and `/2` suffixes, -//! matching `samtools fastq`: the pairing is carried by file position, and suffixes confuse some -//! downstream tools more than they help. +//! Paired FASTQ, as `_1.fastq` and `_2.fastq`, and the two stay in step. A file of singletons sits +//! beside them, for anything with no pair. That covers a library with no pairs, a mate whose +//! partner the code dropped, and a read whose flags disagree with themselves. //! -//! uBAM output (which would preserve `@RG` per-read rather than only in the header) is the other -//! option the design records; FASTQ is the default because every aligner takes it. The writer is -//! isolated in [`writer`] so a uBAM sibling can be added without disturbing collation. +//! The read names go out bare, with no `/1` and `/2` at the end, as `samtools fastq` writes them. +//! The position in the file says which two reads make a pair, and those suffixes confuse some +//! later tools more than they help. +//! +//! The design records one other option: a uBAM output, which would keep the `@RG` at each read, +//! and not in the header alone. FASTQ is the default, because every aligner takes it. [`writer`] +//! holds the writer on its own, so a uBAM writer can go beside it, and the collation does not +//! change. mod collate; mod transform; @@ -51,17 +56,19 @@ use crate::reader; // not: it is the private reason-code that pairs with `revert_record`, which stays internal. pub use transform::{Mate, RevertedRead}; -/// How often the record loop asks whether it has been cancelled. Frequent enough that a click -/// feels immediate, rare enough that the atomic load never shows up in a profile — the same -/// reasoning as the other walkers (see [`crate::cancel`]). +/// How often the record loop asks whether somebody cancelled it. It asks often enough that a click +/// feels immediate, and rarely enough that the atomic load never shows in a profile. That is the +/// same reasoning as in the other walkers. See [`crate::cancel`]. const CANCEL_CHECK_INTERVAL: u64 = 4096; /// What to do with a **primary** record whose CIGAR contains a hard clip. /// -/// Hard clipping means the aligner discarded sequence from the record, so the read cannot be fully -/// recovered. Mainstream aligners hard-clip only supplementary records (which we drop anyway), but -/// some pipelines emit hard-clipped primaries, and emitting those as if whole would silently feed -/// a truncated read to the mapper. +/// A hard clip means that the aligner threw sequence away from the record, so nothing can recover +/// the whole read. +/// +/// A mainstream aligner puts a hard clip on a supplementary record alone, and the code drops those +/// in any case. But some pipelines give a primary record with a hard clip. To emit such a record +/// as if it were whole would give the mapper a short read, and nobody would see it happen. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum HardClipPolicy { /// Drop the read and count it. The default: a missing read is visible in the stats, whereas a @@ -73,10 +80,11 @@ pub enum HardClipPolicy { Emit, } -/// Tuning for [`revert_alignment`]. +/// The controls of [`revert_alignment`]. #[derive(Debug, Clone)] pub struct RevertParams { - /// Bytes of reverted reads held in memory before a sorted run is spilled to scratch. + /// How many bytes of reverted reads stay in memory before the code spills a sorted run to the + /// scratch space. pub sort_buffer_bytes: usize, /// Treatment of hard-clipped primary records. pub hard_clipped: HardClipPolicy, @@ -85,11 +93,12 @@ pub struct RevertParams { } impl Default for RevertParams { - /// The collator is sized from the machine by the same rule as the coordinate sort — see - /// [`navigator_resource::spill_budget`], which also documents `NAVIGATOR_REVERT_SORT_MB`. The - /// constant this replaced was 256 MB, described as keeping the run count low for a WGS; at - /// ~340 bytes a reverted read that is a run every million reads, so a 30x WGS spilled several - /// hundred of them and the merge opened every one. + /// The size of the collator comes from the machine, by the same rule as the coordinate sort. + /// See [`navigator_resource::spill_budget`], which also documents `NAVIGATOR_REVERT_SORT_MB`. + /// + /// The constant before it was 256 MB, and its comment gave the reason: it kept the run count + /// low for a WGS. A reverted read is about 340 bytes, so 256 MB is one run in every million + /// reads. A 30x WGS then spilled some hundreds of runs, and the merge opened every one. fn default() -> Self { Self { sort_buffer_bytes: navigator_resource::spill_budget("NAVIGATOR_REVERT_SORT_MB") as usize, @@ -99,29 +108,34 @@ impl Default for RevertParams { } } -/// What the revert did, for the job log and for the honest reporting the design asks for. +/// What the revert did. It goes into the job log, and into the honest report that the design asks +/// for. /// -/// Every count here exists because something was *dropped or changed*; a revert that silently -/// loses reads is the failure mode this whole struct is here to make impossible. +/// Every count here exists because the code *dropped or changed* something. A revert that loses +/// reads where nobody sees it is the failure that this whole struct makes impossible. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct RevertStats { - /// Records seen in the input, before any filtering. + /// The count of records in the input, before any filter. pub records_read: u64, /// Secondary alignments dropped (`0x100`). pub secondary_dropped: u64, /// Supplementary alignments dropped (`0x800`). pub supplementary_dropped: u64, - /// Primary records dropped or truncated because of hard clipping, per [`HardClipPolicy`]. + /// The count of primary records that the code dropped or cut short because of a hard clip. + /// [`HardClipPolicy`] decides which. pub hard_clipped: u64, - /// Records dropped for having no sequence at all (`SEQ` = `*`) — nothing to revert. + /// The count of records that the code dropped because they hold no sequence at all, where + /// `SEQ` is `*`. There is nothing to revert. pub no_sequence_dropped: u64, - /// Records whose qualities were absent (`QUAL` = `*`) and were therefore **synthesized** at a - /// flat phred 40. The reads are kept because a read with no qualities is still mappable, but - /// the qualities that come out are invented and this count is how that stays visible. + /// The count of records that carried no qualities, where `QUAL` is `*`, and for which the code + /// **made** a flat phred 40. It keeps those reads, because a mapper can still map a read with + /// no qualities. But the code invents the qualities that come out, and this count is what + /// keeps that visible. pub qualities_synthesized: u64, - /// Records whose qualities came from the `OQ` tag rather than `QUAL`. + /// The count of records whose qualities came from the `OQ` tag, and not from `QUAL`. pub original_qualities_used: u64, - /// Reads that were unmapped in the input (`0x4`) — the realignment payoff; see the module docs. + /// The count of reads that had no mapping in the input, at flag `0x4`. Those are the gain of + /// the realignment. See the module documentation. pub unmapped_reads: u64, /// Reads written across all three output files. pub reads_emitted: u64, @@ -144,9 +158,9 @@ pub struct RevertOutput { /// Revert `path` (BAM or CRAM) into paired FASTQ under `out_dir`. /// -/// `reference` is required for CRAM and ignored for BAM, matching [`reader::open_seq`]. `out_dir` -/// receives the three FASTQ files and is also used for the sort's spill files, which are removed -/// before returning. +/// A CRAM needs `reference`, and a BAM ignores it, as in [`reader::open_seq`]. The three FASTQ +/// files go into `out_dir`. The spill files of the sort go there too, and the code removes those +/// before it returns. pub fn revert_alignment( path: &Path, reference: Option<&Path>, @@ -161,8 +175,9 @@ pub fn revert_alignment( /// The container-independent core of [`revert_alignment`], over any source of records. /// -/// Split out so the pipeline can be tested on hand-built records without writing BAM fixtures — -/// the file-format layer is [`reader`]'s job and is already covered there. +/// It is separate so that a test can run the pipeline on records that somebody built by hand, and +/// write no BAM fixture. The layer that reads a file format is the job of [`reader`], and the +/// tests there already cover it. pub fn revert_records( records: impl Iterator>, out_dir: &Path, diff --git a/crates/navigator-analysis/src/revert/tests.rs b/crates/navigator-analysis/src/revert/tests.rs index 21dceb26..87408d16 100644 --- a/crates/navigator-analysis/src/revert/tests.rs +++ b/crates/navigator-analysis/src/revert/tests.rs @@ -1,10 +1,13 @@ //! Tests for the revert stage. //! -//! Records are built by hand rather than read from BAM fixtures: the container layer is -//! [`crate::reader`]'s responsibility and is covered there, and everything interesting here — flag -//! handling, orientation, pairing, the external sort's spill/merge boundary — is independent of -//! how the bytes arrived. Building records directly also lets a test construct the malformed -//! inputs that matter (contradictory flags, absent qualities) which a real aligner rarely emits. +//! These tests build their records by hand, and they do not read a BAM fixture. The container +//! layer belongs to [`crate::reader`], and the tests there cover it. Everything that matters here +//! is independent of how the bytes arrived. That is the flags, the orientation, how two mates +//! pair, and the spill and merge boundary of the external sort. +//! +//! A record that a test builds directly also lets that test make the inputs that are not correct, +//! and those are the ones that matter. A real aligner rarely gives flags that contradict +//! themselves, or a record with no qualities. use noodles::sam::alignment::record::cigar::op::{Kind, Op}; use noodles::sam::alignment::record::data::field::Tag; @@ -17,8 +20,8 @@ use super::*; const OQ: Tag = Tag::new(b'O', b'Q'); -/// Unique scratch dir per test, under the system temp dir (matching the convention elsewhere in -/// the crate — no `tempfile` dependency). +/// A scratch directory of its own for each test, under the temp directory of the system. That is +/// the convention elsewhere in this crate, and it needs no `tempfile` dependency. fn scratch(tag: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!("dun-revert-{}-{tag}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); @@ -26,7 +29,8 @@ fn scratch(tag: &str) -> PathBuf { dir } -/// A minimal record: name, flags, sequence, and qualities matching the sequence length. +/// A small record. It holds a name, the flags, a sequence, and qualities whose length matches +/// that sequence. fn record(name: &str, flags: u16, seq: &str, quals: &[u8]) -> RecordBuf { RecordBuf::builder() .set_name(name) @@ -50,8 +54,8 @@ fn run(records: Vec, dir: &Path, params: &RevertParams) -> RevertOutp fn read_lines(path: &Path) -> Vec { use std::io::BufRead; - // The reverted FASTQ is gzipped (see writer.rs on why), so tests read it the same way the - // mapper does rather than assuming plain text. + // The reverted FASTQ goes through gzip. writer.rs says why. So a test reads it the same way + // that the mapper does, and it does not expect plain text. crate::gzio::open_maybe_gz(path) .unwrap() .lines() @@ -61,8 +65,8 @@ fn read_lines(path: &Path) -> Vec { // ---- the transform -------------------------------------------------------- -/// The core correctness property: a reverse-strand alignment stores the read flipped, and the -/// FASTQ has to carry what the sequencer produced, not what the aligner stored. +/// The core property. A reverse-strand alignment stores the read the other way round. The FASTQ +/// must carry what the sequencer gave, and not what the aligner stored. #[test] fn a_reverse_strand_read_is_restored_to_sequencer_orientation() { let dir = scratch("revcomp"); @@ -81,8 +85,8 @@ fn a_reverse_strand_read_is_restored_to_sequencer_orientation() { ); } -/// A forward-strand read must be passed through untouched — the mirror of the test above, so a -/// bug that reverse-complements unconditionally cannot pass both. +/// A forward-strand read must go through unchanged. This is the mirror of the test above. A bug +/// that takes the reverse complement of every read can then not pass both. #[test] fn a_forward_strand_read_is_left_alone() { let dir = scratch("forward"); @@ -99,8 +103,9 @@ fn a_forward_strand_read_is_left_alone() { ); } -/// Secondary and supplementary records duplicate a read whose full sequence lives on the primary; -/// keeping them would emit the same read more than once and, for supplementaries, truncated. +/// A secondary record and a supplementary one each repeat a read whose full sequence lives on the +/// primary. To keep them would emit the same read more than once, and a supplementary one would +/// come out short. #[test] fn secondary_and_supplementary_records_are_dropped() { let dir = scratch("nonprimary"); @@ -119,8 +124,8 @@ fn secondary_and_supplementary_records_are_dropped() { assert_eq!(out.stats.reads_emitted, 1, "only the primary survives"); } -/// Unmapped reads are the realignment payoff — they must flow through, and be counted so the -/// payoff is measurable. +/// The reads with no mapping are the gain of the realignment. They must come through, and the code +/// must count them, so that somebody can measure the gain. #[test] fn unmapped_reads_are_kept_and_counted() { let dir = scratch("unmapped"); @@ -129,8 +134,9 @@ fn unmapped_reads_are_kept_and_counted() { assert_eq!(out.stats.reads_emitted, 1); } -/// `OQ` holds the qualities from before recalibration overwrote `QUAL`, ASCII-encoded. Preferring -/// it is the difference between reverting to the original read and reverting to a processed one. +/// `OQ` holds the qualities from before a recalibration wrote over `QUAL`, in ASCII. To take `OQ` +/// first is the difference between a revert to the original read and a revert to one that a +/// pipeline already changed. #[test] fn original_qualities_are_preferred_over_recalibrated_ones() { let dir = scratch("oq"); @@ -148,7 +154,7 @@ fn original_qualities_are_preferred_over_recalibrated_ones() { assert_eq!(read_lines(&out.singletons)[3], "IIII", "OQ won, decoded and re-encoded"); } -/// Opting out has to actually opt out, or the flag is decoration. +/// The option that turns this off must turn it off. Else the flag says nothing. #[test] fn original_qualities_can_be_declined() { let dir = scratch("oq-off"); @@ -169,8 +175,8 @@ fn original_qualities_can_be_declined() { assert_eq!(read_lines(&out.singletons)[3], "####", "QUAL 2 == '#', one per base"); } -/// A hard-clipped primary has already lost sequence. Skipping is the default because a dropped -/// read shows up in the stats and a truncated one does not. +/// A primary record with a hard clip has already lost sequence. To skip it is the default. A read +/// that the code drops shows in the statistics, and a read that comes out short does not. #[test] fn hard_clipped_primaries_are_skipped_by_default_and_emittable_on_request() { let rec = || { @@ -196,8 +202,8 @@ fn hard_clipped_primaries_are_skipped_by_default_and_emittable_on_request() { assert_eq!(emitted.stats.reads_emitted, 1); } -/// `QUAL` of `*` is legal. The read is still mappable, so it is kept — but the qualities that come -/// out are invented, and the stat is the only thing that says so. +/// A `QUAL` of `*` is legal. A mapper can still map the read, so the code keeps it. But the code +/// invents the qualities that come out, and the statistic is the one thing that says so. #[test] fn missing_qualities_are_synthesized_and_counted() { let dir = scratch("noqual"); @@ -207,9 +213,10 @@ fn missing_qualities_are_synthesized_and_counted() { assert_eq!(lines[3].len(), 4, "one quality per base, as FASTQ requires"); } -// ---- pairing -------------------------------------------------------------- +// ---- how two mates pair --------------------------------------------------- -/// The headline case: mates arrive far apart in coordinate order and must come back together. +/// The main case. In coordinate order the two mates arrive far apart, and they must come back +/// together. #[test] fn mates_separated_in_the_input_are_paired_in_the_output() { let dir = scratch("pairing"); @@ -232,7 +239,8 @@ fn mates_separated_in_the_input_are_paired_in_the_output() { let r1 = read_lines(&out.read1); let r2 = read_lines(&out.read2); - // Name order, and — the invariant the mapper depends on — R1 and R2 in lockstep. + // The names come in order. And, as the invariant that the mapper depends on says, R1 and R2 + // stay in step. assert_eq!([r1[0].as_str(), r1[4].as_str(), r1[8].as_str()], ["@a", "@b", "@c"]); assert_eq!(r1.len(), r2.len(), "files stay the same length"); for i in (0..r1.len()).step_by(4) { @@ -240,8 +248,8 @@ fn mates_separated_in_the_input_are_paired_in_the_output() { } } -/// A mate whose partner was dropped must not be written into `_1` — doing so would shift every -/// later pair by one and mis-pair the rest of the file. +/// A mate whose partner the code dropped must not go into `_1`. That would move every later pair +/// by one, and the rest of the file would hold the wrong pairs. #[test] fn a_read_whose_mate_was_dropped_becomes_a_singleton() { let dir = scratch("orphan"); @@ -262,7 +270,8 @@ fn a_read_whose_mate_was_dropped_becomes_a_singleton() { assert_eq!(read_lines(&out.singletons)[0], "@b"); } -/// Flags that claim "paired" but not which end cannot be placed in a synchronized file. +/// Flags that say "part of a pair", and that do not say which end, have no place in a file that +/// must stay in step. #[test] fn a_paired_record_with_contradictory_segment_flags_is_a_singleton() { let dir = scratch("contradictory"); @@ -276,8 +285,8 @@ fn a_paired_record_with_contradictory_segment_flags_is_a_singleton() { assert_eq!(out.stats.singletons, 1); } -/// Two records claiming the same name *and* the same end is unresolvable — picking one would -/// silently drop a read and could mis-pair the template. +/// Two records with the same name *and* the same end have no answer. To take one of them would +/// drop a read where nobody sees it, and it could put the wrong reads together. #[test] fn duplicate_segment_bits_under_one_name_do_not_pair() { let dir = scratch("dupe-segment"); @@ -296,9 +305,9 @@ fn duplicate_segment_bits_under_one_name_do_not_pair() { // ---- the external sort ---------------------------------------------------- -/// The property the whole design rests on: the result must not depend on whether the input fit in -/// memory. A budget of 1 byte forces a spill per read and exercises the k-way merge; the output -/// has to be identical to the single-run case. +/// The property that the whole design stands on: the result must not depend on whether the input +/// fit in memory. A budget of 1 byte makes the code spill at every read, and that covers the k-way +/// merge. The output must match the output of the one-run case exactly. #[test] fn spilling_to_disk_produces_the_same_output_as_sorting_in_memory() { let records = || { @@ -341,8 +350,9 @@ fn spilling_to_disk_produces_the_same_output_as_sorting_in_memory() { assert_eq!(read_lines(&spilled.read2), read_lines(&in_memory.read2)); } -/// Names must come out in sorted order after a merge across many runs — the grouping logic reads -/// runs of equal names, so an unsorted merge would silently split templates. +/// The names must come out in sorted order after a merge across many runs. The code that makes the +/// groups reads a run of equal names. A merge that does not sort would then split a template in +/// two, and nobody would see it. #[test] fn the_merge_emits_names_in_sorted_order() { let dir = scratch("sorted"); @@ -366,7 +376,8 @@ fn the_merge_emits_names_in_sorted_order() { assert_eq!(names.len(), 30); } -/// Scratch files are large — tens of GB for a WGS — so they must not outlive the run. +/// The scratch files are large, at tens of GB for a WGS, so none of them must stay after the run +/// ends. #[test] fn run_files_are_cleaned_up() { let dir = scratch("cleanup"); @@ -394,8 +405,8 @@ fn run_files_are_cleaned_up() { // ---- cancellation --------------------------------------------------------- -/// A revert is an hours-long job; an already-cancelled token must stop it rather than run to -/// completion, and must report itself as cancelled rather than as a failure. +/// A revert takes hours. A token that somebody already cancelled must stop it, and the job must +/// not run to its end. The job must also report itself as cancelled, and not as a failure. #[test] fn an_already_cancelled_token_stops_the_revert() { let dir = scratch("cancel"); diff --git a/crates/navigator-analysis/src/revert/transform.rs b/crates/navigator-analysis/src/revert/transform.rs index 27dfb274..d21275b5 100644 --- a/crates/navigator-analysis/src/revert/transform.rs +++ b/crates/navigator-analysis/src/revert/transform.rs @@ -1,10 +1,12 @@ -//! One alignment record → one original read, or a reason it was dropped. +//! One alignment record becomes one original read, or a reason that the code dropped it. //! -//! Everything an aligner did to a read has to be undone here, because the mapper downstream is -//! going to redo it against a different reference. The subtle one is orientation: SAM stores SEQ -//! and QUAL in *reference* orientation, so a read that mapped to the reverse strand is held -//! reverse-complemented relative to what the sequencer produced. Emitting that to FASTQ without -//! flipping it back would hand the mapper a read that never existed. +//! This code must undo everything that an aligner did to a read, because the mapper after it does +//! that work again, against a different reference. +//! +//! The orientation is the part to watch. SAM stores SEQ and QUAL in the orientation of the +//! *reference*. A read that mapped to the reverse strand then sits there as the reverse complement +//! of what the sequencer gave. To put that into a FASTQ, and not turn it back, would give the +//! mapper a read that never existed. use noodles::sam::alignment::record::cigar::op::Kind; use noodles::sam::alignment::record::data::field::Tag; @@ -12,18 +14,21 @@ use noodles::sam::alignment::RecordBuf; use super::{HardClipPolicy, RevertParams, RevertStats}; -/// `OQ` — the original base qualities, before any recalibration overwrote `QUAL`. +/// The `OQ` tag. It holds the original base qualities, from before a recalibration wrote over +/// `QUAL`. const OQ_TAG: Tag = Tag::new(b'O', b'Q'); -/// Phred used when a record carries no qualities at all. See [`RevertStats::qualities_synthesized`] -/// — the reads are worth keeping, but the qualities are invented and must be counted as such. +/// The phred value that the code uses when a record carries no qualities at all. See +/// [`RevertStats::qualities_synthesized`]. The reads are worth a place in the output, but the code +/// invents those qualities, and the count must say so. const SYNTHETIC_PHRED: u8 = 40; /// Which end of a template a read is, from the segment flags. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Mate { - /// Not part of a pair (`0x1` clear), or paired but with neither/both segment bits set — a - /// record whose flags contradict themselves cannot be placed in an R1/R2 file. + /// The record is not part of a pair, where `0x1` is clear. Or it is part of a pair, and it has + /// neither segment bit set, or both. The flags of such a record contradict themselves, so the + /// code can not put it into an R1 or R2 file. Unpaired, /// First segment (`0x40`). One, @@ -34,22 +39,25 @@ pub enum Mate { /// Why a record produced no read. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Skipped { - /// Secondary alignment (`0x100`) — the read's full sequence lives on its primary record. + /// A secondary alignment, at flag `0x100`. The full sequence of the read lives on its primary + /// record. Secondary, - /// Supplementary alignment (`0x800`) — likewise, and usually hard-clipped besides. + /// A supplementary alignment, at flag `0x800`. The same holds, and it usually carries a hard + /// clip as well. Supplementary, /// `SEQ` was `*`; there is no read here to recover. NoSequence, /// A hard-clipped primary under [`HardClipPolicy::Skip`]. HardClipped, - /// No read name, so the record can never be paired with anything. + /// The record has no read name, so nothing can ever pair with it. NoName, } -/// An original, unaligned read: exactly what the sequencer emitted, as far as the record allows. +/// An original, unaligned read. It is exactly what the sequencer gave, as far as the record lets +/// the code recover. /// -/// Ordered by `(name, mate)` so a sort brings mates together with R1 ahead of R2 — which is the -/// entire mechanism [`super::collate`] relies on. +/// The order is by `(name, mate)`. A sort then brings two mates together, with the R1 before the +/// R2. That is the whole mechanism that [`super::collate`] stands on. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct RevertedRead { pub name: Vec, @@ -64,10 +72,12 @@ impl RevertedRead { /// Rough heap footprint, for the sort's memory budget. The two `Vec`s dominate; the constant /// covers the struct itself and allocator overhead closely enough to size a buffer by. pub fn heap_bytes(&self) -> usize { - // The read sits inline in the collator's `Vec`, so its own size counts; the three vectors - // each cost an allocation on top of their contents. The flat 64 this replaced was smaller - // than the struct alone, which was harmless against a fixed 256 MB budget and is not - // against a budget sized from the machine (see `navigator_resource::spill_budget`). + // The read sits inside the `Vec` of the collator, so its own size counts. The three + // vectors each cost an allocation on top of what they hold. + // + // The flat 64 before this was smaller than the struct alone. Against a fixed budget of + // 256 MB that did no harm. Against a budget that comes from the machine it does. See + // `navigator_resource::spill_budget`. std::mem::size_of::() + self.name.len() + self.sequence.len() @@ -76,24 +86,25 @@ impl RevertedRead { } } -/// Record the reason a record was dropped. +/// Note down the reason that the code dropped a record. pub fn count_skip(skipped: Skipped, stats: &mut RevertStats) { match skipped { Skipped::Secondary => stats.secondary_dropped += 1, Skipped::Supplementary => stats.supplementary_dropped += 1, Skipped::NoSequence | Skipped::NoName => stats.no_sequence_dropped += 1, - // Already counted by `revert_record` before it consulted the policy — the stat means - // "records affected by hard clipping" under both policies, so counting it again here - // would double it on the skip path only. + // `revert_record` already counted this, before it read the policy. The statistic means + // "the records that a hard clip affected", under either policy. To count it again here + // would make it twice too large, and only on the skip path. Skipped::HardClipped => {} } } -/// Strip a record back to the read it was made from. +/// Strip a record back to the read that made it. /// -/// Alignment state (position, CIGAR, MAPQ, mate fields, aligner tags like `NM`/`MD`/`AS`) needs no -/// explicit clearing: none of it is carried into [`RevertedRead`], so it is dropped by -/// construction rather than by a list of fields someone has to remember to extend. +/// The alignment state needs no explicit removal. That state is the position, the CIGAR, the MAPQ, +/// the mate fields, and an aligner tag such as `NM`, `MD` or `AS`. None of it goes into a +/// [`RevertedRead`], so the construction itself drops all of it. There is no list of fields that +/// somebody must remember to extend. pub fn revert_record( record: &RecordBuf, params: &RevertParams, @@ -101,8 +112,8 @@ pub fn revert_record( ) -> Result { let flags = record.flags(); - // Primaries only. Both of these carry a partial view of a read whose whole sequence is on - // another record, so keeping them would duplicate reads and truncate some of them. + // Primary records alone. The other two kinds each hold part of a read whose whole sequence + // sits on another record. To keep them would put some reads in twice, and cut others short. if flags.is_secondary() { return Err(Skipped::Secondary); } @@ -116,14 +127,16 @@ pub fn revert_record( return Err(Skipped::NoSequence); } - // A hard clip on a *primary* means sequence was thrown away before we ever saw the record — - // rare, but real, and undetectable downstream once it reaches a FASTQ. + // A hard clip on a *primary* record means that somebody threw sequence away before this code + // saw the record. That is rare, and it is real. Once such a read reaches a FASTQ, no later + // step can find it. let hard_clipped = record.cigar().as_ref().iter().any(|op| op.kind() == Kind::HardClip); if hard_clipped { stats.hard_clipped += 1; if params.hard_clipped == HardClipPolicy::Skip { - // Counted here rather than through `count_skip`, so the stat means "records affected - // by hard clipping" under either policy instead of changing definition with the flag. + // The count goes here, and not through `count_skip`. The statistic then says "the + // records that a hard clip affected" under either policy, and the flag does not move + // what it says. return Err(Skipped::HardClipped); } } @@ -137,8 +150,9 @@ pub fn revert_record( stats.qualities_synthesized += 1; } - // A record whose QUAL disagrees with SEQ is malformed; trust SEQ (the mapper needs one - // quality per base) and pad or truncate rather than emitting an unwritable FASTQ record. + // A record whose QUAL does not agree with SEQ is not correct. Trust SEQ, because the mapper + // needs one quality at each base. Then add to the qualities, or cut them, and do not emit a + // FASTQ record that no writer can write. if qualities.len() != sequence.len() { qualities.resize(sequence.len(), SYNTHETIC_PHRED); } @@ -161,8 +175,9 @@ pub fn revert_record( }) } -/// The `OQ` tag's qualities, if present and wanted. `OQ` is stored as an ASCII phred+33 string — -/// the same encoding FASTQ uses — so it is decoded back to raw phred here to match `QUAL`. +/// The qualities of the `OQ` tag, when that tag is present and the caller wants it. `OQ` holds an +/// ASCII phred+33 string, which is the same encoding that FASTQ uses. So the code decodes it back +/// to a raw phred here, to match `QUAL`. fn original_qualities(record: &RecordBuf, params: &RevertParams, stats: &mut RevertStats) -> Option> { use noodles::sam::alignment::record_buf::data::field::Value; @@ -186,14 +201,16 @@ fn mate_of(flags: noodles::sam::alignment::record::Flags) -> Mate { match (flags.is_first_segment(), flags.is_last_segment()) { (true, false) => Mate::One, (false, true) => Mate::Two, - // Both or neither: the record claims to be paired but won't say which end. It cannot go in - // a synchronized R1/R2 file, so it becomes a singleton rather than corrupting the pairing. + // Both bits, or neither. The record says that it is part of a pair, and it does not say + // which end. It can not go into an R1 or R2 file that must stay in step. So it becomes a + // singleton, and it does not put the wrong reads together. _ => Mate::Unpaired, } } -/// Reverse-complement in place. `N` and any other non-ACGT byte is passed through unchanged rather -/// than normalized — the read is meant to come out of here exactly as the sequencer produced it. +/// Take the reverse complement in place. An `N`, and any other byte that is not ACGT, goes through +/// unchanged. The code does not normalize it, because the read must come out of here exactly as +/// the sequencer gave it. fn reverse_complement(seq: &mut [u8]) { seq.reverse(); for b in seq.iter_mut() { diff --git a/crates/navigator-analysis/src/revert/writer.rs b/crates/navigator-analysis/src/revert/writer.rs index 182b328b..8a22eeec 100644 --- a/crates/navigator-analysis/src/revert/writer.rs +++ b/crates/navigator-analysis/src/revert/writer.rs @@ -1,11 +1,15 @@ //! Emit collated reads as paired FASTQ. //! -//! The invariant the mapper depends on: `_1.fastq` and `_2.fastq` must stay in lockstep, record -//! for record. A template only reaches those files if it has exactly one R1 and exactly one R2; -//! anything else — an unpaired library, a mate lost to hard clipping, flags that don't say which -//! end a read is — goes to singletons. Silently writing an unmatched read into `_1` would shift -//! every later pair by one and mis-pair the entire rest of the file, so the check is per-template -//! rather than a trailing reconciliation. +//! Here is the invariant that the mapper depends on. `_1.fastq` and `_2.fastq` must stay in step, +//! record for record. A template reaches those two files only when it holds exactly one R1 and +//! exactly one R2. +//! +//! Everything else goes to the singletons file. That covers a library with no pairs, a mate that +//! a hard clip removed, and flags that do not say which end a read is. +//! +//! To write a read with no partner into `_1` would move every later pair by one. The whole rest of +//! the file would then hold the wrong pairs, and nobody would see it happen. So the check runs at +//! each template, and not as a reconciliation at the end. use std::fs::File; use std::io::{BufWriter, Write}; @@ -20,23 +24,25 @@ use super::RevertStats; use crate::cancel::CancelToken; use crate::error::AnalysisError; -/// Output buffer per FASTQ file. FASTQ is small records and many of them, so the write path is -/// syscall-bound without a generous buffer. +/// The size of the output buffer of each FASTQ file. FASTQ holds many small records, so without a +/// large buffer the syscalls control the time of the write path. const FASTQ_BUFFER: usize = 1024 * 1024; -/// The reverted reads are written **gzipped**, and that is not an optimisation. +/// The reverted reads go out **through gzip**, and that is not an improvement for its own sake. /// -/// A 30x WGS reverted to plain FASTQ is on the order of 200 GB — larger than the free space on a -/// normal machine, and several times the source alignment it came from, because FASTQ stores one -/// ASCII quality byte per base with none of BAM's packing. Compressed it is nearer 55 GB, which is -/// the difference between the pipeline running and filling the disk in its second stage. +/// A 30x WGS reverted to plain FASTQ is about 200 GB. That is more than the free space on a usual +/// machine, and some times the size of the alignment that it came from. FASTQ stores one ASCII +/// quality byte at each base, and a BAM packs its data where FASTQ does not. Through gzip the same +/// data is nearer to 55 GB. That is the difference between a pipeline that runs and one that fills +/// the disk in its second stage. /// -/// `fast` rather than `default`: this is a temporary file consumed once by the mapper, so trading -/// a few percent of ratio for materially less CPU in a job already measured in hours is the right -/// side of that curve. minimap2 detects gzip from the magic bytes, so nothing downstream changes. +/// The level is `fast`, and not `default`. This is a temporary file, and the mapper reads it once. +/// A few percent of the ratio buys much less CPU, in a job that already takes hours. That is the +/// correct side of the curve. The mapper finds gzip from the first bytes of the file, so +/// nothing after this changes. const FASTQ_COMPRESSION: Compression = Compression::fast(); -/// A gzip-compressing FASTQ sink. +/// A FASTQ sink that puts its output through gzip. type FastqWriter = GzEncoder>; /// Phred offset for FASTQ's ASCII quality encoding (Sanger / Illumina 1.8+). @@ -63,8 +69,9 @@ pub fn write_fastq( let mut ws = open(&ps)?; let mut group: Vec = Vec::new(); - // Reused across every record: the ASCII quality line is the only part that needs building - // rather than copying, and allocating one per read would dominate the write path. + // Every record uses this again. The ASCII quality line is the one part that the code must + // build, and not copy. To allocate one at each read would control the time of the write + // path. let mut qual_scratch: Vec = Vec::new(); let mut templates = 0u64; @@ -98,9 +105,12 @@ pub fn write_fastq( Ok((p1, p2, ps)) } -/// The indices of the R1 and R2 of a complete pair, or `None` if this template is not exactly one -/// of each. Duplicated segment bits (two R1s under one name) are treated as unpaired: which of the -/// two is "the" R1 is unanswerable, and guessing would silently corrupt the pairing. +/// The indices of the R1 and the R2 of a complete pair, or `None` when this template does not hold +/// exactly one of each. +/// +/// A template with the same segment bit twice, which is two R1 records under one name, counts as +/// unpaired. Nobody can say which of the two is "the" R1. A guess would put the wrong reads +/// together, and nobody would see it happen. fn pair_of(group: &[RevertedRead]) -> Option<(usize, usize)> { let mut one = None; let mut two = None; @@ -114,7 +124,7 @@ fn pair_of(group: &[RevertedRead]) -> Option<(usize, usize)> { } } match (one, two) { - // Exactly one of each, and nothing else tagging along under the same name. + // Exactly one of each, and nothing else under the same name. (Some(a), Some(b)) if group.len() == 2 => Some((a, b)), _ => None, } @@ -128,24 +138,28 @@ fn open(path: &Path) -> Result { )) } -/// Finish the gzip stream, not merely flush it. A gzip member without its trailer is truncated, -/// and a reader would stop early rather than error — the same class of bug as a missing BGZF EOF -/// block, and just as quiet. +/// Finish the gzip stream. Do not only flush it. A gzip member with no trailer is a file that +/// stops early. A reader would then stop early too, and it would give no error. That is the same +/// class of bug as a missing BGZF EOF block, and it is as hard to see. fn finish(w: FastqWriter, path: &Path) -> Result<(), AnalysisError> { let mut inner = w.finish().map_err(|e| AnalysisError::io(path, e))?; inner.flush().map_err(|e| AnalysisError::io(path, e)) } -/// One FASTQ record. Names are written bare — no `/1` or `/2` — so R1/R2 pair by position; see the -/// module docs on why. Qualities are shifted into ASCII here, the inverse of the decode in -/// [`super::transform`], through `scratch` so the shift costs no allocation per read. +/// One FASTQ record. The names go out bare, with no `/1` and no `/2`, so an R1 and an R2 pair by +/// their position. The module documentation says why. The code shifts the qualities into ASCII +/// here, which is the inverse of the decode in [`super::transform`]. It does that through +/// `scratch`, so the shift allocates nothing at each read. +/// +/// The code builds the whole record in `scratch`, and it gives that to **one** `write_all`. There +/// were seven of those: `@`, the name, a newline, the sequence, `\n+\n`, the qualities, and a +/// newline. Each one went into the state machine of the gzip encoder on its own. Each read then +/// paid that cost seven times, and not once. /// -/// The whole record is assembled in `scratch` and handed over in **one** `write_all`. It was seven -/// — `@`, name, newline, sequence, `\n+\n`, qualities, newline — and each one entered the gzip -/// encoder's state machine separately, paying that overhead seven times per read rather than once. -/// Measured on this exact stack at 151 bp: **1,882 ns/record against 539 ns**, a 3.5x difference on -/// the write path of the stage that already holds the scratch peak. At ~600 M reads for a 30x WGS -/// that is roughly thirteen minutes of single-threaded CPU per realignment. Identical bytes out. +/// A measurement on this exact stack, at 151 bp, gave **1,882 ns for each record, against +/// 539 ns**. That is a difference of 3.5x, on the write path of the stage that already holds the +/// peak of the scratch space. At about 600M reads for a 30x WGS, that is about thirteen minutes of +/// CPU on one thread, for each realignment. The bytes that come out are the same. fn write_record( w: &mut FastqWriter, read: &RevertedRead, diff --git a/crates/navigator-analysis/src/roh.rs b/crates/navigator-analysis/src/roh.rs index c8e36b2f..82873ef4 100644 --- a/crates/navigator-analysis/src/roh.rs +++ b/crates/navigator-analysis/src/roh.rs @@ -1,23 +1,32 @@ -//! Runs of homozygosity (ROH) / autozygosity detection — the endogamy & consanguinity signal. +//! Detection of runs of homozygosity (ROH), which is autozygosity. That is the signal of endogamy +//! and of consanguinity. //! -//! **Spike (2026-07-22).** A 2-state hidden Markov model (Autozygous / Normal) over a subject's -//! autosomal genotypes, mirroring the [`crate::ancestry::paint_local_ancestry`] HMM idiom -//! (per-contig sorted sites, distance-scaled "reset-to-prior" transitions, log-space Viterbi + -//! forward/backward posteriors). Runs of the Autozygous state are stitched into [`RohSegment`]s and -//! rolled up into an [`RohSummary`] with the genome-wide inbreeding coefficient F_ROH. +//! **This is a spike, from 2026-07-22.** It is a hidden Markov model with two states, Autozygous +//! and Normal, over the autosomal genotypes of a subject. //! -//! **Input substrate.** Feed the subject's autosomal-consensus genotypes -//! (`consensus_genotypes(&DiploidProfile)` in `navigator-app`), which are called at the full 1240k -//! IBD panel — a dense (~1.15M), neutral, biallelic common-SNP set with full 0/1/2 dosages. That is -//! the density class array-based ROH tools (PLINK, BCFtools/RoH, detectRUNS) assume. Segment cM -//! lengths and the F_ROH denominator come from the same [`GeneticMap`] the IBD path already loads. +//! It has the same idiom as the HMM in [`crate::ancestry::paint_local_ancestry`]. That means +//! sorted sites in each contig, and transitions that scale with distance and reset to the prior. +//! It also means a Viterbi and a forward-backward, both in log space. //! -//! **What's deliberately simplified in the spike** (see the module tests + the follow-up notes): -//! - The Normal-state heterozygosity expectation is a single `baseline_het` knob. A production -//! version should derive it per-site from panel allele frequencies (2·f·(1−f)), which -//! `AncestryPanel`/`IbdPanel` already carry, so the emission is properly frequency-aware. -//! - The endogamy-vs-consanguinity [`RohPattern`] classification is a heuristic on the ROH -//! length-class distribution, not a calibrated model. +//! The code joins the runs of the Autozygous state into [`RohSegment`] values. It then rolls those +//! up into an [`RohSummary`], with the genome-wide coefficient F_ROH. +//! +//! **What to give it.** Give it the autosomal consensus genotypes of the subject, from +//! `consensus_genotypes(&DiploidProfile)` in `navigator-app`. The caller calls those at the full +//! 1240k IBD panel, which is a dense set of about 1.15M neutral, biallelic, common SNPs, with full +//! 0/1/2 dosages. That is the density class that a ROH tool for arrays needs, such as PLINK, +//! BCFtools/RoH or detectRUNS. The cM length of a segment, and the denominator of F_ROH, come from +//! the same [`GeneticMap`] that the IBD path already loads. +//! +//! **What this spike leaves simple, deliberately.** See the module tests and the notes that follow +//! them. +//! +//! - One `baseline_het` control holds the heterozygosity that the Normal state expects. A +//! production version must instead derive it at each site from the panel allele frequencies, as +//! 2·f·(1−f). `AncestryPanel` and `IbdPanel` already carry those, so the emission can know the +//! frequency. +//! - The [`RohPattern`] that separates endogamy from consanguinity is a heuristic over the +//! distribution of the ROH length classes. It is not a calibrated model. use crate::caller::SiteGenotype; use crate::ibd::{normalize_chromosome, GeneticMap}; @@ -26,23 +35,30 @@ use std::collections::BTreeMap; /// Detector configuration. Defaults target a 1240k-density common-SNP substrate. #[derive(Debug, Clone, Copy, PartialEq)] pub struct RohConfig { - /// P(heterozygous call | truly autozygous) — i.e. the residual genotyping-error rate that lets a - /// stray het survive inside a homozygous run without breaking it. Small. + /// P(a heterozygous call | the site is truly autozygous). That is the rate of genotype error + /// that stays. It lets one stray het live inside a homozygous run, and that run does not + /// break. The value is small. pub het_error: f64, - /// Normal-state expected heterozygosity P(het | not autozygous). `None` (the default) estimates it - /// from the sample's own autosomal het fraction (clamped), so it tracks the panel's density and - /// ascertainment instead of a fixed guess. A production upgrade is per-site 2·f·(1−f) from allele - /// frequencies; `Some(v)` pins it (mainly for tests / advanced tuning). + /// The heterozygosity that the Normal state expects: P(het | the site is not autozygous). + /// + /// `None` is the default, and it estimates the value from the autosomal het fraction of the + /// sample itself, clamped. It then follows the density of the panel, and its ascertainment, + /// and it is not a fixed guess. The production upgrade is 2·f·(1−f) at each site, from the + /// allele frequencies. `Some(v)` fixes the value, mostly for a test, or for somebody who wants + /// to tune it. pub baseline_het: Option, - /// State-switch hazard per centimorgan. Switch probability over a gap of `d` cM is - /// `1 − exp(−d · switch_rate_per_cm)`. Smaller → longer runs. Default ≈ one switch per ~13 cM. + /// The hazard of a state switch in one centimorgan. The switch probability over a gap of `d` + /// cM is `1 − exp(−d · switch_rate_per_cm)`. A smaller value gives longer runs. The default is + /// about one switch in 13 cM. pub switch_rate_per_cm: f64, - /// Stationary autozygosity fraction — the prior mass on the Autozygous state that a switch - /// resets toward. The classic ROH HMM prior. + /// The stationary autozygosity fraction. It is the prior mass on the Autozygous state, and a + /// switch resets toward it. This is the classic prior of a ROH HMM. pub prior_autozygous: f64, - /// Report runs at least this long in **physical Mb**. PLINK/detectRUNS and the genealogy field - /// threshold ROH on physical length because ROH cluster in low-recombination (pericentromeric) - /// regions where a multi-Mb run spans well under a cM — a genetic (cM) threshold under-reports them. + /// Report a run of this length or more, in **physical Mb**. + /// + /// PLINK, detectRUNS and the genealogy field all threshold a ROH on its physical length. ROH + /// gather in regions of low recombination, near a centromere. There a run of some Mb covers + /// well under one cM, so a threshold in cM reports too few of them. pub min_length_mb: f64, /// Report runs with at least this many genotyped sites (guards sparse-coverage false runs). pub min_sites: usize, @@ -71,26 +87,29 @@ pub struct RohSegment { /// Genetic length from the genetic map (cM); falls back to a 1 cM/Mb estimate if the map lacks /// the chromosome. pub length_cm: f64, - /// Physical span in Mb — the length the report threshold (`min_length_mb`) applies to. + /// The physical span, in Mb. The report threshold `min_length_mb` applies to this length. pub length_mb: f64, /// Number of genotyped sites inside the run. pub n_sites: usize, /// Heterozygous calls inside the run (should be near zero for a clean run). pub n_het: usize, - /// Mean Autozygous posterior over the run's sites (forward/backward) — a confidence in [0,1]. + /// The mean Autozygous posterior over the sites of the run, from the forward-backward pass. + /// It is a confidence in [0,1]. pub mean_posterior: f64, } -/// ROH length classes (physical Mb), used for the endogamy-vs-consanguinity read. Short ROH reflect -/// distant/background relatedness (endogamy); long ROH reflect recent shared ancestry (consanguinity), -/// because longer haplotypes have had fewer generations of recombination to break them up. +/// The length classes of a ROH, in physical Mb. They separate endogamy from consanguinity. +/// +/// A short ROH shows distant, background relatedness, which is endogamy. A long ROH shows recent +/// shared ancestry, which is consanguinity. A longer haplotype has had fewer generations of +/// recombination to break it up. #[derive(Debug, Clone, Copy, PartialEq)] pub enum RohClass { - /// < 5 Mb — deep/background (endogamy). + /// Below 5 Mb. This is deep, background relatedness, which is endogamy. Short, - /// 5–15 Mb — intermediate. + /// From 5 to 15 Mb. This is the middle class. Medium, - /// ≥ 15 Mb — recent (consanguinity). + /// 15 Mb or more. This is recent, and it is consanguinity. Long, } @@ -106,24 +125,31 @@ impl RohClass { } } -/// Coarse pattern read from the ROH length distribution. Heuristic — for narration, not diagnosis. -/// Defined in `navigator-domain` so the Simple-mode brief can consume the verdict [`classify`] -/// reaches here rather than re-deriving one from the raw numbers. +/// A coarse pattern that the code reads from the distribution of the ROH lengths. It is a +/// heuristic. Use it for narration, and not for a diagnosis. +/// +/// It lives in `navigator-domain`, so that the Simple-mode brief can read the answer that +/// [`classify`] reaches here. That brief does not have to derive its own answer from the raw +/// numbers. pub use navigator_domain::roh::RohPattern; -/// Genome-wide rollup. Lengths are **physical Mb** — the canonical (McQuillan) F_ROH is a physical -/// ratio, and it stays consistent with the physical `min_length_mb` run filter (a genetic/cM F_ROH -/// would badly under-count ROH that sit in low-recombination regions). +/// The rollup over the whole genome. Every length is in **physical Mb**. +/// +/// The canonical F_ROH, from McQuillan, is a physical ratio. That also keeps it consistent with +/// `min_length_mb`, which filters the runs by physical length. An F_ROH in cM would count far too +/// few of the ROH that sit in a region of low recombination. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct RohSummary { pub n_segments: usize, pub total_roh_mb: f64, - /// Autosomal physical length covered by the input sites (Mb) — the F_ROH denominator. + /// The autosomal physical length that the input sites cover, in Mb. It is the denominator of + /// F_ROH. pub autosomal_mb: f64, - /// Inbreeding coefficient F_ROH = total ROH length / total autosomal length (both Mb). + /// The coefficient F_ROH. It is the total ROH length divided by the total autosomal length, + /// and both are in Mb. pub f_roh: f64, pub longest_mb: f64, - /// (count, summed Mb) per length class. + /// A (count, total Mb) pair for each length class. pub short: (usize, f64), pub medium: (usize, f64), pub long: (usize, f64), @@ -137,14 +163,16 @@ pub struct RohResult { pub summary: RohSummary, } -/// True for autosomes 1–22 (ROH is computed on autosomes only; X/Y/MT excluded). +/// True for the autosomes 1 to 22. The code computes a ROH on an autosome alone, and it leaves X, +/// Y and MT out. fn is_autosome(contig: &str) -> bool { matches!(normalize_chromosome(contig).parse::(), Ok(1..=22)) } /// Detect runs of homozygosity across the autosomes. pub fn detect_roh(genotypes: &[SiteGenotype], gmap: &GeneticMap, config: &RohConfig) -> RohResult { - // Per-autosome sorted (pos, is_het) over *called* sites (dosage 0/1/2). + // The sorted (pos, is_het) pairs of each autosome, over the sites with a *call*, at dosage 0, + // 1 or 2. let mut by_chr: BTreeMap> = BTreeMap::new(); let (mut called, mut het) = (0u64, 0u64); for g in genotypes { @@ -215,7 +243,7 @@ fn call_chromosome( let pi = [1.0 - cfg.prior_autozygous, cfg.prior_autozygous]; let ln_pi = [ln(pi[0]), ln(pi[1])]; - // Per-site emission log-likelihoods for each state: [normal, auto]. + // The emission log-likelihood at each site, for each state: [normal, auto]. let emit = |is_het: bool| -> [f64; 2] { if is_het { [ln(baseline), ln(cfg.het_error)] @@ -258,7 +286,7 @@ fn call_chromosome( path[t] = back[t + 1][path[t + 1]]; } - // ---- Forward/backward posteriors (for per-run confidence) ---- + // ---- The forward-backward posteriors, which give the confidence of each run ---- let posterior = forward_backward(chr, sites, gmap, cfg, baseline, &ln_pi); // ---- Stitch Autozygous runs ---- @@ -291,7 +319,7 @@ fn call_chromosome( runs } -/// Autozygous-state posterior per site via scaled forward/backward. +/// The posterior of the Autozygous state at each site, from a scaled forward-backward pass. fn forward_backward( chr: &str, sites: &[(i64, bool)], @@ -410,9 +438,11 @@ fn summarize(segments: &[RohSegment], autosomal_mb: f64) -> RohSummary { } } -/// Heuristic pattern read (illustrative — not calibrated). Keyed on the normalized F_ROH so it is -/// independent of how much genome was analyzed; the length-class split then separates recent -/// consanguinity (long-dominated) from endogamy (short-dominated). +/// The heuristic read of the pattern. It is an illustration, and nobody calibrated it. +/// +/// Its key is the normalized F_ROH, so it does not depend on how much of the genome the run +/// covered. The split by length class then separates recent consanguinity, where the long ROH +/// dominate, from endogamy, where the short ones do. fn classify(f_roh: f64, total: f64, short: &(usize, f64), long: &(usize, f64)) -> RohPattern { // Below ~F_ROH 0.02 (roughly a notable-relatedness floor) the sample reads as outbred. if f_roh < 0.02 { @@ -457,7 +487,8 @@ mod tests { } } - /// Sites every `step` bp from 0..count, all homozygous → one ROH spanning the chromosome. + /// A site at every `step` bp, from 0 to count, and all of them homozygous. That gives one ROH + /// over the whole chromosome. #[test] fn all_homozygous_is_one_long_roh() { let step = 20_000i64; @@ -479,7 +510,8 @@ mod tests { fn heterozygous_rich_has_no_roh() { let step = 20_000i64; let count = 1000; - // Every 3rd site het — dense enough to keep the HMM in the Normal state throughout. + // Every third site is het. That is dense enough to hold the HMM in the Normal state all + // the way. let genos: Vec<_> = (0..count).map(|i| site(i * step, i % 3 == 0)).collect(); let gmap = map_chr1((count * step) as i32); let res = detect_roh(&genos, &gmap, &RohConfig::default()); @@ -512,7 +544,7 @@ mod tests { ); } - /// A short homozygous run below `min_length_mb` is filtered out. + /// The filter removes a short homozygous run that is below `min_length_mb`. #[test] fn short_run_below_min_length_is_dropped() { let step = 20_000i64; diff --git a/crates/navigator-analysis/src/scan.rs b/crates/navigator-analysis/src/scan.rs index 2240c304..e1d15e47 100644 --- a/crates/navigator-analysis/src/scan.rs +++ b/crates/navigator-analysis/src/scan.rs @@ -1,11 +1,13 @@ -//! Project-directory scanner (port of the Scala `ProjectDirectoryScanner`). NAS layout: -//! `{projectRoot}/{sampleId}/files…` — each immediate subdirectory is one sample, and the -//! files within are classified by role. The app turns the result into Project → Biosample -//! → SequenceRun → Alignment rows. +//! The scanner over a project directory. It is the port of the Scala `ProjectDirectoryScanner`. //! -//! Pure filesystem classification: no DB, no noodles. Only alignment/index/variant files -//! drive import this slice; `coverage.txt`/`stats.txt`/`*.dragstr.model` are recognized -//! but not consumed (coverage is recomputed from the alignment). +//! The layout on the NAS is `{projectRoot}/{sampleId}/files…`. Each subdirectory of the root is +//! one sample, and the code puts the files inside it into classes by role. The app turns the +//! result into rows: a Project, a Biosample, a SequenceRun and an Alignment. +//! +//! This module classifies files and nothing more. It uses no database and no noodles. In this +//! slice, only an alignment file, an index file and a variant file drive an import. The code +//! recognizes `coverage.txt`, `stats.txt` and `*.dragstr.model`, and it reads none of them, +//! because it computes the coverage again from the alignment. use std::fs; use std::path::{Path, PathBuf}; @@ -21,7 +23,8 @@ pub enum DiscoveredFileType { Index, /// `.vcf` / `.vcf.gz` / `.g.vcf.gz` / `.gvcf.gz`. Variant, - /// A `coverage.txt` (precomputed; ignored — coverage is recomputed). + /// A `coverage.txt`, which somebody computed before. The code ignores it, because it computes + /// the coverage again. Coverage, /// A `stats.txt` (precomputed; ignored). Stats, @@ -31,8 +34,9 @@ pub enum DiscoveredFileType { Other, } -/// Classify a file by its (case-insensitive) name. Multi-part extensions are checked -/// first so `.g.vcf.gz` is a Variant, not matched by a bare `.gz`. +/// Put a file into a class, by its name. The case does not matter. The code checks an extension of +/// more than one part first, so `.g.vcf.gz` gives a Variant, and a bare `.gz` does not match it +/// first. pub fn classify(name: &str) -> DiscoveredFileType { let lower = name.to_ascii_lowercase(); const VARIANT: [&str; 4] = [".g.vcf.gz", ".gvcf.gz", ".vcf.gz", ".vcf"]; @@ -62,37 +66,42 @@ pub struct DiscoveredFile { pub kind: DiscoveredFileType, } -/// The `ytree` pipeline's per-sample sidecars, matched by name suffix. Present only when the -/// sample was processed by that workflow; absent for a plain alignment-only directory. The -/// app's fast-path ingest reads these instead of walking the CRAM. +/// The sidecars that the `ytree` pipeline writes for each sample. The code matches them by the end +/// of the name. They are there only when that workflow ran on the sample, and absent from a +/// directory that holds an alignment alone. The fast-path ingest of the app reads these, and it +/// does not walk the CRAM. /// -/// Serializable so the app can record which files an alignment was ingested from. Discovery is a -/// directory scan performed once at import; without a record of the result, re-running the fast -/// path later — to re-place a haplogroup against a newer tree — would have nothing to run against. +/// This type serializes, so the app can record which files an alignment came from. The code finds +/// the sidecars in a directory scan, and it runs that scan once, at the import. Without a record +/// of the result, a later run of the fast path would have nothing to run against. Such a run +/// places a haplogroup again, against a newer tree. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] pub struct SampleSidecars { - /// `*.chrY.g.vcf.gz` — ploidy-1 chrY GVCF (males). + /// `*.chrY.g.vcf.gz`: the chrY GVCF at ploidy 1, for a male. pub chr_y_gvcf: Option, - /// `*.chrM.g.vcf.gz` — ploidy-1 chrM GVCF. + /// `*.chrM.g.vcf.gz`: the chrM GVCF at ploidy 1. pub chr_m_gvcf: Option, - /// `*.callable.bed` — CallableLoci track. + /// `*.callable.bed`: the CallableLoci track. pub callable_bed: Option, - /// `*.callable.summary.txt` — per-state base counts. + /// `*.callable.summary.txt`: the count of bases in each state. pub callable_summary: Option, - /// `*.sex` — `male` / `female`. + /// `*.sex`: it holds `male` or `female`. pub sex: Option, - /// `coverage.txt` — samtools coverage. + /// `coverage.txt`: the output of samtools coverage. pub coverage: Option, - /// `stats.txt` — samtools stats. + /// `stats.txt`: the output of samtools stats. pub stats: Option, - /// `*.flagstat[.txt]` — samtools flagstat (an alternative read-metrics source). + /// `*.flagstat[.txt]`: the output of samtools flagstat. It is another source of read + /// metrics. pub flagstat: Option, - /// Picard `CollectWgsMetrics` output (`*wgs*metric*`) — the genome-wide depth distribution. + /// The output of Picard `CollectWgsMetrics`, at `*wgs*metric*`. It holds the depth + /// distribution over the whole genome. pub wgs_metrics: Option, /// Picard `CollectAlignmentSummaryMetrics` (`*alignment_summary*`). pub alignment_summary: Option, - /// Build token parsed from the GVCF name (e.g. `chm13`), for confirming the GVCF and the - /// alignment share a build before the liftover-free fast path is taken. + /// The build token that the code read out of the GVCF name, such as `chm13`. Use it to check + /// that the GVCF and the alignment sit on the same build. That check comes before the fast + /// path, which needs no liftover. pub build_hint: Option, } @@ -103,9 +112,9 @@ impl SampleSidecars { } } -/// Detect pipeline sidecars among a sample's files by (case-insensitive) name. Specific -/// multi-part suffixes are matched against the full file name, so `*.chrY.g.vcf.gz.tbi` -/// (an index) does not match `*.chrY.g.vcf.gz`. +/// Find the pipeline sidecars among the files of a sample, by name. The case does not matter. The +/// code matches an exact suffix of more than one part against the whole file name. So +/// `*.chrY.g.vcf.gz.tbi`, which is an index, does not match `*.chrY.g.vcf.gz`. fn detect_sidecars(files: &[DiscoveredFile]) -> SampleSidecars { let by_suffix = |suffix: &str| { files @@ -129,8 +138,8 @@ fn detect_sidecars(files: &[DiscoveredFile]) -> SampleSidecars { }) .map(|f| f.path.clone()) }; - // Picard/flagstat outputs have no fixed name — match a substring of the (lower-cased) filename - // (mirrors the Scala scanner's loose patterns). + // The output of Picard, and that of flagstat, have no fixed name. So the code matches a part + // of the file name, in lower case. The Scala scanner used the same open patterns. let by_pred = |pred: &dyn Fn(&str) -> bool| { files .iter() @@ -143,10 +152,13 @@ fn detect_sidecars(files: &[DiscoveredFile]) -> SampleSidecars { .map(|f| f.path.clone()) }; - // Match the GVCF whether it carries a sample prefix (`HG00096.chm13.chrY.g.vcf.gz`, the ytree - // flat/staging name) or is a bare per-analysis file (`chrY.g.vcf.gz`, the in-repo GATK layout). - // No leading dot, so both match; `chrY.vcf.gz` (called variants, not a GVCF) and the `.tbi` - // index are still excluded. + // Match the GVCF in both forms. One carries a sample name in front, as + // `HG00096.chm13.chrY.g.vcf.gz`, which is the flat name that ytree writes to its work area. + // The other is a bare file of one analysis, as `chrY.g.vcf.gz`, which is the GATK layout + // inside the repo. + // + // The pattern has no dot in front, so both match. It still leaves out `chrY.vcf.gz`, which + // holds called variants and is not a GVCF, and it leaves out the `.tbi` index. let chr_y_gvcf = by_suffix("chry.g.vcf.gz"); let chr_m_gvcf = by_suffix("chrm.g.vcf.gz"); let build_hint = chr_y_gvcf.as_ref().or(chr_m_gvcf.as_ref()).and_then(|p| build_token(p)); @@ -154,8 +166,9 @@ fn detect_sidecars(files: &[DiscoveredFile]) -> SampleSidecars { SampleSidecars { chr_y_gvcf, chr_m_gvcf, - // CallableLoci output is named `*.callable.bed` in staging but `callable_status.bed` in the - // GATK repo layout — match any `.bed` whose name mentions "callable". + // The output of CallableLoci carries the name `*.callable.bed` in the work area, and + // `callable_status.bed` in the GATK repo layout. So match any `.bed` whose name holds + // "callable". callable_bed: by_pred(&|n| n.ends_with(".bed") && n.contains("callable")), callable_summary: by_suffix(".callable.summary.txt"), sex: by_suffix(".sex"), @@ -177,7 +190,7 @@ fn build_token(gvcf: &Path) -> Option { stem.rsplit('.').next().filter(|s| !s.is_empty()).map(|s| s.to_string()) } -/// A sample subdirectory holding at least one alignment or variant file. +/// A subdirectory of one sample, which holds one alignment file or variant file, or more. #[derive(Debug, Clone)] pub struct DiscoveredSample { /// Subdirectory name (typically a sample alias, e.g. `HG00096`). @@ -206,7 +219,8 @@ fn is_hidden(path: &Path) -> bool { .is_some_and(|n| n.starts_with('.')) } -/// Recursively collect files under `dir` up to `max_depth`, skipping hidden directories. +/// Collect the files under `dir`, down to `max_depth`. It walks into a subdirectory, and it skips +/// a directory that is hidden. fn list_files_recursive(dir: &Path, max_depth: usize, depth: usize, out: &mut Vec) { let entries = match fs::read_dir(dir) { Ok(e) => e, @@ -262,8 +276,9 @@ pub fn scan_sample(dir: &Path) -> DiscoveredSample { } } -/// Scan a project directory: each immediate (non-hidden) subdirectory is a sample. Samples -/// with neither an alignment nor a variant file are dropped. Errors if the path is missing, +/// Scan a project directory. Each subdirectory of it that is not hidden holds one sample. The +/// code drops a sample that has no alignment file and no variant file. It returns an error when +/// the path is absent, /// not a directory, has no subdirectories, or yields no samples with data. pub fn scan(project_dir: &Path) -> Result { if !project_dir.exists() { @@ -373,7 +388,8 @@ mod tests { assert_eq!(s.alignment_files.len(), 1); assert_eq!(s.index_files.len(), 1); assert_eq!(s.variant_files.len(), 1); - // coverage.txt / stats.txt are classified but not in the alignment/variant lists. + // The code classifies coverage.txt and stats.txt, and it puts neither into the list of + // alignments or the list of variants. assert!(s.all_files.iter().any(|f| f.kind == DiscoveredFileType::Coverage)); assert!(s.all_files.iter().any(|f| f.kind == DiscoveredFileType::Stats)); @@ -404,7 +420,7 @@ mod tests { let project = scan(&root).unwrap(); let sc = &project.samples[0].sidecars; assert!(sc.has_haplogroup_gvcf()); - // The GVCF is matched, not its .tbi index. + // The code matches the GVCF, and not its .tbi index. assert!(sc .chr_y_gvcf .as_ref() @@ -431,9 +447,10 @@ mod tests { #[test] fn detects_bare_named_gatk_repo_sidecars() { - // The real D2C repo layout: a sample dir with a `CP086569.2/` analysis subtree holding - // bare-named files (`chrY.g.vcf.gz`, `callable_status.bed`) two levels down — not the - // `.chrY.g.vcf.gz` staging names. scan_sample must still find them. + // The real layout of the D2C repo. A sample directory holds a `CP086569.2/` analysis + // subtree, and the bare-named files sit two levels down inside it: `chrY.g.vcf.gz` and + // `callable_status.bed`. Those are not the `.chrY.g.vcf.gz` names of the work + // area. scan_sample must still find them. let dir = scratch("bare-repo").join("1aceb711"); for f in [ "CP086569.2/chrYM.cram", diff --git a/crates/navigator-analysis/src/sex.rs b/crates/navigator-analysis/src/sex.rs index 6a7e7f2c..70abc611 100644 --- a/crates/navigator-analysis/src/sex.rs +++ b/crates/navigator-analysis/src/sex.rs @@ -1,11 +1,14 @@ -//! Sex inference — Rust port of the Scala `SexInference`. Infers biological sex from -//! the chrX:autosome coverage ratio: males (XY) sit near 0.5×, females (XX) near 1.0×. -//! Drives per-contig ploidy for variant calling. +//! Sex inference. It is the Rust port of the Scala `SexInference`. It infers the biological sex +//! from the coverage ratio of chrX against the autosomes. A male, at XY, sits near 0.5x. A female, +//! at XX, sits near 1.0x. The result sets the ploidy of each contig for the variant caller. //! -//! For BAM, uses the **BAI index metadata** (per-reference aligned-record counts) — the -//! Scala fast path — so it is O(contigs), not a read scan; an unindexed BAM is an error. -//! CRAM indexes (`.crai`) carry no per-reference counts, so CRAM falls back to a single -//! record scan tallying mapped reads per chromosome (O(reads), `reference` required). +//! For a BAM it uses the metadata of the **BAI index**, which holds the count of aligned records +//! of each reference. That was the fast path in Scala too, so this costs O(contigs), and it is not +//! a scan over the reads. A BAM with no index gives an error. +//! +//! A CRAM index, which is a `.crai`, holds no count of each reference. So a CRAM falls back to one +//! scan over the records, and that tallies the mapped reads of each chromosome. It costs O(reads), +//! and it needs `reference`. use std::path::Path; @@ -38,7 +41,7 @@ pub enum Confidence { pub struct SexInferenceResult { pub inferred_sex: InferredSex, pub x_autosome_ratio: f64, - /// Autosome reads per 100 bp (the Scala "coverage" proxy). + /// The count of autosome reads in 100 bp. Scala used this as its "coverage" proxy. pub autosome_mean_coverage: f64, pub x_coverage: f64, pub confidence: Confidence, @@ -49,11 +52,13 @@ const MALE_RATIO_THRESHOLD: f64 = 0.65; const FEMALE_RATIO_THRESHOLD: f64 = 0.85; const MIN_AUTOSOME_COVERAGE: f64 = 5.0; -/// Per-chromosome-class accumulators: (autosome reads, autosome length, chrX reads, chrX length). +/// The accumulator of each chromosome class: (autosome reads, autosome length, chrX reads, chrX +/// length). type Tally = (u64, u64, u64, Option); -/// Infer sex from an indexed BAM or CRAM by comparing chrX to autosome read density. -/// `reference` is required for CRAM (the record-scan fallback decodes it). +/// Infer the sex from an indexed BAM or CRAM. It compares the read density of chrX against that of +/// the autosomes. A CRAM needs `reference`, because the fallback that scans the records decodes +/// it. pub fn infer_from_bam(bam_path: &Path, reference: Option<&Path>) -> Result { let tally = match reader::detect_format(bam_path) { Format::Bam => tally_via_bai(bam_path)?, @@ -62,10 +67,12 @@ pub fn infer_from_bam(bam_path: &Path, reference: Option<&Path>) -> Result Result { let (autosome_reads, autosome_length, x_reads, x_length) = tally; @@ -83,7 +90,7 @@ pub(crate) fn result_from_tally(tally: Tally) -> Result 0.0 { @@ -102,10 +109,12 @@ pub(crate) fn result_from_tally(tally: Tally) -> Result class: 0 = other, 1 = autosome, 2 = chrX. class: Vec, @@ -163,7 +172,8 @@ impl SexState { } } -/// BAM fast path: per-reference mapped-record counts from the BAI metadata (O(contigs)). +/// The fast path for a BAM. It takes the count of mapped records of each reference from the BAI +/// metadata, and it costs O(contigs). fn tally_via_bai(bam_path: &Path) -> Result { let header = reader::read_header(bam_path, None)?; let bai_path = bam_path.with_extension("bam.bai"); @@ -190,8 +200,9 @@ fn tally_via_bai(bam_path: &Path) -> Result { Ok((autosome_reads, autosome_length, x_reads, x_length)) } -/// CRAM fallback: a single record scan tallying mapped reads per chromosome class (CRAI -/// has no per-reference counts). Lengths come from the header; reads from `reference_sequence_id`. +/// The fallback for a CRAM. It makes one scan over the records, and it tallies the mapped reads of +/// each chromosome class. A CRAI holds no count of each reference. The lengths come from the +/// header, and the reads come from `reference_sequence_id`. fn tally_via_scan(bam_path: &Path, reference: Option<&Path>) -> Result { let (header, mut reader) = reader::open_seq(bam_path, reference)?; let mut state = SexState::new(&header); @@ -230,25 +241,35 @@ pub fn determine_sex(ratio: f64, autosome_coverage: f64) -> (InferredSex, Confid } } -/// Minimum chrY reads before an alignment can be judged Y-scoped — guards against calling a -/// near-empty file "male" off a handful of stray reads. +/// The count of chrY reads that an alignment needs before the code may call it Y-scoped. It +/// prevents a call of "male" on a file that is almost empty, from a few stray reads. const Y_SCOPED_MIN_Y_READS: u64 = 1_000; -/// chrY reads must exceed this multiple of the autosome + chrX read count for an alignment to -/// count as Y-scoped. A whole-genome male carries ~100× MORE autosome than chrY reads (the -/// autosomes are ~40× the sequence and diploid), so a genuine Y-only extract — chrY in the -/// millions, autosomes a few dozen mismapped reads — clears this by orders of magnitude while -/// WGS and genuine females never approach it. +/// The chrY reads must be this many times the count of autosome plus chrX reads, or more, before +/// an alignment counts as Y-scoped. +/// +/// A whole-genome male carries about 100 times MORE autosome reads than chrY reads. The autosomes +/// hold about 40 times the sequence, and they are diploid. A true Y-only extract has chrY in the +/// millions, and a few dozen autosome reads that the aligner put in the wrong place. So it clears +/// this threshold by orders of magnitude. A WGS run, and a true female, never come near +/// it. const Y_SCOPED_DOMINANCE: u64 = 8; -/// Does this alignment's per-contig read distribution look **Y-scoped** — chrY carrying -/// essentially all the reads while the autosomes and chrX hold only a trace of mismapped ones? -/// That is the shape of a Y-only extract (e.g. GRCh38 chrY reads realigned to hs1) or a Y-Elite / -/// Big Y capture. The chrX:autosome ratio [`determine_sex`] relies on is meaningless for such data -/// and can read as **female** — which would silently disable the entire Y pipeline (the Y -/// haplogroup step skips females *before* it ever downloads the tree). A `true` here means the -/// donor sequenced his Y, so callers should treat him as male regardless of the ratio. +/// Does the read distribution of this alignment, over its contigs, look **Y-scoped**? In that +/// shape, chrY holds almost all of the reads. The autosomes and chrX hold only a trace of reads +/// that the aligner put in the wrong place. +/// +/// That is the shape of a Y-only extract, such as GRCh38 chrY reads that somebody realigned to +/// hs1. It is also the shape of a Y-Elite or Big Y capture. +/// +/// [`determine_sex`] uses the ratio of chrX against the autosomes, and that ratio says nothing +/// about such data. It can read as **female**, which would turn the whole Y pipeline off, and +/// nobody would see it. The Y haplogroup step skips a female *before* it downloads the tree. +/// +/// A `true` here means that the donor sequenced his Y. So a caller must take him as male, whatever +/// the ratio says. /// -/// The discriminator is read *counts*, not depth: pass `(contig_name, mapped_reads)` per contig. +/// This looks at the *count* of reads, and not at the depth. Give it a `(contig_name, +/// mapped_reads)` pair for each contig. pub fn is_y_scoped<'a>(per_contig_reads: impl IntoIterator) -> bool { let (mut y_reads, mut other_reads) = (0u64, 0u64); for (name, reads) in per_contig_reads { @@ -324,7 +345,8 @@ mod tests { ("chrX", 5_000_000), ("chrY", 3_000_000) ])); - // Female WGS: chrY only a trace of mismapping → not Y-scoped. + // A female WGS run. chrY holds only a trace of reads that went to the wrong place, so + // this is not Y-scoped. assert!(!is_y_scoped([ ("chr1", 200_000_000), ("chrX", 10_000_000), diff --git a/crates/navigator-analysis/src/sidecar.rs b/crates/navigator-analysis/src/sidecar.rs index 2937e76b..8bd5222f 100644 --- a/crates/navigator-analysis/src/sidecar.rs +++ b/crates/navigator-analysis/src/sidecar.rs @@ -1,19 +1,21 @@ -//! Parse the `ytree` pipeline's per-sample text sidecars into Navigator's existing result -//! structs, so the fast-path import fills the same caches the CRAM walkers would — without -//! touching the alignment. +//! Parse the text sidecars that the `ytree` pipeline writes for each sample, into the result +//! structs that Navigator already has. The fast-path import then fills the same caches that the +//! CRAM walkers would fill, and it never touches the alignment. //! -//! - `.sex` (`male`/`female`) → [`SexInferenceResult`]. -//! - `stats.txt` (`samtools stats`) → [`ReadMetrics`] — **fully** populated: the `SN` -//! summary gives the scalar counts and the `RL`/`IS` histogram lines give the read-length -//! and insert-size distributions (median/std/min/max), including the `median_insert_size` -//! the project report shows. -//! - `coverage.txt` (`samtools coverage`) + `callable.summary.txt` (GATK `CallableLoci`) → -//! a **lite** [`CoverageResult`]: genome-wide mean depth (length-weighted) + per-contig -//! stats + callable-base counts. The depth histogram and `pct_Nx` / median need the -//! per-base walk, so they're left zeroed and the result is flagged `partial` by the caller. +//! - `.sex`, which holds `male` or `female`, becomes a [`SexInferenceResult`]. +//! - `stats.txt`, from `samtools stats`, becomes a **complete** [`ReadMetrics`]. The `SN` summary +//! gives the scalar counts. The `RL` and `IS` histogram lines give the distributions of the read +//! length and the insert size, with their median, standard deviation, minimum and maximum. That +//! includes the `median_insert_size` that the project report shows. +//! - `coverage.txt`, from `samtools coverage`, together with `callable.summary.txt`, from GATK +//! `CallableLoci`, becomes a **light** [`CoverageResult`]. It holds the genome-wide mean depth, +//! weighted by length, the statistics of each contig, and the counts of callable bases. The +//! depth histogram, the `pct_Nx` values and the median all need the walk over each base. Those +//! stay at zero, and the caller marks the result `partial`. //! -//! Unknown numeric fields are `0.0` (not `NaN`) because the cache round-trips through -//! `serde_json`, which encodes `NaN` as `null` and then fails to read it back. +//! A numeric field whose value nobody knows is `0.0`, and not `NaN`. The cache goes out and back +//! through `serde_json`. That writes a `NaN` as `null`, and it then fails to read the `null` +//! back. use std::collections::{BTreeMap, HashMap}; @@ -51,9 +53,11 @@ pub fn parse_sex(text: &str) -> SexInferenceResult { // ---- stats.txt (samtools stats) ---------------------------------------------- -/// Parse `samtools stats` output into a fully-populated [`ReadMetrics`]. `SN` lines give the -/// scalar counts; `RL`/`IS` lines give the read-length / insert-size histograms (and thus -/// their median/std/min/max). `mean_mapping_quality` isn't emitted by samtools stats → 0.0. +/// Parse the output of `samtools stats` into a complete [`ReadMetrics`]. +/// +/// The `SN` lines give the scalar counts. The `RL` and `IS` lines give the histograms of the read +/// length and the insert size. From those come the median, the standard deviation, the minimum and +/// the maximum. samtools stats writes no `mean_mapping_quality`, so that field is 0.0. pub fn parse_samtools_stats(text: &str) -> ReadMetrics { let mut sn: BTreeMap<&str, f64> = BTreeMap::new(); let mut rl: BTreeMap = BTreeMap::new(); @@ -125,7 +129,7 @@ pub fn parse_samtools_stats(text: &str) -> ReadMetrics { min_insert_size: is_min, max_insert_size: is_max, insert_size_histogram: is, - // samtools stats doesn't classify orientation; Illumina paired-end is FR. + // samtools stats does not classify orientation; Illumina paired-end is FR. pair_orientation: PairOrientation::Fr, // Picard-style chimera rate: read pairs mapping to different chromosomes / total pairs. pct_chimeras: pct(pairs_diff_chrom, total_reads / 2), @@ -175,8 +179,9 @@ fn summarize(hist: &BTreeMap) -> (f64, f64, f64, u32, u32) { // ---- coverage.txt + callable.summary.txt ------------------------------------- -/// Parse `samtools coverage` TSV → per-contig stats + the length-weighted genome-wide mean -/// depth and total territory (sum of contig lengths). Skips the `#rname …` header. +/// Parse the TSV of `samtools coverage`. It gives three things. The statistics of each contig. The +/// genome-wide mean depth, weighted by length. And the total territory, which is the sum of the +/// contig lengths. It skips the `#rname …` header. pub fn parse_samtools_coverage(text: &str) -> (f64, u64, Vec) { let mut contigs = Vec::new(); let mut weighted_depth = 0.0f64; @@ -216,10 +221,11 @@ pub fn parse_samtools_coverage(text: &str) -> (f64, u64, Vec ---` line; the **leading headerless block** in a -/// `.chrYM.` summary is the mitochondrion (`chrM`). Returns total CALLABLE bases and the -/// per-contig breakdown. +/// Parse a summary from GATK `CallableLoci`. It holds `state nBases` blocks, one for each contig. +/// A `--- ---` line opens each block. +/// +/// In a `.chrYM.` summary, the **first block, which has no header**, is the mitochondrion, at +/// `chrM`. Returns the total count of CALLABLE bases, and the breakdown over the contigs. pub fn parse_callable_summary(text: &str) -> (u64, Vec) { let mut out: Vec = Vec::new(); // The first block precedes any `--- … ---` header → mitochondrion in the chrYM summary. @@ -259,8 +265,8 @@ pub fn parse_callable_summary(text: &str) -> (u64, Vec) { started = true; continue; } - // ` ` rows (whitespace-padded); the `state nBases` header has a - // non-numeric second token and is skipped by the parse. + // The rows are ` `, with spaces around them. The `state nBases` header has + // a second token that is not a number, so the parse skips it. let mut it = t.split_whitespace(); let (Some(state), Some(val)) = (it.next(), it.next()) else { continue; @@ -284,9 +290,10 @@ pub fn parse_callable_summary(text: &str) -> (u64, Vec) { (total_callable, out) } -/// Assemble a **lite** [`CoverageResult`] from the coverage + callable sidecars. Mean depth -/// and callable counts are real; median/sd/histogram/`pct_Nx` need the per-base walk and are -/// left zeroed — the caller records this artifact as `partial` so the deep pass upgrades it. +/// Build a **light** [`CoverageResult`] from the coverage sidecar and the callable one. The mean +/// depth and the callable counts are real. The median, the standard deviation, the histogram and +/// the `pct_Nx` values all need the walk over each base, so they stay at zero. The caller records +/// this artifact as `partial`, so that the deep pass replaces it later. pub fn lite_coverage(coverage_txt: &str, callable_summary: Option<&str>) -> CoverageResult { let (mean_coverage, genome_territory, contig_coverage_stats) = parse_samtools_coverage(coverage_txt); let (callable_bases, contig_callable) = callable_summary.map(parse_callable_summary).unwrap_or((0, Vec::new())); @@ -316,10 +323,14 @@ pub fn lite_coverage(coverage_txt: &str, callable_summary: Option<&str>) -> Cove // ---- samtools flagstat ------------------------------------------------------- -/// Parse `samtools flagstat` into a [`ReadMetrics`] — **scalar counts only**. flagstat carries no -/// read-length / insert-size distributions or mapping quality, so those stay 0 (an alternative -/// `ReadMetrics` source when `stats.txt` is absent). Lines are ` + [(…)]`; -/// the first number is the QC-passed count, the category is the text before any `(`. +/// Parse `samtools flagstat` into a [`ReadMetrics`]. It fills the **scalar counts alone**. +/// +/// flagstat carries no distribution of the read length or the insert size, and no mapping quality. +/// Those all stay at 0. Use it as another source of a `ReadMetrics` when `stats.txt` is +/// absent. +/// +/// A line reads ` + [(…)]`. The first number is the count that passed +/// QC, and the category is the text before any `(`. pub fn parse_flagstat(text: &str) -> ReadMetrics { let mut cats: Vec<(String, u64)> = Vec::new(); for line in text.lines() { @@ -328,7 +339,8 @@ pub fn parse_flagstat(text: &str) -> ReadMetrics { continue; }; let Ok(n) = n_str.trim().parse::() else { continue }; - // rest = " (pct…)" — drop the qc-failed number, strip the "(…)" tail. + // The rest is " (pct…)". Drop the number of records that failed QC, + // and cut the "(…)" off the end. let category = rest .trim() .split_once(char::is_whitespace) @@ -362,9 +374,10 @@ pub fn parse_flagstat(text: &str) -> ReadMetrics { // ---- Picard metrics (CollectWgsMetrics / CollectAlignmentSummaryMetrics) ------ -/// Parse a Picard metrics table: skip to the header line beginning with `header_key`, then read the -/// tab-separated data rows until a blank line (Picard appends a histogram section after a blank). -/// Returns `(headers, rows)`. `None` if the header isn't found. +/// Parse a metrics table from Picard. It goes forward to the header line that starts with +/// `header_key`. It then reads the data rows, which have tabs between their fields, until a blank +/// line. Picard puts a histogram section after that blank line. Returns `(headers, rows)`, and +/// `None` when it does not find the header. fn parse_picard_rows(text: &str, header_key: &str) -> Option<(Vec, Vec>)> { let mut lines = text.lines(); let header = lines.by_ref().find(|l| l.trim_start().starts_with(header_key))?; @@ -387,11 +400,16 @@ fn row_map<'a>(keys: &'a [String], row: &'a [String]) -> HashMap<&'a str, &'a st .collect() } -/// Parse Picard `CollectWgsMetrics` → the genome-wide depth distribution of a [`CoverageResult`] -/// (mean/median/sd/MAD, the MAPQ/baseQ exclusion fractions, and the `pct_Nx` depth thresholds — the -/// fields the lite samtools-coverage path leaves at 0). Per-contig stats / histogram stay empty -/// (Picard is genome-wide); the ingest overlays this onto the lite result's contig breakdown. -/// Picard `PCT_*` are 0–1 fractions, matching `CoverageResult`'s convention. `None` if no table. +/// Parse Picard `CollectWgsMetrics`. It fills the genome-wide depth distribution of a +/// [`CoverageResult`]. That covers the mean, the median, the standard deviation and the MAD. It +/// also covers the fractions that the MAPQ and baseQ filters removed, and the `pct_Nx` depth +/// thresholds. Those are the fields that the light path over samtools coverage leaves at 0. +/// +/// The statistics of each contig, and the histogram, stay empty, because Picard works over the +/// whole genome. The ingest puts this on top of the contig breakdown of the light result. +/// +/// The `PCT_*` values of Picard are fractions from 0 to 1, which matches the convention of +/// `CoverageResult`. Returns `None` when there is no table. pub fn parse_wgs_metrics(text: &str) -> Option { let (keys, rows) = parse_picard_rows(text, "GENOME_TERRITORY")?; let row = rows.first()?; @@ -421,7 +439,7 @@ pub fn parse_wgs_metrics(text: &str) -> Option { /// Parse Picard `CollectAlignmentSummaryMetrics` → a [`ReadMetrics`] (the `PAIR` summary row, /// else `UNPAIRED`, else the first). Counts + alignment percentages + mean read length + chimera -/// rate; read-length / insert-size histograms aren't in this metrics class, so they stay 0. Picard +/// rate; read-length / insert-size histograms are not in this metrics class, so they stay 0. Picard /// `PCT_*` are 0–1 fractions → scaled to the `ReadMetrics` 0–100 convention. `None` if no table. pub fn parse_alignment_summary(text: &str) -> Option { let (keys, rows) = parse_picard_rows(text, "CATEGORY")?; @@ -616,14 +634,15 @@ chrY\t1\t500\t20\t400\t80.0\t10.0\t29.0\t40.0 assert_eq!(c.callable_bases, 16249 + 16627537); assert_eq!(c.contig_coverage_stats.len(), 2); assert_eq!(c.contig_callable.len(), 2); - // Deep-walk-only fields stay zeroed (artifact is flagged partial by the caller). + // The fields that need the deep walk stay at zero. The caller marks the artifact + // `partial`. assert_eq!(c.median_coverage, 0.0); assert!(c.coverage_histogram.is_empty()); assert_eq!(c.pct_10x, 0.0); } /// Real-data smoke test: parse HG00096's actual pipeline sidecars off the NAS. No-ops - /// when the share isn't mounted. Run: `cargo test -p navigator-analysis sidecar -- --ignored --nocapture`. + /// when the share is not mounted. Run: `cargo test -p navigator-analysis sidecar -- --ignored --nocapture`. #[test] #[ignore = "reads NAS files; run explicitly"] fn real_sidecars_parse() { diff --git a/crates/navigator-analysis/src/strcaller.rs b/crates/navigator-analysis/src/strcaller.rs index 255e4ce1..9ba5dbbf 100644 --- a/crates/navigator-analysis/src/strcaller.rs +++ b/crates/navigator-analysis/src/strcaller.rs @@ -1,17 +1,24 @@ -//! STR genotyping from aligned reads — the enclosing-read model. +//! STR genotypes from aligned reads. The model works on a read that encloses the tract. //! -//! For STRs shorter than a read (Y-STRs and forensic/genealogical markers all qualify), the -//! informative reads are those that **enclose** the whole repeat tract plus clean flanking sequence -//! on both sides (GangSTR's "enclosing" class — the Spanning/Flanking/FRR classes only matter for -//! expansions longer than a read). For each enclosing read the observed repeat length is read off -//! the **CIGAR** — `tract_bp + (insertions − deletions) within the tract`, measured against the -//! known reference allele (so it carries no systematic offset, unlike counting motif copies in a -//! loose feature region). A geometric **PCR-stutter** model then turns the per-read counts into a -//! maximum-likelihood genotype (haploid for chrY, diploid elsewhere). +//! An STR shorter than a read is the case here, and every Y-STR, and every forensic and +//! genealogical marker, qualifies. The reads that carry information are the ones that **enclose** +//! the whole repeat tract, plus clean sequence on both sides of it. GangSTR calls those the +//! "enclosing" class. Its Spanning, Flanking and FRR classes matter only for an expansion that is +//! longer than a read. //! -//! This is the tractable, principled core of HipSTR/GangSTR — it omits their stutter-EM, HMM -//! realignment, and SNP phasing (a future refinement), trusting the aligner's CIGAR within tight -//! tracts and letting the modal-over-reads genotype absorb per-read misalignment. +//! At each enclosing read the code takes the observed repeat length off the **CIGAR**. It is +//! `tract_bp + (insertions − deletions) inside the tract`, measured against the known reference +//! allele. So it carries no systematic offset, where a count of the motif copies in a loose +//! feature region would. +//! +//! A geometric model of **PCR stutter** then turns the counts of the reads into a +//! maximum-likelihood genotype. That genotype is haploid on chrY, and diploid elsewhere. +//! +//! This is the core of HipSTR and GangSTR that a person can build and defend. It leaves out their +//! stutter EM, their HMM realignment, and their SNP phasing, and those are a later improvement. +//! +//! It trusts the CIGAR of the aligner inside a tight tract. And it lets the genotype, which is the +//! mode over the reads, absorb a read that the aligner put in the wrong place. use std::path::Path; @@ -27,15 +34,17 @@ use crate::strref::StrLocus; /// Tunables for the STR caller. #[derive(Debug, Clone, Copy)] pub struct StrCallerParams { - /// Minimum mapping quality for a read to be used. + /// The mapping quality that a read needs before the code uses it. pub min_mapping_quality: u8, /// Clean, indel-free reference bases required on each side of the tract to count a read. pub flank: i64, - /// Minimum enclosing-read depth to emit a genotype. + /// The depth of reads that enclose the tract that the code needs before it emits a genotype. pub min_depth: u32, - /// `P(read shows the true allele exactly)` — the no-stutter probability (HipSTR default 0.9). + /// `P(a read shows the true allele exactly)`. It is the probability of no stutter, and the + /// HipSTR default is 0.9. pub no_stutter: f64, - /// Geometric decay of stutter magnitude (per extra repeat unit). Smaller → ±1 dominates. + /// The geometric decay of the stutter size, at each extra repeat unit. A smaller value makes + /// ±1 dominate. pub stutter_decay: f64, } @@ -63,7 +72,8 @@ pub enum StrConfidence { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct StrGenotype { pub contig: String, - /// 0-based tract start (BED), for joining back to the reference / a vendor mapping. + /// The 0-based start of the tract, in BED form. Use it to join back to the reference, or to a + /// mapping that a vendor gives. pub start: i64, pub end: i64, pub period: u8, @@ -71,18 +81,21 @@ pub struct StrGenotype { /// HipSTR locus id (the result name until a vendor DYS mapping exists). pub name: String, pub ref_copies: f64, - /// Called allele(s) in **repeat copies** — one for haploid (chrY), one or two for diploid. + /// The called alleles, in **repeat copies**. There is one on a haploid contig, such as chrY, + /// and one or two on a diploid one. pub alleles: Vec, - /// Enclosing reads used (the genotype's depth). + /// The count of enclosing reads that the code used. It is the depth of the genotype. pub depth: u32, - /// Fraction of enclosing reads matching the called allele(s) exactly. + /// The fraction of the enclosing reads that match the called alleles exactly. pub concordance: f64, pub confidence: StrConfidence, } -/// `ln P(observed copies | true allele copies)` under the geometric stutter model: the read shows -/// the allele exactly with probability `no_stutter`; otherwise the magnitude of the deviation (in -/// repeat units) is geometric and symmetric up/down. +/// `ln P(the observed copies | the true allele copies)`, under the geometric stutter model. +/// +/// The read shows the allele exactly, at a probability of `no_stutter`. Otherwise the size of the +/// deviation, in repeat units, follows a geometric distribution, and it is symmetric up and +/// down. fn obs_lnlik(observed: i32, allele: i32, p: &StrCallerParams) -> f64 { if observed == allele { p.no_stutter.ln() @@ -111,8 +124,9 @@ fn call_haploid(observed: &[i32], p: &StrCallerParams) -> Option { .map(|(a, _)| a) } -/// Maximum-likelihood **diploid** genotype `(A,B)` (A<=B): argmax over candidate pairs of the -/// summed `ln[½P(o|A) + ½P(o|B)]` — each read equally likely from either allele. +/// The maximum-likelihood **diploid** genotype `(A,B)`, where A<=B. It is the argmax over the +/// candidate pairs of the sum of `ln[½P(o|A) + ½P(o|B)]`. Each read comes from either allele with +/// equal probability. fn call_diploid(observed: &[i32], p: &StrCallerParams) -> Option<(i32, i32)> { let cands = candidates(observed); let mut best: Option<((i32, i32), f64)> = None; @@ -134,13 +148,17 @@ fn call_diploid(observed: &[i32], p: &StrCallerParams) -> Option<(i32, i32)> { best.map(|(g, _)| g) } -/// The repeat copies observed in one enclosing read at `locus`, or `None` if the read isn't a clean -/// enclosing read (not anchored `flank` bp of indel-free reference on both sides). Reads the length -/// off the CIGAR: `tract_bp + insertions − deletions` within the tract, ÷ period. +/// The repeat copies that one enclosing read shows at `locus`. It is `None` when that read does +/// not enclose the tract cleanly. To do so it must sit on `flank` bp of reference with no indel, +/// on both sides. +/// +/// The code takes the length off the CIGAR, as `tract_bp + insertions − deletions` inside the +/// tract, divided by the period. fn observed_copies(ops: &[(Kind, usize)], aln_start: i64, locus: &StrLocus, flank: i64) -> Option { - // HipSTR tracts are end-INCLUSIVE: ref_copies = (end - start + 1)/period (e.g. Y:2795644-2795670 - // period 4 → 27/4 = 6.75). So the 1-based tract is [start+1, end+1] (length end-start+1), and a - // ref-matching read measures exactly ref_copies — no systematic offset. + // A HipSTR tract INCLUDES its end. So ref_copies = (end - start + 1)/period. For example, + // Y:2795644-2795670 at period 4 gives 27/4 = 6.75. The 1-based tract is then [start+1, end+1], + // and its length is end-start+1. A read that matches the reference then measures exactly + // ref_copies, with no systematic offset. let (ts, te) = (locus.start + 1, locus.end + 1); // 1-based inclusive tract let period = locus.period as i64; if period == 0 { @@ -197,7 +215,8 @@ fn read_passes(r: &RecordBuf, min_mapq: u8) -> bool { && r.mapping_quality().map(|m| m.get()).unwrap_or(0) >= min_mapq } -/// Build a genotype from a locus's enclosing-read counts. `ploidy` 1 = haploid (chrY), else diploid. +/// Build a genotype from the counts of the reads that enclose a locus. A `ploidy` of 1 is haploid, +/// as on chrY. Any other value is diploid. fn genotype_from_counts(locus: &StrLocus, counts: &[i32], ploidy: u8, p: &StrCallerParams) -> Option { let depth = counts.len() as u32; if depth < p.min_depth { @@ -235,9 +254,11 @@ fn genotype_from_counts(locus: &StrLocus, counts: &[i32], ploidy: u8, p: &StrCal }) } -/// Genotype every locus in `loci` (assumed all on `contig`, sorted by start) from `bam`, in one -/// streaming pass: each read contributes its observed copy number to every locus it cleanly -/// encloses. `ploidy` 1 = haploid (chrY). `reference` is required for CRAM. +/// Genotype every locus in `loci` from `bam`, in one streaming pass. Every locus must lie on +/// `contig`, and `loci` must come in order of its start. +/// +/// Each read gives its observed copy number to every locus that it encloses cleanly. A `ploidy` of +/// 1 is haploid, as on chrY. A CRAM needs `reference`. pub fn genotype_str_loci( bam: &Path, contig: &str, @@ -253,7 +274,7 @@ pub fn genotype_str_loci( let region: Region = contig .parse() .map_err(|_| AnalysisError::Message(format!("bad region for contig {contig}")))?; - // Loci sorted by start; collect per-locus observed counts. + // The loci come in order of their start. Collect the observed counts at each locus. let starts: Vec = loci.iter().map(|l| l.start).collect(); let mut counts: Vec> = vec![Vec::new(); loci.len()]; diff --git a/crates/navigator-analysis/src/strmarker.rs b/crates/navigator-analysis/src/strmarker.rs index 8ae39413..e1050d06 100644 --- a/crates/navigator-analysis/src/strmarker.rs +++ b/crates/navigator-analysis/src/strmarker.rs @@ -1,32 +1,44 @@ -//! FTDNA Y-STR convention layer — map the caller's HipSTR-locus repeat counts to FTDNA marker values. +//! The layer that holds the FTDNA Y-STR convention. It maps the repeat count that the caller gives +//! at a HipSTR locus to the value that an FTDNA marker holds. //! -//! The HipSTR reference already names the loci (DYS393, DYS19/DYS394, …; see [`crate::strref`]), so -//! the caller emits DYS names directly. What differs is the **counting convention**: FTDNA reports a -//! per-marker value that is the caller's repeat count plus a fixed offset (0 for most, ±1–3 for some), -//! and a set of markers whose HipSTR tract doesn't correspond 1:1 to the FTDNA marker (large tract -//! mismatches, plus multi-copy/nested markers like DYS385/DYS464/DYS389II) which can't be mapped by a -//! single offset. +//! The HipSTR reference already names the loci, as DYS393, DYS19/DYS394 and so on. See +//! [`crate::strref`]. So the caller emits a DYS name directly. What differs is the **convention +//! for the count**. //! -//! The offset table below was **calibrated against a 216-kit Big Y corpus** (FTDNA R1b project, -//! CHM13-realigned `chrYM.cram` + each kit's FTDNA DYS CSV): per marker, the offset is the modal -//! `ftdna − caller` difference across kits, kept only where it agrees across the corpus (≥70%, and -//! ≥20 kits to be authoritative). The harness is `examples/str_calibrate.rs`. The calibration cross- -//! validated build-independence: where the CHM13 corpus and the earlier 14-kit GRCh38 corpus overlap, -//! the offsets match (DYS438 +2, DYS435 +2, DYS474 −3, DYS442 −3, DYS520 −2, DYS585 −3, DYS615 −2, -//! DYS629 −3). Because offsets are build-independent, markers the CHM13 HipSTR liftover dropped -//! (DYS19, DYS391, DYS426, DYS445, DYS461, DYS512, DYS549, DYS565, DYS567, DYS578, DYS589, DYS632 …) -//! retain their GRCh38-corpus values — they still serve the GRCh38/BAM calling path, and will serve -//! the CHM13 path once those loci are recovered in the lifted reference. Markers absent here report -//! `Uncalibrated`. Re-run the harness over more kits to extend the table. +//! FTDNA reports a value at each marker. That value is the repeat count of the caller, plus a +//! fixed offset. The offset is 0 at most markers, and ±1 to 3 at some. There is also a set of +//! markers whose HipSTR tract does not match the FTDNA marker 1:1. Those are the large tract +//! mismatches, and the markers with more than one copy or with a nest inside, such as DYS385, +//! DYS464 and DYS389II. No single offset can map one of those. +//! +//! The offset table below comes from a **calibration against a corpus of 216 Big Y kits**. That +//! corpus is the FTDNA R1b project, with a `chrYM.cram` realigned to CHM13, and the FTDNA DYS CSV +//! of each kit. At each marker the offset is the modal `ftdna − caller` difference across the +//! kits. The table keeps it only where the corpus agrees, at 70% or more, and over 20 kits or +//! more. The harness is `examples/str_calibrate.rs`. +//! +//! The calibration also checked that the offsets do not depend on the build. Where the CHM13 +//! corpus and the earlier corpus of 14 GRCh38 kits overlap, the offsets match: DYS438 +2, DYS435 +//! +2, DYS474 −3, DYS442 −3, DYS520 −2, DYS585 −3, DYS615 −2, DYS629 −3. +//! +//! The CHM13 HipSTR liftover dropped some markers: DYS19, DYS391, DYS426, DYS445, DYS461, DYS512, +//! DYS549, DYS565, DYS567, DYS578, DYS589, DYS632 and more. Because an offset does not depend on +//! the build, those markers keep the values of the GRCh38 corpus. They still serve the path that +//! calls over a GRCh38 BAM. They will serve the CHM13 path too, once somebody recovers those loci +//! in the lifted reference. +//! +//! A marker that this table does not hold reports `Uncalibrated`. Run the harness over more kits +//! to extend the table. use crate::strcaller::{StrConfidence, StrGenotype}; /// FTDNA value = caller repeat count + offset. Covers the calibrated single-copy markers (offset 0 = /// "reliable", ±1–3 = a real convention). Markers absent here are either in [`EXCLUDE`] or uncalibrated. static OFFSETS: &[(&str, i32)] = &[ - // Reliable (offset 0). Most are CHM13 216-kit ≥100% agreement; a few (DYS388, DYS426, DYS445, - // DYS487, DYS494, DYS505, DYS549, DYS556, DYS565, DYS567, DYS577, DYS578) are GRCh38-corpus - // retentions for loci the CHM13 lift dropped or under-sampled. + // These are reliable, at an offset of 0. Most come from the 216-kit CHM13 corpus, at 100% + // agreement. A few come from the GRCh38 corpus, for loci that the CHM13 lift dropped, or + // sampled too little: DYS388, DYS426, DYS445, DYS487, DYS494, DYS505, DYS549, DYS556, DYS565, + // DYS567, DYS577 and DYS578. ("DYS388", 0), ("DYS390", 0), ("DYS392", 0), @@ -87,9 +99,10 @@ static OFFSETS: &[(&str, i32)] = &[ ("DYS645", 0), ("DYS714", 0), ("Y-GATA-A10", 0), - // Convention offsets (±1–3). DYS19/DYS391/DYS461/DYS512/DYS589/DYS632 are GRCh38-corpus - // retentions (dropped by the CHM13 lift); the rest are CHM13 216-kit. DYS460 was previously - // excluded but the larger corpus resolves it to a clean +1 (n=180, 98%). + // The offsets of the convention, at ±1 to 3. DYS19, DYS391, DYS461, DYS512, DYS589 and DYS632 + // come from the GRCh38 corpus, because the CHM13 lift dropped them. The rest come from the + // 216-kit CHM13 corpus. The table once left DYS460 out. The larger corpus resolves it to a + // clean +1, at n=180 and 98%. ("DYS19", -1), ("DYS389I", 1), ("DYS391", -1), @@ -121,9 +134,12 @@ static OFFSETS: &[(&str, i32)] = &[ ("DYS642", 1), ]; -/// Markers whose HipSTR tract can't be mapped to the FTDNA value by a single offset: large tract -/// mismatches and multi-copy/nested markers (DYS385/DYS464 split sub-loci, DYS389II nesting). Reported -/// as `Excluded` — the enclosing-read caller doesn't yield a vendor-comparable value here (yet). +/// The markers whose HipSTR tract no single offset can map to the FTDNA value. Those are the large +/// tract mismatches, and the markers with more than one copy or with a nest inside. DYS385 and +/// DYS464 split into sub-loci, and DYS389II holds a nest. +/// +/// They report as `Excluded`. The enclosing-read caller does not yet give a value here that you +/// can compare against a vendor. static EXCLUDE: &[&str] = &[ // Multi-copy / nested (split sub-loci, never a single vendor-comparable value). "DYS385", @@ -170,12 +186,14 @@ static EXCLUDE: &[&str] = &[ "Y-GATA-H4", ]; -/// A handful of offsets are **build-dependent**: the CHM13 HipSTR liftover shifted these tract -/// boundaries by one repeat unit, so the enclosing-read count differs by 1 between CHM13 and GRCh38. -/// [`OFFSETS`] holds the CHM13 value (the primary corpus); this delta is *added* on the GRCh38 path. -/// Verified by two independent GRCh38 corpora (a 14-kit set + kit 27520) agreeing against the 216-kit -/// CHM13 corpus: DYS389I/DYS456/DYS525/DYS537/DYS539 are +1 on CHM13 but 0 on GRCh38; DYS714 is the -/// reverse (0 on CHM13, +1 on GRCh38). +/// A few offsets **do depend on the build**. The CHM13 HipSTR liftover moved these tract +/// boundaries by one repeat unit, so the enclosing-read count differs by 1 between CHM13 and +/// GRCh38. [`OFFSETS`] holds the CHM13 value, which is the primary corpus, and the code *adds* +/// this delta on the GRCh38 path. +/// +/// Two independent GRCh38 corpora confirmed this against the 216-kit CHM13 corpus: a 14-kit set, +/// and kit 27520. DYS389I, DYS456, DYS525, DYS537 and DYS539 are +1 on CHM13 and 0 on GRCh38. +/// DYS714 is the reverse: 0 on CHM13 and +1 on GRCh38. static GRCH38_DELTA: &[(&str, i32)] = &[ ("DYS389I", -1), ("DYS456", -1), @@ -185,8 +203,9 @@ static GRCH38_DELTA: &[(&str, i32)] = &[ ("DYS714", 1), ]; -/// Which reference build the caller's repeat counts came from — selects the convention offset for the -/// build-dependent markers (see [`GRCH38_DELTA`]). Default ([`StrBuild::Chm13`]) is the primary corpus. +/// The reference build that the repeat counts of the caller came from. It selects the convention +/// offset of the markers that depend on the build. See [`GRCH38_DELTA`]. The default, +/// [`StrBuild::Chm13`], is the primary corpus. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum StrBuild { #[default] @@ -214,16 +233,19 @@ pub enum MarkerStatus { Reliable, /// Calibrated with a fixed FTDNA convention offset (±1–3). ConventionOffset, - /// Tract mismatch / multi-copy / nested — no 1:1 mapping; value is the raw caller count. + /// The tract does not match, or the marker holds more than one copy, or it holds a nest. + /// There is no 1:1 mapping, and the value is the raw count of the caller. Excluded, - /// Not in the calibration table — value is the raw caller count, pending calibration. + /// The calibration table does not hold this marker. The value is the raw count of the caller, + /// and it waits for a calibration. Uncalibrated, } /// One marker called from sequence, expressed in the FTDNA convention. #[derive(Debug, Clone, PartialEq)] pub struct CalledMarker { - /// FTDNA marker name (normalized — e.g. `DYS19`, not `DYS19/DYS394`). + /// The name of the FTDNA marker, in its normal form. That is `DYS19`, and not + /// `DYS19/DYS394`. pub marker: String, /// FTDNA-convention value (caller count + calibrated offset), or the raw count when not calibrated. pub value: i32, @@ -232,9 +254,10 @@ pub struct CalledMarker { pub depth: u32, } -/// Normalize a caller locus name (from the HipSTR BED) to its base FTDNA marker: take the first of a -/// `/`-alias (`DYS19/DYS394` → `DYS19`), drop a `_N` copy suffix (`DYS385_1` → `DYS385`) and a `.N` -/// partial suffix (`DYS389II.1` → `DYS389II`). +/// Bring the locus name of the caller, which comes from the HipSTR BED, to its base FTDNA marker. +/// It takes the first name of a `/` alias, so `DYS19/DYS394` gives `DYS19`. It drops a `_N` copy +/// suffix, so `DYS385_1` gives `DYS385`. And it drops a `.N` partial suffix, so `DYS389II.1` gives +/// `DYS389II`. pub fn normalize_marker(caller_name: &str) -> String { let n = caller_name.split('/').next().unwrap_or(caller_name); let n = n.split('_').next().unwrap_or(n); @@ -255,9 +278,10 @@ pub fn to_ftdna(caller_name: &str, caller_copies: i32) -> CalledMarker { to_ftdna_build(caller_name, caller_copies, StrBuild::Chm13) } -/// Map one caller locus + its repeat count to the FTDNA convention for a specific build: the marker -/// name, the convention-adjusted value, and how trustworthy that mapping is. The build selects the -/// offset for the build-dependent markers (see [`GRCH38_DELTA`]). +/// Map one locus of the caller, and its repeat count, to the FTDNA convention of one build. It +/// gives the marker name, the value after the convention offset, and how much you can trust that +/// mapping. The build selects the offset of the markers that depend on it. See +/// [`GRCH38_DELTA`]. pub fn to_ftdna_build(caller_name: &str, caller_copies: i32, build: StrBuild) -> CalledMarker { let marker = normalize_marker(caller_name); let (value, status) = if EXCLUDE.contains(&marker.as_str()) { @@ -294,9 +318,13 @@ pub fn called_markers(genotypes: &[StrGenotype]) -> Vec { called_markers_build(genotypes, StrBuild::Chm13) } -/// Convert the caller's genotypes to FTDNA-convention marker calls for a specific build: single-copy -/// (one allele), non-low-confidence loci, deduped per marker keeping the deepest. Multi-copy markers -/// (two alleles) are skipped — they need the (excluded) aggregation/nesting conventions. +/// Turn the genotypes of the caller into marker calls in the FTDNA convention, for one build. +/// +/// It takes a locus with one copy, which holds one allele, and whose confidence is not low. Where +/// two loci give the same marker, it keeps the one with the deepest coverage. +/// +/// It skips a marker with more than one copy, which holds two alleles. Such a marker needs the +/// conventions for aggregation and for a nest, and this module excludes those. pub fn called_markers_build(genotypes: &[StrGenotype], build: StrBuild) -> Vec { use std::collections::HashMap; let mut best: HashMap = HashMap::new(); @@ -357,7 +385,7 @@ mod tests { assert_eq!(to_ftdna("DYS714", 24).value, 24); let h = to_ftdna_build("DYS714", 24, StrBuild::Grch38); assert_eq!((h.value, h.status), (25, MarkerStatus::ConventionOffset)); - // Build-independent marker is unaffected by build. + // A marker that does not depend on the build does not change with the build. assert_eq!(to_ftdna_build("DYS438", 10, StrBuild::Grch38).value, 12); } diff --git a/crates/navigator-analysis/src/strref.rs b/crates/navigator-analysis/src/strref.rs index 96242010..bdd86e68 100644 --- a/crates/navigator-analysis/src/strref.rs +++ b/crates/navigator-analysis/src/strref.rs @@ -1,15 +1,17 @@ //! Short-tandem-repeat reference loci, parsed from a HipSTR-format reference BED. //! -//! The HipSTR reference defines **tight repeat tracts** (not loose feature regions) — the -//! coordinate precision an enclosing-read repeat counter needs. Each line is tab-delimited: +//! The HipSTR reference gives **tight repeat tracts**, and not loose feature regions. That is the +//! coordinate precision that a repeat counter over enclosing reads needs. Each line holds tabs +//! between its fields: //! //! ```text //! chrom start(0-based) end period ref_copies locus_id motif //! Y 10001 10038 6 6.33333 Human_STR_1604566 AACCCT //! ``` //! -//! `motif` is occasionally a `/`-separated alternative set (e.g. `CCTT/CCCT`) — the first is taken -//! as canonical. Contig names are bare (`1`, `Y`); callers normalize against the BAM's naming. +//! Sometimes `motif` holds a set of alternatives, with a `/` between them, as in `CCTT/CCCT`. The +//! code takes the first one as canonical. A contig name is bare, as `1` or `Y`, and a caller +//! normalizes it against the names in the BAM. use std::fs::File; use std::io::{BufRead, BufReader}; @@ -22,7 +24,7 @@ use crate::error::AnalysisError; /// One STR locus: a tight repeat tract with its period (motif length) and reference copy number. #[derive(Debug, Clone, PartialEq)] pub struct StrLocus { - /// Contig as written in the reference BED (bare — `1`, `X`, `Y`). + /// The contig, as the reference BED writes it. It is bare: `1`, `X` or `Y`. pub contig: String, /// 0-based, half-open tract start (BED convention). pub start: i64, @@ -30,7 +32,8 @@ pub struct StrLocus { pub end: i64, /// Repeat-unit length (motif size) in bp. pub period: u8, - /// Copy number in the reference allele (can be fractional — a partial final unit). + /// The copy number of the reference allele. It can hold a fraction, when the last unit is + /// partial. pub ref_copies: f64, /// Locus id (HipSTR `Human_STR_N`), used as the result name until a vendor mapping exists. pub name: String, @@ -39,8 +42,9 @@ pub struct StrLocus { } impl StrLocus { - /// Whether `name`/the BED contig matches `query` after stripping an optional `chr` prefix on - /// either side (the BAM may be `chrY`, the BED `Y`; see the contig-naming convention). + /// True when `name`, which is the contig of the BED, matches `query`. The comparison removes a + /// `chr` prefix from either side first. The BAM can say `chrY` where the BED says `Y`. See the + /// convention for the names of the contigs. pub fn contig_matches(&self, query: &str) -> bool { crate::contig::bare(&self.contig).eq_ignore_ascii_case(crate::contig::bare(query)) } @@ -71,10 +75,16 @@ fn parse_line(line: &str) -> Option { }) } -/// Read STR loci from a (gzipped) HipSTR reference BED, keeping only those on `contig` (matched -/// prefix-insensitively) with `period >= min_period`. Filtering while streaming avoids holding the -/// genome-wide ~1.6M-locus set in memory when only one chromosome is needed. Results are sorted by -/// start. `min_period` of 2 drops homopolymers (period 1) — noisy and not genealogical markers. +/// Read the STR loci from a HipSTR reference BED. That BED may come through gzip. This keeps a +/// locus only when that locus is on `contig`, and when its `period` is `min_period` or more. The +/// match on the contig ignores a `chr` prefix. +/// +/// The filter runs as the code streams the file. It thereby never holds the genome-wide set of +/// about 1.6M loci in memory, when the caller needs one chromosome. The results come back in order +/// of their start. +/// +/// A `min_period` of 2 drops the homopolymers, which have a period of 1. Those are noisy, and they +/// are not genealogical markers. pub fn load_hipstr_contig(bed_gz: &Path, contig: &str, min_period: u8) -> Result, AnalysisError> { let file = File::open(bed_gz).map_err(|e| AnalysisError::io(bed_gz, e))?; let reader = BufReader::new(MultiGzDecoder::new(file)); diff --git a/crates/navigator-analysis/src/sv.rs b/crates/navigator-analysis/src/sv.rs index 0fc5bf45..ba1c6100 100644 --- a/crates/navigator-analysis/src/sv.rs +++ b/crates/navigator-analysis/src/sv.rs @@ -1,14 +1,18 @@ -//! Structural variant calling — port of the Scala `analysis.sv` subsystem (a custom -//! BreakDancer/Pindel/CNV-seq-style caller, not a GATK tool). Pipeline: +//! The call of a structural variant. It is the port of the Scala `analysis.sv` subsystem, which is +//! a caller of its own, in the style of BreakDancer, Pindel and CNV-seq. It is not a GATK tool. //! -//! - [`walker`] gathers evidence in one BAM pass: per-bin depth, discordant pairs -//! (insert-size / orientation / inter-chromosomal), and SA-tag split reads. -//! - [`segmenter`] turns depth bins into CNV segments via z-score analysis. -//! - [`clusterer`] groups PE/SR evidence into breakpoints, infers SV type, and -//! integrates depth segments. -//! - [`caller::call_structural_variants`] orchestrates the above. +//! The pipeline is: //! -//! VCF/artifact output (`SvVcfWriter`) is deferred. Parity target is the Scala caller. +//! - [`walker`] collects the evidence in one pass over the BAM. It takes the depth of each bin. It +//! takes the discordant pairs, from the insert size, from the orientation, and from a pair that +//! crosses two chromosomes. And it takes the split reads, from the SA tag. +//! - [`segmenter`] turns the depth bins into CNV segments, through a z-score. +//! - [`clusterer`] puts the PE and SR evidence into groups at each breakpoint, infers the SV type, +//! and brings in the depth segments. +//! - [`caller::call_structural_variants`] drives all of the above. +//! +//! The write of a VCF or an artifact, which `SvVcfWriter` does, waits for later work. The parity +//! target is the Scala caller. pub mod caller; pub mod clusterer; diff --git a/crates/navigator-analysis/src/sv/caller.rs b/crates/navigator-analysis/src/sv/caller.rs index 93544f99..b0891302 100644 --- a/crates/navigator-analysis/src/sv/caller.rs +++ b/crates/navigator-analysis/src/sv/caller.rs @@ -1,6 +1,7 @@ -//! SV caller orchestration — port of the Scala `SvCaller`: evidence collection -> -//! depth segmentation -> clustering -> result. Artifact/VCF writing (`SvVcfWriter`) is -//! deferred, like the coverage walker's BED output. +//! The orchestration of the SV caller. It is the port of the Scala `SvCaller`. It collects the +//! evidence, segments the depth, puts the evidence into groups, and gives a result. The write of an +//! artifact or a VCF, which `SvVcfWriter` does, waits for later work, as the BED output of the +//! coverage walker does. use std::collections::BTreeMap; use std::path::Path; @@ -11,8 +12,9 @@ use crate::error::AnalysisError; const MERGE_MAX_GAP: i64 = 50_000; -/// Run the full SV pipeline on a BAM or CRAM. Requires >= 10x mean coverage (Scala threshold). -/// `reference` is required for CRAM (ignored for BAM) — see [`walker::collect_evidence`]. +/// Run the whole SV pipeline on a BAM or a CRAM. It needs a mean coverage of 10x or more, which is +/// the threshold that the Scala code used. A CRAM needs `reference`, and a BAM ignores it. See +/// [`walker::collect_evidence`]. #[allow(clippy::too_many_arguments)] pub fn call_structural_variants( bam_path: &Path, diff --git a/crates/navigator-analysis/src/sv/clusterer.rs b/crates/navigator-analysis/src/sv/clusterer.rs index c2714fc3..dcb6db67 100644 --- a/crates/navigator-analysis/src/sv/clusterer.rs +++ b/crates/navigator-analysis/src/sv/clusterer.rs @@ -1,7 +1,9 @@ -//! Evidence clusterer — port of the Scala `SvEvidenceClusterer`. Clusters discordant -//! pairs + split reads into SV calls: inter-chromosomal -> BND, intra-chromosomal -//! positional clustering with orientation/insert-based type inference, then integration -//! with depth-based CNV segments. +//! The step that puts the evidence into groups. It is the port of the Scala +//! `SvEvidenceClusterer`. It turns the discordant pairs and the split reads into SV calls. +//! +//! A pair across two chromosomes gives a BND. A pair inside one chromosome goes into a group by +//! position, and the orientation and the insert size then give the type. Last, the code puts the +//! CNV segments from the depth together with those calls. use std::collections::{BTreeMap, BTreeSet}; @@ -16,7 +18,8 @@ enum EvidencePoint { Split(SplitRead), } -/// Cluster all SV evidence into calls, integrating depth segments. +/// Put all of the SV evidence into groups, and make calls from them. It also brings in the depth +/// segments. pub fn cluster( evidence: &SvEvidenceCollection, depth_segments: &[DepthSegment], @@ -291,7 +294,8 @@ fn breakpoint_cluster_to_call( } } -/// Attach depth evidence to overlapping PE/SR calls; append depth-only calls. +/// Attach the depth evidence to the PE and SR calls that it overlaps. Then add the calls that come +/// from the depth alone. fn integrate_pe_sr_with_depth( pe_sr_calls: Vec, depth_segments: &[DepthSegment], diff --git a/crates/navigator-analysis/src/sv/evidence.rs b/crates/navigator-analysis/src/sv/evidence.rs index 514c66e6..1f4c45ed 100644 --- a/crates/navigator-analysis/src/sv/evidence.rs +++ b/crates/navigator-analysis/src/sv/evidence.rs @@ -1,12 +1,13 @@ -//! SV evidence models — port of the Scala `SvEvidence` (discordant pairs, split reads, -//! depth segments, the evidence collection, and breakpoint clusters). +//! The models of the SV evidence. This is the port of the Scala `SvEvidence`. It holds the +//! discordant pairs, the split reads, the depth segments, the collection of the evidence, and the +//! clusters of breakpoints. use std::collections::BTreeMap; use std::sync::Arc; use super::types::SvType; -/// Why a read pair is considered discordant. +/// The reason that a read pair counts as discordant. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DiscordantReason { InsertSizeOutlier, @@ -16,11 +17,15 @@ pub enum DiscordantReason { /// A discordant read pair (potential SV breakpoint evidence). /// -/// Contig names are `Arc` rather than `String`, and there is no read name. A 30x WGS retains -/// 3–16 M of these (measured across the workspace), and both choices are about that scale: the -/// walker interns one `Arc` per contig and clones a pointer instead of allocating a name per -/// record, and the read name — which nothing downstream ever read — cost an allocation and ~55 -/// bytes each to carry evidence that clustering identifies purely by position. +/// A contig name is an `Arc`, and not a `String`. There is also no read name. A 30x WGS keeps +/// 3M to 16M of these, as a measurement over the workspace showed. Both choices come from that +/// scale. +/// +/// The walker interns one `Arc` for each contig, and it then clones a pointer. It does not allocate +/// a name at each record. +/// +/// The read name cost an allocation, and about 55 bytes, at each record. Nothing after the walker +/// ever read it. The step that groups the evidence finds a breakpoint from the positions alone. #[derive(Debug, Clone, PartialEq)] pub struct DiscordantPair { pub chrom1: Arc, @@ -61,7 +66,8 @@ pub struct DepthSegment { pub sv_type: SvType, } -/// All SV evidence gathered from a BAM. `depth_bins` maps contig -> per-bin read counts. +/// All of the SV evidence that the walker collected from a BAM. `depth_bins` maps a contig to the +/// read count of each bin. #[derive(Debug, Clone, PartialEq)] pub struct SvEvidenceCollection { pub discordant_pairs: Vec, @@ -70,9 +76,12 @@ pub struct SvEvidenceCollection { pub sample_name: String, pub expected_insert_size: f64, pub insert_size_sd: f64, - /// Evidence seen but not retained, because `SvCallerConfig::max_evidence_records` was already - /// met. Zero in every normal run — see that field. Kept so the `total_*` counts below stay the - /// number of items *found*, which is what the walker's stats mean, capped or not. + /// The count of evidence items that the walker saw and did not keep, because the count had + /// already reached `SvCallerConfig::max_evidence_records`. It is zero in every usual run. See + /// that field. + /// + /// This field exists so that the `total_*` counts below stay the count of items that the walker + /// *found*. That is what the statistics of the walker mean, whether the cap fired or not. pub discordant_pairs_dropped: u64, pub split_reads_dropped: u64, } @@ -95,7 +104,7 @@ impl SvEvidenceCollection { } } -/// Grouped evidence supporting a single breakpoint. +/// The evidence behind one breakpoint, in a group. #[derive(Debug, Clone, PartialEq)] pub struct BreakpointCluster { pub chrom: String, diff --git a/crates/navigator-analysis/src/sv/segmenter.rs b/crates/navigator-analysis/src/sv/segmenter.rs index e5f6389e..86d3e975 100644 --- a/crates/navigator-analysis/src/sv/segmenter.rs +++ b/crates/navigator-analysis/src/sv/segmenter.rs @@ -1,12 +1,16 @@ -//! Depth segmenter — port of the Scala `DepthSegmenter`. Z-score CNV detection: expected -//! reads/bin from coverage, Poisson-ish variance, greedy extension of aberrant runs -//! (tolerating short dips), size filter, then merge + conversion to SV calls. +//! The depth segmenter. It is the port of the Scala `DepthSegmenter`. It finds a CNV from a +//! z-score. +//! +//! The coverage gives the count of reads that each bin expects. The variance is near that of a +//! Poisson. The code then extends a run of bins that differ from the expectation, greedily, and it +//! accepts a short dip inside such a run. It filters by size, merges the runs that stay, and turns +//! them into SV calls. use super::evidence::DepthSegment; use super::types::{SvCall, SvCallerConfig, SvType}; use std::collections::BTreeMap; -/// Segment per-contig depth bins into CNV `DepthSegment`s. +/// Turn the depth bins of each contig into CNV `DepthSegment` values. pub fn segment( depth_bins: &BTreeMap>, contig_lengths: &BTreeMap, @@ -52,7 +56,8 @@ pub fn segment( let mut sum_z = z; let mut sum_depth = bins[i] as f64; - // Extend while the next bin is on the same side, or short dips are bridged. + // Extend while the next bin lies on the same side. A short dip does not stop the + // extension, and the run goes over it. loop { let end_bin = start_bin + count - 1; if end_bin + 1 >= z_scores.len() { @@ -113,7 +118,8 @@ pub fn segment( segments } -/// Merge nearby same-type segments (default gap 50 kb), weighting by bin count. +/// Merge two segments of the same type that lie near each other. The default gap is 50 kb. The +/// count of bins gives the weight. pub fn merge_nearby_segments(segments: &[DepthSegment], max_gap: i64) -> Vec { if segments.is_empty() { return Vec::new(); diff --git a/crates/navigator-analysis/src/sv/types.rs b/crates/navigator-analysis/src/sv/types.rs index 0b12c472..a87b6f18 100644 --- a/crates/navigator-analysis/src/sv/types.rs +++ b/crates/navigator-analysis/src/sv/types.rs @@ -1,4 +1,5 @@ -//! SV core types — port of the Scala `SvTypes` (SvType, SvCall, config, confidence). +//! The core SV types. This is the port of the Scala `SvTypes`, which held SvType, SvCall, the +//! configuration and the confidence. use serde::{Deserialize, Serialize}; @@ -45,7 +46,8 @@ pub struct SvCall { pub genotype: String, } -/// Confidence in [0,1] weighting PE / SR / depth evidence (mirrors `calculateConfidence`). +/// The confidence, in [0,1]. It weighs the PE evidence, the SR evidence and the depth evidence. It +/// has the same shape as `calculateConfidence`. pub fn calculate_confidence(call: &SvCall) -> f64 { let pe_weight = 0.3; let sr_weight = 0.4; @@ -74,9 +76,12 @@ pub struct SvCallerConfig { pub min_split_read_support: u32, pub min_total_support: u32, pub min_quality: f64, - /// Ceiling on retained discordant pairs, and separately on retained split reads, for one walk. - /// A safety valve, not a filter: the point is that a pathological library cannot take the whole - /// process — and in a batch, the other 147 samples — down with an OOM. See the default. + /// The upper limit on the discordant pairs that one walk keeps, and, apart from that, on the + /// split reads that it keeps. + /// + /// It is a safety valve, and not a filter. One library of an unusual shape must not take the + /// whole process down with an out-of-memory error. In a batch, that would take the other 147 + /// samples with it. See the default. pub max_evidence_records: u64, } @@ -93,10 +98,13 @@ impl Default for SvCallerConfig { min_split_read_support: 1, min_total_support: 3, min_quality: 10.0, - // Chosen against the real spread, not a round number: across 33 analysed alignments the - // discordant-pair counts run 0.05–15.6 M (split reads 0–1.1 M), so 32 M is ~2x the - // worst case actually observed and does not fire on any of them. What it does is put a - // ~2 GB ceiling on evidence that was previously bounded only by the file. + // This value comes from the real spread, and it is not a round number. Across 33 + // alignments that somebody analysed, the discordant-pair counts run from 0.05M to + // 15.6M, and the split reads from 0 to 1.1M. So 32M is about 2x the worst case that + // anybody has seen, and it fires on none of them. + // + // What it does give is an upper limit of about 2 GB on evidence that the size of the + // file alone used to bound. max_evidence_records: 32_000_000, } } diff --git a/crates/navigator-analysis/src/sv/walker.rs b/crates/navigator-analysis/src/sv/walker.rs index 7a27b73f..1bb7481d 100644 --- a/crates/navigator-analysis/src/sv/walker.rs +++ b/crates/navigator-analysis/src/sv/walker.rs @@ -1,15 +1,20 @@ -//! SV evidence walker — port of the Scala `SvEvidenceWalker`. One pass over the alignment -//! collecting per-bin read depth (CNV), discordant read pairs (BreakDancer-style), and -//! split reads from the SA tag (Pindel-style). +//! The walker over the SV evidence. It is the port of the Scala `SvEvidenceWalker`. It makes one +//! pass over the alignment, and in that pass it collects three things: //! -//! Two walks share one per-record body ([`EvidenceSink::accept_read`]), so they cannot drift: -//! [`collect_evidence_parallel`] fans one region query per contig across a decode-safe rayon pool, -//! and [`collect_evidence`] makes a single sequential pass for files with no coordinate index. -//! Prefer the parallel entry point — it falls back to the sequential one on its own. +//! - the read depth in each bin, for a CNV call; +//! - the discordant read pairs, in the style of BreakDancer; +//! - the split reads from the SA tag, in the style of Pindel. //! -//! It walks records as [`AlnRead`] views rather than a concrete record type: the BAM path stays on -//! the lazy, zero-copy `bam::Record` (this is a whole-genome pass, so a per-read owned copy would -//! be costly), while the CRAM path gets the decoded record it has no cheaper form of. +//! Two walks share one body at the record level, which is [`EvidenceSink::accept_read`]. The two +//! can then never come apart. [`collect_evidence_parallel`] fans one region query for each contig +//! across a rayon pool whose stacks are safe for a decode. [`collect_evidence`] makes one +//! sequential pass, for a file with no coordinate index. Call the parallel one: it falls back to +//! the sequential one by itself. +//! +//! It walks the records as [`AlnRead`] views, and not as one concrete record type. So the BAM path +//! stays on the lazy, zero-copy `bam::Record`. This is a pass over the whole genome, so an +//! owned copy at each read would cost much. The CRAM path gets the decoded record, because it has +//! no cheaper form. use std::collections::BTreeMap; use std::path::Path; @@ -32,17 +37,19 @@ const SA_TAG: Tag = Tag::new(b'S', b'A'); /// How often the record loop polls the cancel token, matching the indexed reader's own cadence. const CANCEL_CHECK_RECORDS: u32 = 4096; -/// The running evidence tally. One instance covers the whole file in the sequential walk and one -/// contig in the parallel fan-out — the difference is only which records get fed to it and how -/// many entries `depth_bins` starts with, so both walks run the identical per-record body. +/// The tally of the evidence as the walk goes on. One instance covers the whole file in the +/// sequential walk, and one contig in the parallel fan-out. The only difference is which records +/// go into it, and how many entries `depth_bins` holds at the start. So both walks run the same +/// body at each record. struct EvidenceSink<'a> { - /// Reference id -> interned name, header order. Needed whole even per contig: a discordant pair - /// names its *mate's* contig, which is routinely a different one. + /// A map from a reference id to an interned name, in header order. Even a walk over one contig + /// needs the whole map. A discordant pair names the contig of its *mate*, and that is often a + /// different contig. names: &'a [Arc], config: &'a SvCallerConfig, budget: &'a EvidenceBudget, - /// The empty name a pair falls back to when its mate's reference id resolves to nothing, - /// interned so that path allocates no more than the normal one. + /// The empty name that a pair takes when the reference id of its mate resolves to nothing. The + /// code interns it, so that path allocates no more than the usual one. unknown_contig: Arc, insert_min: f64, insert_max: f64, @@ -86,8 +93,8 @@ impl<'a> EvidenceSink<'a> { let Some(ref_id) = record.reference_sequence_id() else { return; }; - // `names` is a shared slice, so this borrows the interned name for `'a` — not from `self`, - // which leaves the tallies below free to take `&mut self`. + // `names` is a shared slice, so this borrows the interned name for `'a`. It does not + // borrow from `self`, which leaves the tallies below free to take `&mut self`. let Some(contig) = self.names.get(ref_id) else { return }; let Some(start) = record.alignment_start().map(|p| p as i64) else { return; @@ -128,8 +135,8 @@ impl<'a> EvidenceSink<'a> { } } - /// Drop the borrowed lookup tables, keeping the tally. The fan-out returns this rather than the - /// sink itself so the per-contig results carry no lifetime. + /// Drop the borrowed lookup tables, and keep the tally. The fan-out returns this, and not the + /// sink itself, so that the result of each contig carries no lifetime. fn into_parts(self) -> ContigEvidence { ContigEvidence { depth_bins: self.depth_bins, @@ -141,11 +148,13 @@ impl<'a> EvidenceSink<'a> { } } -/// A ceiling on retained evidence, shared across the fan-out's contig workers so the bound is -/// genome-wide rather than per contig (24 contigs each allowed the full cap would bound nothing). +/// An upper limit on the evidence that the code keeps. The contig workers of the fan-out share it, +/// so the limit covers the whole genome, and not one contig. With 24 contigs, and the full limit +/// for each, the limit would hold nothing back. /// -/// Counts only what is *kept*. Evidence past the cap is still detected and counted as dropped, so -/// the reported totals stay honest and a truncated run is visible rather than silent. +/// It counts what the code *keeps*, and nothing else. The walk still finds the evidence past the +/// limit, and it counts that evidence as dropped. So the totals in the report stay honest, and a +/// run that the limit cut short is visible to the user. struct EvidenceBudget { cap: u64, discordant_kept: AtomicU64, @@ -161,8 +170,9 @@ impl EvidenceBudget { } } - /// Claim one slot in `counter`, or return false once the cap is met. Relaxed ordering: the - /// counters guard memory growth, and nothing is ordered against them. + /// Take one slot in `counter`, or return false once the count reaches the limit. It uses a + /// relaxed ordering. These counters hold the memory down, and nothing orders itself against + /// them. fn claim(&self, counter: &AtomicU64) -> bool { counter .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| { @@ -221,9 +231,9 @@ impl RecordSink for EvidenceSink<'_> { } } -/// Reference id -> name (header order), interned once per walk. Every retained pair and split read -/// holds `Arc`s from this table, so a contig name is stored once for the whole run instead of once -/// per record. +/// A map from a reference id to a name, in header order, interned once for each walk. Every pair +/// and split read that the code keeps holds an `Arc` from this table. A contig name goes into +/// memory once for the whole run, and not once at each record. fn contig_names(header: &noodles::sam::Header) -> Vec> { header .reference_sequences() @@ -254,19 +264,20 @@ fn insert_bounds(expected_insert_size: f64, insert_size_sd: f64, config: &SvCall (min, max) } -/// Collect SV evidence one contig at a time, in parallel. `contig_lengths` selects which contigs -/// get depth bins (and their sizes); `expected_insert_size`/`insert_size_sd` come from -/// read-metrics. `reference` is required for CRAM (ignored for BAM) — SV evidence never consults -/// reference *bases*, but decoding a CRAM record at all does. +/// Collect the SV evidence one contig at a time, in parallel. `contig_lengths` says which contigs +/// get depth bins, and how large those contigs are. `expected_insert_size` and `insert_size_sd` +/// come from read-metrics. A CRAM needs `reference`, and a BAM ignores it. The SV evidence never +/// reads a reference *base*, but the decode of a CRAM record does. /// -/// SV was the last whole-genome analysis still walking the file on one thread, which on a 30x WGS -/// CRAM is hours of single-core decode: 2–5 h per sample, against ~55 min for the per-contig -/// [`crate::unified`] walk over the same files. Region-querying each contig separately spends the -/// same total decode across every core instead of one. +/// SV was the last analysis over the whole genome that still walked the file on one thread. On a +/// 30x WGS CRAM that is hours of decode on one core. It took 2 to 5 h for each sample, against +/// about 55 min for the [`crate::unified`] walk over the contigs, on the same files. A separate +/// region query for each contig spends the same total decode across every core, and not on one. /// -/// Evidence is concatenated in header order, which for a coordinate-sorted file is exactly the -/// order the sequential walk emits — and the clusterer sorts by position regardless, so the calls -/// are identical either way. Falls back to [`collect_evidence`] when there is no `.bai`/`.crai`. +/// The code joins the evidence together in header order. For a file in coordinate order that is +/// exactly the order that the sequential walk gives. And the clusterer sorts by position in any +/// case, so the calls are the same either way. This function falls back to [`collect_evidence`] +/// when there is no `.bai` and no `.crai`. pub fn collect_evidence_parallel( bam_path: &Path, reference: Option<&Path>, @@ -276,8 +287,8 @@ pub fn collect_evidence_parallel( config: &SvCallerConfig, cancel: &crate::cancel::CancelToken, ) -> Result { - // Per-contig region queries need a coordinate index. Without one the only way to reach the - // records is a sequential pass, so take it rather than failing. + // A region query on one contig needs a coordinate index. Without one, a sequential pass is + // the only way to reach the records. So take that pass, and do not fail. if !reader::has_region_index(bam_path) { return collect_evidence( bam_path, @@ -293,18 +304,19 @@ pub fn collect_evidence_parallel( let header = reader::read_header(bam_path, reference)?; let names = contig_names(&header); let (insert_min, insert_max) = insert_bounds(expected_insert_size, insert_size_sd, config); - // Shared across the workers so the ceiling is on the genome, not on each contig. + // The workers share this, so the limit covers the genome, and not each contig. let budget = EvidenceBudget::new(config.max_evidence_records); - // One work item per *header* contig, not per `contig_lengths` entry: depth bins are limited to - // the requested contigs, but discordant pairs and split reads are collected genome-wide (the - // sequential walk sees every record in the file), so every contig has to be visited. + // One work item for each *header* contig, and not for each `contig_lengths` entry. The depth + // bins cover the contigs that the caller asked for. But the discordant pairs and the split + // reads cover the whole genome, because the sequential walk sees every record in the file. So + // the code must visit every contig. // - // Records with no reference position are skipped by both walks — the sequential one via the - // `is_unmapped` guard, this one by never querying for them — so nothing is lost by not sweeping - // the unmapped tail here. + // Both walks skip a record with no reference position. The sequential one does that with its + // `is_unmapped` guard, and this one never queries for such a record. Nothing goes missing when + // this code does not sweep the unmapped tail. let process_contig = |name: &Arc| -> Result { - // Bail before paying for this contig's reader. + // Stop before the cost of the reader of this contig. cancel.check()?; let (h, mut idx) = reader::open_indexed(bam_path, reference)?; let region = Region::new(name.as_bytes().to_vec(), ..); // whole contig @@ -317,9 +329,10 @@ pub fn collect_evidence_parallel( Ok(sink.into_parts()) }; - // noodles' CRAM decoder can recurse deeply enough to blow rayon's default 2 MiB worker stack - // (the main thread's larger stack handles the same file in the sequential walker) — and an - // overflow aborts the whole process, so the workers get a decode-safe stack. + // The CRAM decoder of noodles can recurse deep enough to overflow the default 2 MiB worker + // stack of rayon. In the sequential walker, the larger stack of the main thread holds the same + // file. An overflow aborts the whole process, so the workers get a stack that is safe for a + // decode. let pool = reader::decode_pool(crate::unified::analysis_thread_count())?; let per_contig: Vec = pool.install(|| { names @@ -328,8 +341,9 @@ pub fn collect_evidence_parallel( .collect::, AnalysisError>>() })?; - // Merge. Each contig owns a disjoint depth-bin key, so inserting over the pre-zeroed map is a - // fill rather than a sum; a requested contig with no records keeps its zeros. + // The merge. The depth-bin keys of two contigs never meet. A write over the map, which starts + // at zero, then fills a slot and does not add to one. A contig that the caller asked for, and + // that holds no record, keeps its zeros. let mut merged = ContigEvidence { depth_bins: all_zeroed_bins(contig_lengths, config.bin_size), ..ContigEvidence::default() @@ -344,9 +358,9 @@ pub fn collect_evidence_parallel( Ok(merged.into_collection(expected_insert_size, insert_size_sd, config.max_evidence_records)) } -/// Collect SV evidence in a single sequential pass — the parity reference for -/// [`collect_evidence_parallel`], and the walk used when the alignment has no coordinate index. -/// Arguments are as documented there. +/// Collect the SV evidence in one sequential pass. It is the reference that +/// [`collect_evidence_parallel`] must agree with, and it is the walk for an alignment with no +/// coordinate index. The arguments are the same, and that function documents them. pub fn collect_evidence( bam_path: &Path, reference: Option<&Path>, @@ -384,13 +398,16 @@ pub fn collect_evidence( } impl EvidenceSink<'_> { - /// Classify one primary paired read, building a [`DiscordantPair`] only if it is discordant. + /// Classify one primary read that has a pair. It builds a [`DiscordantPair`] only when that + /// read is discordant. + /// + /// This runs at almost every read in the file, and all but a small fraction of a percent are + /// concordant. So the code builds nothing before it knows the answer. /// - /// This runs for essentially every read in the file and all but a fraction of a percent are - /// concordant, so nothing is built before the verdict is known. It used to allocate the read - /// name and clone the mate contig name up front — two mallocs per read, thrown away almost - /// every time — which is what made a 30x WGS walk malloc-bound. The pair that does get built is - /// now allocation-free: both contigs are `Arc` clones from the interned table. + /// An earlier version allocated the read name, and cloned the name of the mate contig, at the + /// start. That was two mallocs at each read, and it threw away almost all of them. It is what + /// made a 30x WGS walk spend its time in malloc. The pair that the code does build now + /// allocates nothing: both contigs are `Arc` clones from the interned table. fn detect_discordant_pair(&self, record: &impl AlnRead, contig: &Arc, mapq: u8) -> Option { let flags = record.flags(); if flags.is_mate_unmapped() || mapq < self.config.min_mapq { @@ -448,11 +465,13 @@ fn is_expected_orientation(record: &impl AlnRead, pos1: i64, mate_pos: i64) -> b } impl EvidenceSink<'_> { - /// Parse the first SA-tag alignment into a [`SplitRead`]; clip length is the read's own - /// soft/hard-clip total. Only reads that actually carry an `SA` tag get this far, so unlike - /// the discordant-pair path this one is not hot — a genome-wide walk yields ~0.1–1 M of these - /// against billions of reads, which is why interning the supplementary contig off the tag text - /// is not worth a lookup table. + /// Parse the first alignment of the SA tag into a [`SplitRead`]. The clip length is the total + /// soft clip and hard clip of the read itself. + /// + /// Only a read that carries an `SA` tag reaches this code, so this path is not hot, and the + /// discordant-pair path is. A walk over the whole genome gives about 0.1M to 1M of these, + /// against billions of reads. That is why the supplementary contig comes straight off the tag + /// text, and a lookup table to intern it is not worth the code. fn extract_split_read(&self, record: &impl AlnRead, contig: &Arc, mapq: u8) -> Option { let sa = record.string_tag(SA_TAG)?; if sa.is_empty() { diff --git a/crates/navigator-analysis/src/testtype.rs b/crates/navigator-analysis/src/testtype.rs index e78af856..97d8817a 100644 --- a/crates/navigator-analysis/src/testtype.rs +++ b/crates/navigator-analysis/src/testtype.rs @@ -1,12 +1,19 @@ -//! Test-type identification — Rust port of the Scala `TestType` catalog + `TestTypeInference`. +//! Find out which test made an alignment. This is the Rust port of the Scala `TestType` catalog +//! and its `TestTypeInference`. //! -//! The header probe ([`crate::probe`]) only knows *platform* (PacBio→HiFi, Illumina→WGS, …). It -//! can't tell a **targeted** test (FTDNA Big Y, Full Genomes Y Elite, YSEQ, an mtFull run) from a -//! whole-genome one — those look the same in the SAM header. The Scala app distinguished them by -//! **coverage shape**: a Big Y BAM has reads piled on chrY with the autosomes near-empty; an mtFull -//! run piles on chrM. We reproduce that cheaply from the **BAI index** (per-reference mapped-record -//! counts, O(contigs) — the same fast path [`crate::sex`] uses), normalized to a coverage proxy, and -//! combine it with the platform + an optional vendor hint to pick a test-type code. +//! The header probe ([`crate::probe`]) knows the *platform* alone: PacBio gives HiFi, Illumina +//! gives WGS, and so on. It can not separate a **targeted** test from a whole-genome one, because +//! the two look the same in a SAM header. An FTDNA Big Y, a Full Genomes Y Elite, a YSEQ test and +//! an mtFull run are all targeted. +//! +//! The Scala app separated them by the **shape of the coverage**. A Big Y BAM piles its reads on +//! chrY, and its autosomes are almost empty. An mtFull run piles them on chrM. +//! +//! This module gets that shape at low cost, from the **BAI index**. That index holds the count of +//! mapped records of each reference, at O(contigs), and it is the same fast path that +//! [`crate::sex`] uses. The code normalizes those counts to a coverage proxy. It then puts that +//! together with the platform, and with a vendor hint when there is one, and takes a test-type +//! code. use std::path::Path; @@ -15,12 +22,14 @@ use noodles::csi::binning_index::ReferenceSequence as _; use crate::contig; -// Test-type codes are the canonical `navigator_domain::testtype` catalog strings — display names, -// target region, and the UI picker live there. This module only decides *which* code a BAM's -// coverage shape implies, emitting those code literals (validated against the catalog by a test). +// A test-type code is one of the canonical strings of the `navigator_domain::testtype` catalog. +// The display name, the target region and the UI picker all live there. This module decides +// *which* code the coverage shape of a BAM implies, and nothing more. It writes those code +// literals out, and a test checks them against the catalog. -/// Per-chromosome-group coverage proxies (reads × read-length ÷ group length), the same estimate -/// the Scala `ChromosomeCoverageStats` used. `None` group ⇒ no such contig in the reference. +/// The coverage proxy of each chromosome group, as reads × read length ÷ group length. That is the +/// same estimate that the Scala `ChromosomeCoverageStats` used. A group of `None` means that the +/// reference holds no such contig. #[derive(Debug, Clone, Copy, PartialEq)] pub struct CoverageProfile { pub autosome_depth: f64, @@ -30,15 +39,21 @@ pub struct CoverageProfile { pub has_autosomes: bool, } -/// Scala `ASSUMED_READ_LENGTH` — coverage estimate when the true mean read length is unknown. +/// The `ASSUMED_READ_LENGTH` of the Scala code. The estimate uses it when nobody knows the true +/// mean read length. const ASSUMED_READ_LENGTH: u64 = 150; -// Coverage thresholds. The Scala used absolute cutoffs (`yCov>1 && autoCov<1`), but real Big Y -// BAMs aligned to the whole genome carry ~1-2× off-target autosomal reads (a real FTDNA Big Y here -// measured Y 51× / autosome 1.8×), which the absolute test mislabels as low-pass WGS. We instead key -// targeted-Y off the **Y:autosome enrichment ratio** — read-length-independent, so it also survives -// the long-read coverage underestimate — and require autosomes essentially absent for targeted-MT -// (mtDNA is naturally high-copy, so a WGS sample shows huge mt depth without being an mtFull test). +// The coverage thresholds. The Scala code used absolute cutoffs, at `yCov>1 && autoCov<1`. But a +// real Big Y BAM that somebody aligned to the whole genome carries 1 to 2x of autosomal reads that +// are off target. A real FTDNA Big Y here measured Y at 51x and the autosomes at 1.8x. The +// absolute test reads that as a low-pass WGS run. +// +// This code instead keys a targeted-Y test off the **enrichment ratio of Y against the +// autosomes**. That ratio does not depend on the read length, so it also survives the low coverage +// estimate that a long read gives. +// +// For a targeted-MT test it needs the autosomes to be almost absent. mtDNA is naturally +// high-copy. So a WGS sample shows a very large mt depth, and it is not an mtFull test. const Y_PRESENT: f64 = 1.0; // Y depth floor below which we don't call targeted-Y at all const Y_ENRICH: f64 = 5.0; // Y:autosome ratio that marks a Y-targeted capture const MT_PRESENT: f64 = 10.0; @@ -47,10 +62,14 @@ const LONG_READ_LEN: u64 = 1000; const WES_AUTOSOME_DEPTH: f64 = 50.0; const LOW_PASS_AUTOSOME_DEPTH: f64 = 5.0; -/// Build a [`CoverageProfile`] from a BAM's BAI index (no read scan). `mean_read_length` refines -/// the estimate when known (e.g. from `library_stats`); else [`ASSUMED_READ_LENGTH`]. Returns `None` -/// when the index is absent/unreadable (e.g. CRAM — `.crai` carries no per-reference counts), so the -/// caller keeps the header/platform result. +/// Build a [`CoverageProfile`] from the BAI index of a BAM. It scans no read. +/// +/// `mean_read_length` makes the estimate better when somebody knows it, for example from +/// `library_stats`. Without it the code uses [`ASSUMED_READ_LENGTH`]. +/// +/// It returns `None` when the index is absent, or when the code can not read it. A CRAM is one +/// such case, because a `.crai` holds no count of each reference. The caller then keeps the result +/// from the header and the platform. pub fn coverage_profile_from_bai(bam_path: &Path, mean_read_length: Option) -> Option { let header = crate::reader::read_header(bam_path, None).ok()?; let bai_path = bam_path.with_extension("bam.bai"); @@ -100,14 +119,15 @@ pub fn coverage_profile_from_bai(bam_path: &Path, mean_read_length: Option) /// Map a free-text vendor hint to a specific targeted-Y test code (else the honest generic). fn targeted_y_for_vendor(vendor_hint: Option<&str>) -> &'static str { match vendor_hint.map(|v| v.to_lowercase()) { - // FTDNA only sells Big Y, but the *generation* (500 vs 700) isn't in the vendor token — - // it comes from the `@RG LB` label ([`crate::probe`], passed as `big_y_label`) or, on older - // headers that omit it, from the callable-chrY footprint resolved after analysis. Stay - // generic here so neither generation is guessed from the vendor name alone. + // FTDNA sells Big Y alone. But the vendor token does not carry the *generation*, which is + // 500 or 700. That comes from the `@RG LB` label, which [`crate::probe`] reads and passes + // in as `big_y_label`. On an older header that leaves that label out, it comes instead + // from the callable-chrY footprint, which the code resolves after the analysis. So stay + // generic here, and do not guess a generation from the vendor name alone. Some(v) if v.contains("ftdna") || v.contains("familytreedna") => "TARGETED_Y", Some(v) if v.contains("full genomes") || v.contains("fullgenomes") => "Y_ELITE", Some(v) if v.contains("yseq") => "Y_PRIME", - // An unknown vendor isn't mislabeled to a specific product. + // An unknown vendor is not mislabeled to a specific product. _ => "TARGETED_Y", } } @@ -128,10 +148,12 @@ fn wgs_for_platform(platform: Option<&str>, mean_read_length: Option) -> &' } } -/// Infer the test type from coverage shape + platform + vendor hint (Scala `inferFromCoverage`). +/// Infer the test type from the coverage shape, the platform and the vendor hint. This is the +/// Scala `inferFromCoverage`. /// -/// With no coverage profile (CRAM / unindexed), falls back to the platform-only WGS guess — the -/// pre-existing probe behavior. Returns `None` only when nothing at all is known. +/// With no coverage profile it falls back to the WGS guess from the platform alone. That happens +/// for a CRAM, and for a file with no index. The probe behaved that way before this code. It +/// returns `None` only when the code knows nothing at all. pub fn infer_test_type( profile: Option<&CoverageProfile>, platform: Option<&str>, @@ -139,8 +161,8 @@ pub fn infer_test_type( mean_read_length: Option, big_y_label: Option<&str>, ) -> Option { - // An explicit FTDNA Big Y generation from the header (`@RG LB`) is authoritative — it's FTDNA's - // own product label, so it overrides the coverage-shape guess entirely. + // An FTDNA Big Y generation that the header states, in `@RG LB`, is authoritative. It is the + // own product label of FTDNA, so it wins over the guess from the coverage shape. if let Some(code) = big_y_label { return Some(code.to_string()); } @@ -151,7 +173,8 @@ pub fn infer_test_type( // "Autosomal coverage present" = depth above the floor AND autosomal contigs exist at all. let has_autosome = p.has_autosomes && p.autosome_depth > AUTOSOME_PRESENT; - // Y:autosome enrichment — the targeted-Y signal. No autosomes (Y-only reference) ⇒ infinite. + // The enrichment of Y against the autosomes. That is the signal of a targeted-Y test. With no + // autosome at all, on a Y-only reference, it is infinite. let y_ratio = if p.autosome_depth > 0.0 { p.y_depth / p.autosome_depth } else { @@ -161,8 +184,9 @@ pub fn infer_test_type( // Targeted-Y: Y meaningfully covered AND strongly enriched over the autosomes (off-target // autosomal reads are normal), or a Y-only reference (no autosomal contigs). let targeted_y = p.y_depth > Y_PRESENT && (!has_autosome || y_ratio > Y_ENRICH); - // Targeted-MT: only mtDNA covered — autosomes essentially absent and no Y (mtDNA is naturally - // high-copy, so a WGS sample shows huge mt depth without being an mtFull test). + // A targeted-MT test covers mtDNA alone. The autosomes are almost absent, and there is no Y. + // mtDNA is naturally high-copy. So a WGS sample shows a very large mt depth, and it is not an + // mtFull test. let targeted_mt = !targeted_y && p.mt_depth > MT_PRESENT && !has_autosome && p.y_depth <= Y_PRESENT; let code = if targeted_y { @@ -170,7 +194,7 @@ pub fn infer_test_type( } else if targeted_mt { "MT_FULL_SEQUENCE" } else if has_autosome && p.y_depth <= Y_PRESENT && p.autosome_depth > WES_AUTOSOME_DEPTH { - // Very high autosomal depth with no Y signal — exome capture. + // A very high autosomal depth, with no Y signal. That is an exome capture. "WES" } else if has_autosome && p.autosome_depth < LOW_PASS_AUTOSOME_DEPTH { "WGS_LOW_PASS" @@ -195,10 +219,10 @@ mod tests { #[test] fn targeted_y_maps_vendor_or_generic() { - // Y-only reference (no autosomes) — clean targeted-Y. + // A Y-only reference, with no autosome. That is a clean targeted-Y test. let p = prof(0.0, 35.0, 0.0, false); - // FTDNA without a generation label stays generic — 500 vs 700 isn't in the vendor token - // (the header `@RG LB` or the callable-chrY footprint decides it). + // An FTDNA test with no generation label stays generic. The vendor token does not say 500 + // or 700. The header `@RG LB`, or the callable-chrY footprint, decides that. assert_eq!( infer_test_type(Some(&p), Some("ILLUMINA"), Some("FamilyTreeDNA"), None, None).as_deref(), Some("TARGETED_Y") @@ -232,7 +256,8 @@ mod tests { #[test] fn explicit_big_y_label_overrides_coverage_shape() { - // Even a WGS-looking coverage shape yields the header's Big Y generation when @RG LB said so. + // The header wins. Even a coverage shape that looks like WGS gives the Big Y generation + // of the header, when `@RG LB` states one. let wgs_shape = prof(30.0, 15.0, 1000.0, true); assert_eq!( infer_test_type(Some(&wgs_shape), Some("ILLUMINA"), None, None, Some("BIG_Y_700")).as_deref(), @@ -242,8 +267,9 @@ mod tests { #[test] fn targeted_y_by_enrichment_with_offtarget_autosomes() { - // Real Full Genomes Y Elite shape (B6564_Kane.bam): Y 51× over autosome 1.8× off-target — - // absolute "autosome<1" would mislabel this WGS_LOW_PASS; the 28× enrichment marks it. + // The shape of a real Full Genomes Y Elite run, from B6564_Kane.bam. Y is at 51x, over + // autosomes at 1.8x that are off target. An absolute test of "autosome<1" would read this + // as WGS_LOW_PASS. The enrichment of 28x marks it correctly. let p = prof(1.84, 51.0, 7.0, true); assert_eq!( infer_test_type(Some(&p), None, Some("Full Genomes"), None, None).as_deref(), @@ -273,7 +299,8 @@ mod tests { infer_test_type(Some(&male), Some("ILLUMINA"), None, None, None).as_deref(), Some("WGS") ); - // Female WGS: y≈0, mt high — the autosome-present guard keeps it WGS, not targeted-MT. + // A female WGS run. Y is near 0, and mt is high. The guard that needs the autosomes to be + // present keeps this as WGS, and not as targeted-MT. let female = prof(30.0, 0.02, 1200.0, true); assert_eq!( infer_test_type(Some(&female), Some("ILLUMINA"), None, None, None).as_deref(), @@ -312,8 +339,8 @@ mod tests { #[test] fn every_emitted_code_is_in_the_domain_catalog() { - // The codes we emit must be recognized by the canonical catalog (else the UI picker / - // display_name would show a raw code). Exercise every branch's output. + // The canonical catalog must know every code that this module writes. Else the UI picker, + // and display_name, would show a raw code. This test covers the output of every branch. let shapes = [ (prof(0.0, 35.0, 0.0, false), Some("FamilyTreeDNA")), (prof(0.0, 35.0, 0.0, false), Some("Full Genomes")), diff --git a/crates/navigator-analysis/src/unified.rs b/crates/navigator-analysis/src/unified.rs index 8f9d9c39..e82384f3 100644 --- a/crates/navigator-analysis/src/unified.rs +++ b/crates/navigator-analysis/src/unified.rs @@ -1,22 +1,26 @@ -//! Unified quality-metrics walker — one coordinate-ordered pass over a BAM/CRAM that -//! collects coverage + callable loci, read-level QC metrics, and sex inference together. +//! The unified walker over the quality metrics. It makes one pass over a BAM or a CRAM, in +//! coordinate order. In that pass it collects three things together: the coverage and the callable +//! loci, the QC metrics at the read level, and the sex inference. //! -//! The rewrite already had three focused single-pass walkers ([`crate::coverage`], -//! [`crate::read_metrics`], [`crate::sex`]); run separately they read a BAM end-to-end -//! **twice** (coverage pileup + read-metrics scan) and a CRAM **three times** (plus the -//! sex scan, since `.crai` carries no per-reference counts). This walker fuses them into a -//! single record loop — 2→1 for BAM, 3→1 for CRAM (CRAM decode being the expensive case). +//! The rewrite already had three walkers, each with one purpose and one pass: +//! [`crate::coverage`], [`crate::read_metrics`] and [`crate::sex`]. Run apart, they read a BAM +//! from end to end **twice**, for the coverage pileup and the read-metrics scan. They read a CRAM +//! **three times**, because the sex scan is separate: a `.crai` carries no count for each +//! reference. This walker puts them into one record loop. That is 2 passes to 1 for a BAM, and 3 +//! to 1 for a CRAM, and a CRAM decode is the costly case. //! -//! There is **no metric change**: each record is dispatched to the same `*State` accumulators -//! the standalone walkers use (the single source of truth), so the numbers are byte-for-byte -//! identical to running the three separately. The only subtlety is filtering — coverage -//! pre-filters hard (mapped/primary/main-assembly), but read-metrics needs *every* record and -//! sex needs per-contig mapped tallies, so the loop hands every record to all three states and -//! each applies its own filtering internally. +//! **No metric changes.** The loop sends each record to the same `*State` accumulators that the +//! separate walkers use, which are the single source of truth. Every number then matches the +//! number that three separate runs give, to the last digit. //! -//! Sex here is tallied directly from the record stream (no BAI dependency), matching the -//! standalone CRAM path's math; the standalone [`crate::sex::infer_from_bam`] keeps its BAI -//! fast path for the cheap à-la-carte "Sex inference" command. +//! There is one thing to watch, and that is the filter. The coverage pass applies a hard filter, +//! and it keeps only a primary record with a mapping on the main assembly. But +//! read-metrics needs *every* record, and sex needs the mapped tally of each contig. So the loop +//! gives every record to all three states, and each state applies its own filter inside. +//! +//! The sex tally here comes straight from the record stream, and it needs no BAI. The arithmetic +//! is the same as in the separate CRAM path. The separate [`crate::sex::infer_from_bam`] keeps its +//! fast path over the BAI, for the small "Sex inference" command on its own. use std::collections::HashSet; use std::path::Path; @@ -38,27 +42,37 @@ use crate::reader::{self, RecordSink}; use crate::readview::AlnRead; use crate::sex::{self, SexInferenceResult, SexState}; -/// Flush accumulated base-pair progress to the shared counter every this many bp of advance. -/// Small enough that the bar moves smoothly (~1500 ticks over a 3.1 Gb genome) yet coarse enough -/// that the atomic add + progress callback (a mutex-guarded channel send in the GUI) is cheap. +/// Send the base-pair progress that the loop has collected to the shared counter, after this many +/// bp of advance. It is small enough that the bar moves smoothly, at about 1500 ticks over a +/// 3.1 Gb genome. It is also large enough that the atomic add and the progress callback cost +/// little. In the GUI that callback is a channel send behind a mutex. const PROGRESS_FLUSH_BP: u64 = 2_000_000; -/// Per-contig record consumer: feeds each record to read-metrics + coverage and tallies the -/// per-contig mapped counts the sex inference needs. `class`: 1 = autosome, 2 = chrX, 0 = other. -/// Operates on borrowed accumulators so it serves the zero-copy BAM record path. +/// The record consumer for one contig. It gives each record to read-metrics and to coverage, and +/// it tallies the mapped count of each contig, which the sex inference needs. `class` is 1 for an +/// autosome, 2 for chrX, and 0 for anything else. It works on borrowed accumulators, so it serves +/// the zero-copy path for a BAM record. +/// +/// It also drives the **base-pair progress**. The reads inside a contig are in coordinate order, +/// so the alignment start only rises. The difference between one start and the next is the count +/// of bp walked. /// -/// It also drives **base-pair progress**: reads are coordinate-sorted within a contig, so the -/// alignment start advances monotonically — the delta between successive starts is bp walked. -/// Deltas accumulate locally and flush to the shared `processed_bp` counter (and the progress -/// callback) every [`PROGRESS_FLUSH_BP`], so the bar advances continuously *within* a contig -/// instead of only when one finishes (the big autosomes otherwise sit frozen for minutes). +/// The differences add up locally, and go to the shared `processed_bp` counter, and to the +/// progress callback, every [`PROGRESS_FLUSH_BP`]. The bar then advances all the time *inside* a +/// contig. Without that, it moves only when a contig finishes, and the big autosomes leave it +/// frozen for minutes. struct ContigSink<'a> { - /// Header index of the contig this sink is walking. A record is processed only when its own - /// `reference_sequence_id` matches: a CRAM multi-reference slice surfaces in the region query - /// of *every* contig it overlaps, so without this gate its records would be misattributed to - /// the wrong contig (coverage) and counted once per overlapping contig (read-metrics). Each - /// record is processed exactly once — by the query of the contig it actually belongs to — - /// matching the sequential walker's single binned pass. A no-op for single-reference slices. + /// The header index of the contig that this sink walks. The sink takes a record only when the + /// `reference_sequence_id` of that record matches. + /// + /// A CRAM slice with more than one reference comes back from the region query of *every* + /// contig that it overlaps. Without this gate, its records would go to the wrong contig in the + /// coverage pass. Read-metrics would also count them once for each contig that they + /// overlap. + /// + /// With the gate, the code takes each record exactly once, in the query of the contig that it + /// belongs to. That matches the single binned pass of the sequential walker. The gate does + /// nothing for a slice with one reference. ref_id: usize, rm: &'a mut ReadMetricsState, cov: &'a mut Option, @@ -75,8 +89,9 @@ struct ContigSink<'a> { impl RecordSink for ContigSink<'_> { fn accept(&mut self, record: &impl AlnRead) { - // Only this contig's own records (drop a multi-reference slice's foreign records — they're - // processed by their own contig's query). Keeps every per-record tally counted exactly once. + // Take the records of this contig alone. Drop a record of another contig that came in + // with a slice that holds more than one reference. The query of its own contig takes it. + // Every tally over the records then counts each record exactly once. if record.reference_sequence_id() != Some(self.ref_id) { return; } @@ -116,14 +131,18 @@ impl RecordSink for MetricsSink { } } -/// Algorithm version for the unified artifact cache key; bump on any change that alters output. -/// (The three sub-results are persisted under their own existing keys; this is for completeness.) +/// The algorithm version, for the cache key of the unified artifact. Raise it after any change +/// that alters the output. The three sub-results already go into the store under their own keys, +/// so this version exists to make the set complete. pub const UNIFIED_VERSION: &str = "unified-1"; -/// The three quality-metric results collected in one pass. Sex is `None` when inference -/// can't be computed for the input (no autosomes/chrX, or no autosomal reads — e.g. a -/// targeted panel or chrY-only test); coverage + read-metrics are unaffected, mirroring the -/// pipeline where sex is an independent step whose failure doesn't kill the others. +/// The three quality-metric results that one pass collects. +/// +/// Sex is `None` when the code can not infer it for the input. That happens when there is no +/// autosome and no chrX, and when there is no autosomal read. A targeted panel and a chrY-only +/// test are examples. The coverage and the read-metrics do not change. This has the same shape as +/// the pipeline, where sex is an independent step, and a failure there does not kill the other +/// two. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct UnifiedMetricsResult { pub coverage: CoverageResult, @@ -131,10 +150,12 @@ pub struct UnifiedMetricsResult { pub sex: Option, } -/// Single-pass coverage + read-metrics + sex over a coordinate-sorted BAM/CRAM. `reference` -/// is required (CRAM decode + reference-N detection). Equivalent to running +/// The coverage, the read-metrics and the sex, in one pass over a BAM or CRAM in coordinate +/// order. It needs `reference`, both to decode a CRAM and to find the N bases in the reference. +/// +/// The result is the same as three separate steps: /// [`crate::coverage::collect_coverage_callable`], [`crate::read_metrics::collect_read_metrics`], -/// and tallying sex from the same records — but one read of the file. +/// and a tally of the sex over the same records. But it reads the file once. pub fn collect_unified_metrics( bam_path: &Path, reference_path: &Path, @@ -151,9 +172,11 @@ pub fn collect_unified_metrics( ) } -/// Like [`collect_unified_metrics`], reporting `progress(contigs_done, contigs_total)` as the -/// coverage pass finalizes each tracked contig (the slow whole-genome step — so a progress bar -/// can advance instead of sitting frozen for minutes). Assumes a coordinate-sorted BAM/CRAM. +/// The same as [`collect_unified_metrics`], and it also reports +/// `progress(contigs_done, contigs_total)`. It calls that as the coverage pass finishes each +/// contig that it tracks. That pass is the slow step over the whole genome, so a progress bar can +/// then move, and it does not stay frozen for minutes. This function needs a BAM or CRAM in +/// coordinate order. pub fn collect_unified_metrics_with_progress( bam_path: &Path, reference_path: &Path, @@ -168,10 +191,12 @@ pub fn collect_unified_metrics_with_progress( let mut sx = SexState::new(&header); progress(0, cov.total_tracked()); - // Polled on the same cadence as the indexed walker's record loop — often enough that a click - // stops the walk within milliseconds, rare enough to stay invisible next to the per-record - // pileup work. This is the fallback path (unindexed BAM / CRAM), which has no contig boundaries - // to stop at, so without a check here it could not be cancelled at all. + // The code polls this at the same rate as the record loop of the walker that uses an index. + // That rate is often enough that a click stops the walk in milliseconds. It is also rare + // enough to cost nothing next to the pileup work at each record. This is the fallback path, + // for a BAM + // or CRAM with no index. It has no contig boundary to stop at, so without a check here nobody + // could cancel it at all. let mut seen = 0u32; for result in reader.records_lazy(&header) { let record = result?; @@ -197,18 +222,22 @@ pub fn collect_unified_metrics_with_progress( }) } -/// Per-contig parallel unified metrics — the same result as [`collect_unified_metrics`] but -/// computed concurrently across contigs. Coverage is embarrassingly parallel per contig; the -/// per-position pileup compute (not decompression) is the bottleneck a sequential pass hits. +/// The unified metrics, with one task for each contig. The result is the same as +/// [`collect_unified_metrics`], and the contigs run at the same time. The coverage pass over one +/// contig is independent of every other contig. The compute of the pileup at each position, and +/// not the decompression, is what limits a sequential pass. +/// +/// This needs an **indexed BAM**, for the region query of each contig and for a sweep over the +/// unmapped tail. Anything else falls back to the sequential [`collect_unified_metrics`], and the +/// caller sees no difference. That covers a CRAM, because a `.crai` has no query for the unmapped +/// reads, and a BAM with no `.bai`. A caller can then always ask for this function. /// -/// Requires an **indexed BAM** (per-contig region queries + an unmapped-tail sweep). Anything -/// else — CRAM (no `.crai` unmapped query), or a BAM without a `.bai` — transparently falls -/// back to the sequential [`collect_unified_metrics`], so callers can always prefer this. +/// The output matches that of the sequential walker to the last digit. At each contig it runs the +/// same `*State` accumulators. The merge is over sums that commute, and over outputs that follow +/// the header order of the contigs. /// -/// Output is byte-identical to the sequential walker: per contig it runs the same `*State` -/// accumulators, and the merge is over commutative sums / header-ordered per-contig outputs. -/// Read-metrics covers **every** contig (not just main-assembly) plus the unmapped tail — the -/// same record set the sequential pass sees — so totals match exactly. +/// Read-metrics covers **every** contig, and not the main assembly alone, plus the unmapped tail. +/// That is the same set of records that the sequential pass sees, so the totals agree exactly. pub fn collect_unified_metrics_parallel( bam_path: &Path, reference_path: &Path, @@ -225,10 +254,10 @@ pub fn collect_unified_metrics_parallel( ) } -/// Worker threads for the per-contig fan-out. Defaults to all available cores capped at 12 — -/// past that the wall time is floored by the largest contig + the unmapped sweep, so more -/// threads only add memory. Override with `NAVIGATOR_ANALYSIS_THREADS`. Shared with the -/// de-novo caller's region fan-out. +/// The count of worker threads for the fan-out over the contigs. The default is every available +/// core, up to 12. Above that, the largest contig plus the unmapped sweep set the floor on the +/// wall time, so more threads only add memory. `NAVIGATOR_ANALYSIS_THREADS` overrides it. The +/// region fan-out of the de-novo caller uses the same value. pub(crate) fn analysis_thread_count() -> usize { std::env::var("NAVIGATOR_ANALYSIS_THREADS") .ok() @@ -242,9 +271,10 @@ pub(crate) fn analysis_thread_count() -> usize { .max(1) } -/// A token from the reference-load semaphore; returns itself to the pool on drop (including on -/// the error path). Bounds how many contigs hold their full reference buffer at once — the peak -/// memory driver, since the per-contig N-mask is tiny once built. +/// A token from the semaphore that limits the reference loads. It goes back into the pool when it +/// drops, and that includes the error path. It bounds how many contigs hold their full reference +/// buffer at one time. That is what sets the peak memory, because the N-mask of a contig is very +/// small once the code has built it. struct LoadPermit<'a> { tx: &'a std::sync::mpsc::Sender<()>, } @@ -263,11 +293,12 @@ struct ContigPartial { x_reads: u64, } -/// Like [`collect_unified_metrics_parallel`], reporting `progress(megabases_done, megabases_total)` -/// — base-pair position walked across all contigs, so the bar advances continuously rather than -/// stepping once per finished contig (the big autosomes run first and finish in a late burst, -/// freezing a contig-count bar at 0 for ~half the run). The callback is `Fn + Sync` because it's -/// invoked concurrently from worker threads. +/// The same as [`collect_unified_metrics_parallel`], and it also reports +/// `progress(megabases_done, megabases_total)`. Those are the base-pair positions walked over all +/// of the contigs, so the bar advances all the time. It does not take one step at each finished +/// contig. The big autosomes start first and finish together late, which holds a bar over the +/// contig count at 0 for about half of the run. The callback is `Fn + Sync`, because the worker +/// threads call it at the same time. pub fn collect_unified_metrics_parallel_with_progress( bam_path: &Path, reference_path: &Path, @@ -276,10 +307,13 @@ pub fn collect_unified_metrics_parallel_with_progress( progress: &(dyn Fn(usize, usize) + Sync), cancel: &CancelToken, ) -> Result { - // The parallel path needs a coordinate index for per-contig region queries — a BAM `.bai` or a - // CRAM `.crai`. Without one (unindexed BAM/CRAM) fall back to the sequential walker. A CRAM can't - // region-query its unmapped tail, so its read-metrics totals exclude unmapped-only reads (those - // carry no coverage/sex signal) — the per-contig coverage + mapped read-metrics still parallelize. + // The parallel path needs a coordinate index for the region query of each contig. That is a + // `.bai` for a BAM, or a `.crai` for a CRAM. Without one, fall back to the sequential walker. + // + // A CRAM has no region query for its unmapped tail. So the read-metrics totals of a CRAM + // leave out the reads that have no mapping and nothing else. Those reads carry no coverage + // signal and no sex signal. The coverage of each contig, and the read-metrics over the mapped reads, + // still run in parallel. if !reader::has_region_index(bam_path) { return collect_unified_metrics_with_progress( bam_path, @@ -294,8 +328,9 @@ pub fn collect_unified_metrics_parallel_with_progress( let header = reader::read_header(bam_path, Some(reference_path))?; - // Work items: one per reference sequence (read-metrics + sex span all contigs). Coverage - // runs only for tracked = main-assembly ∩ allowlist contigs (matching the sequential walker). + // The work items: one for each reference sequence, because read-metrics and sex cover every + // contig. The coverage runs only for the contigs that the code tracks, which are the ones on + // the main assembly that are also in the allowlist. That matches the sequential walker. struct Work { ref_id: usize, name: String, @@ -327,20 +362,24 @@ pub fn collect_unified_metrics_parallel_with_progress( }); } - // Progress is reported in **megabases of reference walked** rather than contigs finished, so - // the bar advances continuously from the first seconds — the big autosomes (scheduled first by - // rayon) otherwise complete in a late burst, leaving the bar frozen at 0 for ~half the run. - // Denominator = every walked contig's length (read-metrics covers all contigs); the dominant - // main-assembly contigs make the count track genomic position closely enough. + // The progress goes out in **megabases of reference walked**, and not in contigs finished. + // The bar then advances all the time, from the first seconds. rayon schedules the big + // autosomes first, and they otherwise finish together late. That leaves the bar frozen at 0 + // for about half of the run. + // + // The denominator is the length of every contig that the code walks, because read-metrics + // covers all of them. The contigs of the main assembly dominate that sum, so the count follows + // the position in the genome closely enough. let total_bp: u64 = works.iter().map(|w| w.length as u64).sum(); let total_mb = (total_bp / 1_000_000).max(1) as usize; let processed_bp = AtomicU64::new(0); progress(0, total_mb); let n_threads = analysis_thread_count(); - // Bound concurrent full-reference loads (the peak-memory driver) independently of compute - // parallelism: at most a few contigs hold their raw reference at once while building the - // compact N-mask. A token pool implements the counting semaphore. + // Limit how many full-reference loads run at one time, and set that limit apart from the + // compute parallelism. Those loads are what set the peak memory. At most a few contigs hold + // their raw reference at one time, while the code builds the compact N-mask. A pool of tokens + // is the counting semaphore. let load_permits = n_threads.min(4); let (perm_tx, perm_rx) = std::sync::mpsc::channel::<()>(); for _ in 0..load_permits { @@ -349,15 +388,15 @@ pub fn collect_unified_metrics_parallel_with_progress( let perm_rx = std::sync::Mutex::new(perm_rx); let process_contig = |w: &Work| -> Result { - // Bail before paying for this contig's reader + reference load. In-flight contigs stop at - // their own record-loop check inside `for_each`. + // Stop before the cost of the reader and the reference load of this contig. A contig that + // already runs stops at its own check in the record loop inside `for_each`. cancel.check()?; let (h, mut idx) = reader::open_indexed(bam_path, Some(reference_path))?; let region = Region::new(w.name.as_bytes().to_vec(), ..); // whole contig let mut cov_accum = if w.tracked { - // Hold a load permit only across the raw-reference load + mask build; release before - // the long pileup (which keeps just the small mask). + // Hold a load token across the raw-reference load and the build of the mask, and no + // longer. Release it before the long pileup, which keeps the small mask alone. let _permit = { let _ = perm_rx.lock().unwrap().recv(); LoadPermit { tx: &perm_tx } @@ -401,8 +440,9 @@ pub fn collect_unified_metrics_parallel_with_progress( }) }; - // The unmapped tail (no reference position) is invisible to region queries but the - // sequential read-metrics counts it (total/pf reads, read-length) — sweep it separately. + // A region query can not see the unmapped tail, because it has no reference position. But the + // sequential read-metrics counts it, in the total reads, the pf reads and the read length. So + // sweep it on its own. let process_unmapped = || -> Result { let (_h, mut idx) = reader::open_indexed(bam_path, Some(reference_path))?; let mut sink = MetricsSink { @@ -412,11 +452,14 @@ pub fn collect_unified_metrics_parallel_with_progress( Ok(sink.rm) }; - // noodles' CRAM decoder can recurse deeply enough to blow rayon's default 2 MiB worker stack - // (the main thread's larger stack handles the same file in the sequential walker). CRAM 3.1 - // files (new range/arithmetic + fqzcomp + name-tokenizer codecs) recurse deeper still. Give the - // workers a generous decode-safe stack so the per-contig CRAM decode doesn't overflow — an - // overflow aborts the whole process, so this must not be marginal. + // The CRAM decoder of noodles can recurse deep enough to overflow the default 2 MiB worker + // stack of rayon. In the sequential walker, the larger stack of the main thread holds the same + // file. A CRAM 3.1 file recurses deeper still, because of its new range and arithmetic codecs, + // fqzcomp, and the name tokenizer. + // + // So give the workers a large stack that is safe for a decode, and the CRAM decode of one + // contig then does not overflow. An overflow aborts the whole process, so the margin here must + // be wide. let pool = reader::decode_pool(n_threads)?; let (contig_results, unmapped_rm) = pool.install(|| { @@ -439,8 +482,8 @@ pub fn collect_unified_metrics_parallel_with_progress( let contig_results = contig_results?; let unmapped_rm = unmapped_rm?; - // Merge: read-metrics is a commutative fold; coverage merges per-contig (header order); - // sex sums per-contig class counts into one tally. + // The merge. Read-metrics is a fold that commutes. Coverage merges one contig at a time, in + // header order. Sex adds the class counts of each contig into one tally. let mut rm_total = ReadMetricsState::default(); let mut cov_partials: Vec = Vec::new(); let (mut autosome_reads, mut x_reads) = (0u64, 0u64); @@ -465,22 +508,25 @@ pub fn collect_unified_metrics_parallel_with_progress( }) } -/// Per-pass timings from [`profile_contig`] over one contig. +/// The time of each pass from [`profile_contig`], over one contig. #[doc(hidden)] #[derive(Debug, Clone, Copy)] pub struct ContigProfile { pub reads: u64, - /// Raw decode only: BGZF + BAM record decode, minimal lazy field touch (no `RecordBuf`). + /// The raw decode alone: BGZF and the BAM record decode. It touches as few lazy fields as it + /// can, and it makes no `RecordBuf`. pub raw: std::time::Duration, - /// Raw decode + `RecordBuf::try_from_alignment_record` (the owned per-read copy). + /// The raw decode, plus `RecordBuf::try_from_alignment_record`, which is the owned copy of + /// each read. pub recordbuf: std::time::Duration, /// The full production work: `RecordBuf` + read-metrics + coverage pileup. pub full: std::time::Duration, } -/// Diagnostic: time the per-read loop over a **single** contig in three passes so the cost splits -/// out — raw decode vs the `RecordBuf` owned copy vs the metrics/pileup work. Profiles the hot loop -/// without walking the whole genome. Not used in production. +/// A diagnostic. It times the loop over the reads of a **single** contig, in three passes. The +/// cost then separates into its parts: the raw decode, the owned `RecordBuf` copy, and the metrics +/// and pileup work. It profiles the hot loop, and it does not walk the whole genome. Production +/// does not use it. #[doc(hidden)] pub fn profile_contig( bam_path: &Path, @@ -492,7 +538,8 @@ pub fn profile_contig( let region = Region::new(contig.as_bytes().to_vec(), ..); - // Pass 1 — raw decode: iterate the lazy bam::Record, touch flags + sequence length, no RecordBuf. + // Pass 1, the raw decode. Walk the lazy bam::Record, touch the flags and the sequence length, + // and make no RecordBuf. let mut raw_reads = 0u64; let raw = { let mut inner = bam::io::indexed_reader::Builder::default() @@ -512,7 +559,7 @@ pub fn profile_contig( start.elapsed() }; - // Pass 2 — + RecordBuf conversion (no accepts). + // Pass 2. The same, plus the conversion to a RecordBuf. It accepts nothing. let recordbuf = { let (h, mut idx) = reader::open_indexed(bam_path, Some(reference_path))?; let start = std::time::Instant::now(); @@ -523,7 +570,7 @@ pub fn profile_contig( start.elapsed() }; - // Pass 3 — the full production per-read work. + // Pass 3. The full work at each read, as production does it. let length = { let (h, _) = reader::open_indexed(bam_path, Some(reference_path))?; h.reference_sequences() @@ -554,10 +601,13 @@ pub fn profile_contig( }) } -/// Diagnostic: run the full per-read work on each of `contigs` concurrently (mirroring the real -/// parallel walker's per-contig fan-out) and return `(total_reads, wall_clock)`. Comparing the -/// aggregate throughput against the single-threaded [`profile_contig`] rate exposes contention -/// (e.g. allocator thrash on per-read `RecordBuf` allocations). Not used in production. +/// A diagnostic. It runs the full work at each read, on every contig in `contigs`, at the same +/// time. That has the same shape as the fan-out over contigs in the real parallel walker. It +/// returns `(total_reads, wall_clock)`. +/// +/// Compare that throughput against the rate of [`profile_contig`], which runs on one thread. The +/// difference shows contention, for example an allocator under pressure from the `RecordBuf` +/// allocation at each read. Production does not use this. #[doc(hidden)] pub fn profile_contigs_parallel( bam_path: &Path, @@ -615,9 +665,10 @@ mod tests { let cov = coverage::collect_coverage_callable(&bam, &reference, ¶ms, None).unwrap(); let rm = read_metrics::collect_read_metrics(&bam, Some(&reference)).unwrap(); - // The fixture is chrM-only (no autosomes/chrX), so sex inference can't be computed — - // the fused walker reports it as `None` (best-effort) while still returning coverage + - // read-metrics, and the standalone walker errors. Both agree sex is unavailable here. + // The fixture holds chrM alone, with no autosome and no chrX, so the code can not infer + // the sex. The fused walker reports `None`, and it still gives back the coverage and the + // read-metrics. The separate walker returns an error. Both agree that there is no sex + // here. assert!(sex::infer_from_bam(&bam, Some(&reference)).is_err()); assert_eq!(unified.coverage, cov, "coverage diverged"); diff --git a/crates/navigator-analysis/src/vcf.rs b/crates/navigator-analysis/src/vcf.rs index 005e0e75..e7d88e7b 100644 --- a/crates/navigator-analysis/src/vcf.rs +++ b/crates/navigator-analysis/src/vcf.rs @@ -1,6 +1,9 @@ -//! Minimal VCF 4.2 writer for diploid genotype calls (the de-novo diploid caller + known-site -//! genotyping output). Emits one variant record per [`SiteGenotype`] with `FORMAT GT:AD:DP:GQ:PL`. -//! Records are written in the order given (the caller returns ascending position per contig). +//! A small writer of VCF 4.2, for diploid genotype calls. It serves the de-novo diploid caller, and +//! the output of a genotype run at known sites. +//! +//! It emits one variant record for each [`SiteGenotype`], with `FORMAT GT:AD:DP:GQ:PL`. The records +//! go out in the order that the caller gave them. That caller returns each contig in order of the +//! position, from the lowest up. use crate::caller::SiteGenotype; @@ -36,8 +39,9 @@ pub fn write_diploid_vcf(sample: &str, calls: &[SiteGenotype]) -> String { } else { c.pls.iter().map(|p| p.to_string()).collect::>().join(",") }; - // Multiallelic sites carry an explicit GT string + per-allele AD; biallelic sites derive - // both from `dosage` / `ref_depth,alt_depth`. + // A site with more than two alleles carries an explicit GT string, and an AD for each + // allele. A site with two alleles gets both from `dosage`, and from `ref_depth` and + // `alt_depth`. let gt = c.gt.clone().unwrap_or_else(|| genotype_field(c.dosage).to_string()); let ad = match &c.allele_depths { Some(d) => d.iter().map(|x| x.to_string()).collect::>().join(","), diff --git a/crates/navigator-analysis/tests/caller.rs b/crates/navigator-analysis/tests/caller.rs index f7713b7f..1c6070fc 100644 --- a/crates/navigator-analysis/tests/caller.rs +++ b/crates/navigator-analysis/tests/caller.rs @@ -1,11 +1,17 @@ -//! Haploid caller tests against the shared coverage fixture (tests/fixtures). +//! Tests of the haploid caller, against the coverage fixture that the tests share, in +//! tests/fixtures. //! -//! The fixture's reads are all `A` over reference chrM = `ACGTACGT...` (N at 25): -//! pos 1-10 depth 4 MAPQ 60 -> pass filters, consensus A -//! pos 11-20 depth 2 MAPQ 60 -> below min_depth 4 -//! pos 26-30 depth 5 MAPQ 0 -> dropped by min_mapping_quality 20 -//! With default params, de-novo calls land only at 1-10 where ref != A: -//! ref bases 1..10 = A C G T A C G T A C -> SNPs at {2,3,4,6,7,8,10}. +//! Every read of that fixture is `A`, over the reference chrM, which is `ACGTACGT...` with an N at +//! 25: +//! +//! ```text +//! pos 1-10 depth 4 MAPQ 60 -> passes the filters, consensus A +//! pos 11-20 depth 2 MAPQ 60 -> below the min_depth of 4 +//! pos 26-30 depth 5 MAPQ 0 -> the min_mapping_quality of 20 drops it +//! ``` +//! +//! With the default parameters, a de-novo call lands only at 1 to 10, where the reference base is +//! not `A`. Those bases are A C G T A C G T A C, so the SNPs are at {2,3,4,6,7,8,10}. use std::collections::HashSet; use std::path::PathBuf; @@ -79,7 +85,7 @@ fn force_call_genotypes_known_sites() { ) .expect("force-call should succeed"); - // indel + off-contig are dropped; 5 SNP sites remain. + // The code drops the indel and the site off the contig. 5 SNP sites stay. assert_eq!(calls.len(), 5); let by_name: std::collections::HashMap<&str, &_> = calls.iter().map(|c| (c.name.as_str(), c)).collect(); @@ -142,7 +148,7 @@ fn private_set_subtracts_known_tree_positions() { ) .unwrap(); - // Pretend positions 2 and 3 are known tree sites. + // This test treats positions 2 and 3 as known tree sites. let known: HashSet = [2, 3].into_iter().collect(); let private = subtract_known(&calls, &known); assert_eq!( diff --git a/crates/navigator-analysis/tests/cancel_real.rs b/crates/navigator-analysis/tests/cancel_real.rs index a8bb1952..8d283341 100644 --- a/crates/navigator-analysis/tests/cancel_real.rs +++ b/crates/navigator-analysis/tests/cancel_real.rs @@ -1,9 +1,13 @@ -//! Cancellation against a real alignment — the claim that matters is wall-clock, not a flag. +//! A cancel against a real alignment. The claim that matters is the wall time, and not the state +//! of a flag. //! -//! `#[ignore]` (live file, like the other `*_real` harnesses). Point it at a BAM/CRAM: +//! This test carries `#[ignore]`, because it needs a live file, as the other `*_real` harnesses do. +//! Point it at a BAM or a CRAM: //! -//! NAV_CANCEL_BAM=/path/sample.cram NAV_CANCEL_REF=/path/GRCh38.fa \ -//! cargo test -p navigator-analysis --test cancel_real -- --ignored --nocapture +//! ```text +//! NAV_CANCEL_BAM=/path/sample.cram NAV_CANCEL_REF=/path/GRCh38.fa \ +//! cargo test -p navigator-analysis --test cancel_real -- --ignored --nocapture +//! ``` use std::path::PathBuf; use std::time::{Duration, Instant}; @@ -14,9 +18,9 @@ fn env_path(key: &str) -> Option { std::env::var(key).ok().map(PathBuf::from) } -/// A whole-genome walk over a real WGS file takes minutes. Cancel it a second in and assert it -/// returns in well under that — the entire point of threading the token into the walkers, and the -/// thing that a unit test on the token alone cannot demonstrate. +/// A walk over the whole genome, on a real WGS file, takes minutes. This test cancels it one second +/// in, and it asserts that the walk returns well below that time. That is the whole point of the +/// token inside the walkers, and a unit test on the token alone can not show it. #[test] #[ignore] fn cancelling_a_whole_genome_walk_returns_promptly() { @@ -52,9 +56,9 @@ fn cancelling_a_whole_genome_walk_returns_promptly() { matches!(result, Err(navigator_analysis::AnalysisError::Cancelled)), "must report cancellation, not a generic failure" ); - // Generous bound: the contigs already in flight finish their current record batch, and rayon - // has to unwind the fan-out. Anything near the full walk time means the token is not reaching - // the record loops. + // The bound is generous. The contigs that already run finish their current batch of records, + // and rayon must then unwind the fan-out. A time near that of the full walk means that the + // token does not get to the record loops. assert!( elapsed < Duration::from_secs(30), "cancel took {elapsed:.1?} — the walk is not polling the token" diff --git a/crates/navigator-analysis/tests/coverage.rs b/crates/navigator-analysis/tests/coverage.rs index 95f3c88d..e13b9fea 100644 --- a/crates/navigator-analysis/tests/coverage.rs +++ b/crates/navigator-analysis/tests/coverage.rs @@ -1,13 +1,18 @@ -//! Coverage-walker tests against the synthetic fixture (see tests/fixtures/make_fixture.sh). +//! Tests of the coverage walker, against the synthetic fixture. See +//! tests/fixtures/make_fixture.sh. //! -//! Reference chrM (50 bp, N at position 25). Reads: -//! pos 1-10 depth 4 MAPQ 60 -> CALLABLE (10 bp) -//! pos 11-20 depth 2 MAPQ 60 -> LOW_COVERAGE (10 bp) -//! pos 21-24 depth 0 -> NO_COVERAGE (4 bp) -//! pos 25 depth 0 ref N -> REF_N (1 bp) -//! pos 26-30 depth 5 MAPQ 0 -> POOR_MAPPING_QUALITY (5 bp) -//! pos 31-50 depth 0 -> NO_COVERAGE (20 bp) -//! Base quality is Phred 40 throughout. +//! The reference is chrM, at 50 bp, with an N at position 25. The reads give: +//! +//! ```text +//! pos 1-10 depth 4 MAPQ 60 -> CALLABLE (10 bp) +//! pos 11-20 depth 2 MAPQ 60 -> LOW_COVERAGE (10 bp) +//! pos 21-24 depth 0 -> NO_COVERAGE (4 bp) +//! pos 25 depth 0 ref N -> REF_N (1 bp) +//! pos 26-30 depth 5 MAPQ 0 -> POOR_MAPPING_QUALITY (5 bp) +//! pos 31-50 depth 0 -> NO_COVERAGE (20 bp) +//! ``` +//! +//! Every base carries a quality of Phred 40. use std::path::PathBuf; @@ -65,14 +70,16 @@ fn coverage_matches_hand_computed_values() { ); assert_eq!(result.callable_bases, 10); - // --- samtools-style per-contig stats (per-base-observation averaging) --- + // --- The statistics of each contig, in the style of samtools. The mean goes over the base + // observations. --- let cs = &result.contig_coverage_stats[0]; assert_eq!(cs.num_reads, 11); // 4 + 2 + 5 assert_eq!(cs.cov_bases, 25); approx(cs.coverage, 50.0); approx(cs.mean_depth, 1.7); approx(cs.mean_base_q, 40.0); // all bases Phred 40 - // map quality per base obs: (60 obs * 60 + 25 obs * 0) / 85 + // The map quality over the base observations: + // (60 obs * 60 + 25 obs * 0) / 85 approx(cs.mean_map_q, 3600.0 / 85.0); } diff --git a/crates/navigator-analysis/tests/genotype.rs b/crates/navigator-analysis/tests/genotype.rs index a3dc26d9..20b2656e 100644 --- a/crates/navigator-analysis/tests/genotype.rs +++ b/crates/navigator-analysis/tests/genotype.rs @@ -19,7 +19,8 @@ fn chr1_reference() -> PathBuf { std::fs::create_dir_all(&dir).unwrap(); let fa = dir.join("chr1.fa"); std::fs::write(&fa, b">chr1\nACGTACGTAC\n").unwrap(); - // .fai: name, length, offset-of-first-base, bases-per-line, bytes-per-line. + // The `.fai` fields: the name, the length, the offset of the first base, the bases in a line, + // and the bytes in a line. std::fs::write(dir.join("chr1.fa.fai"), b"chr1\t10\t6\t10\t11\n").unwrap(); fa } @@ -93,7 +94,8 @@ fn denovo_diploid_calls_het_and_hom_alt_then_writes_vcf() { ) .unwrap(); - // Only the variant sites are emitted (the 7 hom-ref positions are not), in position order. + // The code emits the variant sites alone, in position order. It does not emit the 7 hom-ref + // positions. let by_pos = |p: i64| calls.iter().find(|c| c.position == p).cloned(); assert_eq!( calls.len(), @@ -152,9 +154,10 @@ fn denovo_diploid_calls_a_heterozygous_deletion() { #[test] fn call_indels_at_confirms_a_present_deletion() { - // indel_multi.bam (chrM): 8 reads carry the 2 bp deletion at anchor pos 5 (VCF REF=ACG, ALT=A), - // 6 carry a 3 bp deletion, and there are NO reference-spanning reads. Targeting the 2 bp deletion - // as a tree indel locus, the sample clearly carries it → the derived sentinel. + // indel_multi.bam, on chrM. 8 reads carry the 2 bp deletion at the anchor position 5. There + // the VCF REF is ACG, and the ALT is A. 6 reads carry a 3 bp deletion. NO read covers the + // reference there. With the 2 bp deletion as the target tree indel locus, the sample clearly + // carries it, so the result is the derived sentinel. let dir = fixtures(); let calls = call_indels_at( &dir.join("indel_multi.bam"), @@ -186,7 +189,8 @@ fn call_indels_at_is_additive_only_no_ancestral_call() { #[test] fn call_indels_at_without_reference_is_empty() { - // No reference → can't left-normalize or know deleted bases → indels are skipped (SNPs unaffected). + // With no reference, the code can not left-normalize, and it can not know the deleted bases. + // So it skips every indel. A SNP does not change. let dir = fixtures(); let calls = call_indels_at( &dir.join("indel_multi.bam"), @@ -203,7 +207,7 @@ fn call_indels_at_without_reference_is_empty() { fn denovo_diploid_calls_a_multiallelic_snv() { // snv_multi.bam (chr1): 10 reads carry G at pos 2, 10 carry T (ref C) → compound het 1/2. let dir = fixtures(); - // A private chr1 reference dir (distinct from chr1_reference()'s, so parallel tests don't race). + // A private chr1 reference dir (distinct from chr1_reference()'s, so parallel tests do not race). let refdir = std::env::temp_dir().join(format!("dun-snvmulti-ref-{}", std::process::id())); std::fs::create_dir_all(&refdir).unwrap(); let reference = refdir.join("chr1.fa"); @@ -233,9 +237,10 @@ fn denovo_diploid_calls_a_multiallelic_snv() { #[test] fn denovo_diploid_calls_a_multiallelic_indel() { - // indel_multi.bam (chrM): 8 reads delete ref pos 6-7 (2 bp) + 6 reads delete pos 6-8 (3 bp), - // both left-normalizing to emit pos 5. ref ACGTAC… → REF=ACGT, ALTs AT (2 bp del) and A (3 bp - // del), a compound het 1/2. + // indel_multi.bam, on chrM. 8 reads delete the reference positions 6 and 7, which is 2 bp. 6 + // reads delete positions 6 to 8, which is 3 bp. Both left-normalize to the emit position 5. + // The reference reads ACGTAC…, so REF is ACGT, and the two ALTs are AT for the 2 bp deletion + // and A for the 3 bp one. That is a compound het, 1/2. let dir = fixtures(); let calls = call_denovo_diploid( &dir.join("indel_multi.bam"), diff --git a/crates/navigator-analysis/tests/heteroplasmy.rs b/crates/navigator-analysis/tests/heteroplasmy.rs index 24616ae3..58eff57c 100644 --- a/crates/navigator-analysis/tests/heteroplasmy.rs +++ b/crates/navigator-analysis/tests/heteroplasmy.rs @@ -1,11 +1,18 @@ -//! Heteroplasmy-detection tests against the `diploid.bam` fixture (tests/fixtures). +//! Tests of the heteroplasmy detection, against the `diploid.bam` fixture in tests/fixtures. //! -//! That fixture is two haplotypes on chr1 at depth 20 (10 reads each): -//! H1 = ACGTACGAAC, H2 = AGGTTCGAAC -//! so the per-position pileup carries two alleles only at pos2 (C/G) and pos5 (A/T); -//! every other position is homozygous. With the default screening params (min_depth 20, -//! minor fraction ≥ 0.03, ≥3 minor reads) detection must flag exactly those two sites, -//! each at a 50% minor fraction. +//! That fixture holds two haplotypes on chr1, at a depth of 20, with 10 reads for each: +//! +//! ```text +//! H1 = ACGTACGAAC +//! H2 = AGGTTCGAAC +//! ``` +//! +//! The pileup thereby carries two alleles at two positions alone: pos2, at C and G, and pos5, at A +//! and T. Every other position is homozygous. +//! +//! The default parameters are a min_depth of 20, a minor fraction of 0.03 or more, and 3 minor +//! reads or more. With those, the detection must flag exactly those two sites, and each one must +//! show a minor fraction of 50%. use std::path::PathBuf; @@ -47,7 +54,7 @@ fn detects_the_two_mixed_sites_on_the_diploid_fixture() { #[test] fn min_minor_count_suppresses_low_support() { - // Demanding more minor reads than the fixture supplies (10) yields nothing. + // A request for more minor reads than the fixture holds, which is 10, gives nothing. let strict = HeteroplasmyParams { min_minor_count: 11, ..HeteroplasmyParams::default() diff --git a/crates/navigator-analysis/tests/index.rs b/crates/navigator-analysis/tests/index.rs index 1127142c..7d706557 100644 --- a/crates/navigator-analysis/tests/index.rs +++ b/crates/navigator-analysis/tests/index.rs @@ -1,5 +1,6 @@ -//! Building a coordinate index (`.bai`/`.crai`) for a fixture that has none, and confirming the -//! result makes the file region-queryable and matches the fixture's checked-in index. +//! Build a coordinate index, a `.bai` or a `.crai`, for a fixture that has none. Then confirm two +//! things: a region query on that file now works, and the index matches the one that the repo holds +//! beside the fixture. use std::fs; use std::path::{Path, PathBuf}; @@ -19,7 +20,8 @@ fn scratch(tag: &str) -> PathBuf { dir } -/// Copy just the alignment (not its sibling index) into `dir` so `ensure_index` has to build one. +/// Copy the alignment into `dir`, and not the index that sits beside it. `ensure_index` must then +/// build one. fn copy_without_index(name: &str, dir: &Path) -> PathBuf { let dst = dir.join(name); fs::copy(fixtures().join(name), &dst).unwrap(); diff --git a/crates/navigator-analysis/tests/mastervar_real.rs b/crates/navigator-analysis/tests/mastervar_real.rs index 3be191d0..24c2785f 100644 --- a/crates/navigator-analysis/tests/mastervar_real.rs +++ b/crates/navigator-analysis/tests/mastervar_real.rs @@ -1,12 +1,19 @@ -//! Real CompleteGenomics masterVar parse check (ignored by default; needs a local dump). +//! A check of the parse of a real CompleteGenomics masterVar file. It carries `#[ignore]`, and it +//! needs a local dump. //! -//! Run against a real file (compressed or plain): -//! MASTERVAR_TSV=/path/to/var-GS00253-DNA_A01_200_37-ASM.tsv.bz2 \ -//! cargo test -p navigator-analysis --test mastervar_real -- --ignored --nocapture +//! Run it against a real file, compressed or plain: //! -//! It streams the whole genome and prints the sample id, reference build, loci/SNP tallies, a -//! per-contig call count, and a genotype-shape breakdown — enough to confirm the two-allele rows -//! collapse sanely and the haploid contigs (chrY/chrM) come through as hemizygous. +//! ```text +//! MASTERVAR_TSV=/path/to/var-GS00253-DNA_A01_200_37-ASM.tsv.bz2 \ +//! cargo test -p navigator-analysis --test mastervar_real -- --ignored --nocapture +//! ``` +//! +//! It streams the whole genome. It then prints the sample id and the reference build, the tallies +//! of the loci and the SNPs, and the call count of each contig. Last, it prints a breakdown of the +//! genotype shapes. +//! +//! That is enough to confirm two things. The rows that hold two alleles come together correctly. +//! And the haploid contigs, chrY and chrM, come through as hemizygous. use std::collections::BTreeMap; @@ -41,7 +48,7 @@ fn parse_real_master_var() { println!("per-contig: {per_contig:?}"); println!("genotype shapes: {gt:?}"); - // chrY / chrM must be hemizygous (genotype "1") — never diploid. + // chrY and chrM must both be hemizygous, at the genotype "1". Neither may be diploid. for c in out.calls.iter().filter(|c| c.contig == "chrY" || c.contig == "chrM") { assert_eq!( c.genotype.as_deref(), diff --git a/crates/navigator-analysis/tests/parity_real.rs b/crates/navigator-analysis/tests/parity_real.rs index 7ee60f79..3322f3b7 100644 --- a/crates/navigator-analysis/tests/parity_real.rs +++ b/crates/navigator-analysis/tests/parity_real.rs @@ -1,12 +1,15 @@ -//! Real-data smoke test (seed of the phase-3 §4c parity harness). Ignored by default; -//! runs only when pointed at a local BAM + reference via env vars: +//! A smoke test on real data. It is the seed of the parity harness of phase 3, §4c. The test +//! carries `#[ignore]`, and it runs only when the environment variables point it at a local BAM +//! and a reference: //! -//! HG002_CHRM_BAM=/tmp/hg002.chrM.bam CHM13_REF=/Users/.../chm13v2.0.fa \ -//! cargo test -p navigator-analysis --test parity_real -- --ignored --nocapture +//! ```text +//! HG002_CHRM_BAM=/tmp/hg002.chrM.bam CHM13_REF=/Users/.../chm13v2.0.fa \ +//! cargo test -p navigator-analysis --test parity_real -- --ignored --nocapture +//! ``` //! -//! This is a sanity check that noodles handles a real BAM (varied CIGARs/MAPQ) and the -//! chrM numbers are plausible — NOT strict parity, which is measured against the Scala -//! walker / GATK in phase 3. +//! This test checks two things. noodles handles a real BAM, which carries CIGARs and MAPQ values +//! of every shape. And the chrM numbers look reasonable. It is NOT a strict parity check. Phase 3 +//! measures that against the Scala walker and against GATK. use std::collections::HashSet; use std::path::PathBuf; @@ -54,7 +57,7 @@ fn hg002_chrm_smoke() { eprintln!("callable metrics = {:?}", result.contig_callable); eprintln!("coverage stats = {:?}", result.contig_coverage_stats); - // chrM should be fully covered at high depth. + // The reads must cover the whole of chrM, at a high depth. assert_eq!(result.genome_territory, 16569); let cs = &result.contig_coverage_stats[0]; assert_eq!(cs.contig, "chrM"); @@ -90,8 +93,10 @@ fn hg002_chrm_denovo_smoke() { ); } - // HG002 mtDNA vs CHM13 chrM: a handful to a few dozen real differences at high - // depth — never thousands (that would mean the consensus/fraction gate is broken). + // The mtDNA of HG002 against the chrM of CHM13. There are a few real differences, from a + // handful up to a few dozen, at a high depth. There are never thousands. Thousands would mean + // that a gate no longer works: either the one on the consensus, or the one on the + // fraction. assert!(!calls.is_empty(), "expected some mtDNA variants"); assert!(calls.len() < 1000, "implausibly many calls: {}", calls.len()); for c in &calls { @@ -101,8 +106,8 @@ fn hg002_chrm_denovo_smoke() { } } -/// De-novo calling on chrY (57 Mb) against the full BAM — exercises the chunked tally -/// on a large contig. Memory is bounded by the chunk, not chrY's length. +/// A de-novo call over chrY, at 57 Mb, against the full BAM. It covers the tally that works in +/// chunks, on a large contig. The chunk bounds the memory, and the length of chrY does not. #[test] #[ignore = "requires HG002_BAM + CHM13_REF (chrY de-novo, chunked)"] fn hg002_chry_denovo_streams() { @@ -125,9 +130,10 @@ fn hg002_chry_denovo_streams() { } } -/// Whole-genome coverage over the full BAM (no allowlist). Only feasible because the -/// walker streams a sliding window — the old dense version allocated per-position -/// arrays for every main-assembly contig at once (~84 GB). +/// The coverage over the whole genome, from the full BAM, with no allowlist. This is possible only +/// because the walker streams a window that slides. The dense version before it allocated an array +/// at each position. It held one for every contig of the main assembly at one time, and that came +/// to about 84 GB. #[test] #[ignore = "requires HG002_BAM + CHM13_REF (whole-genome streaming coverage)"] fn hg002_wgs_coverage_streams_all_contigs() { @@ -205,9 +211,9 @@ fn hg002_read_metrics_smoke() { assert!(m.proper_pairs > 0); } -/// The fused unified walker must produce, in one whole-genome pass, exactly what the three -/// standalone walkers produce separately — coverage, read-metrics, and sex, field for field. -/// This is the parity guard for the single-pass optimization on real data. +/// In one pass over the whole genome, the fused unified walker must give exactly what the three +/// separate walkers give in three passes. That covers the coverage, the read-metrics and the sex, +/// field for field. This test guards the parity of the one-pass change, on real data. #[test] #[ignore = "requires HG002_BAM + CHM13_REF (whole-genome single-pass parity)"] fn hg002_unified_matches_standalone() { @@ -227,16 +233,18 @@ fn hg002_unified_matches_standalone() { assert_eq!(unified.coverage, cov, "coverage diverged from standalone"); assert_eq!(unified.read_metrics, rm, "read metrics diverged from standalone"); - // Standalone BAM sex uses the BAI fast path; the fused path tallies from the record stream. - // They classify identically; the read-density floats can differ in the last ULP if the BAI - // mapped-record count differs from the streamed mapped-read count, so compare the call. + // The separate sex path on a BAM uses the fast path over the BAI. The fused path tallies from + // the record stream. The two put the sample into the same class. But the read-density floats + // can differ in the last ULP. That happens when the mapped-record count of the BAI differs + // from the mapped-read count of the stream. So compare the call itself. let fused_sex = unified.sex.expect("HG002 has autosomes + chrX"); assert_eq!(fused_sex.inferred_sex, sex.inferred_sex, "sex call diverged"); assert_eq!(fused_sex.inferred_sex, InferredSex::Male); } -/// The per-contig parallel walker must produce a result byte-identical to the sequential one. -/// Needs an indexed BAM (else it falls back to sequential and the test is trivially true). +/// The parallel walker over the contigs must give a result that matches the sequential one to the +/// last byte. It needs an indexed BAM. Without one it falls back to the sequential path, and the +/// test then proves nothing. #[test] #[ignore = "requires indexed HG002_BAM + CHM13_REF (parallel == sequential parity)"] fn hg002_unified_parallel_matches_sequential() { @@ -251,14 +259,16 @@ fn hg002_unified_parallel_matches_sequential() { let seq = collect_unified_metrics(&bam, &reference, ¶ms, None).unwrap(); let par = collect_unified_metrics_parallel(&bam, &reference, ¶ms, None).unwrap(); - // Whole struct equality: coverage (incl. per-contig + histogram), read-metrics, sex. + // The whole struct must be equal: the coverage, which holds the values of each contig and the + // histogram, the read-metrics, and the sex. assert_eq!(par.coverage, seq.coverage, "parallel coverage diverged"); assert_eq!(par.read_metrics, seq.read_metrics, "parallel read-metrics diverged"); assert_eq!(par.sex, seq.sex, "parallel sex diverged"); } -/// Wall-clock comparison of sequential vs per-contig parallel unified metrics (not a -/// correctness assertion — see the parity test for that). Prints both times + speedup. +/// A comparison of the wall time of the sequential unified metrics against the parallel ones, +/// which run one task at each contig. It asserts nothing about correctness, and the parity test +/// covers that. It prints both times, and the speedup. #[test] #[ignore = "perf smoke: requires indexed HG002_BAM + CHM13_REF"] fn hg002_unified_parallel_timing() { @@ -285,10 +295,13 @@ fn hg002_unified_parallel_timing() { ); } -/// §4c parity gate: Rust de-novo SNP calls vs a GATK truth VCF on HG002 chrM. -/// Generate the truth with: -/// gatk HaplotypeCaller -I hg002.chrM.bam -R chm13v2.0.fa -L chrM \ -/// --sample-ploidy 1 -O hg002.chrM.gatk.vcf.gz && bgzip -d hg002.chrM.gatk.vcf.gz +/// The parity gate of §4c. It puts the de-novo SNP calls of the Rust caller against a truth VCF +/// from GATK, on the chrM of HG002. Make that truth file with: +/// +/// ```text +/// gatk HaplotypeCaller -I hg002.chrM.bam -R chm13v2.0.fa -L chrM \ +/// --sample-ploidy 1 -O hg002.chrM.gatk.vcf.gz && bgzip -d hg002.chrM.gatk.vcf.gz +/// ``` #[test] #[ignore = "requires GATK_CHRM_VCF + HG002_CHRM_BAM + CHM13_REF env vars"] fn hg002_chrm_gatk_parity() { diff --git a/crates/navigator-analysis/tests/read_metrics.rs b/crates/navigator-analysis/tests/read_metrics.rs index 7f5dfa11..ac86cbbb 100644 --- a/crates/navigator-analysis/tests/read_metrics.rs +++ b/crates/navigator-analysis/tests/read_metrics.rs @@ -51,7 +51,8 @@ fn read_metrics_match_hand_computed_values() { #[test] fn cram_read_metrics_match_bam() { - // Same pairs as paired.bam, ref-compressed — the unified reader must give identical metrics. + // The same pairs as in paired.bam, compressed against the reference. The unified reader must + // give the same metrics. let dir = fixtures(); let bam = collect_read_metrics(&dir.join("paired.bam"), None).unwrap(); let cram = collect_read_metrics(&dir.join("paired.cram"), Some(&dir.join("ref.fa"))).unwrap(); diff --git a/crates/navigator-analysis/tests/revert.rs b/crates/navigator-analysis/tests/revert.rs index c98b6c4b..b5bc07b0 100644 --- a/crates/navigator-analysis/tests/revert.rs +++ b/crates/navigator-analysis/tests/revert.rs @@ -1,13 +1,16 @@ //! End-to-end revert over real containers, against `paired.bam` / `paired.cram`. //! -//! The unit tests in `src/revert/tests.rs` drive records directly; this closes the remaining gap — -//! that the same pipeline works when the records come out of an actual BAM or CRAM decode. +//! The unit tests in `src/revert/tests.rs` give the records directly. This test closes the gap +//! that they leave: the same pipeline must work when the records come out of a real BAM or CRAM +//! decode. //! -//! `paired.bam` is a good fixture for this by accident of how it was built: two FR pairs at -//! chrM:1/31 and chrM:5/25, coordinate-sorted, so the file order is pairA, pairB, pairB, pairA and -//! neither template's mates are adjacent. That is the exact condition collation exists for. The -//! `/2` records carry flag 147, which includes `0x10`, so the reverse-complement restore runs on -//! real decoded records too. +//! `paired.bam` suits this by accident of how somebody built it. It holds two FR pairs, at chrM:1 +//! and 31, and at chrM:5 and 25, in coordinate order. The file order is then pairA, pairB, pairB, +//! pairA, and the two mates of a template never sit beside each other. That is the exact condition +//! that the collation exists for. +//! +//! The `/2` records carry flag 147, and that flag holds `0x10`. So the restore of the reverse +//! complement also runs on real decoded records. use std::path::{Path, PathBuf}; @@ -59,16 +62,18 @@ fn reverts_a_coordinate_sorted_bam_into_synchronized_pairs() { assert_eq!([r1[0].as_str(), r1[4].as_str()], ["@pairA", "@pairB"]); assert_eq!([r2[0].as_str(), r2[4].as_str()], ["@pairA", "@pairB"]); - // The fixture's reads are poly-A stored forward on /1; the /2 records are flagged reverse, so - // what was stored as poly-A comes back as poly-T once restored to sequencer orientation. + // The reads of the fixture are poly-A, and the /1 records store them forward. The /2 records + // carry the reverse flag. What the file holds as poly-A comes back as poly-T, once the code + // restores the orientation of the sequencer. assert_eq!(r1[1], "AAAAAAAAAA", "/1 is forward, passed through"); assert_eq!(r2[1], "TTTTTTTTTT", "/2 is reverse-flagged, so it is complemented back"); } #[test] fn cram_reverts_identically_to_bam() { - // Same reads, different container: the revert must not be able to tell. CRAM needs the - // reference to decode, which is the one path difference worth covering. + // The same reads, in a different container. The revert must not be able to see a difference. A + // CRAM needs the reference to decode, and that is the one difference in the path that this + // test must cover. let dir = fixtures(); let from_bam = revert_alignment( &dir.join("paired.bam"), @@ -92,8 +97,8 @@ fn cram_reverts_identically_to_bam() { assert_eq!(lines(&from_cram.read2), lines(&from_bam.read2)); } -/// The same input reverted through a one-byte sort budget must produce the same FASTQ — on real -/// decoded records, not just synthetic ones. +/// The same input, through a sort budget of one byte, must give the same FASTQ. This holds on real +/// decoded records, and not on synthetic ones alone. #[test] fn spilling_does_not_change_the_result_on_a_real_bam() { let bam = fixtures().join("paired.bam"); diff --git a/crates/navigator-analysis/tests/sex.rs b/crates/navigator-analysis/tests/sex.rs index 7447c944..5efa20d8 100644 --- a/crates/navigator-analysis/tests/sex.rs +++ b/crates/navigator-analysis/tests/sex.rs @@ -22,8 +22,8 @@ fn infers_male_from_low_x_coverage() { #[test] fn cram_sex_inference_matches_bam() { - // CRAM has no per-reference counts in the index, so this exercises the record-scan - // fallback; same reads as sex.bam, so the result must match. + // The index of a CRAM holds no count for each reference. So this test covers the fallback that + // scans the records. It holds the same reads as sex.bam, so the result must match. let dir = fixtures(); let bam = infer_from_bam(&dir.join("sex.bam"), None).unwrap(); let cram = infer_from_bam(&dir.join("sex.cram"), Some(&dir.join("sexref.fa"))).unwrap(); diff --git a/crates/navigator-analysis/tests/sv.rs b/crates/navigator-analysis/tests/sv.rs index cb782b13..1924b43b 100644 --- a/crates/navigator-analysis/tests/sv.rs +++ b/crates/navigator-analysis/tests/sv.rs @@ -28,7 +28,8 @@ fn walker_extracts_discordant_pairs_split_reads_and_depth() { ) .expect("walker should succeed"); - // 2 inter-chromosomal (one per mate) + 2 insert-size outliers. + // 2 pairs across two chromosomes, one for each mate, and 2 pairs whose insert size is an + // outlier. assert_eq!(ev.total_discordant_pairs(), 4); let inter = ev.inter_chromosomal_pairs(); assert_eq!(inter.len(), 2); @@ -55,9 +56,10 @@ fn walker_extracts_discordant_pairs_split_reads_and_depth() { assert_eq!(ev.depth_bins["chr2"], vec![1, 0, 0, 0, 0]); } -/// The same reads stored as CRAM must yield byte-identical evidence. The walker used to open every -/// file with the BAM (BGZF) reader, so a CRAM failed at open with "invalid BGZF header" and SV -/// calling was silently unavailable for every CRAM in the workspace. +/// The same reads, stored as a CRAM, must give evidence that matches to the last byte. The walker +/// once opened every file with the BAM reader, which reads BGZF. So a CRAM failed at the open, +/// with `invalid BGZF header`. Every CRAM in a workspace then had no SV call at all, and nobody saw +/// it. #[test] fn walker_reads_cram_with_the_same_result_as_bam() { let lengths = BTreeMap::from([("chr1".to_string(), 5000i64), ("chr2".to_string(), 5000)]); @@ -89,8 +91,9 @@ fn walker_reads_cram_with_the_same_result_as_bam() { assert_eq!(from_cram.total_discordant_pairs(), from_bam.total_discordant_pairs()); assert_eq!(from_cram.total_split_reads(), from_bam.total_split_reads()); - // Compare the evidence itself, not just the counts — the split read carries the fields that - // come from the accessors CRAM implements differently (SA tag, CIGAR clip length). + // Compare the evidence itself, and not the counts alone. The split read carries the fields + // that come from the accessors that a CRAM implements in a different way. Those are the SA tag + // and the clip length of the CIGAR. let (b, c) = (&from_bam.split_reads[0], &from_cram.split_reads[0]); assert_eq!( (c.clip_length, &c.supp_chrom, c.supp_pos, c.primary_pos), @@ -108,10 +111,15 @@ fn walker_reads_cram_with_the_same_result_as_bam() { assert_eq!(placed(&from_cram), placed(&from_bam)); } -/// The per-contig parallel fan-out must be a pure speedup: identical depth bins, discordant pairs -/// and split reads, in identical order. SV was the last whole-genome analysis still decoding on one -/// thread (2–5 h per 30x CRAM in a batch), so the fan-out is only worth having if it changes nothing -/// but the wall clock. Run over BAM *and* CRAM — they take different decode paths into the same sink. +/// The parallel fan-out over the contigs must give speed and nothing else. The depth bins, the +/// discordant pairs and the split reads must all match, and they must come in the same order. +/// +/// SV was the last analysis over the whole genome that still decoded on one thread, at 2 to 5 h for +/// each 30x CRAM in a batch. So the fan-out is worth its code only when it changes the wall time +/// and nothing more. +/// +/// This runs over a BAM *and* over a CRAM. The two take different decode paths into the same +/// sink. #[test] fn parallel_walk_matches_sequential_on_bam_and_cram() { let lengths = BTreeMap::from([("chr1".to_string(), 5000i64), ("chr2".to_string(), 5000)]); @@ -120,8 +128,9 @@ fn parallel_walk_matches_sequential_on_bam_and_cram() { for (file, reference) in cases { let path = fixtures().join(file); - // Without an index the parallel entry point falls back to the sequential walk, which would - // make this test compare a walk against itself and pass no matter what the fan-out does. + // With no index, the parallel entry point falls back to the sequential walk. This test + // would then compare a walk against itself, and it would pass whatever the fan-out + // does. assert!( navigator_analysis::reader::has_region_index(&path), "{file} needs its .bai/.crai for this test to exercise the parallel path" @@ -154,10 +163,11 @@ fn parallel_walk_matches_sequential_on_bam_and_cram() { } } -/// The evidence cap must truncate what is *retained* without falsifying what was *found*. A cap -/// that quietly lowered `total_discordant_pairs` would make a truncated run read as a clean sample, -/// which is the one failure mode a safety valve must not have. Exercised on both walks, since each -/// claims against the shared budget separately. +/// The evidence cap must cut short what the code *keeps*, and it must not change what the code +/// *found*. A cap that lowered `total_discordant_pairs`, where nobody saw it, would make a run that +/// the cap cut short read as a clean sample. That is the one failure that a safety valve must not +/// have. This test covers both walks, because each one claims against the shared budget on its +/// own. #[test] fn evidence_cap_truncates_retained_evidence_but_not_the_reported_totals() { let lengths = BTreeMap::from([("chr1".to_string(), 5000i64), ("chr2".to_string(), 5000)]); @@ -187,7 +197,8 @@ fn evidence_cap_truncates_retained_evidence_but_not_the_reported_totals() { assert_eq!(ev.discordant_pairs.len(), 1, "parallel={parallel} retained"); assert_eq!(ev.discordant_pairs_dropped, 3, "parallel={parallel} dropped"); assert_eq!(ev.total_discordant_pairs(), 4, "parallel={parallel} reported total"); - // Only 1 split read exists, so its cap is met exactly and nothing is dropped. + // There is only 1 split read. So its count reaches the cap exactly, and the code drops + // nothing. assert_eq!(ev.split_reads.len(), 1, "parallel={parallel} split retained"); assert_eq!(ev.split_reads_dropped, 0, "parallel={parallel} split dropped"); // Depth bins are not evidence records and are never capped. diff --git a/crates/navigator-analysis/tests/testtype_real.rs b/crates/navigator-analysis/tests/testtype_real.rs index 27214287..f5b02312 100644 --- a/crates/navigator-analysis/tests/testtype_real.rs +++ b/crates/navigator-analysis/tests/testtype_real.rs @@ -1,11 +1,15 @@ -//! Real-BAM test-type inference check (ignored by default; needs a local indexed BAM). +//! A check of the test-type inference on a real BAM. It carries `#[ignore]`, and it needs a local +//! indexed BAM. //! -//! Run against a specific BAM: -//! TESTTYPE_BAM=/path/to.bam [TESTTYPE_PLATFORM=ILLUMINA] [TESTTYPE_VENDOR=FamilyTreeDNA] \ -//! cargo test -p navigator-analysis --test testtype_real -- --ignored --nocapture +//! Run it against one BAM: //! -//! It prints the BAI-derived coverage profile + the inferred test-type code so a real Big Y / Y -//! Elite / mtFull BAM can be confirmed positively and a WGS BAM negatively. +//! ```text +//! TESTTYPE_BAM=/path/to.bam [TESTTYPE_PLATFORM=ILLUMINA] [TESTTYPE_VENDOR=FamilyTreeDNA] \ +//! cargo test -p navigator-analysis --test testtype_real -- --ignored --nocapture +//! ``` +//! +//! It prints the coverage profile from the BAI, and the test-type code that it inferred. A real +//! Big Y, Y Elite or mtFull BAM must come out with that type, and a WGS BAM must not. use navigator_analysis::testtype::{coverage_profile_from_bai, infer_test_type}; @@ -19,7 +23,8 @@ fn print_inferred_test_type() { let platform = std::env::var("TESTTYPE_PLATFORM").ok(); let vendor = std::env::var("TESTTYPE_VENDOR").ok(); - // Show what the header probe scrapes (platform + vendor hint) — env overrides for testing. + // Show what the header probe takes out: the platform, and the hint about the vendor. An + // environment variable can override each one, for a test. let probe = navigator_analysis::probe::probe_alignment(std::path::Path::new(&bam)).ok(); let probe_platform = probe.as_ref().and_then(|p| p.platform.clone()); let probe_vendor = probe.as_ref().and_then(|p| p.vendor_hint.clone()); diff --git a/crates/navigator-app/examples/blocktree_check.rs b/crates/navigator-app/examples/blocktree_check.rs index d438e9e3..2a86ef8e 100644 --- a/crates/navigator-app/examples/blocktree_check.rs +++ b/crates/navigator-app/examples/blocktree_check.rs @@ -1,6 +1,8 @@ -//! Throwaway validation for the project block tree: run `App::project_block_tree` against the live -//! workspace and print the result as an indented tree. Proves the real data path — tree fetch, name -//! index, induced subtree, collapse — on an actual multi-thousand-member cohort. +//! A temporary check for the project block tree. It calls `App::project_block_tree` on the live +//! workspace and prints the result as a tree with an indent for each level. +//! +//! The check covers the real data path on a cohort with some thousands of members. That path is the +//! tree fetch, the name index, the induced subtree, and the collapse step. //! //! ```bash //! cargo run -p navigator-app --example blocktree_check -- @@ -71,8 +73,10 @@ async fn main() -> Result<(), Box> { ); } - // Split the unplaced: "no placement at all" is expected (STR-only kits), but "has a terminal - // this tree doesn't carry" is provider/build skew worth naming. + // Separate the two groups of subjects with no place in the tree. A subject with no placement + // is normal, because an STR-only kit has none. The second group has a terminal node that this + // tree does not hold. That group shows a difference between the provider and the build, and + // the report must give its count. let (skew, unplaced_none): (Vec<_>, Vec<_>) = tree.unplaced.iter().partition(|u| u.terminal.is_some()); println!( "unplaced: {} with no Y placement · {} with a terminal absent from this tree", @@ -89,7 +93,8 @@ async fn main() -> Result<(), Box> { }); } - // Pre-order with depth as indent is exactly how the aggregate is ordered, so this prints itself. + // The aggregate is already in pre-order, and the depth gives the indent. So the code prints + // the rows in the order that it receives them. for b in tree.blocks.iter().take(60) { let indent = " ".repeat(b.depth); let folded = if b.collapsed.is_empty() { diff --git a/crates/navigator-app/examples/realign_wgs.rs b/crates/navigator-app/examples/realign_wgs.rs index ccb9feb0..3160d77c 100644 --- a/crates/navigator-app/examples/realign_wgs.rs +++ b/crates/navigator-app/examples/realign_wgs.rs @@ -1,9 +1,11 @@ //! Drive a whole-genome realignment headlessly, for the phase 5 WGS-scale validation. //! -//! The GUI can start this job, but a run measured in hours should not depend on a window staying -//! open — and the validation wants a timestamped log of where the time went, which the progress -//! cards do not keep. This is the same `App::realign_alignment` the UI calls, with the stage -//! reports printed instead of drawn. +//! The GUI can start this job. But a run of many hours must not depend on an open window. The +//! validation also needs a log with a timestamp for each stage, and the progress cards do not keep +//! one. +//! +//! This example calls the same `App::realign_alignment` function that the UI calls. It prints each +//! stage report, and the UI draws it. //! //! ```bash //! cargo run --release -p navigator-app --example realign_wgs -- @@ -13,8 +15,8 @@ //! # to find; a run that is killed outright leaves them regardless //! ``` //! -//! Ctrl-C cancels through the job's own token rather than killing the process, so the scratch -//! directory — hundreds of GB at WGS scale — is still cleaned up on the way out. +//! Ctrl-C stops the job through the cancel token of that job. It does not stop the process. So the +//! example still removes the scratch directory, which holds hundreds of GB for a WGS sample. use std::path::PathBuf; use std::sync::Mutex; @@ -34,9 +36,10 @@ async fn main() -> Result<(), Box> { .map(PathBuf::from) .unwrap_or_else(|_| home.join(".decodingus/references/chm13v2.0.fa")); let scratch_root = std::env::var("SCRATCH").ok().map(PathBuf::from); - // `PRESET` overrides the technology inference, which refuses any test type it does not know - // rather than guessing — correct for the app, but it puts real vendor products (`Y_ELITE`) - // out of reach of a smoke test. + // `PRESET` replaces the value that the code deduces from the technology. That code refuses a + // test type that it does not know, and it never makes an estimate. This behaviour is correct + // for the app. But it also puts a real vendor product, such as `Y_ELITE`, out of the reach of + // a quick test. let preset = match std::env::var("PRESET") { Ok(p) => Some(Preset::parse(&p).map_err(|e| format!("PRESET={p}: {e}"))?), Err(_) => None, @@ -113,9 +116,9 @@ async fn main() -> Result<(), Box> { match app.realign_alignment(alignment_id, params, cancel, progress).await { Ok(outcome) => { - // A resumed run did not necessarily run the stage that counts a given figure, and the - // earlier attempt may have been killed before it wrote one down. "not measured" is the - // honest rendering; a zero here would read as a result. + // A run that continues an earlier run does not always do the stage that counts a + // figure. The earlier run can also stop before it writes that figure. So the report + // shows "not measured". A zero value here looks like a result. let count = |n: Option| n.map(|n| n.to_string()).unwrap_or_else(|| "not measured".into()); println!( "\ndone in {:.1} min\n alignment #{} at {}\n reads written: {}\n duplicates marked: {}\n source unmapped reads (had a chance to place): {}", diff --git a/crates/navigator-app/src/analysis.rs b/crates/navigator-app/src/analysis.rs index f775195d..c72df2a2 100644 --- a/crates/navigator-app/src/analysis.rs +++ b/crates/navigator-app/src/analysis.rs @@ -6,9 +6,10 @@ use navigator_analysis::{contig, CancelToken}; impl App { // ---- analysis (compute + persist) -------------------------------------- - /// Run the coverage + callable walker on an alignment's BAM and persist the result - /// as a versioned `coverage` artifact. The blocking noodles I/O runs on a blocking - /// thread so the async runtime is not stalled. + /// Run the coverage walker and the callable walker on the BAM file of an alignment. The method + /// writes the result as a `coverage` artifact with a version. + /// + /// The I/O of noodles blocks, so it runs on its own thread. The async runtime then continues. pub async fn run_coverage( &self, alignment_id: i64, @@ -34,17 +35,20 @@ impl App { .await } - /// Run coverage using the alignment's own stored BAM/reference paths, then persist. - /// Errors if the alignment is unknown or has no paths recorded. + /// Calculate the coverage with the BAM path and the reference path of the alignment, and then + /// write the result. The method fails when the store holds no such alignment, and when that + /// alignment holds no path. pub async fn run_coverage_for_alignment(&self, alignment_id: i64) -> Result { self.run_coverage_for_alignment_with_progress(alignment_id, |_, _| {}, CancelToken::none()) .await } - /// Like [`run_coverage_for_alignment`], reporting `progress(contigs_done, contigs_total)` as - /// the whole-genome pass walks each contig (the slow step — minutes on a real WGS BAM — so a - /// progress bar can advance instead of sitting frozen). The callback runs on the blocking - /// thread. + /// The same work as [`run_coverage_for_alignment`], with a progress report. The method calls + /// `progress(contigs_done, contigs_total)` as the whole-genome pass reads each contig. + /// + /// That pass is the slow step, and it needs some minutes on a real WGS BAM file. So a progress + /// bar can move, and the app does not look stopped. The callback runs on the thread that + /// blocks. pub async fn run_coverage_for_alignment_with_progress( &self, alignment_id: i64, @@ -53,8 +57,9 @@ impl App { ) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let bam = Self::alignment_file(&aln)?; - // The reference isn't asked for at import — resolve the alignment's build via the gateway - // (cached, else download) when no FASTA was stored. + // The import step does not ask the user for the reference. When the alignment holds no + // FASTA path, the gateway finds the build of that alignment. It reads the cache, and it + // downloads the file when the cache holds none. let reference = match aln.reference_path { Some(p) => PathBuf::from(p), None => { @@ -63,8 +68,9 @@ impl App { .await? } }; - // For a targeted test (Big Y, etc.) restrict the walk to the target chromosome(s) so the - // headline depth reflects the target rather than being diluted to ~0 by the empty genome. + // For a targeted test, such as Big Y, read only the target chromosomes. The depth that the + // app reports then describes the target. Across the full genome, most contigs hold no read, + // and they make that value almost zero. let allowlist = self.coverage_target_allowlist(alignment_id).await?; let mut params = CallableLociParams::default(); let result = tokio::task::spawn_blocking(move || { @@ -92,21 +98,29 @@ impl App { Ok(result) } - /// The coverage contig allowlist for a targeted test, or `None` (whole genome) for WGS/autosomal. - /// A Y-targeted test (FTDNA Big Y, Y Elite, …) walks chrY only — plus chrM so the "has mtDNA - /// reads" signal survives for the few Big Ys that retained mitochondrial reads (the UI hides the - /// mtDNA sections when chrM has none). An mtDNA-targeted test walks chrM only. Build-agnostic - /// (both `chr`-prefixed and bare contig names are listed). + /// The list of contigs that the coverage walk reads, for a targeted test. The method returns + /// `None` for a WGS test and an autosomal test, and the walk then reads the full genome. + /// + /// A Y test, such as FTDNA Big Y or Y Elite, reads chrY. It also reads chrM, so the signal "this + /// test holds mtDNA reads" survives. A few Big Y files hold mitochondrial reads, and the UI + /// hides the mtDNA sections when chrM holds none. + /// + /// An mtDNA test reads chrM only. + /// + /// The list does not depend on the build. It holds each contig name with the `chr` prefix and + /// each name without it. async fn coverage_target_allowlist(&self, alignment_id: i64) -> Result>, AppError> { use navigator_domain::testtype::TargetType; let aln = self.alignment_or_err(alignment_id).await?; let Some(run) = sequence_run::get(self.store.pool(), aln.sequence_run_id).await? else { return Ok(None); }; - // `target_of` (not bare `by_code`) so a stored human label like "Big Y" — which a bulk - // import / --test-type override writes instead of BIG_Y_500/700 — still scopes the walk to - // chrY+chrM. Otherwise coverage walks the whole genome, which on a targeted multi-reference - // CRAM is the ~1-hour batch-analysis stall. + // The code calls `target_of` and not `by_code`. So a label that a person wrote, such as + // "Big Y", still limits the walk to chrY and chrM. A bulk import writes such a label, and + // the `--test-type` option also writes one, in place of BIG_Y_500 or BIG_Y_700. + // + // Without this call, the walk reads the full genome. On a targeted CRAM file with many + // references, that walk is the stop of about one hour in a batch analysis. let contigs: &[&str] = match navigator_domain::testtype::target_of(&run.test_type) { Some(TargetType::YChromosome) => &["chrY", "Y", "chrM", "chrMT", "M", "MT"], Some(TargetType::MtDna) => &["chrM", "chrMT", "M", "MT"], @@ -115,10 +129,14 @@ impl App { Ok(Some(contigs.iter().map(|s| s.to_string()).collect())) } - /// Whether a cached coverage result was computed at the right scope for the alignment's test. - /// A targeted test (Big Y, mtFull) must cover only its target contig(s); a whole-genome cached - /// result for it is stale — the headline depth was diluted across the empty genome — and must - /// be recomputed. Whole-genome tests (no allowlist) are always in scope. + /// Shows whether a cached coverage result covers the correct contigs for the test of this + /// alignment. + /// + /// A targeted test, such as Big Y or mtFull, must cover its target contigs only. A cached + /// whole-genome result for such a test is wrong, because the depth is small across the contigs + /// with no read. The app must calculate that result again. + /// + /// A whole-genome test has no list of contigs, and its result is always correct. pub(crate) async fn coverage_is_correctly_scoped( &self, alignment_id: i64, @@ -130,9 +148,12 @@ impl App { } } - /// Cached coverage for analysis reuse: the stored result, but only when it was computed at the - /// right scope for the test (see [`Self::coverage_is_correctly_scoped`]). A stale whole-genome - /// result for a targeted test reads as a cache miss so the caller recomputes it correctly. + /// The cached coverage result, for a later analysis. The method returns the stored result only + /// when that result covers the correct contigs for the test. See + /// [`Self::coverage_is_correctly_scoped`]. + /// + /// A whole-genome result for a targeted test is wrong. The method then returns nothing, and the + /// caller calculates the correct result. pub async fn cached_coverage_for_analysis(&self, alignment_id: i64) -> Result, AppError> { match self.cached_coverage(alignment_id).await? { Some(cov) if self.coverage_is_correctly_scoped(alignment_id, &cov).await? => Ok(Some(cov)), @@ -140,9 +161,11 @@ impl App { } } - /// Infer biological sex from the alignment's chrX:autosome read-density ratio, persisting - /// the result as a `sex` artifact. Cheap (BAI fast-path for BAM). `reference` is used only - /// for CRAM decode. + /// Find the biological sex from the ratio between the read density of chrX and the read density + /// of the autosomes. The method writes the result as a `sex` artifact. + /// + /// The step is fast, because a BAM file has a BAI index. The code uses `reference` only to + /// decode a CRAM file. pub async fn run_sex(&self, alignment_id: i64) -> Result { let (bam, reference) = self.alignment_paths(alignment_id).await?; let result = @@ -153,9 +176,11 @@ impl App { Ok(result) } - /// Write the inferred sex back to the biosample when the user didn't provide one, so it - /// shows in the subjects table + header instead of "Unknown". No-op for Unknown sex or - /// when the biosample already carries a sex. + /// Write the sex that the code found to the biosample, when the user gave none. The subjects + /// table and the header then show that value in place of "Unknown". + /// + /// The method does nothing when the code found no sex, and when the biosample already holds + /// one. pub(crate) async fn write_back_inferred_sex( &self, alignment_id: i64, @@ -200,11 +225,14 @@ impl App { Ok(result) } - /// Mirror an alignment's library-level read stats onto its owning sequence run (`total_reads`, - /// `mean_read_length`, `mean_insert_size`) so the Data Sources run card shows them without - /// re-walking. Best-effort: a missing alignment/run is ignored. When a run has several - /// alignments the last write wins — these are per-library properties, so any pass is - /// representative. + /// Copy the library-level read statistics of an alignment to its sequence run. Those values are + /// `total_reads`, `mean_read_length`, and `mean_insert_size`. The run card of the Data Sources + /// tab then shows them, and the app reads no file again. + /// + /// The step is optional, and the method ignores an absent alignment and an absent run. + /// + /// When a run holds more than one alignment, the last write wins. These values describe the + /// library, so each alignment gives the same answer. pub(crate) async fn write_back_read_stats( &self, alignment_id: i64, @@ -242,51 +270,65 @@ impl App { self.load_analysis(alignment_id, "read_metrics", "1").await } - /// Scratch directory for alignments copied off a slow/removable volume (see [`localize`]). - /// Entries are owned by a [`LocalAlignment`] and removed when the last holder drops. + /// The scratch directory for an alignment that the code copied from a slow volume or a + /// removable volume. See [`localize`]. A [`LocalAlignment`] value owns each entry, and the code + /// removes that entry after the last holder drops it. pub(crate) fn align_cache_dir() -> std::path::PathBuf { navigator_refgenome::cache::base_dir().join("cache").join("aln") } - /// If `remote` lives on a slow/removable volume (a `/Volumes/…` mount), copy it — and its `.crai` - /// / `.bai` index — into the local cache and return the *local* path; otherwise return `remote` - /// unchanged. The analysis walkers do random-access record iteration (region seeks, per-read - /// decode), which is pathologically slow over a network/USB mount even though a plain sequential - /// **copy** of the same file is fast — so we pay one fast bulk copy up front and let every - /// subsequent pass read from local disk. The copy is reused across a subject's passes and cleared - /// per subject by [`clear_align_cache`]. A copy failure falls back to the remote path (slow, but - /// still works). + /// Copy `remote` to the local cache and return the *local* path, when that file sits on a slow + /// volume or a removable volume. Such a volume has a `/Volumes/…` mount point. The method also + /// copies the `.crai` index or the `.bai` index. For any other path, it returns `remote` with no + /// change. + /// + /// An analysis walker reads records at random positions. It seeks to a region, and it decodes + /// each read. That access is very slow over a network mount or a USB mount. A plain sequential + /// **copy** of the same file is fast. + /// + /// So the code pays for one fast copy first, and each later pass reads from the local disk. The + /// passes of one subject share that copy, and [`clear_align_cache`] removes it for each subject. + /// + /// A failed copy gives the remote path. The analysis is then slow, and it still works. pub(crate) async fn localize(&self, remote: &Path) -> LocalAlignment { if std::env::var_os("NAVIGATOR_NO_LOCALIZE").is_some() || !is_removable_volume(remote) { return LocalAlignment::borrowed(remote); } let local = Self::align_cache_dir().join(local_cache_name(remote)); - // Serialize the cache-check-then-copy per destination. The worker `tokio::spawn`s every - // command, so a batch walk and a per-alignment command genuinely overlap on one alignment; - // without this both miss the cache and both copy, writing a second full 40 GB pull over the - // network for nothing, after which the loser of the rename reads from the remote anyway. + // Order the two steps, the cache test and the copy, for each destination. The worker calls + // `tokio::spawn` for each command. So a batch walk and a command for one alignment do + // overlap on the same alignment. + // + // Without this lock, both find no cache entry and both copy. The second copy reads another + // 40 GB over the network for no result. The task that loses the rename then reads the + // remote file. // - // The gate is held across the copy, so it must not be taken re-entrantly — no `localize` - // may be called while another is outstanding *on the same path in the same task*. The three - // call sites are sequential today (`debug_y_calls` awaits `base_calls` to completion before - // localizing itself); keep it that way. + // The code holds the lock across the copy. So it must not take that lock a second time. No + // call of `localize` can run while another call is open *on the same path in the same + // task*. + // + // The three call sites run in sequence today. `debug_y_calls` waits for `base_calls` to + // complete before it localizes its own file. Keep that order. let gate = copy_gate(&local); let _copying = gate.lock().await; - // Size the remote once: it decides both whether an existing copy can be trusted and whether - // the one we make arrived whole. + // Read the size of the remote file one time. That value answers two questions. It shows + // whether the code can trust a copy that exists, and it shows whether the new copy is + // complete. let remote_len = tokio::fs::metadata(remote).await.ok().map(|m| m.len()); - // Another holder is already using this copy — share it and bump the count. + // Another holder already uses this copy. Share it, and add one to the count. if LocalAlignment::retain(&local, remote_len) { return LocalAlignment::owned(local); } let (remote_owned, local2) = (remote.to_path_buf(), local.clone()); match tokio::task::spawn_blocking(move || copy_with_index(&remote_owned, &local2, remote_len)).await { - // Registering can still fail if the copy was removed in the gap (a concurrent holder - // finishing and dropping to zero). Returning an `owned` handle to a missing path would - // fail the walk with a confusing ENOENT and then "clean up" a file that isn't there. + // This step can still fail, because another holder can remove the copy in the time + // between the two steps. That holder completes its work and drops the count to zero. + // + // An `owned` handle to an absent path fails the walk with an ENOENT error that no user + // can read. The code then also tries to remove a file that is not there. Ok(Ok(())) if LocalAlignment::retain(&local, remote_len) => LocalAlignment::owned(local), Ok(Ok(())) => { eprintln!( @@ -306,21 +348,32 @@ impl App { } } - /// Run the unified quality-metrics walker — coverage + callable, read-level QC metrics, and - /// sex inference in **one pass** over the alignment's BAM/CRAM (vs. the separate passes - /// `run_coverage` + `run_read_metrics` + `run_sex` cost: 2 reads for BAM, 3 for CRAM). All - /// three sub-results are persisted under their existing artifact keys (`coverage`/ - /// `COVERAGE_VERSION`, `read_metrics`/`"1"`, `sex`/`"1"`), so `cached_coverage`/ - /// `cached_read_metrics`/`cached_sex` and the SV step's reuse logic keep working unchanged. + /// Run the unified quality-metrics walker. It makes **one pass** over the BAM file or the CRAM + /// file of the alignment. That pass gives three results: the coverage with the callable regions, + /// the quality metrics of each read, and the sex. + /// + /// The separate calls `run_coverage`, `run_read_metrics`, and `run_sex` cost more. They read a + /// BAM file two times, and a CRAM file three times. + /// + /// The method writes each of the three results under its existing artifact key. Those keys are + /// `coverage` with `COVERAGE_VERSION`, `read_metrics` with `"1"`, and `sex` with `"1"`. + /// + /// So `cached_coverage`, `cached_read_metrics`, `cached_sex`, and the reuse rule of the SV step + /// each work with no change. pub async fn run_unified_metrics(&self, alignment_id: i64) -> Result { self.run_unified_metrics_with_progress(alignment_id, |_, _| {}, CancelToken::none()) .await } - /// Like [`run_unified_metrics`], reporting `progress(contigs_done, contigs_total)` as the - /// (slow) whole-genome coverage portion finalizes each contig. Uses the per-contig parallel - /// walker (falling back to a sequential pass for CRAM / unindexed BAM); the callback is - /// `Fn + Sync` because it's invoked concurrently from the fan-out's worker threads. + /// The same work as [`run_unified_metrics`], with a progress report. The method calls + /// `progress(contigs_done, contigs_total)` as the whole-genome coverage step completes each + /// contig. That step is the slow one. + /// + /// The method uses the parallel walker, which works on each contig at the same time. For a CRAM + /// file, and for a BAM file with no index, it reads the file from start to end instead. + /// + /// The callback is `Fn + Sync`, because the worker threads of the parallel walker call it at the + /// same time. pub async fn run_unified_metrics_with_progress( &self, alignment_id: i64, @@ -329,13 +382,16 @@ impl App { ) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let run_id = aln.sequence_run_id; - // Copy off a slow/removable volume to local disk first — the walker's random-access record - // iteration is far slower over a network/USB mount than a one-shot bulk copy. - // Held for the whole walk: dropping it removes the local copy. + // Copy the file from a slow volume or a removable volume to the local disk first. The + // walker reads records at random positions, and that access is much slower over a network + // mount or a USB mount than one bulk copy. + // + // The code holds this value for the full walk. A drop of it removes the local copy. let bam = self.localize(&Self::alignment_file(&aln)?).await; let bam = bam.path().to_path_buf(); - // The walker requires a reference (CRAM decode + reference-N detection); resolve the - // build via the gateway when no FASTA was stored at import. + // The walker needs a reference. It decodes the CRAM file with that reference, and it finds + // each N base of the reference. When the import stored no FASTA path, the gateway finds the + // build. let reference = match aln.reference_path { Some(p) => PathBuf::from(p), None => { @@ -344,9 +400,12 @@ impl App { .await? } }; - // Restrict a targeted test (Big Y, mtFull) to its target contig(s), exactly like the - // standalone coverage walker — otherwise the headline depth is diluted across the empty - // genome (a Big Y reads as ~0.2× instead of ~50× on chrY). WGS keeps the whole-genome walk. + // Limit a targeted test, such as Big Y or mtFull, to its target contigs. The separate + // coverage walker uses the same rule. + // + // Across the full genome, most contigs hold no read, and they make the depth small. A Big Y + // test then reads as about 0.2x, and its true depth on chrY is about 50x. A WGS test keeps + // the whole-genome walk. let allowlist = self.coverage_target_allowlist(alignment_id).await?; let mut params = CallableLociParams::default(); let result = tokio::task::spawn_blocking(move || { @@ -376,11 +435,16 @@ impl App { self.save_analysis(alignment_id, "read_metrics", "1", &result.read_metrics) .await?; self.write_back_read_stats(alignment_id, &result.read_metrics).await?; - // Sex: a Y-targeted test (Big Y, Y Elite, …) sequences the donor's Y chromosome — he is male - // by definition. The chrX/autosome ratio the inference needs isn't present in a chrY-scoped - // walk, and is unreliable even whole-genome (a Big Y's off-target chrX ≈ autosome ≈ 0.4× - // reads as *female*). So force Male for a Y-targeted test, overriding the inference + any - // prior auto-assignment; WGS / mt-targeted keep the walk's result. + // The sex. A Y test, such as Big Y or Y Elite, reads the Y chromosome of the donor. So that + // donor is male, by definition. + // + // A walk of chrY alone holds no ratio between chrX and the autosomes, and the code needs + // that ratio. The ratio is also wrong across the full genome. In a Big Y file, chrX and the + // autosomes each hold about 0.4x, and the code then reads the donor as *female*. + // + // So the code writes Male for a Y test. That value replaces the result of the ratio, and it + // replaces a value from an earlier run. A WGS test and an mt test keep the result of the + // walk. let y_targeted = matches!( sequence_run::get(self.store.pool(), run_id) .await? @@ -388,12 +452,18 @@ impl App { .and_then(|r| navigator_domain::testtype::target_of(&r.test_type)), Some(navigator_domain::testtype::TargetType::YChromosome) ); - // A Y-scoped alignment reads as male the same way a Y-targeted test does — chrY carries - // essentially all the reads while the autosomes hold only a few dozen mismapped ones (a - // Y-only extract, e.g. GRCh38 chrY reads realigned to hs1, or a Y-Elite/Big Y capture that - // came in mislabeled WGS). The ratio walk can then read it as *female*, which silently - // disables the whole Y pipeline (assign_y_haplogroup skips females before it ever fetches - // the tree). Detect it from the per-contig read counts and force male, exactly like a Y test. + // An alignment with reads on chrY only is male, as a Y test is. Its chrY contig holds + // almost each read, and its autosomes hold a few reads that the mapper placed wrongly. + // + // Two files have that shape. One is a chrY extract, such as GRCh38 chrY reads that the app + // realigned to hs1. The other is a Y Elite capture, or a Big Y capture, that arrived with a + // WGS label. + // + // The ratio can read such a file as *female*. That value stops the full Y pipeline with no + // message, because `assign_y_haplogroup` skips a female subject before it reads the tree. + // + // So the code finds this shape from the read count of each contig, and it writes Male, as it + // does for a Y test. let y_scoped = navigator_analysis::sex::is_y_scoped( result .coverage @@ -416,8 +486,9 @@ impl App { if let Some(sex) = &sex { self.save_analysis(alignment_id, "sex", "1", sex).await?; if male_by_scope { - // Definitive (Y test / Y-scoped ⇒ male): override any prior auto-inferred sex — - // including a stale false "Female" — rather than write-if-empty. + // This value is definite: a Y test, or an alignment with reads on chrY only, is + // male. So the code replaces a sex from an earlier run, and that set holds a wrong + // "Female" value. It does not only write into an empty field. if let Ok(guid) = self.biosample_of_alignment(alignment_id).await { biosample::set_sex(self.store.pool(), guid, "Male").await?; } @@ -436,15 +507,19 @@ impl App { alignment_id: i64, cancel: CancelToken, ) -> Result { - // Resume: a fresh cached SV result (source unchanged) is reused rather than recomputed. + // A new SV result in the cache, from a source file that did not change, is correct. The + // code uses that result and calculates nothing. if let Some(c) = self.cached_sv(alignment_id).await? { return Ok(c); } let aln = self.alignment_or_err(alignment_id).await?; let reference_build = aln.reference_build.clone(); - // Resolve the reference for decode (see alignment_reference_for_decode): required for a CRAM, - // None for a BAM. SV never consults reference *bases* — but decoding a CRAM record does, so - // the walker needs it too, not just the header-lengths probe. + // Find the reference for the decoder. See alignment_reference_for_decode. A CRAM file needs + // it, and a BAM file uses None. + // + // The SV step reads no reference *base*. But a decode of a CRAM record does read one. So the + // walker also needs the reference, and not only the step that reads the contig lengths from + // the header. let (bam, reference) = self.alignment_reference_for_decode(alignment_id).await?; let cov = match self.cached_coverage(alignment_id).await? { @@ -508,11 +583,16 @@ impl App { p.exists().then_some(p) } - /// Genotype short tandem repeats on `contig` from the alignment, via the enclosing-read caller - /// over the HipSTR reference tracts (haploid for chrY/chrM, diploid elsewhere). Persisted as a - /// `str:{contig}` artifact (so it's cached + source-invalidated like other analyses). Errors if - /// no STR reference is configured for the alignment's build (the tracts are build-specific — - /// CHM13/GRCh37 need their own reference or liftover, not yet wired). + /// Genotype the short tandem repeats on `contig` from the alignment. The caller reads each + /// record that covers a full tract, and it uses the HipSTR reference tracts. It calls chrY and + /// chrM as haploid, and each other contig as diploid. + /// + /// The method writes the result as a `str:{contig}` artifact. So the cache holds it, and a + /// change to the source file makes it invalid, as it does for another analysis. + /// + /// The method fails when no STR reference exists for the build of the alignment. The tracts + /// belong to one build. CHM13 and GRCh37 each need their own reference, or a liftover, and no + /// code does that work yet. pub async fn run_str_calls( &self, alignment_id: i64, @@ -531,10 +611,13 @@ impl App { )) })?; // Resolve the reference for decode (see alignment_reference_for_decode): required for a CRAM, - // None for a BAM. STR region-genotyping reads the alignment; it doesn't consult reference bases. + // None for a BAM. STR region-genotyping reads the alignment; it does not consult reference bases. let (bam, reference) = self.alignment_reference_for_decode(alignment_id).await?; - // chrY / chrM are haploid (one allele); autosomes + chrX (in a female) are diploid. We - // genotype chrY/chrM haploid and everything else diploid — sex-aware chrX is a refinement. + // A cell holds one copy of chrY and one copy of chrM, so each has one allele. It holds two + // copies of each autosome, and a female cell holds two copies of chrX. + // + // So the code calls chrY and chrM as haploid, and each other contig as diploid. A rule for + // chrX that reads the sex is a later improvement. let ploidy: u8 = if contig::is_haploid(&contig) { 1 } else { 2 }; let params = navigator_analysis::strcaller::StrCallerParams::default(); let genos = tokio::task::spawn_blocking(move || { @@ -553,16 +636,23 @@ impl App { Ok(genos) } - /// Compare the STR markers called from sequence (mapped to the FTDNA convention via the - /// corpus-calibrated [`navigator_analysis::strmarker`] table) against the subject's imported - /// vendor Y-STR profile — the By-Panel concordance view. One row per marker present in either - /// source: the called value + its calibration status, the imported value, and whether they agree. - /// `contig` is typically `chrY`. Reuses the cached `str:{contig}` calls. + /// Compare the STR markers from the sequence data with the vendor Y-STR profile that the user + /// imported. The By-Panel view shows this comparison. + /// + /// The [`navigator_analysis::strmarker`] table changes each called value to the FTDNA + /// convention. A corpus of real kits calibrated that table. + /// + /// The result holds one row for each marker in either source. A row holds the called value with + /// its calibration state, the imported value, and a flag that shows whether the two agree. + /// + /// The `contig` value is usually `chrY`. The method reads the `str:{contig}` calls from the + /// cache. pub async fn str_concordance(&self, alignment_id: i64, contig: String) -> Result, AppError> { use navigator_analysis::strmarker::{called_markers_build, normalize_marker, MarkerStatus, StrBuild}; - // The FTDNA convention offset is build-dependent for a few markers (the CHM13 liftover shifted - // some tract boundaries) — select the offsets for this alignment's build. + // For a few markers, the offset of the FTDNA convention changes with the build. The CHM13 + // liftover moved the boundary of some tracts. So the code reads the offsets of the build of + // this alignment. let build = alignment::get(self.store.pool(), alignment_id) .await? .map(|a| StrBuild::from_build_str(&a.reference_build)) @@ -625,11 +715,18 @@ impl App { Ok(out) } - /// Pick the subject's best STR-capable alignment and run the Y-STR concordance on chrY — the - /// entry point the UI calls. "STR-capable" = an alignment whose reference build has a HipSTR - /// reference present ([`str_reference_path`](Self::str_reference_path)); highest mean coverage - /// wins. A CRAM needs no stored reference here — [`run_str_calls`](Self::run_str_calls) resolves - /// it for decode. Errors with guidance when none qualifies (no HipSTR reference / no alignment). + /// Select the best alignment of the subject for STR work, and compare the Y-STR markers on + /// chrY. The UI calls this method. + /// + /// An alignment can do STR work when a HipSTR reference exists for its build. See + /// [`str_reference_path`](Self::str_reference_path). Among those alignments, the one with the + /// highest mean coverage wins. + /// + /// A CRAM file needs no stored reference here, because + /// [`run_str_calls`](Self::run_str_calls) finds one for the decoder. + /// + /// The method fails with a hint when no alignment passes. The two causes are an absent HipSTR + /// reference and a subject with no alignment. pub async fn str_concordance_for_subject( &self, biosample_guid: SampleGuid, @@ -640,7 +737,8 @@ impl App { if Self::str_reference_path(&a.reference_build).is_none() { continue; // no HipSTR reference for this build } - // A CRAM with no stored reference is fine — run_str_calls resolves it via the gateway. + // A CRAM file with no stored reference is acceptable, because run_str_calls finds one + // through the gateway. let cov = self .cached_coverage(a.id) .await @@ -705,9 +803,13 @@ impl App { .await } - /// Whole-contig **de-novo diploid** SNV calling (het 0/1 + hom-alt 1/1) on `contig`, cached per - /// alignment+contig. Reuses the alignment's BAM + reference (resolved from the build). Returns - /// [`SiteGenotype`]s in position order — feed to [`Self::diploid_vcf`]. + /// Call the **de-novo diploid** SNVs across the full `contig`. The caller writes a heterozygous + /// call as 0/1 and a homozygous alternate call as 1/1. The cache key is the alignment with the + /// contig. + /// + /// The method reads the BAM file of the alignment and its reference, which the code finds from + /// the build. It returns the [`SiteGenotype`] values in the order of their positions. Give them + /// to [`Self::diploid_vcf`]. pub async fn run_diploid_calls( &self, alignment_id: i64, @@ -734,8 +836,9 @@ impl App { Ok(calls) } - /// A diploid VCF (VCFv4.2, `GT:AD:DP:GQ:PL`) of the de-novo diploid SNV calls for `contig` - /// (computing + caching them if needed). The sample column is `aln`. + /// A diploid VCF file of the de-novo diploid SNV calls of `contig`. The file uses VCFv4.2, and + /// its format field is `GT:AD:DP:GQ:PL`. The method calculates those calls and writes them to + /// the cache when the cache holds none. The sample column is `aln`. pub async fn diploid_vcf( &self, alignment_id: i64, @@ -749,11 +852,16 @@ impl App { )) } - /// A **whole-genome** diploid VCF: de-novo SNV + indel calls over the diploid primary - /// chromosomes (1–22, X) of the alignment, per-contig cached. chrY and chrM are **excluded** — - /// they're haploid, so the diploid (het 0/1) model is wrong for them; their variants come from - /// the haploid caller and the Y/mt haplogroup + mtDNA-mutation features. Heavy (a real WGS - /// calling pass); the caller runs it off the UI thread (the export path). + /// A **whole-genome** diploid VCF file. It holds the de-novo SNV calls and indel calls across + /// the diploid primary chromosomes of the alignment, which are 1 to 22 and X. The cache holds + /// the result of each contig. + /// + /// The file holds **no** chrY data and **no** chrM data. A cell holds one copy of each, so the + /// diploid model, with its 0/1 calls, is wrong for them. Their variants come from the haploid + /// caller, and from the Y and mt haplogroup features with the mtDNA mutation list. + /// + /// This method is a full WGS calling pass, and it costs much. The caller runs it away from the + /// UI thread, on the export path. pub async fn diploid_vcf_genome(&self, alignment_id: i64, cancel: CancelToken) -> Result { let (bam, reference) = self.alignment_bam_reference(alignment_id).await?; let contigs = @@ -771,10 +879,14 @@ impl App { )) } - /// The subject's alignments on the **dominant reference build** (the build the most alignments - /// share, compared on the canonical build so `chm13v2`/`hs1` agree). The consensus diploid - /// genotype pools only same-build alignments — de-novo variant coordinates can't be merged - /// across builds by position without genome-wide liftover (out of scope). `None` if no alignments. + /// The alignments of the subject on the **most frequent reference build**. The code compares the + /// canonical build, so `chm13v2` and `hs1` count as one build here. + /// + /// The consensus diploid genotype pools the alignments of one build only. The position of a + /// de-novo variant does not compare across two builds, and a join by position needs a liftover + /// of the full genome. That work is not in this feature. + /// + /// The method returns `None` when the subject has no alignment. pub(crate) async fn consensus_diploid_alignments(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alns = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; if alns.is_empty() { @@ -796,13 +908,23 @@ impl App { .collect()) } - /// **Subject-level consensus** diploid genotype across the subject's same-build WGS alignments — - /// the joint genotype (opportunity #3). Per [`reconcile_site_genotypes`]: call each alignment's - /// variants (cached [`run_diploid_calls`]), union the SNV sites, force-genotype **every** - /// alignment at the union (so a site absent from one run is its real hom-ref / no-call), and vote - /// a depth-weighted 0/1/2 dosage per site. Returns the variant (het/hom-alt) consensus sites. - /// `contigs` limits the scan (None = all primary chromosomes). Heavy (a call pass + a force-call - /// pass per alignment) — an explicit export action; nothing is persisted. + /// The **consensus diploid genotype of a subject**, across its WGS alignments on one build. This + /// value is the joint genotype, which is opportunity #3. + /// + /// [`reconcile_site_genotypes`] does the work in four steps. + /// + /// It calls the variants of each alignment, and [`run_diploid_calls`] gives those calls from the + /// cache. It joins the SNV sites of each alignment into one set. It then genotypes **each** + /// alignment at each site of that set. So a site that one run does not hold gets its real + /// hom-ref call or no-call. It then votes a dosage of 0, 1, or 2 at each site, and a deeper run + /// has more weight. + /// + /// The method returns the consensus sites with a variant, which are the heterozygous sites and + /// the homozygous alternate sites. The `contigs` value limits the scan, and `None` reads each + /// primary chromosome. + /// + /// The method costs much: one call pass and one forced-call pass for each alignment. The user + /// starts it from the export screen, and the method stores nothing. pub async fn consensus_diploid_calls( &self, biosample_guid: SampleGuid, @@ -814,7 +936,8 @@ impl App { return Ok(Vec::new()); } - // (bam, reference) per same-build alignment, resolved once. + // The pair (bam, reference) of each alignment on this build. The code finds each pair one + // time. let mut paths = Vec::new(); for id in &aln_ids { paths.push((*id, self.alignment_bam_reference(*id).await?)); @@ -836,8 +959,8 @@ impl App { } }; for contig in clist { - // Tolerate a contig absent from this alignment's header (heterogeneous inputs) — - // skip it for this source rather than aborting the whole consensus. + // The header of this alignment can hold no such contig, because the inputs differ. + // Skip that contig for this source, and do not stop the full consensus. let Ok(variants) = self.run_diploid_calls(*id, contig, cancel.clone()).await else { continue; }; @@ -874,14 +997,15 @@ impl App { per_aln.push(g); } - // 4. Vote per site → consensus. min_depth = 2: a run abstains only when essentially - // uncovered; depth-weighting lets deep runs dominate the rest. + // 4. Vote at each site to get the consensus. The value min_depth = 2 means that a run + // gives no vote only when it has almost no coverage there. A deeper run has more weight than + // a shallow one. Ok(caller::reconcile_site_genotypes(&per_aln, 2)) } - /// A **consensus** diploid VCF (VCFv4.2) for the subject — the joint genotype across same-build - /// alignments (see [`consensus_diploid_calls`]), sample column `consensus`. Heavy; the export - /// path runs it off the UI thread. + /// A **consensus** diploid VCF file for the subject, in VCFv4.2. It holds the joint genotype + /// across the alignments on one build. See [`consensus_diploid_calls`]. The sample column is + /// `consensus`. The method costs much, and the export path runs it away from the UI thread. pub async fn consensus_diploid_vcf(&self, biosample_guid: SampleGuid) -> Result { let calls = self .consensus_diploid_calls(biosample_guid, None, CancelToken::none()) @@ -889,11 +1013,14 @@ impl App { Ok(navigator_analysis::vcf::write_diploid_vcf("consensus", &calls)) } - /// Run de-novo calling on `contig` using the alignment's own stored paths. - /// The alignment's BAM + a usable reference FASTA: the stored path, else resolved from the - /// alignment's build via the gateway (cached, else downloaded). Errors only if no BAM is - /// recorded. Use this in steps that *require* the reference, so the user never has to supply - /// one (it follows from the header-detected build). + /// Call the de-novo variants on `contig` with the stored paths of the alignment. + /// + /// The method returns the BAM path of the alignment and a reference FASTA path that the code can + /// use. That reference is the stored path. When the alignment holds none, the gateway finds one + /// from the build of the alignment, from the cache or by a download. + /// + /// The method fails only when the alignment holds no BAM path. Use it in a step that *needs* the + /// reference. The user then supplies no reference, because the build in the header gives it. pub(crate) async fn alignment_bam_reference(&self, alignment_id: i64) -> Result<(PathBuf, PathBuf), AppError> { let aln = self.alignment_or_err(alignment_id).await?; let bam = Self::alignment_file(&aln)?; @@ -908,12 +1035,20 @@ impl App { Ok((bam, reference)) } - /// The alignment's path and a reference suitable for **decoding** it: a CRAM can't be read - /// without the reference, so resolve it (stored path, else from the build via the gateway, - /// cache-first); a BAM decodes without one, so return the stored path as-is (usually `None`) and - /// never force a reference download. Use this for record/pileup reads and SNP-site genotyping - /// that don't consult reference bases; use [`alignment_bam_reference`](Self::alignment_bam_reference) - /// for calling paths (de-novo SNV/indel) that need the reference even on a BAM. + /// The path of the alignment, and a reference that the code can use to **decode** it. + /// + /// No reader can open a CRAM file without its reference. So for a CRAM file the method takes + /// the stored path first, and then the build through the gateway. It reads the cache before it + /// starts a download. + /// + /// A reader can open a BAM file with no reference. So for a BAM file the method returns the + /// stored path with no change, and that value is usually `None`. It never starts a download. + /// + /// Use this method to read records, to read a pileup, and to genotype a SNP site. None of those + /// steps reads a reference base. + /// + /// Use [`alignment_bam_reference`](Self::alignment_bam_reference) for a caller path, such as a + /// de-novo SNV call or indel call. Those paths need the reference for a BAM file also. pub(crate) async fn alignment_reference_for_decode( &self, alignment_id: i64, @@ -939,9 +1074,12 @@ impl App { self.gateway.cached_reference(build).is_some() } - /// The distinct reference builds across a subject's alignments — the builds whose FASTA an - /// analysis of this subject may need. Used to pre-resolve references (with a progress bar) after - /// import and before a subject-level analysis, so on-demand downloads aren't silent. + /// Each distinct reference build across the alignments of a subject. An analysis of that subject + /// can need the FASTA file of any of them. + /// + /// The code reads this list after an import, and before an analysis of the subject. It then + /// downloads each file with a progress bar. So a download during the analysis never surprises + /// the user. pub async fn reference_builds_for_subject(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alns = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut builds: Vec = alns.into_iter().map(|a| a.reference_build).collect(); @@ -950,16 +1088,18 @@ impl App { Ok(builds) } - /// The reference build of a single alignment (`None` if it no longer exists) — for pre-resolving - /// that alignment's reference before a per-alignment analysis. + /// The reference build of one alignment. The method returns `None` when the store holds no such + /// alignment. The code reads this value to find the reference before it analyzes that + /// alignment. pub async fn reference_build_of_alignment(&self, alignment_id: i64) -> Result, AppError> { Ok(alignment::get(self.store.pool(), alignment_id) .await? .map(|a| a.reference_build)) } - /// The alignment IDs (BAM/CRAM only) across a subject's alignments — for pre-building each one's - /// coordinate index (with a progress bar) after import and before a subject-level analysis. + /// The id of each alignment of a subject that has a BAM file or a CRAM file. The code reads this + /// list to make the coordinate index of each one, with a progress bar. It does that work after an + /// import, and before an analysis of the subject. pub async fn alignment_ids_for_subject(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alns = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; Ok(alns @@ -969,12 +1109,21 @@ impl App { .collect()) } - /// Ensure the alignment's coordinate index (`.bai`/`.crai`) exists, **building it if missing** so - /// the query-driven analyses (the per-contig walker, callable intervals, the de-novo / STR - /// callers) can seek by region instead of erroring or degrading to a whole-file linear scan. - /// Returns the index path if one was built, `None` if it was already present. `progress(done, - /// total)` reports a byte fraction for a BAM and indeterminate progress (`total = None`) for a - /// CRAM. The build is a single sequential pass, run on a decode-safe blocking thread. + /// Make sure that the coordinate index of the alignment exists. That index is a `.bai` file or a + /// `.crai` file, and the method **makes it** when the disk holds none. + /// + /// Each analysis that queries a region needs that index. Those analyses are the walker that + /// works on one contig, the step that finds the callable intervals, the de-novo caller, and the + /// STR caller. Without an index, such a step fails, or it reads the full file from start to end. + /// + /// The method returns the path of the index when it made one. It returns `None` when the index + /// already existed. + /// + /// The method calls `progress(done, total)`. For a BAM file, that call gives a fraction of the + /// bytes. For a CRAM file, the `total` value is `None`, and the progress has no end value. + /// + /// The method reads the file one time, from start to end, on a thread that can decode + /// safely. pub async fn ensure_alignment_index( &self, alignment_id: i64, @@ -990,15 +1139,18 @@ impl App { Ok(built) } - /// Diagnose why an alignment can't be read, naming the **exact file** at fault rather than the - /// one the failing call happened to be handed. See [`navigator_analysis::preflight`] for why - /// that distinction is the whole point: an unreadable `.crai` and an unreadable CRAM produce - /// the same `io error on …cram` message today, and on macOS a privacy (TCC) denial and a Unix - /// permission denial are told apart only by the raw errno. + /// Report the reason that the app can not read an alignment. The report names the **exact + /// file** at fault, and not the file that the failed call received. + /// + /// [`navigator_analysis::preflight`] gives the reason for that rule. A `.crai` file that the app + /// can not read, and a CRAM file that it can not read, give the same `io error on …cram` message + /// today. On macOS, only the raw errno separates a privacy denial from TCC and a Unix permission + /// denial. /// - /// Deliberately **cache-only** for the reference: a diagnostic has to describe the machine as - /// it is, so resolving (and silently downloading) a missing FASTA here would paper over exactly - /// the state we were asked to report. A CRAM with no cached reference is a finding, not a task. + /// For the reference, the method reads the **cache only**, by design. A diagnostic must describe + /// the machine as it is. A download of an absent FASTA file here hides the exact state that the + /// user asked about. A CRAM file with no reference in the cache is a result of this check. It + /// is not a task for this method. pub async fn diagnose_alignment( &self, alignment_id: i64, @@ -1028,9 +1180,11 @@ impl App { .await } - /// The [`PublishGate`] for an alignment, adapted to its mean read length (HiFi relaxes the - /// supporting-read floor — see [`PublishGate::for_read_len`]). Samples the BAM head; any error - /// falls back to the short-read default. + /// The [`PublishGate`] of an alignment, for its mean read length. A HiFi read needs fewer reads + /// at a site than a short read. See [`PublishGate::for_read_len`]. + /// + /// The method reads the first records of the BAM file. After any error, it returns the default + /// gate for a short read. pub async fn publish_gate_for_alignment(&self, alignment_id: i64) -> Result { let (bam, reference) = self.alignment_bam_reference(alignment_id).await?; let read_len = tokio::task::spawn_blocking(move || { @@ -1043,15 +1197,20 @@ impl App { } } -/// True for a path on a removable/network mount (macOS `/Volumes/…`), where per-record random access -/// is slow but a bulk sequential copy is fast — the case [`App::localize`] copies to local disk. +/// True for a path on a removable mount or a network mount. On macOS such a mount is below +/// `/Volumes/…`. +/// +/// A read of one record at a random position is slow on that mount, and a bulk copy from start to +/// end is fast. [`App::localize`] copies such a file to the local disk. fn is_removable_volume(p: &Path) -> bool { p.starts_with("/Volumes/") } -/// A collision-free local filename for a remote alignment. Every kit's file is named `chrYM.cram`, -/// so the basename alone collides; hash the full remote path and keep the extension so the reader -/// still finds the sibling index at `.crai` / `.bai`. +/// A local file name for a remote alignment. Two such names are never the same. +/// +/// The file of each kit has the name `chrYM.cram`, so the base name alone gives the same local name +/// for two kits. The function hashes the full remote path and keeps the extension. So the reader +/// still finds the index beside the file, at `.crai` or `.bai`. fn local_cache_name(remote: &Path) -> String { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -1060,11 +1219,16 @@ fn local_cache_name(remote: &Path) -> String { format!("{:016x}.{ext}", h.finish()) } -/// A scratch name for one caller's in-progress copy of `local`. **Unique per call**: a temp path -/// derived from the destination alone (`.partial`) is shared by every concurrent copier of -/// that alignment, which lets two of them open the same inode — one truncating what the other is -/// writing, and continuing to write into it after the other has renamed it into place and started -/// reading. Uniqueness makes that unrepresentable rather than merely unlikely. +/// A scratch name for the copy of `local` that one caller is writing. Each call gives a **different +/// name**. +/// +/// A name from the destination alone, such as `.partial`, is the same name for each caller +/// that copies that alignment at the same time. Two callers then open the same inode. +/// +/// One caller empties the file that the other one writes. It also continues to write into that file +/// after the other caller renames it into place and starts to read it. +/// +/// A different name for each call makes that state impossible. It does not only make it rare. fn partial_path(local: &Path) -> PathBuf { static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -1072,18 +1236,22 @@ fn partial_path(local: &Path) -> PathBuf { local.with_file_name(format!("{stem}.partial.{}.{n}", std::process::id())) } -/// Copy `remote` → `local` plus its index sibling. The index is copied **first** and the main file -/// last (via a temp + rename), so a present `local` always implies its index is present too — the -/// cache check in [`App::localize`] can't see a half-copied pair. +/// Copy `remote` to `local`, and copy the index beside it. +/// +/// The function copies the index **first** and the main file last. For the main file it writes a +/// temporary file and then renames it. So a `local` file that exists always has its index. The cache +/// test in [`App::localize`] can then never see one file of the pair. +/// +/// `expect_len` is the size of the remote file. When the caller gives that value, the function +/// refuses a copy with a different size and publishes nothing. /// -/// `expect_len` is the remote's size; when known, a copy that doesn't match it is rejected rather -/// than published. A short copy is otherwise indistinguishable from corrupt data: it surfaces as a -/// decode error ("unexpected end of file", a bad container checksum) tens of gigabytes into a walk, -/// naming the cache path, and the copy is deleted on drop before anyone can look at it. +/// A short copy looks the same as damaged data. It gives a decode error tens of GB into a walk, such +/// as "unexpected end of file" or a bad container checksum. That error names the cache path, and the +/// code deletes the copy at the drop, before a user can look at it. /// -/// Nothing partial survives a failure — neither the temp nor the index copied ahead of it. A -/// leftover `.partial` used to sit in the cache indefinitely, occupying the disk while satisfying -/// no one. +/// No partial file survives a failure. That rule covers the temporary file and the index that the +/// function copied first. An old `.partial` file stayed in the cache for all time, and it filled the +/// disk with data that no code used. fn copy_with_index(remote: &Path, local: &Path, expect_len: Option) -> std::io::Result<()> { if let Some(parent) = local.parent() { std::fs::create_dir_all(parent)?; @@ -1129,19 +1297,22 @@ fn copy_with_index(remote: &Path, local: &Path, expect_len: Option) -> std: result } -/// One step of a full analysis of a single alignment, in the order [`App::plan_full_analysis`] -/// returns them. The variants carry whatever the step needs, so a caller's dispatch is total and it -/// cannot silently run a step the plan excluded. +/// One step of a full analysis of one alignment, in the order that [`App::plan_full_analysis`] +/// gives. Each variant carries the values that its step needs. So a caller must handle each variant, +/// and it can not run a step that the plan left out. #[derive(Debug, Clone, PartialEq)] pub enum AnalysisStep { /// Coverage + callable, read-level QC, and sex inference in one pass over the alignment. QualityMetrics, - /// CNV + discordant pairs. Needs ≥10× — the step itself reports when the depth is too low. + /// The CNV calls and the discordant pairs. The step needs a depth of 10x or more, and it + /// reports a depth below that value. /// - /// **Opt-in only.** SV is experimental and, alone among the steps, walks every read in the file - /// for a result nothing else consumes — hours per whole-genome sample. It is planned only when - /// a caller asks for it (`include_sv`); the GUI's "Call SV" button and `analyze --sv` are the - /// ways in. Nothing runs it unattended. + /// **The user must select this step.** It is experimental. It is also the one step that reads + /// each record in the file for a result that no other step uses. It needs hours for one + /// whole-genome sample. + /// + /// The plan holds this step only when a caller sets `include_sv`. The "Call SV" button of the + /// GUI and the `analyze --sv` command do that. No code runs it without a user. StructuralVariants, /// De-novo calling on the mitochondrial contig (small and fully callable, unlike whole chrY). MitoDenovo { contig: String }, @@ -1153,7 +1324,8 @@ pub enum AnalysisStep { YSignature { biosample_guid: SampleGuid }, /// Genotype the ancestry markers into the autosomal consensus profile. AutosomalProfile { biosample_guid: SampleGuid }, - /// Estimate admixture/PCA *from* the autosomal profile — must follow [`Self::AutosomalProfile`]. + /// Estimate the admixture and the PCA values *from* the autosomal profile. This step must come + /// after [`Self::AutosomalProfile`]. Ancestry { biosample_guid: SampleGuid }, } @@ -1188,20 +1360,26 @@ impl AnalysisStep { } impl App { - /// The ordered steps a full analysis of `alignment_id` should run. + /// The steps of a full analysis of `alignment_id`, in their order. + /// + /// This function is the **one** definition of that pipeline. The GUI sends progress events, and + /// the CLI writes a log. So the two report a step in different ways. + /// + /// But the set of steps, and each condition that skips one, must be the same. The two did become + /// different. The copy in the CLI genotyped the Y chromosome at each run, and it replaced a + /// trusted external call. On ancient DNA it replaced that call with a worse one. /// - /// This is the **one** definition of that pipeline. The GUI streams progress events and the CLI - /// prints a log, so how a step is reported differs — but which steps run, and the conditions - /// under which one is skipped, must not: the two had drifted, and the CLI's copy re-genotyped Y - /// unconditionally, overwriting a trusted external call (on ancient DNA, with a worse one). + /// The `coverage` value is the result that [`AnalysisStep::QualityMetrics`] calculated, when + /// that step already ran. The mitochondrial decision then has the correct data. Before that + /// step, pass `None`. The function then reads the cached coverage. With no cached value, it + /// assumes that the file holds chrM reads. /// - /// `coverage` is the just-computed result when [`AnalysisStep::QualityMetrics`] has already run, - /// which makes the mitochondrial decision authoritative; pass `None` before that and the cached - /// coverage (or, with none, the assumption that chrM is present) is used instead. Callers that - /// show a step count should re-plan after the metrics step, as the count can drop. + /// A caller that shows a count of steps must call this function again after the metrics step, + /// because that count can become smaller. /// - /// `include_sv` adds the experimental [`AnalysisStep::StructuralVariants`]; see that variant for - /// why it is off by default. `include_ancestry` likewise gates the two heaviest ancestry steps. + /// `include_sv` adds the experimental step [`AnalysisStep::StructuralVariants`]. That variant + /// gives the reason for its default. `include_ancestry` adds the two ancestry steps that cost + /// the most. pub async fn plan_full_analysis( &self, alignment_id: i64, @@ -1209,9 +1387,12 @@ impl App { include_sv: bool, coverage: Option<&CoverageResult>, ) -> Result, AppError> { - // Skip the mitochondrial steps when the alignment has no chrM reads (e.g. an FTDNA Big Y): - // scoring zero chrM data just records a meaningless RSRS root. Unknown coverage (never run) - // keeps them rather than silently skipping. + // Leave out the mitochondrial steps when the alignment holds no chrM read. An FTDNA Big Y + // file is one example. A placement with no chrM data writes the RSRS root, and that result + // has no value. + // + // With no coverage result, the plan keeps those steps. The code must not remove a step with + // no message. let has_mtdna = match coverage { Some(c) => chrm_has_reads(c), None => self @@ -1222,7 +1403,8 @@ impl App { .map(|c| chrm_has_reads(&c)) .unwrap_or(true), }; - // Subject-level steps need the owning subject; an unattached alignment simply skips them. + // A step at the level of a subject needs that subject. The plan leaves out each such step + // for an alignment with no subject. let guid = self.biosample_of_alignment(alignment_id).await.ok(); let mut steps = vec![AnalysisStep::QualityMetrics]; @@ -1232,10 +1414,14 @@ impl App { if has_mtdna { steps.push(AnalysisStep::MitoDenovo { contig: "chrM".into() }); } - // Skip the internal Y/mt genotyping when a trusted external caller (GATK4 GVCF, sidecar fast - // path) already placed this alignment and the user prefers it — re-walking would only produce - // a secondary call that loses the vote, and on ancient DNA a wrong one. The assign_* commands - // guard this too; planning around it also avoids the wasted decode. See external-caller-precedence. + // Leave out the internal Y step and the internal mt step under two conditions. A trusted + // external caller already placed this alignment, and the user prefers that caller. Such a + // caller is a GATK4 GVCF file through the sidecar fast path. + // + // A second walk gives a call that loses the vote. On ancient DNA it also gives a wrong call. + // + // Each `assign_*` command has the same guard. A plan without the step also saves the decode. + // See external-caller-precedence. if !self .has_preferred_external_call(alignment_id, DnaType::Y) .await @@ -1251,9 +1437,12 @@ impl App { { steps.push(AnalysisStep::MtHaplogroup); } - // The Y signature makes the descent report ready without an explicit click (the button stays - // for an on-demand rebuild). Built once — an existing profile is left alone — and it needs no - // extra read of the file, since the Y assignment above just cached the chrY genotypes. + // The Y signature makes the descent report ready, and the user presses no button. That + // button stays, and it rebuilds the report at any time. + // + // The code builds the signature one time, and it does not change a profile that exists. It + // reads the file no more times, because the Y step above wrote the chrY genotypes to the + // cache. if let Some(guid) = guid { if self.cached_y_profile(guid).await?.is_none() { steps.push(AnalysisStep::YSignature { biosample_guid: guid }); @@ -1270,27 +1459,32 @@ impl App { } } -/// Whether a coverage result shows any reads on the mitochondrial contig, under either naming. +/// Shows whether a coverage result holds a read on the mitochondrial contig. The function accepts +/// both names of that contig. fn chrm_has_reads(c: &CoverageResult) -> bool { c.contig_coverage_stats .iter() .any(|s| contig::is_chr_m(&s.contig) && s.num_reads > 0) } -/// Live localized copies: local path → how many [`LocalAlignment`]s hold it. A copy is removed when -/// the count reaches zero, so its lifetime belongs to the copy rather than to a caller remembering -/// to clean up. +/// The local copies that exist now. The map takes a local path and gives the count of +/// [`LocalAlignment`] values that hold it. +/// +/// The code removes a copy when its count reaches zero. So the copy itself controls its life, and no +/// caller must remember to remove it. fn localized_registry() -> &'static std::sync::Mutex> { static REG: std::sync::OnceLock>> = std::sync::OnceLock::new(); REG.get_or_init(|| std::sync::Mutex::new(HashMap::new())) } -/// The lock serializing cache copies for one destination path — see [`App::localize`], which holds -/// it across the copy so a second caller waits for the first rather than duplicating it. +/// The lock that orders the cache copies for one destination path. [`App::localize`] holds it across +/// the copy, so a second caller waits for the first one and makes no second copy. /// -/// Async, because it is held across the copy's `await`. Entries are never removed: one small entry -/// per distinct alignment localized in this process, bounded by the workspace's alignment count, -/// which is cheaper than the bookkeeping needed to retire them safely. +/// The lock is async, because the code holds it across the `await` of the copy. +/// +/// The map never removes an entry. It holds one small entry for each alignment that this process +/// copies, and the count of alignments in the workspace limits that number. The code to remove an +/// entry safely costs more than those entries. fn copy_gate(local: &Path) -> std::sync::Arc> { #[allow(clippy::type_complexity)] static GATES: std::sync::OnceLock>>>> = @@ -1300,17 +1494,22 @@ fn copy_gate(local: &Path) -> std::sync::Arc> { std::sync::Arc::clone(gates.entry(local.to_path_buf()).or_default()) } -/// An alignment path to read from, owning any local copy made for it. +/// A path to read an alignment from. This value owns the local copy of that alignment, when one +/// exists. +/// +/// The earlier design kept each copy in a directory that one caller emptied, and that caller was +/// `analyze_biosample`. But three call sites made a copy. /// -/// The previous design cached copies in a directory cleared by one caller — `analyze_biosample` — -/// while three call sites created them. Every other path (notably the Y genotyping a batch drives) -/// copied ~400 MB per alignment and never cleaned up; that reached 687 files and 145 GB, and filled -/// the volume mid-run. Tying removal to `Drop` makes that leak unrepresentable, and the refcount -/// keeps the original benefit: a subject's several passes still share one copy instead of re-copying -/// per pass. +/// Each other path copied about 400 MB for one alignment and removed nothing. The Y genotype step of +/// a batch is the main example. That fault reached 687 files and 145 GB, and it filled the volume +/// during a run. +/// +/// A removal at the `Drop` call makes that fault impossible. The count of holders keeps the first +/// advantage. The passes of one subject still share one copy, and the code copies that file one +/// time. pub(crate) struct LocalAlignment { path: PathBuf, - /// False when `path` is the original (no copy was made, so nothing to remove). + /// False when `path` is the original file. The code made no copy, so it removes nothing. owned: bool, } @@ -1326,17 +1525,22 @@ impl LocalAlignment { Self { path, owned: true } } - /// Register interest in an existing copy; `true` when one was present and is now retained. + /// Take a share of a copy that exists. The method returns `true` when such a copy was there and + /// now has one more holder. + /// + /// A file with no entry in the registry is a **file from an earlier process**. No `Drop` call + /// ran there, so something stopped that run. + /// + /// The method compares such a file with `expect_len`, which is the size of the remote file. It + /// removes the file when the two differ. /// - /// A file with no registry entry is a **leftover from an earlier process** — `Drop` never ran, - /// so the run was killed — and is validated against `expect_len` (the remote's size) before - /// being trusted, then discarded if it doesn't match. Adopting a leftover on its existence - /// alone is how a truncated copy gets read as though it were the alignment: the failure then - /// appears as a decode error deep into a walk, pointing at a cache path whose file is deleted - /// moments later. A wrong-sized copy is worth exactly one re-copy to be rid of. + /// A method that took such a file on its existence alone would read a short copy as the + /// alignment. The fault then appears as a decode error deep in a walk. It names a cache path, + /// and the code deletes that file a moment later. One more copy is a small price to remove a + /// file of the wrong size. /// - /// An entry that *is* registered belongs to a live holder in this process and was validated - /// when it was made, so it is shared without re-statting. + /// A file *with* an entry belongs to a holder in this process, and the code checked it when it + /// made that copy. So the method shares it and reads no metadata. fn retain(local: &Path, expect_len: Option) -> bool { let mut reg = localized_registry().lock().unwrap(); if let Some(n) = reg.get_mut(local) { @@ -1364,7 +1568,8 @@ impl LocalAlignment { true } - /// The path to read from — the local copy when one was made, else the original. + /// The path to read from. The value is the local copy when the code made one, and the original + /// path when it made none. pub(crate) fn path(&self) -> &Path { &self.path } @@ -1387,8 +1592,8 @@ impl Drop for LocalAlignment { return; } reg.remove(&self.path); - // Best-effort: a copy left behind is a disk-space problem, not a correctness one, and a - // panic here would mask whatever the caller was actually doing. + // The step is optional. A copy that stays on the disk costs space, and it gives no wrong + // result. A panic here also hides the work of the caller. let _ = std::fs::remove_file(&self.path); let p = self.path.to_string_lossy().into_owned(); for suffix in [".crai", ".bai"] { @@ -1424,8 +1629,9 @@ mod local_alignment_tests { #[test] fn a_shared_copy_survives_until_the_last_holder_drops() { - // The reason for the refcount: a subject's passes each localize the same alignment, and - // removing it when the first finishes would force the rest to re-copy ~400 MB. + // The reason for the count of holders. Each pass of one subject localizes the same + // alignment. A removal after the first pass makes each later pass copy about 400 MB + // again. let d = scratch("shared"); let cram = d.join("b.cram"); std::fs::write(&cram, "x").unwrap(); @@ -1441,9 +1647,11 @@ mod local_alignment_tests { assert!(!cram.is_file(), "removed once nobody holds it"); } - /// A leftover from a killed run must be checked, not trusted. Adopting a short copy is how a - /// truncated cache entry gets read as though it were the alignment — surfacing as a decode - /// failure tens of gigabytes into a walk, blamed on a cache file that is deleted moments later. + /// The code must check a file from a run that stopped. It must not trust that file. + /// + /// A method that took a short copy would read that copy as the alignment. The fault then appears + /// as a decode failure tens of GB into a walk. The message names a cache file, and the code + /// deletes that file a moment later. #[test] fn a_wrong_sized_leftover_is_discarded_rather_than_adopted() { let d = scratch("stale"); @@ -1467,9 +1675,11 @@ mod local_alignment_tests { drop(LocalAlignment::owned(cram.clone())); } - /// Two concurrent copiers must never be able to name the same scratch file: sharing one let - /// them open a single inode, where one truncates what the other is writing — and keeps writing - /// into it after the other renames it into place and starts reading. + /// Two callers that copy at the same time must never use the same scratch file name. + /// + /// With one name, the two open the same inode. One caller then empties the file that the other + /// one writes. It also continues to write into that file after the other caller renames it into + /// place and starts to read it. #[test] fn each_copy_gets_its_own_partial_path() { let local = PathBuf::from("/tmp/nav-cache/abc.cram"); @@ -1482,8 +1692,9 @@ mod local_alignment_tests { } } - /// A copy that arrives short is rejected instead of published, and leaves nothing behind — not - /// the scratch file, and not the index copied ahead of it. Both used to accumulate in the cache. + /// The code refuses a copy that is too short, and it publishes nothing. It also leaves no file + /// behind. That rule covers the scratch file and the index that it copied first. The cache used + /// to collect both of them. #[test] fn a_short_copy_is_rejected_and_leaves_no_debris() { let d = scratch("short"); @@ -1491,7 +1702,8 @@ mod local_alignment_tests { std::fs::write(&remote, "0123456789").unwrap(); std::fs::write(d.join("src.cram.crai"), "idx").unwrap(); - // Claim the remote is larger than it is — the same shape as a copy cut short. + // Give a remote size that is larger than the real size. The result has the same shape as a + // copy that stopped early. let err = copy_with_index(&remote, &local, Some(64)).expect_err("a short copy must fail"); assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof, "{err}"); @@ -1505,9 +1717,10 @@ mod local_alignment_tests { assert!(debris.is_empty(), "scratch files left behind: {debris:?}"); } - /// The gate is what stops two overlapping callers from both copying the same alignment. Every - /// worker command is `tokio::spawn`ed, so that overlap is real, and the duplicate was a second - /// full pull of a 40 GB CRAM over the network whose only outcome was a failed rename. + /// The lock stops two callers that overlap, so only one of them copies the alignment. Each + /// worker command + /// runs under `tokio::spawn`, so that overlap does occur. The second copy read a 40 GB CRAM file + /// over the network again, and its only result was a failed rename. #[tokio::test] async fn one_destination_copies_at_a_time_and_others_are_not_blocked() { use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1559,8 +1772,9 @@ mod local_alignment_tests { #[test] fn the_original_is_never_removed() { - // A path we didn't copy (local disk, or a failed copy falling back to the remote) must be - // left alone — deleting the user's own alignment would be catastrophic. + // The code must not change a path that it did not copy. Such a path is a file on the local + // disk, or the remote file after a failed copy. A delete of the alignment of the user is a + // very bad fault. let d = scratch("borrowed"); let original = d.join("c.cram"); std::fs::write(&original, "x").unwrap(); diff --git a/crates/navigator-app/src/appview.rs b/crates/navigator-app/src/appview.rs index 926fce29..cd36c7ca 100644 --- a/crates/navigator-app/src/appview.rs +++ b/crates/navigator-app/src/appview.rs @@ -1,29 +1,35 @@ -//! The one way Navigator talks to the AppView's `/api/v1/*` Edge API. +//! The one way that Navigator speaks to the `/api/v1/*` Edge API of the AppView. //! -//! Three clients grew here independently — IBD exchange, social, recruitment — and each arrived at -//! the same two shapes: an unauthenticated-looking POST whose body carries the device-key -//! signature, and a replay-guarded signed GET whose `did`/`ts`/`sig` ride on the query string. The -//! IBD and social versions were byte-for-byte identical, and the remaining one-off calls in -//! `sync.rs` / `matching.rs` open-coded the same thing a fourth and fifth time. They are all this -//! module now, so the error mapping, the signing-query layout, and the non-2xx classification are -//! decided once. +//! Three clients grew here separately: IBD exchange, social, and recruitment. Each client made the +//! same two request shapes. //! -//! What travels: a DID, a timestamp, a signature, and whatever the caller chose to send. Never -//! genotypes, never coordinates. +//! The first shape is a POST. The body of the POST holds the device-key signature, so the request +//! looks unauthenticated. The second shape is a signed GET with a replay guard. Its `did`, `ts`, +//! and `sig` values go on the query string. +//! +//! The IBD version and the social version were the same code. The single calls in `sync.rs` and +//! `matching.rs` wrote the same request a fourth time and a fifth time. All of this code is now in +//! this module. So the error map, the layout of the signature query, and the class of a non-2xx +//! response have one definition. +//! +//! These values cross the network: a DID, a timestamp, a signature, and the content that the caller +//! chose to send. A genotype never crosses. A coordinate never crosses. use super::*; -/// A transport failure (connection refused, timeout, TLS) on an AppView call. +/// A transport failure on a call to the AppView. Examples are a refused connection, a timeout, and +/// a TLS fault. /// -/// The AppView is reached with a bare `reqwest` client rather than through the sync engine, but a -/// network failure means the same thing either way, so it lands in the same error variant the PDS -/// paths use and the offline indicator already understands. +/// A plain `reqwest` client makes these calls. The calls do not go through the sync engine. But a +/// network failure has the same result on both paths. So this function returns the error variant +/// that the PDS paths use, and the offline indicator already knows that variant. pub(crate) fn transport(e: reqwest::Error) -> AppError { AppError::Sync(navigator_sync::SyncError::from(e)) } -/// Classify a non-2xx AppView response into a user-facing [`AppError::AppView`]. Consumes `resp` to -/// read the body (so capture the status first at the call site if it is also needed). +/// Change a non-2xx response from the AppView into an [`AppError::AppView`] for the user. The +/// function consumes `resp` to read the body. So the caller must keep the status first, if the +/// caller also needs it. pub(crate) async fn status_error(api: &str, resp: reqwest::Response) -> AppError { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); @@ -44,12 +50,13 @@ impl App { format!("{}/api/v1/{path}", decodingus_appview_url()) } - /// POST a JSON body to an `/api/v1/` endpoint and return the decoded response. + /// Send a JSON body to an `/api/v1/` endpoint with POST, and return the decoded + /// response. /// - /// The signature (and the DID it is over) belongs in `body` — these endpoints authenticate the - /// device key per call, not the HTTP request — so this deliberately takes an already-signed - /// body rather than signing on the caller's behalf: the canonical string differs per endpoint - /// and only the caller knows it. + /// The signature and the DID for that signature belong in `body`. These endpoints authenticate + /// the device key for each call. They do not authenticate the HTTP request. For this reason the + /// function takes a body that the caller signed. It does not sign the body, because the + /// canonical string is different at each endpoint and only the caller knows it. pub(crate) async fn appview_post( &self, path: &str, @@ -69,11 +76,13 @@ impl App { resp.json().await.map_err(transport) } - /// Device-key-signed GET to an `/api/v1/` endpoint, decoded into `T`. + /// Send a GET to an `/api/v1/` endpoint with a device-key signature, and decode the + /// response into `T`. /// - /// `build_msg(did, ts)` produces the canonical string to sign — the one thing that varies - /// between a poll, a thread read, and an exchange pull. `did`/`ts`/`sig` plus `extra` go on the - /// query; the timestamp is what makes the signature replay-guarded. + /// `build_msg(did, ts)` makes the canonical string for the signature. That string is the only + /// difference between a poll, a thread read, and an exchange pull. The `did`, `ts`, and `sig` + /// values go on the query, together with `extra`. The timestamp gives the signature its replay + /// guard. pub(crate) async fn appview_get_signed( &self, path: &str, diff --git a/crates/navigator-app/src/auth.rs b/crates/navigator-app/src/auth.rs index 3ed35f40..9a78c79a 100644 --- a/crates/navigator-app/src/auth.rs +++ b/crates/navigator-app/src/auth.rs @@ -5,9 +5,11 @@ use super::*; impl App { // ---- authentication ---------------------------------------------------- - /// Run the public-client OAuth login for `handle` (handle or DID): browser authorize → - /// loopback callback → token exchange. On success the DPoP-bound session is persisted - /// to the OS keychain and becomes the active account. Returns the authenticated DID. + /// Do the public-client OAuth login for `handle`. The value can be a handle or a DID. + /// + /// The sequence is: the browser authorizes, the loopback receives the callback, and the client + /// exchanges the token. After a good login, the code writes the DPoP-bound session to the OS + /// keychain, and that session becomes the active account. The function returns the DID. pub async fn login(&self, handle: &str) -> Result { let session = login_default(&self.auth.http, &self.auth.config, handle).await?; let did = session.did.clone(); @@ -22,17 +24,20 @@ impl App { self.auth.active.lock().unwrap().clone() } - /// The signed-in account's DID, or [`AppError::NotAuthenticated`] — the cheap auth guard publish - /// methods run before building a record / touching the DB. + /// The DID of the active account, or [`AppError::NotAuthenticated`]. This is the small guard + /// that the publish methods call first, before they make a record or read the database. pub(crate) fn require_account(&self) -> Result { self.current_account().ok_or(AppError::NotAuthenticated) } - /// Adopt a **local `did:key` identity** as the active account: the device key *is* the identity, - /// so AppView calls self-certify (`verify_signed` accepts `did:key` directly — no PDS record). - /// This is the desktop bootstrap for the federated edge: device-key-signed calls (IBD suggestions, - /// the encrypted exchange) work with no OAuth/PDS. Reuses an existing local identity if one is - /// active; otherwise generates + persists a fresh device key. Returns the `did:key`. + /// Use a **local `did:key` identity** as the active account. The device key is the identity. + /// So a call to the AppView certifies itself, because `verify_signed` accepts a `did:key` + /// directly and needs no PDS record. + /// + /// This function is the desktop start point for the federated edge. Calls that the device key + /// signs, such as IBD suggestions and the encrypted exchange, then work with no OAuth and no + /// PDS. If a local identity is already active, this function uses it. If not, it makes a new + /// device key and writes it to the keychain. It returns the `did:key`. pub fn use_local_identity(&self) -> Result { if let Some(did) = self.current_account() { if did.starts_with("did:key:") && DeviceKey::load(KEYCHAIN_SERVICE, &did)?.is_some() { @@ -47,8 +52,9 @@ impl App { Ok(did) } - /// Switch the active account to an already-known DID (in-memory; the keychain marker too). For - /// multi-identity flows — e.g. driving both sides of an exchange from one process. + /// Change the active account to a known DID. The change applies to memory and to the keychain + /// marker. Use this for a flow with more than one identity. One example is a test that operates + /// both sides of an exchange in one process. pub fn set_active_account(&self, did: &str) { let _ = self.auth.tokens.set_active(did); *self.auth.active.lock().unwrap() = Some(did.to_string()); @@ -64,9 +70,11 @@ impl App { Ok(()) } - /// Build the resilient sync engine for the active account, loading its session from the - /// keychain. Errors with [`AppError::NotAuthenticated`] when no one is signed in. The - /// engine auto-refreshes on 401 and retries transient failures with backoff. + /// Make the sync engine for the active account and read its session from the keychain. + /// + /// The function returns [`AppError::NotAuthenticated`] if no account is active. On a 401 + /// response, the engine refreshes the token. It also tries again after a temporary failure. + /// The delay becomes longer after each try. pub(crate) fn sync_engine(&self) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let session = self.auth.tokens.load(&did)?.ok_or(AppError::NotAuthenticated)?; diff --git a/crates/navigator-app/src/blocktree.rs b/crates/navigator-app/src/blocktree.rs index c0f0c7d3..d7b59b24 100644 --- a/crates/navigator-app/src/blocktree.rs +++ b/crates/navigator-app/src/blocktree.rs @@ -1,36 +1,46 @@ -//! Project **block tree** — the cohort counterpart to the per-subject descent report. +//! The project **block tree**. It is the cohort form of the descent report of one subject. //! -//! Given a project, build the induced subtree of the haplotree spanning its members' terminal -//! haplogroups: every branch that any member lies on, each carrying the run of defining SNPs that -//! are phylogenetically equivalent on it (a *block*), with members hanging off their own terminal. -//! This is the FTDNA "Block Tree" surface, over placements Navigator already computed. +//! For a project, this module builds the part of the haplotree that covers the terminal haplogroups +//! of its members. That subtree holds each branch that a member lies on. //! -//! Two rules shape everything here: +//! Each branch carries a *block*, which is the run of SNPs that define it and that the tree treats +//! as equivalent. Each member appears below its own terminal branch. //! -//! - **It reads placements, never re-places.** Terminals come from `haplogroup_terminals` — the same -//! reconciliation the subjects table and project report use. Nothing here can move a subject. -//! - **A member that can't be placed is reported, not dropped** ([`UnplacedMember`]). On a multi-lab -//! cohort provider/build skew is expected; silently omitting those members would make the tree -//! look like it accounts for the whole project when it doesn't. +//! This surface is the FTDNA "Block Tree", and it uses the placements that Navigator already made. //! -//! Design: `documents/design/project-block-tree.md`. +//! Two rules control this module: +//! +//! - **It reads a placement, and it never makes one.** Each terminal comes from +//! `haplogroup_terminals`. The subjects table and the project report use the same +//! reconciliation. No code here can move a subject. +//! - **The report names a member with no placement** ([`UnplacedMember`]). It does not remove that +//! member. In a cohort from many laboratories, a difference between providers and builds is +//! normal. Without those members, the tree looks complete when it is not. +//! +//! The design is in `documents/design/project-block-tree.md`. use std::collections::BTreeSet; use super::*; -/// Collapse a run of member-less single-child branches only when it is at least this long. A lone -/// intermediate branch is worth naming; a run of two or more is noise between the splits the cohort -/// actually resolves. +/// The minimum length of a run of branches that the code joins into one. Each branch in the run has +/// one child and no member. +/// +/// One branch between two splits has a name that the reader needs. A run of two or more such +/// branches only fills space between the splits that the cohort resolves. pub const COLLAPSE_MIN_RUN: usize = 2; impl App { - /// Build the [`ProjectBlockTree`] for `project_id`. + /// Build the [`ProjectBlockTree`] of `project_id`. + /// + /// The method returns `Ok(None)` only when the project has no member. + /// + /// A project with no placed member still gives a tree. That tree has an empty `blocks` list, + /// and it holds each member in `unplaced`. This answer is a useful one, because it tells the + /// user that the app placed nothing yet. /// - /// `Ok(None)` only when the project has no members at all. A project whose members are all - /// unplaced still yields a tree — empty `blocks`, everyone in `unplaced` — because that is a - /// meaningful answer ("nothing here is placed yet"), and it costs no tree fetch: the multi-MB - /// download + parse is skipped entirely when no member has a terminal. + /// That answer also costs nothing. When no member has a terminal, the method does not download + /// the tree and does not parse it. That document is many MB. pub async fn project_block_tree( &self, project_id: i64, @@ -41,11 +51,12 @@ impl App { return Ok(None); } - // One bulk reconciliation for the whole workspace rather than a query per member — the same - // call `project_report` and `project_str_overview` make. + // One reconciliation covers the full workspace. The code does not send one query for each + // member. `project_report` and `project_str_overview` call the same function. let terminals = self.haplogroup_terminals().await?; - // (guid, display name, terminal) per member, for the requested lineage. + // The guid, the display name, and the terminal of each member, for the lineage that the + // caller requested. let wanted: Vec<(SampleGuid, String, Option)> = members .iter() .map(|b| { @@ -57,9 +68,9 @@ impl App { }) .collect(); - // Cheap first, as `descent_report` does: with nothing placed there is no tree to draw, and - // fetching + parsing a multi-MB document to discover that is pure waste. Common on a - // freshly imported project. + // Do the fast test first, as `descent_report` does. With no placement there is no tree to + // draw. A download and a parse of a document of many MB, only to learn that, is work with + // no result. This state is common in a project that a user imported a moment ago. if wanted.iter().all(|(_, _, t)| t.is_none()) { let mut unplaced: Vec = wanted .into_iter() @@ -70,8 +81,9 @@ impl App { dna, blocks: Vec::new(), unplaced, - // The *configured* provider: no tree was fetched, so no runtime fallback happened - // either. `build_key` stays empty for the same reason — nothing was parsed. + // This value is the provider in the settings. The code downloaded no tree, so it + // also used no second provider. `build_key` stays empty for the same reason, + // because the code parsed nothing. provider: match y_tree_provider() { YTreeProvider::DecodingUs => "decodingus".to_string(), YTreeProvider::Ftdna => "ftdna".to_string(), @@ -82,9 +94,9 @@ impl App { })); } - // Provider and build key are whatever the fetch actually resolved to, not what was - // configured: the mtDNA path falls back to FTDNA at runtime when the DecodingUs tree can't be - // remapped, and its loci are rCRS either way — not the Y coordinate space. + // The provider and the build key come from the download, and not from the settings. The + // mtDNA path uses the FTDNA tree when the code can not remap the DecodingUs tree. The loci + // of that path are rCRS loci in each case, and they are not in the Y coordinate space. let (tree, provider, build_key) = match dna { DnaType::Y => match y_tree_provider() { YTreeProvider::DecodingUs => { @@ -97,7 +109,8 @@ impl App { YTreeProvider::Ftdna => { let json = self.fetch_ftdna_y_tree().await?; let tree = navigator_analysis::haplo::parse_ftdna_json(&json).map_err(AppError::Import)?; - // The FTDNA Y tree is published on GRCh38, whatever the members are aligned to. + // FTDNA publishes its Y tree on GRCh38. The build of each member does not + // change that. (tree, "ftdna", "GRCh38") } }, @@ -107,12 +120,12 @@ impl App { } }; - // One name index for the whole cohort. The per-subject path scans the node map linearly for - // its single terminal, which would be quadratic here. + // One name index covers the full cohort. The path for one subject reads the node map from + // start to end to find its terminal. Here that method would cost O(n²). let index = navigator_analysis::haplo::name_index(&tree); - // Resolve placement first, so the private-Y load below covers only members who actually - // appear on the tree — on a real cohort that is a fraction of the roster (243 of 1881 on - // R1b-CTS4466Plus), and an unplaced member's private variants can't be drawn anywhere. + // Find the placements first. The private-Y read below then covers only the members that + // appear on the tree. In a real cohort that group is small: 243 of 1,881 members on + // R1b-CTS4466Plus. The view can draw no private variant of a member with no placement. let mut placed: Vec<(SampleGuid, String, i64)> = Vec::new(); let mut unplaced = Vec::new(); for (guid, name, terminal) in wanted { @@ -156,15 +169,15 @@ impl App { evidence: Vec::new(), }) .collect(); - // Stable leaf order, so the layout doesn't reshuffle between opens. + // Stable leaf order, so the layout does not reshuffle between opens. for b in &mut blocks { b.members.sort_by(|x, y| (&x.name, x.guid.0).cmp(&(&y.name, y.guid.0))); } roll_up_subtree_members(&mut blocks); let blocks = collapse_blocks(blocks, COLLAPSE_MIN_RUN); - // Candidates go in *after* the collapse: they are leaves with members, so they could never - // be absorbed, and inserting them earlier would only make the collapse reason about - // synthetic nodes. + // The code adds each candidate *after* the collapse step. A candidate is a leaf with + // members, so the collapse can never absorb it. An earlier insert only makes the collapse + // examine nodes that the code made. let (blocks, candidate_conflicts, candidate_recurrent) = insert_candidate_branches(blocks, &private); unplaced.sort_by(|a, b| (&a.name, a.guid.0).cmp(&(&b.name, b.guid.0))); @@ -179,13 +192,18 @@ impl App { })) } - /// The one coordinate space the cohort's tree is parsed under: the **modal** DecodingUs build key - /// across the members' alignments, falling back to `hs1`. + /// The one coordinate space for the tree of the cohort. It is the most frequent DecodingUs + /// build key across the alignments of the members. The default is `hs1`. /// - /// A cohort spans builds, so there is no per-subject answer as there is in `descent_report`. - /// Picking one is safe because node names and topology are build-independent — only the loci - /// *positions* are, and the aggregate carries the key so the view can say which it means. Ties - /// break on the key name, so the choice doesn't depend on map iteration order. + /// A cohort holds more than one build, so there is no answer for one subject, as there is in + /// `descent_report`. + /// + /// One choice is safe, because the node names and the shape of the tree do not depend on the + /// build. Only the *positions* of the loci depend on it. The aggregate carries the key, so the + /// view can name the space that it shows. + /// + /// Two keys with the same count give the key that sorts first. So the choice does not depend on + /// the order of a map. async fn project_build_key(&self, members: &[Biosample]) -> &'static str { let guids: Vec = members.iter().map(|b| b.guid).collect(); let Ok(alns) = alignment::list_for_biosamples(self.store.pool(), &guids).await else { @@ -201,10 +219,10 @@ impl App { } } -/// Fill `subtree_members` — members at or below each block. +/// Fill `subtree_members`, which holds the members at each block and below it. /// -/// `blocks` is in pre-order, so walking it **backwards** visits every child before its parent and one -/// pass suffices. +/// The `blocks` list is in pre-order. So a read from the end to the start reaches each child before +/// its parent, and one pass is enough. fn roll_up_subtree_members(blocks: &mut [Block]) { let mut from_children: HashMap = HashMap::with_capacity(blocks.len()); for i in (0..blocks.len()).rev() { @@ -216,8 +234,8 @@ fn roll_up_subtree_members(blocks: &mut [Block]) { } } -/// Synthesize a [`Locus`] standing for a private (unnamed) variant, so a candidate branch's shared -/// variants render through exactly the same path as a named branch's defining SNPs. +/// Make a [`Locus`] value for a private variant, which has no name. The view then draws the shared +/// variants of a candidate branch through the same code as the SNPs of a named branch. fn private_locus(v: &PrivateVariant) -> Locus { Locus { position: v.position, @@ -227,14 +245,19 @@ fn private_locus(v: &PrivateVariant) -> Locus { } } -/// The positions a subject carries as **high-confidence new-branch candidates**: novel (not in the -/// tree at all) *and* in unique sequence. +/// The positions that a subject carries as **new-branch candidates with high confidence**. Such a +/// position is new, so the tree does not hold it, and it is in unique sequence. +/// +/// The function removes a *known* variant that is off the path, by design. That variant supports a +/// finer branch that already exists. It asks a question about the placement, and not about a new +/// branch. +/// +/// The function also removes a call in a structural region. A palindrome and an amplicon on chrY +/// hold paralogs. Two men with the "same" call in such a region share a mapping artefact more often +/// than they share an ancestor. /// -/// Off-path-*known* variants are excluded on purpose — those support an existing finer branch, which -/// is a placement question, not a new one. Structural-region calls are excluded because chrY -/// palindromes and amplicons are paralog-prone: two men "sharing" a call there are far more likely to -/// share a mapping artefact than an ancestor. Sharing noise would manufacture branches, which is the -/// one failure mode this feature must not have. +/// Shared noise makes a branch that does not exist. That result is the one fault that this feature +/// must not produce. fn candidate_positions(bucket: &PrivateBucket) -> BTreeSet { let novel: BTreeSet = bucket .variants @@ -245,18 +268,20 @@ fn candidate_positions(bucket: &PrivateBucket) -> BTreeSet { drop_clustered(&novel) } -/// How far apart two novel calls must be to count as independent mutations. +/// The minimum distance between two new calls that come from separate mutations. /// -/// Real Y mutations are scattered across megabases; a handful of "novel" calls within tens of bases -/// is one misaligned read smearing several false SNVs, which is why the GVCF path already imposes a -/// depth floor for the same reason. On the CTS4466 cohort the first candidate branches were built -/// almost entirely from such clusters — six positions inside 32 bp, gaps of 5–8 bp. +/// A real Y mutation is far from the next one, at a distance of megabases. A group of "new" calls +/// inside tens of bases comes from one read that the mapper placed wrongly, and that read gives +/// some false SNVs. The GVCF path applies a depth limit for the same reason. +/// +/// In the CTS4466 cohort, almost every first candidate branch came from such a group. One group +/// held six positions inside 32 bp, with gaps of 5 bp to 8 bp. const CANDIDATE_MIN_SEPARATION_BP: i64 = 100; -/// Drop every position that has another candidate within [`CANDIDATE_MIN_SEPARATION_BP`]. +/// Remove each position that has another candidate inside [`CANDIDATE_MIN_SEPARATION_BP`]. /// -/// The whole cluster goes, not the extras: when several calls share one mapping event there is no -/// basis for electing one of them the real mutation. +/// The function removes the full group, and not only the extra positions. When some calls come from +/// one mapping event, no rule can select one of them as the real mutation. fn drop_clustered(positions: &BTreeSet) -> BTreeSet { let ordered: Vec = positions.iter().copied().collect(); ordered @@ -271,22 +296,27 @@ fn drop_clustered(positions: &BTreeSet) -> BTreeSet { .collect() } -/// Share of the cohort's private-Y-bearing members above which a position is treated as -/// **population-shared** rather than private. +/// The share of the members with private-Y data above which the code treats a position as +/// **shared by the population**, and not as private. +/// +/// A variant that most of a cohort carries did not start on one branch of that cohort. On +/// R1b-CTS4466Plus, *all* 111 donors with private-Y data carried five positions. Those positions +/// are differences between the reference and the population. They are real, but they are not +/// private. /// -/// A variant carried by most of a cohort did not arise on one branch of it. On R1b-CTS4466Plus five -/// positions were carried by *all* 111 donors with private-Y — those are reference-vs-population -/// differences, real but not private, and the bundled cohort-shared blocklist (derived from a -/// 3,352-sample CHM13 cohort that predates this collection) does not list them. Deriving the -/// exclusion from the cohort in hand catches what a bundled list cannot anticipate. +/// The blocklist in the application bundle does not name them. That list comes from a CHM13 cohort +/// of 3,352 samples, and that cohort is older than this collection. A rule that reads the cohort in +/// the workspace finds what a fixed list can not. const COHORT_SHARED_FRACTION: f64 = 0.25; -/// Donors required before the frequency rule engages at all. +/// The count of donors that the frequency rule needs before it applies. +/// +/// The rule examines what a *large* population shares. A candidate branch needs two carriers, by +/// definition. So in a small cohort, two carriers are already a large share. With four donors, each +/// true branch goes past a limit of 25% and the code removes it. /// -/// The rule reasons about what a *large* population shares. A candidate branch needs two carriers by -/// definition, so in a small cohort two carriers are already a large share — at four donors, every -/// genuine branch would exceed a 25% ceiling and be thrown away. Below this many donors there is no -/// population to argue from, so the rule abstains rather than guessing. +/// Below this count of donors there is no population for the rule to examine. So the rule does +/// nothing, and it makes no estimate. const COHORT_SHARED_MIN_DONORS: usize = 20; /// Positions carried by more than [`COHORT_SHARED_FRACTION`] of the members that have private-Y. @@ -313,15 +343,18 @@ fn population_shared_positions(blocks: &[Block], private: &HashMap) -> BTreeSet { let defining: Vec = candidate_defining_positions(blocks, private).into_iter().collect(); defining @@ -361,12 +395,14 @@ fn clustered_candidate_positions(blocks: &[Block], private: &HashMap) -> BTreeSet { let mut blocks_per_position: HashMap> = HashMap::new(); for block in blocks { @@ -393,26 +429,32 @@ fn recurrent_positions(blocks: &[Block], private: &HashMap, private: &HashMap, ) -> (Vec, usize, usize) { - // Computed across all blocks before any group is accepted — a position defining branches under - // two parents is disqualified everywhere, not just wherever it happens to be seen second. + // The code calculates this set across each block, before it accepts any group. A position + // that defines a branch below two parents fails everywhere. It does not fail only at the + // second place where the code reads it. let recurrent: BTreeSet = recurrent_positions(&blocks, private) .into_iter() .chain(population_shared_positions(&blocks, private)) @@ -427,7 +469,8 @@ pub(crate) fn insert_candidate_branches( out.push(block); continue; } - // position → the members of *this block* carrying it, keyed by index into `block.members`. + // A map from a position to the members of *this block* that hold it. The key of a member + // is its index in `block.members`. let mut carriers: HashMap> = HashMap::new(); for (i, m) in block.members.iter().enumerate() { let Some(bucket) = private.get(&m.guid) else { continue }; @@ -451,8 +494,9 @@ pub(crate) fn insert_candidate_branches( continue; } - // Largest first, so a broader branch is accepted before the finer ones nested inside it. - // Ties broken deterministically: more shared variants, then lowest position. + // Take the largest group first, so the code accepts a wide branch before the finer + // branches inside it. Two groups of the same size compare on the count of shared variants, + // and then on the lowest position. So the order is always the same. let mut ordered: Vec<(BTreeSet, Vec)> = groups.into_iter().collect(); for (_, positions) in &mut ordered { positions.sort_unstable(); @@ -490,7 +534,7 @@ pub(crate) fn insert_candidate_branches( .map(|(_, (_, _, id))| *id) }; - // Each member goes to the smallest accepted set containing it. + // Each member goes to the smallest accepted set that holds it. let owner: HashMap = (0..block.members.len()) .filter_map(|m| { accepted @@ -598,15 +642,18 @@ pub(crate) fn insert_candidate_branches( /// Fold runs of **member-less single-child** branches into the branch below them, when the run is at /// least `min_run` long. /// -/// An induced subtree over a deep haplotree is mostly such chains: intermediate branches no member -/// sits on that split nothing within this cohort. Merging them is not a display trick — within this -/// cohort those branches genuinely are one undivided block, so the absorbed loci join the survivor's -/// own (root-most first) and the absorbed names are kept in [`Block::collapsed`]. +/// A subtree over a deep haplotree holds mostly such chains. Those are the branches that no member +/// sits on, and that divide nothing inside this cohort. /// -/// `subtree_members` survives untouched: an absorbed node has no members of its own and exactly one -/// child, so its count already equals the survivor's. +/// The join is not only a change to the display. Inside this cohort those branches are one block +/// that nothing divides. So the loci of an absorbed branch go to the branch that stays, with the +/// loci nearest the root first. [`Block::collapsed`] keeps the names of the absorbed branches. /// -/// Pure — no tree, no I/O — so it is unit-testable on a hand-built `Vec`. +/// The function does not change `subtree_members`. An absorbed node has no member of its own and +/// has one child. So its count is already the count of the branch that stays. +/// +/// The function is pure. It reads no tree and does no I/O, so a unit test can call it with a +/// `Vec` that the test builds. pub(crate) fn collapse_blocks(blocks: Vec, min_run: usize) -> Vec { if blocks.is_empty() || min_run == 0 { return blocks; @@ -626,8 +673,9 @@ pub(crate) fn collapse_blocks(blocks: Vec, min_run: usize) -> Vec let mut by_id: HashMap = blocks.iter().map(|b| (b.node_id, b.clone())).collect(); let mut absorbed: HashSet = HashSet::new(); - // `blocks` is pre-order, so a run's root-most node is always reached first. A node whose parent - // is itself absorbable is therefore mid-run and already handled by the run's head. + // The `blocks` list is in pre-order, so the code always reaches the node of a run that is + // nearest the root first. A node whose parent the code can also absorb is in the middle of a + // run. So the head of that run already covers it. for b in &blocks { if !absorbable(b) { continue; @@ -671,8 +719,9 @@ pub(crate) fn collapse_blocks(blocks: Vec, min_run: usize) -> Vec target.parent = head_parent; } - // Re-emit in the original pre-order minus the absorbed nodes, depth recomputed against the - // surviving parents. Rewiring only ever moves a node *up* to an ancestor, so pre-order holds. + // Write the list again in the first pre-order, with no absorbed node. The code calculates each + // depth against the parents that stay. A change to a link only moves a node *up* to an + // ancestor, so the list stays in pre-order. let mut depth: HashMap = HashMap::new(); let mut out = Vec::with_capacity(blocks.len() - absorbed.len()); for b in &blocks { @@ -764,10 +813,12 @@ mod tests { /// └─> D(smith) /// ``` /// - /// A run of exactly two member-less single-child branches (`A`, `B`) above a placed one. `D` - /// keeps `root` a branch point, so the run's head is `A` — otherwise `root` would be absorbable - /// too and the whole spine would fold (which is correct, and is what - /// `collapse_stops_a_run_at_a_placed_branch` covers). + /// A run of two branches, `A` and `B`, above a branch with a member. Each branch in the run + /// has one child and no member. + /// + /// The member `D` keeps `root` a point where the tree divides. So the head of the run is `A`. + /// Without `D`, the code could also absorb `root`, and the full line would become one block. + /// That result is correct, and `collapse_stops_a_run_at_a_placed_branch` covers it. fn chain() -> Vec { let mut b = vec![ block(1, "root", 0, &[]), @@ -814,7 +865,7 @@ mod tests { #[test] fn collapse_respects_min_run() { - // With a threshold of 3, the run of 2 is left alone. + // With a limit of 3, the code does not change the run of 2. let out = collapse_blocks(chain(), 3); let names: Vec<&str> = out.iter().map(|b| b.name.as_str()).collect(); assert_eq!(names, vec!["root", "A", "B", "C", "D"]); @@ -897,7 +948,8 @@ mod tests { assert!(cand.name.is_empty(), "the view localizes a candidate's label"); assert_eq!(cand.parent, Some(1)); assert_eq!(cand.depth, 1); - // Both shared positions are equivalent on this branch — one block, two loci. + // The two shared positions are equivalent on this branch. They give one block with two + // loci. let mut pos: Vec = cand.loci.iter().map(|l| l.position).collect(); pos.sort_unstable(); assert_eq!(pos, vec![100_000, 200_000]); @@ -926,8 +978,9 @@ mod tests { #[test] fn structural_region_and_off_path_calls_never_form_a_branch() { let b = block(1, "R-X", 0, &["kane", "smith"]); - // Both men "share" a palindrome call and a known off-path SNP. Neither is evidence of a - // shared ancestor — the first is a paralog artefact, the second an existing branch. + // The two men "share" a call in a palindrome and a known SNP that is off the path. + // Neither call shows a shared ancestor. The first is a paralog artefact. The second marks + // a branch that already exists. let shared = PrivateBucket { terminal: "R-X".into(), variants: vec![ @@ -943,7 +996,8 @@ mod tests { #[test] fn nested_sharing_nests_the_candidate_branches() { let b = block(1, "R-X", 0, &["a", "b", "c"]); - // All three share 100; a and b additionally share 200 — a finer branch inside the broader one. + // Each of the three members shares position 100. Members a and b also share position 200, + // which gives a finer branch inside the wider one. let p = privates( &b, &[ @@ -971,7 +1025,7 @@ mod tests { let mut fine_names: Vec<&str> = fine.members.iter().map(|m| m.name.as_str()).collect(); fine_names.sort_unstable(); assert_eq!(fine_names, vec!["a", "b"]); - // Pre-order: a parent is emitted before its child. + // The list is in pre-order, so a parent comes before its child. let pos = |id: i64| out.iter().position(|x| x.node_id == id).unwrap(); assert!(pos(broad.node_id) < pos(fine.node_id)); } @@ -979,8 +1033,9 @@ mod tests { #[test] fn overlapping_non_nested_sharing_is_counted_as_a_conflict_not_forced() { let b = block(1, "R-X", 0, &["a", "b", "c"]); - // {a,b} share 100; {b,c} share 200. Neither set contains the other, so they cannot both be - // branches of one tree — the smaller-ranked one is dropped and counted. + // The set {a,b} shares position 100, and the set {b,c} shares position 200. Neither set + // holds the other. So one tree can not hold both as a branch. The code removes the set with + // the lower rank and counts it. let p = privates( &b, &[bucket(&[100_000]), bucket(&[100_000, 200_000]), bucket(&[200_000])], @@ -997,8 +1052,8 @@ mod tests { #[test] fn a_member_with_no_computed_private_y_is_simply_not_grouped() { let b = block(1, "R-X", 0, &["kane", "smith"]); - // Only `kane` has a bucket at all — `smith` was never analyzed, which is not the same as - // having no private variants. + // Only `kane` has a bucket. No analysis ran for `smith`, and that state is not the same + // as a subject with no private variant. let p: HashMap = [(b.members[0].guid, bucket(&[100]))].into_iter().collect(); let (out, _, _) = insert_candidate_branches(vec![b], &p); assert_eq!(out.len(), 1); @@ -1009,15 +1064,16 @@ mod tests { #[test] fn clustered_calls_are_dropped_whole() { - // Six "novel" calls inside 32 bp is one misaligned read, not six mutations — the shape that - // produced most of the first candidate branches on the CTS4466 cohort. + // Six "new" calls inside 32 bp come from one read that the mapper placed wrongly. They are + // not six mutations. This shape produced most of the first candidate branches on the + // CTS4466 cohort. let cluster = bucket(&[16342231, 16342238, 16342245, 16342253, 16342258, 16342263]); assert!( candidate_positions(&cluster).is_empty(), "the whole cluster goes; there is no basis for electing one call the real one" ); - // A lone call keeps its place, and a distant neighbour doesn't drag it down. + // A lone call keeps its place, and a distant neighbour does not drag it down. let spread = bucket(&[1_000_000, 2_000_000, 2_000_050]); let kept: Vec = candidate_positions(&spread).into_iter().collect(); assert_eq!(kept, vec![1_000_000], "only the pair within 100 bp is dropped"); @@ -1025,9 +1081,9 @@ mod tests { #[test] fn a_position_defining_branches_under_two_parents_is_rejected() { - // 11311865 was shared by two members under one block *and* two under another. A variant that - // arose twice cannot mark a new branch, and the laminar check can't see it — it reasons - // inside a single block. + // Two members below one block shared position 11311865, and two members below another + // block also shared it. A variant that occurred two times can not mark a new branch. The + // laminar test can not find this state, because it examines one block only. let mut left = block(1, "R-A", 0, &["a", "b"]); left.subtree_members = 2; let mut right = block(2, "R-B", 0, &["c", "d"]); @@ -1068,9 +1124,9 @@ mod tests { #[test] fn candidates_clustering_across_the_cohort_are_all_rejected() { - // The 56.83 Mb case: three branches with *different* member sets inside 567 bp. Three - // independent lineage events in under a kilobase is not a thing; one repeat unit - // mis-mapping across several donors is. + // The case at 56.83 Mb. Three branches with *different* member sets are inside 567 bp. + // Three separate lineage events inside one kilobase do not occur. One repeat unit that the + // mapper places wrongly across some donors does occur. let mut a = block(1, "R-A", 0, &["a", "b"]); a.subtree_members = 2; let mut b = block(2, "R-B", 0, &["c", "d"]); @@ -1108,8 +1164,8 @@ mod tests { // ---- export --------------------------------------------------------------- - /// A two-block tree with one candidate branch and one unplaced member — enough to exercise - /// every column the export has to get right. + /// A tree with two blocks, one candidate branch, and one member with no placement. This shape + /// covers each column that the export must write correctly. fn exportable() -> ProjectBlockTree { let mut named = block(1, "R-X", 0, &["kane"]); named.subtree_members = 3; diff --git a/crates/navigator-app/src/brief.rs b/crates/navigator-app/src/brief.rs index 820aa43d..c454d626 100644 --- a/crates/navigator-app/src/brief.rs +++ b/crates/navigator-app/src/brief.rs @@ -1,12 +1,16 @@ -//! Composition of a casual-reader [`SubjectBrief`]: pull the existing analysis signals for one -//! subject, load the narrative reference pack, and assemble the render-ready model via the pure -//! templating in `navigator_domain::brief`. +//! This module makes a [`SubjectBrief`] for a reader who is not a specialist. //! -//! The reference pack is loaded with **graceful fallback** (decided 2026-06-22): a bundled seed is -//! the always-available floor; a CDN-hosted pack refreshes/augments it when reachable; a stale cache -//! covers a failed refresh. A brief is never blocked by a missing pack — sections degrade to the -//! structured facts the analysis already provides, and [`SubjectBrief::pack_status`] records how -//! fresh the narrative is. +//! It reads the analysis signals of one subject, reads the narrative reference pack, and builds the +//! model that the UI draws. The template code in `navigator_domain::brief` does the last step, and +//! that code is pure. +//! +//! The module reads the reference pack in three steps, and each step has a fallback. The team +//! decided this on 2026-06-22. The app always holds a seed pack, which is the lowest step. A pack +//! on the CDN refreshes and extends the seed when the network permits it. An old cache covers a +//! failed refresh. +//! +//! An absent pack never stops a brief. Each section then falls back to the structured facts of the +//! analysis. [`SubjectBrief::pack_status`] records the age of the narrative. use crate::{decodingus_appview_url, App, AppError}; use navigator_domain::ancestry::AncestryResult; @@ -20,14 +24,16 @@ use navigator_domain::reconciliation::{CompatibilityLevel, Consensus, DnaType}; use navigator_domain::testtype::{self, TargetType}; use navigator_refgenome::cache as refgenome_cache; -/// The bundled seed pack — the offline floor. Authored in `assets/brief-pack.seed.json`. +/// The seed pack in the application bundle. It is the lowest step, and it works offline. The file +/// is `assets/brief-pack.seed.json`. const SEED_PACK: &str = include_str!("../assets/brief-pack.seed.json"); /// Default CDN location of the refreshable reference pack. Override with `NAVIGATOR_BRIEF_PACK_URL`. /// A 404 / unreachable host falls back gracefully to the cache, then the bundled seed. const DEFAULT_BRIEF_PACK_URL: &str = "https://assets.decodingus.org/briefs/brief-pack.json"; -/// How long a downloaded pack is trusted before a refresh is attempted (days). +/// The count of days that the app trusts a downloaded pack. After this time, the app tries a +/// refresh. const BRIEF_PACK_TTL_DAYS: u64 = 7; fn brief_pack_url() -> String { @@ -53,12 +59,14 @@ pub(crate) fn cache_is_fresh(path: &std::path::Path, ttl_days: u64) -> bool { .unwrap_or(false) } -/// How long a per-haplogroup enrichment record is trusted before a refresh is attempted (days). +/// The count of days that the app trusts the extra record of one haplogroup. After this time, the +/// app tries a refresh. const HAPLO_ENRICH_TTL_DAYS: u64 = 30; -/// Live haplogroup content fetched from the AppView, cached per (dna-type, name). `found = false` is -/// a negative-cache marker (the endpoint answered but had nothing) so a definitively-absent -/// haplogroup isn't re-requested every rebuild. +/// Haplogroup content from the AppView. The cache key is the DNA type together with the name. +/// +/// A `found = false` value marks an absent record. The endpoint answered, but it held nothing. So +/// the app does not request that haplogroup again at each rebuild. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] struct HaploEnrichment { found: bool, @@ -73,7 +81,7 @@ struct HaploEnrichment { } impl HaploEnrichment { - /// Does this carry any narrative/age content worth folding in? + /// Shows that this record holds narrative content or age content for the brief. fn has_content(&self) -> bool { self.found && (self.formed_ybp.is_some() || self.origin.is_some() || self.story.is_some()) } @@ -99,9 +107,13 @@ fn haplo_enrich_cache_path(dna_type: DnaType, name: &str) -> std::path::PathBuf } impl App { - /// Build the plain-language brief for one subject. Pulls the consensus haplogroups, the best - /// alignment's coverage, and the run's test type, and joins them to the reference pack. Always - /// returns a brief (degrading per-section); only a store error propagates. + /// Build the plain-language brief for one subject. + /// + /// The method reads the consensus haplogroups, the coverage of the best alignment, and the test + /// type of the run. It then joins these values to the reference pack. + /// + /// The method always returns a brief. A section with no data falls back to a simpler form. Only + /// a store error stops the method. pub async fn subject_brief(&self, biosample_guid: SampleGuid) -> Result { let bio = navigator_store::biosample::get(self.store.pool(), biosample_guid) .await? @@ -120,14 +132,17 @@ impl App { let test_code = run.as_ref().map(|r| r.test_type.clone()); let (pack, pack_status) = self.load_brief_pack().await; - // The brief is prose written for the reader, so it is built in *their* language. Resolved - // here rather than passed in because every consumer wants the same thing: the UI renders it, - // the HTML export writes it to a file the user keeps, and the local-LLM prompt hands it to a - // model that should answer in the language the user is reading. + // The brief is prose for the reader, so the code builds it in the language of that + // reader. The code finds the language here, and the caller does not supply it, because + // each caller needs the same language. The UI draws the brief. The HTML export writes it + // to a file that the user keeps. The local-LLM prompt gives it to a model, and that model + // must answer in the language that the user reads. let lang = i18n::load_lang().unwrap_or(Lang::En); - // Consensus lineages (None when not placed yet, or N/A for the test). Each terminal is - // enriched best-effort from the live haplogroup endpoint (cached); pack values stand offline. + // The consensus lineages. A value is None when the app did not place the subject, or when + // the test does not cover that lineage. The code tries to add content for each terminal + // node from the haplogroup endpoint, and it caches the result. Offline, the pack values + // apply. let cons_y = self.haplogroup_consensus(biosample_guid, DnaType::Y).await?; let cons_mt = self.haplogroup_consensus(biosample_guid, DnaType::Mt).await?; let mut enriched = false; @@ -157,11 +172,14 @@ impl App { .await .ok() .flatten(); - // Deep (ancient) components. Reading *only* `ANCIENT_ADMIXTURE` is also what keeps a - // stale `PCA_PROJECTION_GMM` / `G25_NMONTE` row — persisted by the build whose - // fabricated numbers prompted this rebuild — from resurfacing in the brief, the - // DNA-story HTML export, or the LLM facts. Absent when the three ancient sources - // can't express the sample's ancestry: no card beats a wrong card. + // The deep, or ancient, components. The code reads *only* `ANCIENT_ADMIXTURE`. + // An older build wrote incorrect numbers to a `PCA_PROJECTION_GMM` row and a + // `G25_NMONTE` row, and that fault caused this rebuild. A read of only one source + // keeps those old rows out of three places. They are the brief, the DNA-story HTML + // export, and the facts for the LLM. + // + // The value is absent when the three ancient sources can not express the ancestry + // of the sample. No card is better than a wrong card. let ancient = if crate::ANCIENT_ANCESTRY_ENABLED { self.consensus_ancestry(biosample_guid, navigator_analysis::ancestry::ANCIENT_ADMIXTURE) .await @@ -175,8 +193,10 @@ impl App { _ => None, }; - // Runs-of-homozygosity (relatedness / endogamy). Read-only: only surfaced when it's already - // been computed and cached (the brief must stay cheap — ROH computation is on-demand). + // The runs of homozygosity, which show relatedness and endogamy. The code only reads + // them. It shows a value only when an earlier run calculated it and wrote it to the cache. + // The brief must stay fast, and the ROH calculation runs only at the request of the + // user. let roh = self.cached_roh(biosample_guid).await?.map(|r| { brief::roh_brief( lang, @@ -188,8 +208,9 @@ impl App { ) }); - // Archaic (Neanderthal) markers — same contract as ROH: read-only, surfaced only once the - // count has been computed and cached, so the brief stays cheap. + // The archaic markers for Neanderthal. The rule is the same as the rule for ROH. The code + // only reads them, and it shows a value only after an earlier run calculated the count and + // wrote it to the cache. The brief must stay fast. let archaic = self.cached_archaic(biosample_guid).await?.map(|a| { brief::archaic_brief( lang, @@ -217,7 +238,8 @@ impl App { summary: headline_summary(lang, &bio.donor_identifier, paternal.as_ref(), maternal.as_ref()), }; - // Computed before the brief is assembled, because `paternal` is moved into it. + // The code calculates this value first, because the next step moves `paternal` into the + // brief. let realign_offer = self .realign_offer(biosample_guid, paternal.is_some(), default_aln.map(|(_, aln)| aln)) .await?; @@ -240,27 +262,32 @@ impl App { }) } - /// Whether to offer this subject a realignment, and which alignment to offer. + /// Shows whether to offer a realignment to this subject, and which alignment to offer. /// - /// Every condition here exists to avoid proposing hours of work that would change nothing: + /// Each condition below stops an offer of many hours of work that would change nothing. /// - /// - **A paternal line to improve.** Realignment buys Y-chromosome discovery and essentially - /// nothing else — ancestry, IBD and the autosomes already handle GRCh37/38 and give the same - /// answer either way. With no Y placed there is no payoff, so no offer. - /// - **Reads to re-map.** A chip or VCF-only subject has no alignment; an alignment row without - /// a file cannot be read. - /// - **No CHM13 alignment already.** The offer claims part of their paternal line cannot - /// currently be read; for someone who already has data on the complete assembly, by any route, - /// that claim is simply false — even if some older file of theirs has never been realigned. - /// - **Reads the job would actually act on** — not already on CHM13, not itself a realignment, - /// and not already realigned once. Those three are not re-implemented here: they are - /// [`crate::realign::realignable_for_subject`], the same rule the batch count and the job - /// itself use. Offering work the job would then refuse is worse than not offering it. + /// - **The subject has a paternal line to improve.** A realignment gives discovery on the Y + /// chromosome and almost nothing more. The ancestry, the IBD, and the autosomes already work + /// on GRCh37 and GRCh38, and they give the same answer on CHM13. With no Y placement there is + /// no gain, so the app makes no offer. + /// - **The subject has reads to map again.** A subject with only a chip or a VCF has no + /// alignment. An alignment row with no file has no reads. + /// - **The subject has no CHM13 alignment.** The offer states that the app can not read part of + /// the paternal line of the user. That statement is false for a user who already has data on + /// the complete assembly, by any route. An older file of that user can still be without a + /// realignment, and the statement stays false. + /// - **The job would act on the reads.** Three tests apply. The alignment must not be on + /// CHM13. It must not be a realignment. It must not have a realignment already. /// - /// Among qualifying alignments it prefers the subject's default — the widest, then deepest test, - /// per [`Self::default_alignment_for_subject`] — and otherwise takes the first. That matters for - /// someone holding both a whole genome and a Y-only test: the whole genome is the one whose - /// realignment answers more. + /// This code does not repeat those tests. [`crate::realign::realignable_for_subject`] holds + /// them, and the batch count and the job call the same function. An offer of work that the + /// job then refuses is worse than no offer. + /// + /// Among the alignments that pass, the code selects the default alignment of the subject. That + /// is the test with the largest breadth, and then the largest depth, from + /// [`Self::default_alignment_for_subject`]. If there is none, the code takes the first + /// alignment. This choice matters for a user with a whole genome and a Y-only test. The + /// realignment of the whole genome answers more questions. async fn realign_offer( &self, biosample_guid: SampleGuid, @@ -273,12 +300,16 @@ impl App { let alignments = navigator_store::alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; - // Already reading this person's Y against the complete assembly — however they got there. - // The per-alignment rule below would still find, say, an old GRCh37 file nobody has - // realigned, and the *job* would indeed act on it; but the offer's promise is about the - // subject's paternal line rather than about one file, and for this reader that promise is - // already kept. Observed on a donor holding four CHM13 alignments and being told their - // father's line had nowhere to be read from. + // The app already reads the Y chromosome of this person against the complete assembly. + // The route to that state does not matter. + // + // The rule below, which looks at one alignment, can still find an old GRCh37 file with no + // realignment. The *job* would act on that file. But the offer makes a promise about the + // paternal line of the subject, not about one file. For this reader the app already keeps + // that promise. + // + // A donor with four CHM13 alignments saw this fault. The app told that donor that it could + // read nothing of the line of their father. if alignments .iter() .any(|a| crate::realign::is_target_build(&a.reference_build)) @@ -317,7 +348,7 @@ impl App { .ok() .and_then(|s| serde_json::from_str(&s).ok()); - // Fresh cache → use it without touching the network. + // The cache is new enough. Use it, and do not use the network. if let Some(cp) = &cached { if cache_is_fresh(&cache_path, BRIEF_PACK_TTL_DAYS) { pack.merge(cp.clone()); @@ -325,7 +356,8 @@ impl App { } } - // Stale / absent → try a refresh, falling back to the stale cache (then the seed). + // The cache is old or absent. Try a refresh. If the refresh fails, use the old cache, and + // then the seed pack. let url = brief_pack_url(); let fetched: Result = async { let resp = self @@ -368,11 +400,16 @@ impl App { (pack, status) } - /// Best-effort live enrichment for one haplogroup: cache-first (30-day TTL), else a short-timeout - /// `GET {appview}/api/v1/haplogroup/{name}`. A definitive answer (200 / 404) is cached — including - /// "not found" — so it isn't re-requested each rebuild; a transient network error is *not* cached, - /// so enrichment self-heals once connectivity returns. Returns content only when there's something - /// worth folding in (an age or narrative). + /// Read the extra content for one haplogroup, if the network permits it. + /// + /// The method reads the cache first, and the cache entry is valid for 30 days. If the cache has + /// no entry, the method sends `GET {appview}/api/v1/haplogroup/{name}` with a short timeout. + /// + /// The method caches a definite answer, which is a 200 response or a 404 response. It caches + /// the "not found" result also, so it does not send the request again at each rebuild. It does + /// not cache a temporary network error, so the content appears after the network returns. + /// + /// The method returns content only when that content has an age or a narrative. async fn enrich_haplogroup(&self, name: &str, dna_type: DnaType) -> Option { if name.trim().is_empty() { return None; @@ -404,7 +441,7 @@ impl App { } // The endpoint answered but had nothing (404 etc.) → cache a negative result. Ok(_) => HaploEnrichment::default(), - // Network/timeout error → don't cache (retry next time). + // Network/timeout error → do not cache (retry next time). Err(_) => return None, }; @@ -453,8 +490,9 @@ fn parse_haplo_enrichment(body: &str) -> HaploEnrichment { } } -/// Assemble a lineage section from the consensus + pack content, overlaying live `enrich`ment when -/// present (it wins over pack values for age/origin/story). `is_paternal` only chooses the lookup. +/// Build a lineage section from the consensus and the pack content. The `enrich` value, when it is +/// present, replaces the age, the origin, and the story of the pack. `is_paternal` only selects the +/// lookup. fn build_lineage( lang: Lang, kind: LineageKind, @@ -542,18 +580,20 @@ fn build_ancestry( .iter() .filter(|c| c.percentage >= 0.5) .map(|c| { - // Pack content (by code, then display name) supplies an optional friendlier name - // and the explanation — so a bare code like "ANF" reads as "Anatolian Farmer". + // The pack content gives a clearer name and the explanation. The code looks + // up the pack by the code first, and then by the display name. So a plain code + // such as "ANF" becomes "Anatolian Farmer". let direct = pack .population(&c.population_code) .or_else(|| pack.population(&c.population_name)); let name = direct .and_then(|p| p.name.clone()) .unwrap_or_else(|| c.population_name.clone()); - // The model's reference set mixes ancient and *modern* populations; the modern - // ones (e.g. Colombian/Puerto Rican standing in for Native American ancestry) - // rarely have their own blurb, so fall back to the continental description rather - // than leaving real non-European signal unexplained. + // The reference set of the model holds ancient populations and *modern* + // populations. A modern population usually has no text of its own. One example + // is a Colombian or Puerto Rican population, which represents Native American + // ancestry. So the code uses the continental description. Without it, a real + // signal from outside Europe has no explanation. let blurb = direct.and_then(|p| p.blurb.clone()).or_else(|| { population_super(&c.population_code) .map(population_name) @@ -625,7 +665,7 @@ fn build_test( } } -/// Plain-language test description when the pack doesn't cover the code, derived from what the test +/// Plain-language test description when the pack does not cover the code, derived from what the test /// targets. fn fallback_test_text(lang: Lang, target: TargetType) -> (String, Option) { let (what, limits) = match target { diff --git a/crates/navigator-app/src/commands.rs b/crates/navigator-app/src/commands.rs index b557c0af..d4140d32 100644 --- a/crates/navigator-app/src/commands.rs +++ b/crates/navigator-app/src/commands.rs @@ -34,15 +34,22 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("project {id}")))) } - /// Delete a project, detaching its members first. Subjects are first-class and shared across - /// projects, so deleting the grouping keeps the subjects — it only removes their membership in - /// this project (and clears the legacy home column for subjects homed here). + /// Delete a project, and remove its members from it first. + /// + /// A subject is an independent record, and many projects can hold the same subject. So a delete + /// of the project keeps each subject. It removes only the membership of that subject in this + /// project. It also clears the old home column of a subject whose home is this project. pub async fn delete_project(&self, id: i64) -> Result<(), AppError> { - // A project is only a grouping — subjects are first-class and shared across projects. Deleting - // it detaches its members (drops the M:N memberships + clears the legacy home column for - // subjects homed here) and removes the project; the subjects themselves remain in the - // workspace. This lets a mis-targeted import be undone: delete the project and re-import - // cleanly, rather than being stuck because "N subjects still belong to it". + // A project is only a group. A subject is an independent record, and many projects can + // hold the same subject. + // + // A delete does three steps. It removes each membership from the M:N table. It clears the + // old home column of a subject whose home is this project. It then removes the project. + // Each subject stays in the workspace. + // + // So a user can undo an import that went to the wrong project. The user deletes the project + // and imports again. Without this behaviour, the message "N subjects still belong to it" + // stops the user. biosample_project::remove_all_for_project(self.store.pool(), id).await?; biosample::clear_home_project(self.store.pool(), id).await?; if !project::delete(self.store.pool(), id).await? { @@ -51,9 +58,9 @@ impl App { Ok(()) } - /// Register a biosample, assigning its stable `SampleGuid` here (identity is an - /// app-layer decision, not the UI's). Verifies the target project exists first so - /// the caller gets a clear `NotFound` rather than a raw foreign-key error. + /// Add a biosample and give it a stable `SampleGuid` here. The app layer decides the identity, + /// and the UI does not. The method checks that the target project exists first. So the caller + /// receives a clear `NotFound` error and not a raw foreign-key error. pub async fn add_biosample( &self, project_id: Option, @@ -76,8 +83,9 @@ impl App { Ok(b) } - /// Update a subject's editable fields (identity, accession, description, center, sex). - /// Empty strings are normalized to NULL. Returns the updated record. + /// Change the fields of a subject that the user can edit. They are the identity, the + /// accession, the description, the center, and the sex. The method changes an empty string to + /// NULL. It returns the new record. pub async fn update_biosample( &self, guid: SampleGuid, @@ -111,7 +119,8 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("biosample {}", guid.0)))) } - /// Assign a subject to a project (validating the project exists). `None` clears it. + /// Add a subject to a project. The method checks that the project exists. A value of `None` + /// removes the subject from its project. pub async fn add_biosample_to_project(&self, guid: SampleGuid, project_id: Option) -> Result<(), AppError> { if let Some(pid) = project_id { if project::get(self.store.pool(), pid).await?.is_none() { @@ -124,9 +133,11 @@ impl App { Ok(()) } - /// Delete a subject. Refused (with a clear message) when it still has dependent data — - /// sequencing runs or any imported profile — so the user removes data first rather than - /// silently orphaning rows. + /// Delete a subject. + /// + /// The method refuses, and gives a clear message, when the subject still has data. That data is + /// a sequence run or an imported profile. So the user removes the data first. Without this + /// guard, the delete leaves rows with no subject and gives no message. pub async fn delete_biosample(&self, guid: SampleGuid) -> Result<(), AppError> { let runs = self.list_sequence_runs(guid).await?.len(); let strs = self.list_str_profiles(guid).await?.len(); @@ -140,9 +151,10 @@ impl App { {variants} variant-set, {chips} chip, {mt} mtDNA record(s) — remove its data first" ))); } - // The guard above ensures no runs/profiles remain; sweep any derived-only orphans - // (stale haplogroup/consensus/reconciliation/ancestry/IBD rows from an earlier - // incomplete delete) so removing the subject can never leave dangling rows. + // The guard above makes sure that no run and no profile stays. This step then removes + // each derived row with no owner. Such a row is an old haplogroup, consensus, + // reconciliation, ancestry, or IBD row from a delete that did not complete. So the delete + // of the subject can never leave a row behind. biosample::clear_data(self.store.pool(), guid).await?; if !biosample::delete(self.store.pool(), guid).await? { return Err(AppError::Store(StoreError::NotFound(format!("biosample {}", guid.0)))); @@ -157,10 +169,14 @@ impl App { Ok(created) } - /// A Y-targeted test (Big Y, Targeted Y, a Y-SNP pack, …) or any Y-STR profile is definitive - /// evidence of a male subject. Set the biosample's sex to "Male" when such data is present and - /// it isn't already recorded as male. Best-effort and idempotent — safe to call after any run - /// or STR-profile import (it re-derives the verdict from the stored data each time). + /// A Y test or a Y-STR profile is proof that the subject is male. A Y test is a Big Y test, a + /// Targeted Y test, or a Y-SNP pack. + /// + /// The method sets the sex of the biosample to "Male" when such data exists and the record does + /// not already hold that value. + /// + /// The step is optional, and a second call is safe. Call it after any run import or STR-profile + /// import. It reads the stored data and decides again at each call. pub(crate) async fn assign_male_for_y_evidence(&self, guid: SampleGuid) -> Result<(), AppError> { use navigator_domain::testtype::{by_code, TargetType}; let has_y_test = self @@ -186,9 +202,9 @@ impl App { Ok(alignment::create(self.store.pool(), &aln).await?) } - /// Update a sequence run's descriptive fields (test type required; platform defaults to - /// "UNKNOWN" when blank; instrument/layout optional). Read metrics are preserved. Returns - /// the updated record. + /// Change the descriptive fields of a sequence run. The test type is necessary. A blank + /// platform becomes "UNKNOWN". The instrument and the layout are optional. The method keeps the + /// read metrics and returns the new record. pub async fn update_sequence_run( &self, id: i64, @@ -223,8 +239,9 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("sequence run {id}")))) } - /// Update an alignment's descriptive fields (reference build + aligner required; variant - /// caller optional). File paths are managed by import/probe. Returns the updated record. + /// Change the descriptive fields of an alignment. The reference build and the aligner are + /// necessary, and the variant caller is optional. The import step and the probe step control + /// the file paths. The method returns the new record. pub async fn update_alignment( &self, id: i64, @@ -245,16 +262,18 @@ impl App { self.alignment_or_err(id).await } - /// An alignment by id, or `None` if there is no such row. + /// The alignment with this id, or `None` when the store holds no such row. /// - /// Public because provenance made alignments something callers ask about directly — the UI - /// needs the row to say "realigned to hs1 from alignment #N" rather than just listing files. + /// This method is public because a caller now asks about one alignment directly. The provenance + /// feature caused that change. The UI needs the row to write "realigned to hs1 from alignment + /// #N". Before, it only listed the files. pub async fn alignment(&self, id: i64) -> Result, AppError> { Ok(alignment::get(self.store.pool(), id).await?) } - /// Fetch an alignment by id, mapping a missing row to a `NotFound` error. The standard way - /// the analysis/query methods resolve an `alignment_id` before touching its BAM/CRAM. + /// Read the alignment with this id, and change an absent row into a `NotFound` error. Each + /// analysis method and each query method uses this method to resolve an `alignment_id` before + /// it opens the BAM file or the CRAM file. pub(crate) async fn alignment_or_err(&self, id: i64) -> Result { alignment::get(self.store.pool(), id) .await? @@ -264,17 +283,23 @@ impl App { /// The alignment's BAM/CRAM path, confirmed to still resolve on disk. The standard way a read /// path turns an [`Alignment`] into a path to open. /// - /// Both checks belong together and belong *early*. A recorded path that no longer resolves is - /// routine in a long-lived workspace — vendor downloads get cleaned out, volumes get unmounted — - /// but nothing checked for it, so the failure surfaced as a bare `No such file or directory` - /// from inside the reader, after the caller had already fetched a multi-MB haplotree. Worse, one - /// caller read that io error as *the tree* being unavailable and fell back to the FTDNA tree, - /// which then failed on the same absent file: a misleading log line, a wasted download, and a - /// silent change of tree provider, all from a deleted BAM. + /// The two checks belong together, and they belong *early*. + /// + /// A recorded path that no longer points to a file is normal in a workspace with a long life. A + /// user removes old vendor downloads, and a user disconnects a volume. + /// + /// No code checked for that state. So the fault appeared as a plain `No such file or directory` + /// error from inside the reader. By that point the caller had already downloaded a haplotree of + /// many MB. /// - /// The existence check races anything that deletes the file a microsecond later; that is fine. - /// It is here to name the common case correctly and cheaply, not to make opening infallible — - /// callers still handle a read error from the open itself. + /// The result was worse than one unclear message. One caller read that io error as an absent + /// *tree*. It then used the FTDNA tree, and that read failed on the same absent file. One + /// deleted BAM file gave an incorrect log line, a download with no purpose, and a change of + /// tree provider with no message. + /// + /// Another process can delete the file directly after this check. That result is acceptable. + /// The check names the common case correctly and at a low cost. It does not make the open + /// operation safe, and each caller still handles a read error from that operation. pub(crate) fn alignment_file(aln: &Alignment) -> Result { let path = aln.bam_path.clone().ok_or(AppError::MissingPaths(aln.id))?; let p = PathBuf::from(&path); @@ -287,8 +312,9 @@ impl App { /// Delete a sequence run and everything beneath it (its alignments + cached analysis /// artifacts). This is how a mistaken BAM/CRAM import is undone. pub async fn delete_sequence_run(&self, id: i64) -> Result<(), AppError> { - // Capture the run's subject + alignments before the cascade so we can purge any derived - // haplogroup/consensus data keyed on those alignments (it would otherwise go stale). + // Read the subject and the alignments of the run before the cascade. The code then + // removes each derived haplogroup row and consensus row with a key on those alignments. + // Without this step, those rows stay and become incorrect. let biosample = sequence_run::get(self.store.pool(), id) .await? .map(|r| r.biosample_guid); @@ -306,10 +332,15 @@ impl App { Ok(()) } - /// Merge `secondary` sequence run into `primary` (both must belong to `biosample_guid`): - /// reparent the secondary run's alignments onto the primary, then delete the now-empty secondary - /// (its analysis artifacts travel with the alignments — they're alignment-keyed). Destructive + - /// irreversible. Returns the number of alignments moved. + /// Join the `secondary` sequence run to the `primary` run. Both runs must belong to + /// `biosample_guid`. + /// + /// The method moves each alignment of the secondary run to the primary run. It then deletes the + /// secondary run, which is now empty. Each analysis artifact moves with its alignment, because + /// the key of an artifact is the alignment. + /// + /// This method destroys data, and the user can not undo it. It returns the count of the + /// alignments that it moved. pub async fn merge_sequence_runs( &self, biosample_guid: SampleGuid, @@ -335,14 +366,17 @@ impl App { count += 1; } } - // The secondary is now empty; delete it (cascade is a no-op for alignments — already moved). + // The secondary run is now empty, so delete it. The cascade does nothing for the + // alignments, because the code already moved them. sequence_run::delete(self.store.pool(), secondary).await?; Ok(count) } - /// Delete a single alignment and its cached analysis artifacts (the parent run is kept). + /// Delete one alignment and each analysis artifact in its cache. The method keeps the parent + /// run. pub async fn delete_alignment(&self, id: i64) -> Result<(), AppError> { - // Resolve the subject (via run) before deleting, to purge derived haplogroup/consensus data. + // Find the subject through the run before the delete. The code then removes each derived + // haplogroup row and consensus row. let biosample = match alignment::get(self.store.pool(), id).await? { Some(a) => sequence_run::get(self.store.pool(), a.sequence_run_id) .await? @@ -358,53 +392,73 @@ impl App { Ok(()) } - /// Remove derived data keyed on now-deleted alignments: each alignment's Y + mt haplogroup calls - /// (`aln:` / `aln::mt`), and the subject's genome-level consensus profiles + painting - /// (Y/mt/Auto), which were pooled from sources that may no longer exist. The consensus is - /// recomputable on demand; clearing it makes the displayed haplogroup fall back to reconciling the - /// remaining cached calls (or nothing), rather than showing a stale placement. A user manual - /// override is left intact. + /// Remove the derived data whose key is an alignment that the app deleted. + /// + /// That data is the Y haplogroup call and the mt haplogroup call of each alignment, which use + /// the keys `aln:` and `aln::mt`. It is also the genome-level consensus profiles and + /// the painting of the subject, for Y, mt, and Auto. The app pooled those results from sources + /// that can now be absent. + /// + /// The app can calculate a consensus again at any time. After this method clears it, the + /// displayed haplogroup comes from the cached calls that remain, or from nothing. Without this + /// step, the app shows an old placement. + /// + /// The method keeps a value that the user set. async fn purge_alignment_derived(&self, biosample: SampleGuid, alignment_ids: &[i64]) -> Result<(), AppError> { let pool = self.store.pool(); for &aln in alignment_ids { haplogroup_call::delete_one(pool, biosample, DnaType::Y, &format!("aln:{aln}")).await?; haplogroup_call::delete_one(pool, biosample, DnaType::Mt, &format!("aln:{aln}:mt")).await?; - // The per-alignment ancestry estimates die with the alignment. + // The ancestry estimate of each alignment goes with that alignment. ancestry_result::delete_for_alignment(pool, aln).await?; } for dna in ["Y", "Mt", "Auto"] { consensus_profile::delete(pool, biosample, dna).await?; } - // Every signature-keyed cache, from the one list — this used to name three of the four by - // hand and leave the Tier-B archaic segments behind, still keyed to a deleted alignment. + // Each signature-keyed cache, from the one list. Before this list, the code named three + // of the four caches by hand. It left the Tier-B archaic segments in the store, with a key + // on an alignment that the app had deleted. for cache in sig_cache::ALL { cache.delete(pool, biosample).await?; } - // The audit log describes the consensus we just wiped; clear it so deleting the last run - // can't leave a stale RUN_RECORDED history pointing at gone alignments. It is re-appended - // when the consensus is next rebuilt from any remaining calls. + // The audit log describes the consensus that this method removed. Clear the log also. + // Without that step, a delete of the last run leaves an old RUN_RECORDED entry that names + // absent alignments. The app writes the log again at the next rebuild of the consensus, + // from the calls that remain. recon_store::clear_audit(pool, biosample, DnaType::Y).await?; recon_store::clear_audit(pool, biosample, DnaType::Mt).await?; Ok(()) } - /// Reset a subject's analysis: clear **all** sequencing + derived/imported data (runs, - /// alignments, cached artifacts, Y/mt haplogroups + consensus + reconciliation, ancestry, IBD - /// results, and chip/STR/variant/mtDNA profiles) while keeping the subject itself — its - /// identity (name/sex/center), vendor IDs, project memberships, and MDKA genealogy. The - /// recovery tool for a botched import: clears orphaned/garbage rows so the subject can be - /// re-imported cleanly. Atomic ([`biosample::clear_data`] runs in one transaction). + /// Reset the analysis of a subject. The method clears **all** sequence data and each derived + /// or imported result. + /// + /// It removes the runs, the alignments, and the cached artifacts. It removes the Y and mt + /// haplogroups with their consensus and reconciliation rows. It also removes the ancestry, the + /// IBD results, and the chip, STR, variant, and mtDNA profiles. + /// + /// It keeps the subject. It also keeps the identity of that subject, which is the name, the + /// sex, and the center. It keeps the vendor IDs, the project memberships, and the MDKA + /// genealogy. + /// + /// This method is the recovery tool for an import that went wrong. It removes each row with no + /// owner, so the user can import the subject again. The work is atomic, because + /// [`biosample::clear_data`] runs in one transaction. pub async fn clear_biosample_data(&self, guid: SampleGuid) -> Result<(), AppError> { biosample::clear_data(self.store.pool(), guid).await?; - // Imported external autosomal call-set dosages live in their own table (outside the - // biosample cascade) — drop them too so a cleared subject starts truly empty. + // The dosages of an imported external autosomal call set are in their own table, outside + // the cascade of the biosample. Remove them also, so a subject that the user clears holds + // nothing. navigator_store::external_panel_dosage::delete_for_biosample(self.store.pool(), guid).await?; Ok(()) } - /// Reset only the subject's haplogroup placement (calls + consensus + override/audit, Y & mt), - /// keeping coverage/ancestry/imported data. Drops a stale legacy lineage so re-analysis re-places - /// it; the placement repopulates on the next full analysis (WGS) or re-import (vendor data). + /// Reset only the haplogroup placement of the subject. That placement is the calls, the + /// consensus, and the override and audit rows, for Y and for mt. + /// + /// The method keeps the coverage, the ancestry, and the imported data. It removes an old + /// lineage, so the next analysis places the subject again. The placement returns at the next + /// full analysis of a WGS sample, or at the next import of vendor data. pub async fn clear_haplogroup_data(&self, guid: SampleGuid) -> Result<(), AppError> { biosample::clear_haplogroup_data(self.store.pool(), guid).await?; Ok(()) @@ -457,10 +511,13 @@ impl App { .await } - /// Like [`save_analysis`] but stamps provenance: `source` (`navigator-walk` | - /// `pipeline-sidecar`) and `completeness` (`full` | `partial`). The fast-path sidecar - /// ingest uses this so the manual deep pass can tell a sidecar/partial result apart from a - /// full walk and upgrade it rather than skip it. + /// The same work as [`save_analysis`], but the method also writes the provenance. The + /// provenance is `source`, which is `navigator-walk` or `pipeline-sidecar`, and `completeness`, + /// which is `full` or `partial`. + /// + /// The fast-path sidecar import uses this method. The manual deep pass can then see the + /// difference between a partial sidecar result and a full walk. It replaces the partial result + /// and does not skip it. pub async fn save_analysis_with_provenance( &self, alignment_id: i64, @@ -488,12 +545,18 @@ impl App { .await?) } - /// Like [`save_analysis_with_provenance`] but refuses to **downgrade** an existing artifact: - /// if a result is already stored for this `(kind, version)` whose completeness is at least the - /// incoming one (e.g. a full `navigator-walk` scan vs an incoming `partial` sidecar), the - /// existing artifact is kept untouched. The fast-path sidecar ingest uses this so re-importing - /// a project folder can't clobber real deep scans with lite sidecar stats. Returns whether the - /// write actually happened (`false` = kept the existing, equal-or-fuller result). + /// The same work as [`save_analysis_with_provenance`], but the method never replaces a better + /// artifact with a worse one. + /// + /// The store can already hold a result for this `(kind, version)` pair. When the completeness + /// of that result is the same as the new one, or higher, the method keeps the stored artifact. + /// One example is a full `navigator-walk` scan against a new `partial` sidecar result. + /// + /// The fast-path sidecar import uses this method. So a second import of a project folder can + /// not replace a real deep scan with the smaller statistics of a sidecar. + /// + /// The method returns `true` when it wrote the artifact. It returns `false` when it kept the + /// stored result, which was the same or better. pub async fn save_analysis_no_downgrade( &self, alignment_id: i64, @@ -513,10 +576,15 @@ impl App { Ok(true) } - /// Persist a marker that a Navigator walk failed for this alignment (e.g. an undecodable / - /// corrupt CRAM). Stored as the `error`/`"1"` artifact so the project report can surface a - /// "Failed" cell instead of a silent blank; cleared by [`clear_analysis_error`] on the next - /// successful walk. Best-effort — a failure to record the marker is swallowed (it's diagnostic). + /// Write a mark that shows a failed Navigator walk for this alignment. One cause is a CRAM + /// file that the reader can not decode. + /// + /// The store holds the mark as the `error` artifact with the value `"1"`. The project report + /// then shows a "Failed" cell and not an empty cell. [`clear_analysis_error`] removes the mark + /// after the next good walk. + /// + /// The step is optional. The code hides a failure to write the mark, because the mark is only a + /// diagnostic. pub async fn record_analysis_error(&self, alignment_id: i64, step: &str, message: &str) { let mut message = message.to_string(); message.truncate(500); // keep the payload small; the head carries the cause @@ -543,17 +611,23 @@ impl App { } } - /// The alignment's source-file signature (`mtime:size`) for cache staleness. `None` when the - /// alignment / its path is gone or unstattable — then the cache is trusted (nothing to - /// recompute against). Cheap: a metadata stat, no file read (content hashing is the separate, - /// deferred federation-identity path). + /// The signature of the source file of the alignment, as `mtime:size`. The code uses it to + /// find an old cache entry. + /// + /// The value is `None` when the alignment is absent, when its path is absent, or when the + /// operating system can not read the metadata. The code then trusts the cache, because it has + /// no value to compare. + /// + /// The call is fast. It reads the metadata and does not read the file. The content hash is a + /// separate path for the federation identity, and it runs later. async fn bam_source_sig(&self, alignment_id: i64) -> Option { let aln = alignment::get(self.store.pool(), alignment_id).await.ok().flatten()?; file_signature(Path::new(&aln.bam_path?)) } - /// `(source, completeness)` of a cached artifact, defaulting `None` columns to - /// `("navigator-walk", "full")` (pre-provenance rows). `None` when no artifact exists. + /// The `(source, completeness)` pair of a cached artifact. A `None` column becomes + /// `("navigator-walk", "full")`, because a row from before the provenance change holds no + /// value. The method returns `None` when no artifact exists. pub async fn analysis_provenance( &self, alignment_id: i64, @@ -579,8 +653,9 @@ impl App { ) -> Result, AppError> { match artifact::get(self.store.pool(), alignment_id, kind, algorithm_version).await? { Some(a) => { - // Treat a cached result as a miss when the source file changed since it was computed - // (BAM-mtime invalidation) — the caller then recomputes + re-stamps it. + // Treat a cached result as absent when the source file changed after the + // calculation. The mtime of the BAM file shows that change. The caller then + // calculates the result again and writes a new signature. let current = self.bam_source_sig(alignment_id).await; if !artifact_is_fresh(a.source_sig.as_deref(), current.as_deref()) { return Ok(None); diff --git a/crates/navigator-app/src/dm.rs b/crates/navigator-app/src/dm.rs index 14126625..d8ec4091 100644 --- a/crates/navigator-app/src/dm.rs +++ b/crates/navigator-app/src/dm.rs @@ -1,15 +1,21 @@ -//! `impl App` methods for **peer direct messages** (social roadmap 3a) — a thin DM layer over the -//! generic D1 encrypted exchange (`ibd_exchange.rs` + `navigator_sync::exchange`). The crypto, -//! discovery, consent, and relay are reused unchanged; this adds the DM `purpose`, persistence of the -//! established session key (so a conversation is async + survives restart), and the per-message -//! store. The AppView only ever relays ciphertext — message plaintext never leaves the device. +//! `impl App` methods for **peer direct messages** (social roadmap 3a). +//! +//! This module is a thin DM layer on the generic D1 encrypted exchange, which is `ibd_exchange.rs` +//! and `navigator_sync::exchange`. It uses the cryptography, the discovery, the consent, and the +//! relay of that exchange without a change. +//! +//! This module adds three parts: the DM `purpose`, storage for the session key, and a store for +//! each message. Storage of the session key makes a conversation asynchronous, and the conversation +//! continues after a restart. The AppView relays only ciphertext. The plaintext of a message never +//! leaves the device. use super::*; use base64::engine::general_purpose::STANDARD; use base64::Engine; -/// Exchange `purpose` tag for a peer DM (the AppView titles its consent notification from this; IBD -/// requests use `IBD_*`, so filtering on it keeps the DM inbox separate from the IBD tab). +/// The exchange `purpose` tag for a peer DM. The AppView makes the title of its consent +/// notification from this tag. An IBD request uses a tag that starts with `IBD_`. So a filter on +/// the tag keeps the DM inbox separate from the IBD tab. pub const DM_PURPOSE: &str = "GENEALOGY_PII"; impl App { @@ -19,7 +25,8 @@ impl App { self.exchange_request(partner_did, DM_PURPOSE, None).await } - /// Inbound DM requests awaiting our consent (symmetric-blind; DM-purpose only). + /// The DM requests that arrived and that need our consent. The view is symmetric-blind, and it + /// holds only requests with the DM purpose. pub async fn dm_incoming(&self) -> Result, AppError> { Ok(self .exchange_incoming() @@ -29,8 +36,9 @@ impl App { .collect()) } - /// Consent-ready DM sessions (both parties consented) that we have not yet connected — i.e. no - /// persisted conversation/key yet. DM-purpose only. + /// The DM sessions that both parties agreed to, but that we did not connect. These sessions + /// have no conversation and no key in the store. The list holds only sessions with the DM + /// purpose. pub async fn dm_ready(&self) -> Result, AppError> { let mut out = Vec::new(); for info in self.exchange_pending().await? { @@ -52,10 +60,14 @@ impl App { self.exchange_consent(request_uri, given).await } - /// Connect a consent-ready DM session: run the X3DH-lite handshake (both peers must be online for - /// this one step) and persist the derived session key + partner, so all later send/receive is - /// async and restart-safe. Idempotent — re-connecting refreshes the key without resetting the - /// conversation's seq counters. + /// Connect a DM session that both parties agreed to. + /// + /// The function does the X3DH-lite handshake. Both peers must be online for this one step. It + /// then writes the derived session key and the partner to the store. So each later send and + /// receive is asynchronous, and it continues after a restart. + /// + /// A second call is safe. It makes a new key, and the seq counters of the conversation keep + /// their values. pub async fn dm_connect(&self, info: &ExchangeSessionInfo) -> Result<(), AppError> { let did = self.require_account()?; let session = self.open_exchange_session(info).await?; @@ -93,8 +105,8 @@ impl App { Ok(navigator_store::dm::messages(self.store.pool(), session_id).await?) } - /// Encrypt + relay a message on an established conversation, persisting it locally. Returns the - /// seq it was sent under. + /// Encrypt a message, relay it on an open conversation, and write it to the local store. The + /// function returns the seq of the message. pub async fn dm_send(&self, session_id: &str, text: &str) -> Result { let convo = self.dm_conversation_or_err(session_id).await?; let session = self.rebuild_session(&convo)?; @@ -107,8 +119,8 @@ impl App { Ok(seq) } - /// Pull, decrypt, persist, and ack any messages waiting on a conversation. Returns the count of - /// newly-stored (non-duplicate) messages. + /// Read, decrypt, store, and acknowledge each message that a conversation holds. The function + /// returns the count of new messages. It does not count a duplicate message. pub async fn dm_sync(&self, session_id: &str) -> Result { let did = self.require_account()?; let convo = self.dm_conversation_or_err(session_id).await?; @@ -118,12 +130,13 @@ impl App { let Ok(parsed) = exchange::Envelope::from_blob(&env.blob) else { continue; }; - // A leftover handshake (seq 0) can't decrypt as data — ack and drop it. + // An old handshake has seq 0, and the code can not decrypt it as data. Acknowledge + // the handshake and remove it. if matches!(parsed, exchange::Envelope::Handshake { .. }) { let _ = self.exchange_relay_ack(env.id).await; continue; } - // AAD binds the sender's routing: from = the partner (sender), to = us. + // The AAD binds the route of the sender. `from` is the partner, and `to` is us. let aad = exchange::relay_aad(session_id, &env.from_did, &did, env.seq); let Ok(pt) = exchange::open(&session.key, &aad, &parsed) else { continue; // not for this session key / tampered — leave un-acked @@ -182,7 +195,8 @@ mod tests { let back: [u8; 32] = STANDARD.decode(&stored).unwrap().try_into().unwrap(); assert_eq!(key, back); - // And it actually works as an AES session key: seal here, open with the rebuilt key. + // The value also works as an AES session key. Seal the data here, then open the data with + // the key that the code makes again. let aad = exchange::relay_aad("sess", "did:plc:a", "did:plc:b", 1); let blob = exchange::seal(&key, &aad, b"hello").and_then(|e| e.to_blob()).unwrap(); let parsed = exchange::Envelope::from_blob(&blob).unwrap(); diff --git a/crates/navigator-app/src/error.rs b/crates/navigator-app/src/error.rs index 794d934c..1f49e980 100644 --- a/crates/navigator-app/src/error.rs +++ b/crates/navigator-app/src/error.rs @@ -8,36 +8,45 @@ pub enum AppError { #[error(transparent)] Analysis(#[from] navigator_analysis::AnalysisError), - /// Read mapping (realignment stage B). Its own variant rather than folded into `Analysis` - /// because `navigator-align` is a separate crate with its own error type, and a mapping - /// failure points somewhere different from an analysis one. + /// A fault in read mapping, which is realignment stage B. + /// + /// This is a separate variant, not part of `Analysis`. `navigator-align` is a separate crate + /// with its own error type. Also, a mapping fault has a different cause from an analysis + /// fault. #[error("{0}")] Align(#[from] navigator_align::AlignError), #[error("serialization error: {0}")] Serde(#[from] serde_json::Error), - /// A blocking analysis task failed to join (panicked or was cancelled). + /// An analysis task on a blocked thread did not join. The task had a panic, or the user + /// stopped it. #[error("analysis task failed: {0}")] Join(String), #[error("alignment {0} has no BAM/reference path recorded")] MissingPaths(i64), - /// The alignment's recorded file is no longer on disk — a superseded vendor download, a - /// deleted import, an unmounted volume. Distinct from [`AppError::MissingPaths`], which means - /// no path was ever recorded: here there is one, it just no longer resolves. + /// The file of the alignment is no longer on disk. The cause can be a newer vendor download, a + /// deleted import, or a volume that the user removed. + /// + /// This variant is different from [`AppError::MissingPaths`]. That variant means that the + /// alignment never had a path. Here the alignment has a path, but the path no longer points to + /// a file. /// - /// Worth its own variant because it is the one read failure that is *expected* in a long-lived - /// workspace and is nobody's fault. Sweeps over many alignments skip on it - /// ([`AppError::is_missing_alignment_file`]) rather than counting a failure, and it is raised - /// before the expensive setup a walk implies rather than surfacing as an opaque io error from - /// deep inside the reader — where it had been misread as the *haplotree* being unavailable. + /// This fault has its own variant for two reasons. First, it is the one read fault that a long + /// life workspace can expect, and no user made a mistake. So a sweep across many alignments + /// skips the alignment with [`AppError::is_missing_alignment_file`] and does not count a + /// failure. + /// + /// Second, the code raises this error before the large setup that a walk needs. Before this + /// variant, the reader gave an unclear io error from deep in its code, and a user read that + /// error as an absent haplotree. #[error("alignment {id} file is no longer at {path}")] AlignmentFileMissing { id: i64, path: String }, - /// The ancestry reference panel file is missing — build it with `navigator-panelbuild` - /// and install it (or set `$NAVIGATOR_ANCESTRY_PANEL`). + /// The ancestry reference panel file is absent. Make the file with `navigator-panelbuild` and + /// install it. As an alternative, set `$NAVIGATOR_ANCESTRY_PANEL`. #[error("ancestry panel not found at {0} — build it with navigator-panelbuild")] AncestryPanelMissing(std::path::PathBuf), @@ -52,7 +61,7 @@ pub enum AppError { #[error("not signed in — log in to a PDS account first")] NotAuthenticated, - /// An AppView API call failed (e.g. federated IBD). 403 → the device key isn't + /// An AppView API call failed (e.g. federated IBD). 403 → the device key is not /// registered/verified yet; 422 → clock skew; otherwise the server's reason. #[error("appview error: {0}")] AppView(String), @@ -69,13 +78,14 @@ pub enum AppError { #[error(transparent)] Refgenome(#[from] navigator_refgenome::RefgenomeError), - /// Import needs reference build(s) that aren't cached — the UI prompts, downloads via - /// the gateway, then retries. No DB writes happened. + /// The import needs one or more reference builds that the cache does not hold. The UI asks the + /// user, downloads the builds through the gateway, and then tries again. The code wrote nothing + /// to the database. #[error("reference download required: {0:?}")] ReferenceNeeded(Vec), - /// A requested mutation is refused because of current state (e.g. deleting a subject that - /// still has sequencing data or profiles). + /// The app refuses a change because of the current state. One example is a request to delete a + /// subject that still has sequence data or a profile. #[error("{0}")] Conflict(String), @@ -91,18 +101,21 @@ pub enum AppError { } impl AppError { - /// Whether this is a user-requested cancellation rather than a genuine failure. + /// Shows that the user stopped the job. It does not show a fault. + /// + /// A stop moves through the code as an error, because an error unwinds the walk from any point. + /// But the caller must know the difference between the two. A run that the user stopped holds a + /// partial result, and the app must not write that result to the store. The UI must also show + /// the word "cancelled", not an error. /// - /// Cancellation travels as an error so it unwinds the walk from wherever it was, but callers - /// must be able to tell the two apart: a cancelled run holds a partial result that must not be - /// persisted, and the UI has to say "cancelled" instead of showing an error. Lives here rather - /// than in the UI so the layers above never have to reach past `navigator-app` for it. + /// This method is in this crate, not in the UI. So a layer above `navigator-app` never needs to + /// look inside this crate. pub fn is_cancellation(&self) -> bool { matches!(self, AppError::Analysis(navigator_analysis::AnalysisError::Cancelled)) } - /// Whether this failure is only "the alignment's file is gone", which a sweep should skip past - /// rather than record as a failure. See [`AppError::AlignmentFileMissing`]. + /// Shows that the only fault is an absent alignment file. A sweep must skip such an alignment + /// and must not count a failure. See [`AppError::AlignmentFileMissing`]. pub fn is_missing_alignment_file(&self) -> bool { matches!(self, AppError::AlignmentFileMissing { .. }) } @@ -115,11 +128,11 @@ impl From for AppError { } impl AppError { - /// Whether this is a user-requested stop rather than a failure. + /// Shows that the user stopped the job. It does not show a fault. /// - /// Long jobs have to tell the two apart — reporting someone's own Cancel click as an error is - /// both wrong and alarming — and the distinction lives here so callers do not resort to - /// matching on message text. + /// A long job must know the difference between the two. To report the Cancel click of the user + /// as an error is incorrect, and it makes the user afraid. This method holds the difference, so + /// a caller does not compare the text of a message. pub fn is_cancelled(&self) -> bool { matches!( self, diff --git a/crates/navigator-app/src/export.rs b/crates/navigator-app/src/export.rs index 251655c4..b29d9b78 100644 --- a/crates/navigator-app/src/export.rs +++ b/crates/navigator-app/src/export.rs @@ -1,6 +1,9 @@ -//! Result exports (gap §6): pure formatters that turn a cached analysis result into a shareable -//! file body — TSV / HTML / BED. Kept free of I/O and `App` so they're trivially unit-testable; the -//! app layer loads the result and writes the returned `String` to the user-chosen path. +//! Result exports (gap §6). Each function here is a formatter. It changes a cached analysis result +//! into the body of a file that the user can share. The formats are TSV, HTML, and BED. +//! +//! These functions do no I/O and do not use `App`, so a unit test can call them directly. The app +//! layer reads the result, calls a formatter, and writes the `String` to the path that the user +//! chose. use navigator_analysis::coverage::CoverageResult; use navigator_analysis::haplo::CallState; @@ -13,12 +16,13 @@ use navigator_domain::reconciliation::DnaType; use crate::{Block, BranchReport, DescentReport, ProjectBlockTree}; -/// Minimal HTML text escaping for the small, controlled strings we embed (population names etc.). +/// A small HTML escape for the short strings that the app puts in a page. One example is the name +/// of a population. fn esc(s: &str) -> String { s.replace('&', "&").replace('<', "<").replace('>', ">") } -/// Shared inline stylesheet for the HTML exports (self-contained — no external assets). +/// The stylesheet that each HTML export holds. The page is complete and needs no other file. const HTML_STYLE: &str = "body{font-family:-apple-system,Segoe UI,Roboto,sans-serif;margin:2rem;color:#222}\ h1{font-size:1.3rem}h2{font-size:1rem;margin-top:1.5rem}\ table{border-collapse:collapse;margin-top:.5rem}\ @@ -29,8 +33,9 @@ th{background:#f3f3f3}td:first-child,th:first-child{text-align:left}\ // ---- coverage ---------------------------------------------------------------- -/// Coverage as TSV: a `#`-commented genome-wide metrics header, then a per-contig table (joining the -/// samtools-style depth stats with the GATK-style callable breakdown). +/// The coverage as TSV. The file starts with a header of genome-wide metrics, and each header line +/// starts with `#`. A table for each contig follows. The table holds the depth values in the +/// samtools form together with the callable values in the GATK form. pub fn coverage_tsv(cov: &CoverageResult) -> String { let mut out = String::new(); out.push_str("# DUNavigator coverage export\n"); @@ -83,7 +88,8 @@ pub fn coverage_tsv(cov: &CoverageResult) -> String { out } -/// Coverage as a self-contained HTML page (genome-wide summary + per-contig table). +/// The coverage as a complete HTML page. The page holds a genome-wide summary and a table for each +/// contig. pub fn coverage_html(cov: &CoverageResult, label: &str) -> String { let mut rows = String::new(); for s in &cov.contig_coverage_stats { @@ -169,8 +175,9 @@ pub fn read_metrics_tsv(m: &ReadMetrics) -> String { // ---- ancestry ---------------------------------------------------------------- -/// Ancestry as a single TSV table — super-population then fine-population rows distinguished by a -/// `level` column — under a `#`-commented metadata header. +/// The ancestry as one TSV table. The super-population rows come first, and the fine-population +/// rows follow. The `level` column shows the difference. A metadata header is above the table, and +/// each header line starts with `#`. pub fn ancestry_tsv(a: &AncestryResult) -> String { let mut out = String::new(); out.push_str("# DUNavigator ancestry export\n"); @@ -257,15 +264,12 @@ pub fn ancestry_html(a: &AncestryResult) -> String { ) } -// ---- mtDNA variants ---------------------------------------------------------- +// ---- descent and branch reports ---------------------------------------------- -/// mtDNA variants (vs rCRS) as TSV: position, compact notation, region, ref/alt, type. -/// The Y-DNA / mtDNA **descent report** (root→terminal lineage) as TSV: one row per defining SNP of -/// each node on the path, with the subject's call state and observed base. Mirrors the on-screen -/// per-node descent grid so it can be shared / diffed outside the app. -/// TSV for a [`BranchReport`] — one row per defining marker in the reported subtree, with the -/// sample's observed base + call state + evidence. Shareable for placement spot-checks / researcher -/// exchange. Missing evidence renders as `.` (VCF convention). +/// TSV for a [`BranchReport`]. The file holds one row for each marker that defines a node in the +/// reported subtree. Each row holds the observed base of the sample, the call state, and the +/// evidence. A user can share the file to check a placement or to send it to a researcher. A row +/// with no evidence shows `.`, which is the VCF convention. pub fn branch_report_tsv(report: &BranchReport) -> String { let dna = match report.dna { DnaType::Y => "Y-DNA", @@ -316,6 +320,10 @@ pub fn branch_report_tsv(report: &BranchReport) -> String { out } +/// The Y-DNA or mtDNA **descent report** as TSV, from the root to the terminal lineage. The file +/// holds one row for each SNP that defines a node on the path. Each row also holds the call state +/// of the subject and the observed base. The file has the same content as the descent grid on the +/// screen, so a user can share it or compare it outside the app. pub fn descent_tsv(report: &DescentReport) -> String { let dna = match report.dna { DnaType::Y => "Y-DNA", @@ -348,13 +356,17 @@ pub fn descent_tsv(report: &DescentReport) -> String { // ---- project block tree ------------------------------------------------------ -/// The cohort block tree as TSV: one row per block, in the aggregate's own pre-order, with `depth` -/// carrying the shape. Candidate branches (inferred from shared private variants, not named in the -/// published tree) are marked in the `kind` column and named `candidate` rather than left blank, so -/// a reader of the file alone can't mistake one for a published haplogroup. +/// The cohort block tree as TSV. The file holds one row for each block, in the pre-order of the +/// aggregate. The `depth` column gives the shape of the tree. +/// +/// A candidate branch comes from private variants that members share. The published tree does not +/// name it. The `kind` column marks such a branch with the word `candidate`, and the column is +/// never blank. So a reader of the file alone can not mistake a candidate for a published +/// haplogroup. /// -/// Members are a comma-joined cell rather than one row each: the unit a researcher shares from this -/// view is the *branch*, and exploding it per member would bury the tree shape. +/// One cell holds all members, with a comma between them. The file does not use one row for each +/// member. A researcher shares the *branch* from this view. One row for each member hides the shape +/// of the tree. pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { let dna = match tree.dna { DnaType::Y => "Y-DNA", @@ -399,9 +411,9 @@ pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { names.join(","), )); } - // Per-carrier evidence for every candidate. A candidate is an inference, and a shared export - // that showed only "1 SNP, 3 members" would ask the reader to trust it — the depth and derived - // fraction behind each call are what let them judge it instead. + // The evidence of each carrier, for every candidate. A candidate is a deduction. An export + // that showed only "1 SNP, 3 members" asks the reader to trust it. The depth and the derived + // fraction behind each call let the reader judge it. let candidates: Vec<&Block> = tree .blocks .iter() @@ -427,8 +439,8 @@ pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { } } - // The members the tree does not account for belong in the same file — a shared export that - // silently covered only the placed fraction would misrepresent the cohort. + // The same file must hold the members that the tree does not cover. An export that held only + // the placed members, with no note, gives a false picture of the cohort. if !tree.unplaced.is_empty() { out.push_str("\n# not on this tree\nname\tterminal\treason\n"); for u in &tree.unplaced { @@ -442,8 +454,9 @@ pub fn block_tree_tsv(tree: &ProjectBlockTree) -> String { out } -/// The cohort block tree as a self-contained HTML page: the same rows, indented by depth so the -/// shape reads at a glance, with candidate branches called out. +/// The cohort block tree as a complete HTML page. The page holds the same rows. Each row has an +/// indent for its depth, so the reader sees the shape quickly. The page marks each candidate +/// branch. pub fn block_tree_html(tree: &ProjectBlockTree, project: &str) -> String { let dna = match tree.dna { DnaType::Y => "Y-DNA", @@ -511,6 +524,10 @@ pub fn block_tree_html(tree: &ProjectBlockTree, project: &str) -> String { ) } +// ---- mtDNA variants ---------------------------------------------------------- + +/// The mtDNA variants against rCRS as TSV. Each row holds the position, the short notation, the +/// region, the reference allele, the alternate allele, and the type. pub fn mtdna_variants_tsv(variants: &[MtVariant]) -> String { let mut out = String::from("# DUNavigator mtDNA variants vs rCRS (NC_012920.1)\n"); out.push_str("position\tnotation\tregion\tref\talt\ttype\n"); @@ -530,8 +547,9 @@ pub fn mtdna_variants_tsv(variants: &[MtVariant]) -> String { // ---- IBD segments ------------------------------------------------------------ -/// IBD segments as TSV (`chromosome start end length_cm snp_count`), 1-based bp. The match -/// browser's "Export segments CSV" — a tab-delimited table for downstream analysis / sharing. +/// The IBD segments as TSV, with the columns `chromosome`, `start`, `end`, `length_cm`, and +/// `snp_count`. The positions are 1-based. The "Export segments CSV" button of the match browser +/// makes this file. A user can read the table into another tool or send it to a partner. pub fn ibd_segments_tsv(segments: &[IbdSegment]) -> String { let mut out = String::from("# DUNavigator IBD segments export\n"); out.push_str("chromosome\tstart_position\tend_position\tlength_cm\tsnp_count\thalf_identical\n"); @@ -593,10 +611,13 @@ fn lineage_html(lb: &LineageBrief, title: &str) -> String { s } -/// The subject brief as a self-contained "DNA Story" HTML document — the casual-reader report a user -/// can save or print. Mirrors the Simple-mode card stack. When an AI narration is provided (a cached -/// "Polish with AI" result), it leads the document as a clearly-labelled, additive section above the -/// structured facts. +/// The subject brief as a complete "DNA Story" HTML document. This report is for a reader who is +/// not a specialist, and the user can save it or print it. It has the same content as the card +/// stack of Simple mode. +/// +/// The caller can supply an AI narration, which is a cached result of "Polish with AI". That text +/// comes first in the document, above the structured facts, with a clear label. It adds to those +/// facts and does not replace them. pub fn subject_brief_html(b: &SubjectBrief, narration: Option<&crate::NarratedBrief>) -> String { let mut body = String::new(); body.push_str(&format!("

{} — Your DNA Story

\n", esc(&b.headline.name))); @@ -674,8 +695,9 @@ pub fn subject_brief_html(b: &SubjectBrief, narration: Option<&crate::NarratedBr body.push_str("

Neanderthal ancestry

\n"); body.push_str(&format!("

{}

\n", esc(&a.pattern))); body.push_str(&format!("

{}

\n", esc(&a.summary_phrase))); - // A count over what was assayed, never a "percent Neanderthal" (design S1/S7) — the export - // has to hold the same line as the UI or the two disagree about what was measured. + // Report a count of the markers that the test measured. Never report a "percent + // Neanderthal" figure. See design S1 and S7. The export must give the same statement as + // the UI. If not, the two disagree about the measurement. body.push_str(&format!( "

{} of {} marker copies

\n", a.total_copies, a.possible_copies @@ -844,7 +866,7 @@ mod tests { assert!(tsv .lines() .any(|l| l.starts_with("chr1\t500\t42\t480\t96.00\t30.10\t35.0\t58.0\t470\t10\t15\t0\t0\t5"))); - // HTML variant renders without panicking and includes the title. + // The HTML function completes with no panic, and the page holds the title. assert!(coverage_html(&cov, "KANE-0001").contains("Coverage — KANE-0001")); } diff --git a/crates/navigator-app/src/fastpath.rs b/crates/navigator-app/src/fastpath.rs index 9bcd6109..ed45e9c0 100644 --- a/crates/navigator-app/src/fastpath.rs +++ b/crates/navigator-app/src/fastpath.rs @@ -2,10 +2,15 @@ //! 2026-06 simplification round; `use super::*` reaches the crate-root types + free helpers. use super::*; -/// Load a bundled chrY position mask/blocklist BED for a build (best-effort). `env_var` overrides the -/// path; otherwise the seeded `/masks/..bed`, trying the gzipped -/// `.bed.gz` first (how the bundled assets ship) then a plain `.bed`. Returns `None` if absent, -/// unparseable, or empty — so a missing cohort asset simply skips that filter rather than blocking. +/// Read a chrY position mask, or blocklist, from a BED file in the application bundle, for one +/// build. The step is optional. +/// +/// The variable `env_var` gives the path when the user sets it. If not, the code reads +/// `/masks/..bed`. It tries the `.bed.gz` file first, because the +/// bundle holds that form, and then a plain `.bed` file. +/// +/// The function returns `None` when the file is absent, when the parser refuses it, and when it +/// holds no row. An absent cohort asset then removes that filter, and it stops no work. fn load_y_position_bed(env_var: &str, stem: &str, build_token: &str) -> Option { let candidates: Vec = if let Ok(p) = std::env::var(env_var) { vec![PathBuf::from(p)] @@ -21,20 +26,26 @@ fn load_y_position_bed(env_var: &str, stem: &str, build_token: &str) -> Option.chrY.g.vcf.gz` next to the CRAM; a -/// per-run pipeline emits `gatk4/chrY.g.vcf.gz`, whose name has no sample prefix at all. Matching -/// only the dotted suffix missed every file of the second kind — and since finding the GVCF is what -/// lets placement skip decoding the CRAM, missing it silently turns a seconds-long read into a -/// minutes-long whole-chromosome walk. +/// The function reads the directory of the alignment first, where the ytree layout puts a +/// `*.chrY.g.vcf.gz` file. It then reads the known directories of each caller, where the usual name +/// is a plain `chrY.g.vcf.gz`. +/// +/// Both names matter. The flat ytree layout writes `.chrY.g.vcf.gz` beside the CRAM file. A +/// pipeline that works on one run writes `gatk4/chrY.g.vcf.gz`, and that name holds no sample +/// prefix. +/// +/// An earlier version matched the dotted name only, and it found no file of the second kind. The +/// GVCF file is what lets the placement skip the CRAM decode. So a search that fails changes a read +/// of some seconds into a walk of the full chromosome, which needs some minutes. fn gvcf_beside_alignment(aln: &Alignment, contig_token: &str) -> Option { let dotted = format!(".{contig_token}.g.vcf.gz"); let bare = format!("{contig_token}.g.vcf.gz"); @@ -52,9 +63,11 @@ fn gvcf_beside_alignment(aln: &Alignment, contig_token: &str) -> Option scan(dir).or_else(|| CALLER_SUBDIRS.iter().find_map(|sub| scan(&dir.join(sub)))) } -/// Locate a per-sample chrY GVCF for an alignment: the `NAVIGATOR_Y_GVCF` path override, else -/// [`gvcf_beside_alignment`]. `None` when absent — the private-Y path then falls back to the pileup -/// caller, and placement to a full CRAM walk. +/// Find the chrY GVCF file of one sample, for an alignment. The `NAVIGATOR_Y_GVCF` variable gives +/// the path when the user sets it. If not, the code calls [`gvcf_beside_alignment`]. +/// +/// The function returns `None` when it finds no file. The private-Y path then uses the pileup +/// caller, and the placement walks the full CRAM file. pub(crate) fn chr_y_gvcf_for_alignment(aln: &Alignment) -> Option { if let Ok(p) = std::env::var("NAVIGATOR_Y_GVCF") { let p = PathBuf::from(p); @@ -65,8 +78,9 @@ pub(crate) fn chr_y_gvcf_for_alignment(aln: &Alignment) -> Option { gvcf_beside_alignment(aln, "chry") } -/// Locate a per-sample chrM GVCF for an alignment: the `NAVIGATOR_M_GVCF` path override, else -/// [`gvcf_beside_alignment`]. The mtDNA counterpart to [`chr_y_gvcf_for_alignment`]. +/// Find the chrM GVCF file of one sample, for an alignment. The `NAVIGATOR_M_GVCF` variable gives +/// the path when the user sets it. If not, the code calls [`gvcf_beside_alignment`]. This function +/// is the mtDNA form of [`chr_y_gvcf_for_alignment`]. pub(crate) fn chr_m_gvcf_for_alignment(aln: &Alignment) -> Option { if let Ok(p) = std::env::var("NAVIGATOR_M_GVCF") { let p = PathBuf::from(p); @@ -77,9 +91,12 @@ pub(crate) fn chr_m_gvcf_for_alignment(aln: &Alignment) -> Option { gvcf_beside_alignment(aln, "chrm") } -/// The bundled-mask filename token for an alignment's reference build, or `None` when no chrY masks -/// ship for it. CHM13 masks are native (hs1); the GRCh38 masks are lifted from them (CrossMap -/// hs1→hg38). GRCh37 has no masks yet (bare-`Y` contig naming + no lifted set). +/// The token in the mask file name for the reference build of an alignment. The function returns +/// `None` when the bundle holds no chrY mask for that build. +/// +/// The CHM13 masks are native, on hs1. CrossMap moves them from hs1 to hg38, and those files are the +/// GRCh38 masks. There is no mask for GRCh37 yet. That build names its contig `Y`, and no code moved +/// the masks to it. fn y_mask_build_token(build: &str) -> Option<&'static str> { match canonical_build(build) { Some(ReferenceBuild::Chm13v2 | ReferenceBuild::Chm13v2MaskedRcrs) => Some("chm13v2"), @@ -95,10 +112,12 @@ type YRegionsHandle = std::sync::Arc, ) -> Result, AppError> { let aln = self.alignment_or_err(alignment_id).await?; - // The reference is required: a GVCF hom-ref site means "the sample's base == the - // reference base" — and the reference (e.g. CHM13 = HG002/J1 Y) is itself deep in the - // tree, so its base there is often the *derived* allele, not the ancestral. We read the - // reference base at every callable tree position (exactly what call_bases_at observes). + // The method needs the reference. A hom-ref site in a GVCF file states that the base of + // the sample equals the base of the reference. + // + // The reference is itself deep in the tree. The CHM13 reference holds the Y chromosome of + // HG002, which is in haplogroup J1. So its base at a tree position is often the *derived* + // allele and not the ancestral one. + // + // So the code reads the reference base at each callable tree position, which is the value + // that `call_bases_at` also reads. let reference = match aln.reference_path { Some(p) => PathBuf::from(p), None => { @@ -153,8 +177,9 @@ impl App { let ref_base = self.reference_bases(&reference, contig, &called.callable).await?; Ok(gvcf::assemble_calls(&called, &ref_base)) } - // Lifted: read the GVCF at each lifted contig + the reference bases there, then map - // observations back to tree positions (reverse-complementing minus-strand lifts). + // The code moved the positions. It reads the GVCF file at each new contig, and it + // reads the reference bases there. It then maps each observation back to a tree + // position. For a position on the minus strand, it takes the reverse complement. Some(lifted) => { let mut by_contig: HashMap> = HashMap::new(); for lp in &lifted { @@ -178,9 +203,13 @@ impl App { } } - /// Reference genome bases (uppercase A/C/G/T) at `positions` on `contig`. Reads the contig - /// sequence once off-thread; positions are 1-based. Non-ACGT / out-of-range positions are - /// omitted. Used by the GVCF fast path to resolve hom-ref tree sites to the actual base. + /// The reference genome bases at each of the `positions` on `contig`. Each base is an upper-case + /// A, C, G, or T. + /// + /// The method reads the contig sequence one time, on another thread. Each position is 1-based. + /// The result holds no position with another base, and no position outside the contig. + /// + /// The GVCF fast path calls this method. It needs the real base at a hom-ref tree site. async fn reference_bases( &self, reference: &Path, @@ -212,19 +241,27 @@ impl App { Ok(map) } - /// Fingerprint of a GVCF-sourced placement: the GVCF's content hash ⊕ the tree's hash. - /// Distinct from the CRAM-based [`Self::y_score_fingerprint`] (`gv:` vs `f:` prefix) so a - /// later deep analyze can tell the call came from a sidecar (phase: deep-pass skip logic). + /// The fingerprint of a placement that came from a GVCF file. The value joins the content hash + /// of that file with the hash of the tree. + /// + /// The value is different from the fingerprint of [`Self::y_score_fingerprint`], which the CRAM + /// path writes. This one starts with `gv:`, and that one starts with `f:`. So a later deep + /// analysis can see that the call came from a sidecar file, and it can then skip a step. async fn gvcf_fingerprint(&self, gvcf: &Path, tree_json: &str, tag: &str) -> Result { let h = sha256_file_async(gvcf.to_path_buf()).await?; Ok(format!("gv:{}|{}:{}", &h[..16], tag, &sha256_str(tree_json)[..16])) } - /// Assign a Y haplogroup from a precomputed chrY GVCF — no CRAM walk. Places against the - /// DecodingUs tree at the alignment's native build (liftover-free), records the call under - /// the same source key as the CRAM path (`aln:{id}`) with a `gv:`-prefixed fingerprint. - /// Errors if the build has no DecodingUs coordinates or the tree is unreachable; the caller - /// (`ingest_sidecars`) treats that as "leave Y for the deep pass". + /// Assign a Y haplogroup from a chrY GVCF file that another tool made. The method walks no CRAM + /// file. + /// + /// It places the sample against the DecodingUs tree, at the native build of the alignment, and + /// it needs no liftover. It writes the call under the same source key as the CRAM path, which is + /// `aln:{id}`, with a fingerprint that starts with `gv:`. + /// + /// The method fails when the DecodingUs tree has no coordinates for that build, and when it can + /// not read the tree. The caller is `ingest_sidecars`, and it then leaves the Y haplogroup for + /// the deep pass. pub async fn assign_y_from_gvcf(&self, alignment_id: i64, gvcf: &Path) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let build_key = decodingus_build_key(&aln.reference_build).ok_or_else(|| { @@ -236,13 +273,19 @@ impl App { let tree_json = self.fetch_decodingus_y_tree().await?; let tree = navigator_analysis::haplo::parse_decodingus_json(&tree_json, build_key).map_err(AppError::Import)?; let calls = self.gvcf_base_calls(alignment_id, "chrY", gvcf, &tree, None).await?; - // Robust (proportional-top) selection, not the strict alignment-tuned guard. A - // joint-genotyped GVCF gives confident calls that include a few stray ancestral - // contradictions on the deep backbone (recurrent sites, the CHM13=J1 reference, joint - // hard-filters); strict `path_admissible` then vetoes the genuine deep lineage and - // drops to a shallow node (HG00096 → A1b instead of its true R1b1a1b1a1a, which `score` - // ranks top at 344/364). This is the same confident-but-sparse-contradiction regime as - // BISDNA chip data — see [`assemble_assignment_robust`]. + // Use the proportional-top selection, and not the strict guard that the alignment path + // needs. + // + // A GVCF file from a joint genotype step gives confident calls. A few of those calls + // contradict the deep backbone with an ancestral state. The causes are a recurrent site, + // the CHM13 reference, which is in haplogroup J1, and the hard filters of the joint step. + // + // The strict `path_admissible` rule then refuses the true deep lineage and takes a node + // near the root. Sample HG00096 gave A1b, and its true terminal is R1b1a1b1a1a. The `score` + // function ranks that terminal first, at 344 of 364. + // + // The data has the same shape as BISDNA chip data: confident, with a few contradictions. + // See [`assemble_assignment_robust`]. let assignment = assemble_assignment_robust(&tree, &calls); if let Ok(bio) = self.biosample_of_alignment(alignment_id).await { let fp = self.gvcf_fingerprint(gvcf, &tree_json, "yt").await.ok(); @@ -260,10 +303,15 @@ impl App { Ok(assignment) } - /// Assign an mtDNA haplogroup from a precomputed chrM GVCF — no CRAM walk. Places against - /// the FTDNA mt tree; on CHM13 the tree's rCRS positions are lifted onto `chrM` (the cheap - /// self-generated rCRS↔chrM map), on GRCh38 they're read directly. Recorded under the CRAM - /// path's mt source key (`aln:{id}:mt`) with a `gv:`-prefixed fingerprint. + /// Assign an mtDNA haplogroup from a chrM GVCF file that another tool made. The method walks no + /// CRAM file. + /// + /// It places the sample against the mt tree of FTDNA. On GRCh38, it reads the rCRS positions of + /// that tree directly. On CHM13, it moves those positions onto the `chrM` contig. The code makes + /// that map itself, at a low cost. + /// + /// The method writes the call under the mt source key of the CRAM path, which is `aln:{id}:mt`, + /// with a fingerprint that starts with `gv:`. pub async fn assign_mt_from_gvcf(&self, alignment_id: i64, gvcf: &Path) -> Result { let tree_json = self.fetch_ftdna_mt_tree().await?; let tree = navigator_analysis::haplo::parse_ftdna_json(&tree_json).map_err(AppError::Import)?; @@ -271,8 +319,8 @@ impl App { let calls = self .gvcf_base_calls(alignment_id, "chrM", gvcf, &tree, source_build) .await?; - // Robust selection, as for Y (see assign_y_from_gvcf) — the GVCF's confident calls fit - // the proportional-top regime better than the strict alignment guard. + // Use the proportional-top selection, as the Y path does. See assign_y_from_gvcf. The + // confident calls of a GVCF file fit that rule better than the strict alignment guard. let assignment = assemble_assignment_robust(&tree, &calls); if let Ok(bio) = self.biosample_of_alignment(alignment_id).await { let fp = self.gvcf_fingerprint(gvcf, &tree_json, "mt").await.ok(); @@ -290,11 +338,14 @@ impl App { Ok(assignment) } - /// The sidecar paths this alignment was ingested from, as recorded by [`Self::ingest_sidecars`]. + /// The sidecar paths that this alignment came from. [`Self::ingest_sidecars`] writes them. + /// + /// The method reads the value and does not compare the mtime of the source. This value records + /// *what the app used*. It is not a derived result, so a change to the CRAM file does not make + /// it wrong. /// - /// Read directly, with no source-mtime freshness check: this is a record of *what was used*, - /// not a derived result that a changed CRAM invalidates. `None` for an alignment that never - /// went through the fast path (imported before this was recorded, or with no sidecars at all). + /// The method returns `None` for an alignment that never used the fast path. Such an alignment + /// arrived before this record existed, or it had no sidecar file. pub async fn recorded_sidecars(&self, alignment_id: i64) -> Result, AppError> { match artifact::get(self.store.pool(), alignment_id, SIDECARS_KIND, SIDECARS_VERSION).await? { Some(a) => Ok(serde_json::from_str(&a.payload).ok()), @@ -302,11 +353,17 @@ impl App { } } - /// Fast-path ingest of a sample's pipeline sidecars onto one alignment: place Y + mt from - /// the GVCFs, and fill sex / read-metrics / lite-coverage from the text sidecars — all - /// without touching the CRAM. Each step is independent and best-effort: a failure is - /// recorded in the returned report and the rest proceed (a missing/!matching sidecar just - /// leaves that result for the deep pass). Returns what it managed to fill. + /// Read the pipeline sidecar files of a sample onto one alignment, on the fast path. + /// + /// The method places the Y haplogroup and the mt haplogroup from the GVCF files. It fills the + /// sex, the read metrics, and a small coverage result from the text sidecar files. It reads no + /// CRAM file. + /// + /// Each step is independent, and each one is optional. A failure goes into the report that the + /// method returns, and the other steps continue. An absent sidecar file, or one that does not + /// match, leaves that result for the deep pass. + /// + /// The method returns the values that it filled. pub async fn ingest_sidecars( &self, alignment_id: i64, @@ -314,12 +371,17 @@ impl App { ) -> Result { let mut out = SidecarIngest::default(); - // Record which files this alignment was ingested from, before using them. Discovery is a - // directory scan done once at import, so without this the fast path is a one-shot: a Y - // placement made from a GVCF against the tree of the day could never be re-derived, and the - // resulting `haplogroup_call` row outlived every tree it was placed against. See - // `App::replace_against_current_tree`, which replays this. Best-effort — a workspace that - // cannot record the paths should still get the ingest. + // Record the files that this alignment came from, before the code reads them. + // + // The code finds those files with one directory scan, at the import. Without this record, + // the fast path runs one time only. A Y placement from a GVCF file, against the tree of + // that day, could never run again. The `haplogroup_call` row then stayed after each later + // tree. + // + // `App::replace_against_current_tree` reads this record and runs the placement again. + // + // The step is optional. A workspace that can not write the paths must still receive the + // data. let _ = self .save_analysis_with_provenance( alignment_id, @@ -350,16 +412,20 @@ impl App { Err(e) => out.errors.push(format!("sex: {e}")), } } - // Read metrics: richest source wins — samtools `stats` (full, with histograms) > Picard - // AlignmentSummaryMetrics > samtools `flagstat` (counts only). + // The read metrics. The source with the most data wins. The order is samtools `stats`, + // which holds the full data with each histogram, then Picard AlignmentSummaryMetrics, then + // samtools `flagstat`, which holds counts only. match self.ingest_read_metrics(alignment_id, sidecars).await { Ok(true) => out.read_metrics = true, Ok(false) => {} Err(e) => out.errors.push(format!("read metrics: {e}")), } - // Coverage: samtools `coverage` gives per-contig stats; Picard CollectWgsMetrics gives the - // genome-wide depth distribution (median/sd/MAD, exclusion fractions, pct_Nx). Use whichever - // are present, overlaying the distribution onto the per-contig breakdown. + // The coverage. The samtools `coverage` output holds the statistics of each contig. The + // Picard CollectWgsMetrics output holds the depth distribution of the full genome. That + // distribution is the median, the sd, the MAD, the fractions that the tool excluded, and + // the pct_Nx values. + // + // Use each file that exists, and write the distribution onto the table of contigs. if sidecars.coverage.is_some() || sidecars.wgs_metrics.is_some() { match self.ingest_coverage_sidecar(alignment_id, sidecars).await { Ok(wrote) => out.lite_coverage = wrote, @@ -418,7 +484,8 @@ impl App { } else { return Ok(false); }; - // Don't downgrade a full deep walk on reimport — keep it if it's already equal-or-fuller. + // A second import must not replace a full deep walk with a smaller result. Keep the stored + // result when it is the same or better. let wrote = self .save_analysis_no_downgrade( alignment_id, @@ -432,8 +499,11 @@ impl App { Ok(wrote) } - /// Ingest lite coverage from the sidecar(s). Returns whether it was written (`false` = an - /// equal-or-fuller coverage artifact already exists, e.g. a deep walk on reimport). + /// Read the small coverage result from the sidecar files. The method returns `true` when it + /// wrote the result. + /// + /// It returns `false` when the store already holds a coverage artifact that is the same or + /// better. A deep walk from an earlier run gives such an artifact. async fn ingest_coverage_sidecar(&self, alignment_id: i64, sidecars: &SampleSidecars) -> Result { let read = |p: &Path| { let p = p.to_path_buf(); @@ -443,7 +513,8 @@ impl App { .map_err(|e| AppError::Import(format!("{}: {e}", p.display()))) } }; - // Per-contig stats + callable counts from samtools coverage (empty base if absent). + // The statistics of each contig, and the callable counts, from the samtools coverage + // output. The code starts from an empty value when that file is absent. let lite = match &sidecars.coverage { Some(cp) => { let cov = read(cp).await?; @@ -455,8 +526,9 @@ impl App { } None => CoverageResult::default(), }; - // Overlay Picard's genome-wide depth distribution onto the per-contig breakdown: start from - // the Picard result (median/sd/MAD, exclusion fractions, pct_Nx) and graft the contig stats. + // Write the genome-wide depth distribution of Picard onto the table of contigs. Start from + // the Picard result, which holds the median, the sd, the MAD, the fractions that the tool + // excluded, and the pct_Nx values. Then add the statistics of each contig. let result = match &sidecars.wgs_metrics { Some(wp) => match sidecar::parse_wgs_metrics(&read(wp).await?) { Some(mut w) => { @@ -475,9 +547,11 @@ impl App { }, None => lite, }; - // Still `partial`: no per-base depth histogram (only the deep walk produces that), so the - // deep pass still upgrades this. Stored under the standard coverage key. Never downgrade a - // full deep-walk coverage on reimport — keep it if one is already present. + // The result keeps the `partial` mark. It holds no depth histogram for each base, because + // only the deep walk makes one. So the deep pass still replaces this result. + // + // The store holds it under the standard coverage key. A second import must never replace a + // full deep-walk result with this one. Keep the stored result when it exists. let wrote = self .save_analysis_no_downgrade( alignment_id, @@ -491,14 +565,20 @@ impl App { Ok(wrote) } - /// Self-referential callable intervals (BED 0-based half-open) for `contig` from the - /// alignment's own reads. Parameters adapt to the sample: long reads (HiFi) earn - /// callability at lower depth, and the CALLABLE-run gate scales with molecule length - /// (`f`·fragment), so long molecules clear it over far more of chrY. Requires the BAM. + /// The callable intervals of `contig`, from the reads of the alignment itself. The intervals + /// are in the BED form, which is 0-based and half open. + /// + /// The parameters change with the sample. A long read from a HiFi test becomes callable at a + /// lower depth. The limit on a CALLABLE run also grows with the length of the molecule, at + /// `f` times the fragment length. So a long molecule passes that limit across much more of + /// chrY. + /// + /// The method needs the BAM file. pub async fn callable_chr_intervals(&self, alignment_id: i64, contig: &str) -> Result, AppError> { - // Resolve the reference via the gateway when the alignment has no stored path — a CRAM can't - // be decoded without one, and most imported alignments leave `reference_path` null (the build - // alone is recorded). Same resolution the de-novo caller uses. + // Find the reference through the gateway when the alignment holds no path. No reader can + // decode a CRAM file without a reference, and most imported alignments hold a NULL + // `reference_path` value. The row then records the build only. The de-novo caller finds the + // reference in the same way. let (bam, reference) = self.alignment_bam_reference(alignment_id).await?; let reference = Some(reference); let contig = contig.to_string(); @@ -515,10 +595,14 @@ impl App { .map_err(Into::into) } - /// The **private bucket**: de-novo SNP calls on chrY that the Y placement doesn't - /// explain (not on the assigned backbone), classified as off-path-known (a finer/ - /// sibling FTDNA branch) or novel (a new-branch candidate). With `callable_bed` (e.g. - /// the Poznik/1KG `b38_sites.bed`), calls outside reliable regions are dropped. + /// The **private bucket**. It holds the de-novo SNP calls on chrY that the Y placement does not + /// explain. Those calls are not on the backbone that the code assigned. + /// + /// The method puts each call in one of two groups. A known call off the path marks a finer FTDNA + /// branch, or a branch beside the assigned one. A new call is a candidate for a new branch. + /// + /// With `callable_bed`, such as the Poznik file `b38_sites.bed` from 1KG, the method removes + /// each call outside a reliable region. pub async fn private_y_variants( &self, alignment_id: i64, @@ -531,13 +615,18 @@ impl App { self.private_y_core(alignment_id, mask).await } - /// [`private_y_variants`] using the sample's **own** callable-Y BED as the mask - /// (self-referential — adapts to the sample's depth and read tech; no external file). + /// The work of [`private_y_variants`], with the callable-Y BED of the sample itself as the + /// mask. That mask changes with the depth and the read technology of the sample, and it needs no + /// other file. + /// + /// With a GVCF sidecar for the sample, the method **does not** apply that mask. The confidence + /// values in the GVCF file are the evidence that a site is callable. /// - /// With a per-sample GVCF sidecar the self-mask is **skipped**: the GVCF's own confidence - /// gating is the callable evidence (re-imposing Navigator's callable-loci depth threshold would - /// discard GATK calls the whole point is to trust), and skipping it avoids a CRAM walk — so the - /// GVCF fast path stays fast. Reliability then comes from the cohort callable mask + GVCF GQ. + /// A second depth limit from Navigator would remove GATK calls, and the purpose of this path is + /// to trust those calls. The method also avoids a CRAM walk, so the GVCF fast path stays fast. + /// + /// The reliability then comes from the callable mask of the cohort and the GQ value of the GVCF + /// file. pub async fn private_y_variants_self_masked(&self, alignment_id: i64) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let mask = if chr_y_gvcf_for_alignment(&aln).is_some() { @@ -547,11 +636,15 @@ impl App { Some(navigator_analysis::mask::RegionMask::from_intervals(intervals)) }; let bucket = self.private_y_core(alignment_id, mask).await?; - // Persist the self-masked bucket so it reloads instead of recomputing next session. Version - // "3": prefers a per-sample GVCF sidecar as the derived-call source (was pileup-only in v2), - // so v2 blobs must recompute rather than reload. - // Version 4: private variants are now classified against structural masks lifted to the - // alignment's own build. A v3 bucket on a GRCh38 alignment saw no mask at all. + // Write the masked bucket to the store, so the next session reads it and calculates + // nothing. + // + // Version 3 takes the GVCF sidecar of the sample as its source of derived calls. Version 2 + // used the pileup only. So the code must calculate a version 2 value again, and it must not + // read it. + // + // Version 4 classifies each private variant against the structural masks in the build of + // the alignment. A version 3 bucket on a GRCh38 alignment used no mask. self.save_analysis(alignment_id, "private_y", "4", &bucket).await?; Ok(bucket) } @@ -561,27 +654,36 @@ impl App { self.load_analysis(alignment_id, "private_y", "4").await } - /// Shared core: assign Y, de-novo chrY, subtract the backbone, optionally mask, classify. - /// The curated CHM13 chrY structural regions (palindrome/amplicon/AZF-DYZ), resolving + - /// caching the three BEDs on first use. Best-effort: any download/parse failure yields - /// `None` so the annotation never blocks the analysis. - /// Genome-region metadata (centromere/telomere/cytoband/PAR) for a build, via the gateway's - /// 2-layer cache (fetches the UCSC cytoBand table on a cold miss). For QC / display context. + /// The shared core. It runs five steps. It assigns the Y haplogroup and calls de-novo variants + /// on chrY. It then removes the backbone calls, applies a mask when the caller asks for one, and + /// classifies each remaining call. + /// + /// It also reads the curated structural regions of chrY on CHM13, which are the palindromes, the + /// amplicons, and the AZF-DYZ regions. It finds and caches the three BED files at the first use. + /// Each step is optional, and a failed download or a failed parse gives `None`. So this + /// annotation never stops the analysis. + /// + /// It also reads the genome-region metadata of a build, which is the centromere, the telomeres, + /// the cytobands, and the PAR regions. Those values come from the two-layer cache of the + /// gateway, and the gateway reads the UCSC cytoBand table when the cache holds nothing. The UI + /// uses them for quality checks and for context. pub async fn genome_regions(&self, build: &str) -> Result, AppError> { Ok(self.gateway.genome_regions(build, &mut |_, _| {}).await?) } - /// Region annotation for a 1-based `position` on `contig` in `build` (centromere/telomere/PAR - /// membership + cytoband name). Uses the cached regions only — `None` if not yet fetched. + /// The region of a 1-based `position` on `contig` in `build`. The value states whether the + /// position is in a centromere, a telomere, or a PAR region, and it gives the cytoband name. The + /// method reads the cache only, and it returns `None` when the cache holds nothing. pub fn region_annotation(&self, build: &str, contig: &str, position: i64) -> Option { self.gateway .cached_genome_regions(build) .map(|r| r.annotate(contig, position)) } - /// Memo for [`y_structural_regions_for`]: lifting parses the whole chain file, and a project - /// pass over thousands of subjects would otherwise repeat that per subject — the same trap the - /// tree fetch fell into. The masks are static within a process, so resolve each build once. + /// A memory for [`y_structural_regions_for`]. A liftover parses the full chain file. A project + /// pass across thousands of subjects would repeat that parse for each subject, and the tree + /// fetch had the same fault. The masks do not change inside one process, so the code resolves + /// each build one time. fn y_regions_memo() -> &'static std::sync::Mutex>> { static MEMO: std::sync::OnceLock>>> = std::sync::OnceLock::new(); @@ -590,25 +692,30 @@ impl App { /// The curated chrY structural regions **in `build`'s coordinates**. /// - /// The three BEDs are CHM13-native, so anything else is lifted. That matters more than it - /// sounds: without it, a GRCh38 or GRCh37 source has no structural mask at all, every - /// palindromic and amplicon call counts as unique sequence, and private-variant counts inflate - /// into the hundreds — the difference between a donor averaging 4 and one averaging 661. + /// The three BED files are native to CHM13, so the code moves them to any other build. /// - /// Best-effort throughout: any download / chain / parse failure yields `None` so the annotation - /// never blocks the analysis, exactly as before. + /// That step is important. Without it, a GRCh38 source or a GRCh37 source has no structural + /// mask. Each call in a palindrome and in an amplicon then counts as unique sequence, and the + /// count of private variants grows into the hundreds. One donor gave a mean of 4 with the mask + /// and a mean of 661 without it. + /// + /// Each step is optional. A failed download, a failed liftover, and a failed parse each give + /// `None`, so this annotation never stops the analysis. async fn y_structural_regions_for(&self, build: &str) -> Option { - // Keyed by the *canonical* build: `hs1`, `CHM13v2.0` and the masked variant share coordinates - // and must share one entry rather than lifting three times. + // The key is the *canonical* build. The builds `hs1`, `CHM13v2.0`, and the masked build + // use the same coordinates. So they must share one entry, and the code must not do three + // liftovers. let key = canonical_build(build)?.as_str().to_string(); if let Some(hit) = Self::y_regions_memo().lock().unwrap().get(&key) { return hit.clone(); } let built = self.build_y_structural_regions(build).await.map(std::sync::Arc::new); if built.is_none() { - // Cached so a batch does not retry a failing download per subject — but *said*, because - // "no structural mask" is the condition that inflated private-variant counts into the - // hundreds in the first place, and it must never be reached silently again. + // The code caches this state, so a batch does not try a failed download again for + // each subject. It also writes a message. + // + // The state "no structural mask" grew the count of private variants into the hundreds. + // The app must never reach that state again with no message. eprintln!( "no chrY structural mask available for {key} — private-variant counts will include \ palindromic and amplicon calls" @@ -642,8 +749,9 @@ impl App { if matches!(target, ReferenceBuild::Chm13v2 | ReferenceBuild::Chm13v2MaskedRcrs) { return Some(native); } - // PAR and heterochromatin are taken natively per build rather than lifted — a chain is least - // trustworthy in exactly those places (PAR is shared with chrX, Yq12 is satellite). + // The code reads the PAR regions and the heterochromatin natively for each build. It does + // not move them from CHM13. A chain file is least reliable in those places. Both chrX and + // chrY hold the PAR regions, and Yq12 is satellite sequence. let landmarks = navigator_analysis::mask::y_landmarks(build)?; self.gateway @@ -656,8 +764,9 @@ impl App { .lift_intervals(ReferenceBuild::Chm13v2.as_str(), target.as_str(), "chrY", m.intervals()) .ok()?; if iv.is_empty() { - // A mask that lifted to nothing is not a mask; better to annotate nothing than to - // report "no structural regions here" as though it had been checked. + // A mask that gives no interval after the liftover is not a mask. The code then + // writes no annotation. It must not report "no structural regions here" as a + // result of a real check. eprintln!("chrY {what} mask lifted CHM13→{} to nothing; skipping", target.as_str()); return None; } @@ -676,23 +785,28 @@ impl App { RegionMask::from_intervals(landmarks.par.to_vec()), lift(palindrome_m, "palindrome")?, lift(amplicon_m, "amplicon")?, - // The satellite arrays rarely survive a chain, so the build's heterochromatin bound is - // the load-bearing part here and the lifted AZF/DYZ intervals only refine it. + // A satellite array rarely survives a chain file. So the heterochromatin bound of the + // build carries this test, and the AZF and DYZ intervals from the liftover only make it + // more exact. lift(native.heterochromatin_mask(), "AZF/DYZ") .unwrap_or_else(|| RegionMask::from_intervals(vec![])) .union(&[landmarks.heterochromatin]), )) } - /// Derive chrY private-variant candidates from a per-sample GVCF, returning the same - /// [`VariantCall`] shape the pileup de-novo path produces so `private_y_core`'s downstream - /// classification is identical. GATK's reassembly recovers SNVs the pileup caller misses. + /// Find the private-variant candidates on chrY from the GVCF file of one sample. + /// + /// The method returns the same [`VariantCall`] shape that the pileup de-novo path returns. So + /// the classification in `private_y_core` is the same for both paths. + /// + /// The reassembly step of GATK finds SNVs that the pileup caller does not find. async fn run_denovo_from_gvcf(&self, gvcf: &Path) -> Result, AppError> { let gvcf = gvcf.to_path_buf(); let snvs = tokio::task::spawn_blocking(move || { - // min_dp 4 (over the reader's permissive default 2): a real private SNV is covered by ≥4 - // reads, whereas a misaligned-read cluster smears 2–3 reads across many nearby false SNVs - // — so the depth floor removes those artifact clusters without touching the (DP≥4) truth. + // Use min_dp 4, and not the default of 2 that the reader allows. A real private SNV + // has 4 reads or more. A group of misaligned reads gives 2 or 3 reads across many false + // SNVs that are near each other. So this depth limit removes each artefact group, and + // it keeps each true call with a DP of 4 or more. let params = navigator_analysis::gvcf::GvcfReadParams { min_dp: 4, min_gq: 20 }; navigator_analysis::gvcf::read_derived_snvs(&gvcf, "chrY", ¶ms) }) @@ -717,15 +831,21 @@ impl App { alignment_id: i64, mask: Option, ) -> Result { - // Classify novels against the **DecodingUs** tree — the app's placement authority, which - // folds in the cohort-derived branches (from the de-novo tree pipeline). A shared lineage - // variant is named there, so it reads as OffPathKnown, not a false "novel"; a variant absent - // from this tree yet shared across the cohort is genuinely suspect. FTDNA fallback keeps the - // report working when the AppView tree is unavailable or the build has no DecodingUs coords. + // Classify each new call against the **DecodingUs** tree. That tree has the authority for + // a placement in this app, and it holds the branches that the de-novo tree pipeline found + // in the cohort. + // + // A variant of a shared lineage has a name in that tree, so the code marks it OffPathKnown + // and not "novel". A variant that the tree does not hold, and that the cohort shares, is + // doubtful. + // + // The FTDNA tree is the second choice. It keeps the report correct when the code can not + // read the AppView tree, and when the build has no DecodingUs coordinates. let (tree, tree_calls) = match self.y_decodingus_tree_calls(alignment_id).await { Ok(tc) => tc, - // A gone alignment file is not a tree problem: the fallback reads the same absent file, - // so it can only fail again while logging a tree provider that was never at fault. + // An absent alignment file is not a fault of the tree. The second path reads the same + // absent file, so it also fails. It then writes a log entry that names a tree provider + // with no fault. Err(e) if e.is_missing_alignment_file() => return Err(e), Err(e) => { eprintln!("DecodingUs Y tree unavailable ({e}); private-Y classifying against FTDNA"); @@ -741,27 +861,42 @@ impl App { let path = navigator_analysis::haplo::path_positions(&tree, terminal.id); let known = navigator_analysis::haplo::tree_positions(&tree); - // The structural BEDs are in CHM13 chrY coordinates, so they only annotate a CHM13 alignment. - // The cohort masks apply per build: native for CHM13, CrossMap-lifted (hs1→hg38) for GRCh38. + // The structural BED files hold CHM13 chrY coordinates, so they annotate a CHM13 alignment + // only. The cohort masks apply to each build: the CHM13 files are native, and CrossMap + // moved the GRCh38 files from hs1 to hg38. let aln = self.alignment_or_err(alignment_id).await?; let regions = self.y_structural_regions_for(&aln.reference_build).await; - // L2: the cohort **callable mask** (Poznik-style, CALLABLE in ≥90% of a ~3k-male cohort) — - // only ~25% of non-PAR chrY is reliably callable cohort-wide. L3: a **cohort-shared-sites** - // blocklist — every position that varies with ≥2 carriers across the cohort (plus homoplasy - // hotspots). A real shared lineage variant belongs in the DecodingUs tree (and so classifies - // as off-path-known above); one that is cohort-shared yet *absent* from the tree is a suspect - // recurrent artifact, not a private SNP. A truly private variant has a single cohort carrier, - // so it survives this filter. This is the single-sample stand-in for the de-novo pipeline's - // cohort carrier filter. Bundled per build (CHM13 native, GRCh38 lifted); absent ⇒ skipped. + // Layer 2 is the **callable mask** of the cohort, in the Poznik form. A position is in that + // mask when it is CALLABLE in 90% or more of a cohort of about 3,000 men. Only about 25% of + // chrY outside the PAR regions is reliably callable across a cohort. + // + // Layer 3 is a blocklist of the **sites that the cohort shares**. It holds each position + // that varies with two carriers or more across the cohort. It also holds the homoplasy + // hotspots. + // + // A real variant of a shared lineage is in the DecodingUs tree, and the step above marks it + // off-path-known. A variant that the cohort shares and the tree does not hold is a + // recurrent artefact. It is not a private SNP. + // + // A true private variant has one carrier in the cohort, so it passes this filter. This + // layer takes the place of the cohort carrier filter of the de-novo pipeline, for one + // sample. + // + // The bundle holds a file for each build. The CHM13 file is native, and CrossMap moved the + // GRCh38 file. The code skips this layer when the file is absent. let mask_token = y_mask_build_token(&aln.reference_build); let cohort_mask = mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_CALLABLE_MASK", "chrY_callable_mask", t)); let cohort_shared = mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_COHORT_SHARED", "chrY_cohort_shared_sites", t)); - // Derived-call source. Prefer a per-sample chrY GVCF (GATK HaplotypeCaller's local haplotype - // reassembly resolves misaligned-ref ~50/50 sites the pileup caller drops — see the WGS229 - // recall gap); fall back to Navigator's de-novo pileup caller when no sidecar is present. + // The source of the derived calls. Take the chrY GVCF file of the sample first. + // + // GATK HaplotypeCaller builds the local haplotypes again. Take a site with about 50% + // reference reads, where the mapper placed the reference reads wrongly. That step gives an + // answer at such a site, and the pileup caller removes it. See the recall gap of WGS229. + // + // When the sample has no sidecar file, use the de-novo pileup caller of Navigator. let denovo = match chr_y_gvcf_for_alignment(&aln) { Some(gvcf) => { eprintln!("private-Y: sourcing chrY calls from GVCF sidecar {}", gvcf.display()); @@ -817,7 +952,7 @@ mod gvcf_discovery_tests { } } - /// Each case gets its own directory — these run in parallel. + /// Each case has its own directory, because these tests run at the same time. fn scratch(name: &str) -> PathBuf { let d = std::env::temp_dir().join(format!("nav-gvcf-{}-{name}", std::process::id())); let _ = std::fs::remove_dir_all(&d); @@ -836,9 +971,11 @@ mod gvcf_discovery_tests { #[test] fn finds_a_bare_named_gvcf_in_a_caller_subdirectory() { - // The D2C per-run layout: `/CP086569.2/gatk4/chrY.g.vcf.gz`, with no sample prefix and - // one directory down. Matching only `*.chry.g.vcf.gz` beside the CRAM found none of these, - // so every subject fell back to decoding the whole chromosome. + // The D2C layout for one run: `/CP086569.2/gatk4/chrY.g.vcf.gz`. That name holds no + // sample prefix, and the file is one directory below the alignment. + // + // An earlier version matched `*.chry.g.vcf.gz` beside the CRAM file only, and it found no + // file of this kind. So each subject decoded the full chromosome. let d = scratch("subdir"); std::fs::write(d.join("chrYM.cram"), "").unwrap(); std::fs::create_dir_all(d.join("gatk4")).unwrap(); @@ -885,72 +1022,100 @@ mod gvcf_discovery_tests { /// path's floor: below this, a call is far more likely a misaligned-read cluster than a real SNV. const VCF_PRIVATE_MIN_DP: u32 = 4; -/// Genotype-quality floor, matching the GVCF path's `min_gq`. Aligns the two sources' gates as far -/// as the evidence allows — though it does not make them comparable: see -/// [`App::private_y_from_variant_set`] on why a vendor caller's call set is a different instrument. +/// The lowest genotype quality that a call can have. The value equals the `min_gq` value of the GVCF +/// path. +/// +/// This value makes the gates of the two sources as similar as the evidence permits. It does not +/// make their results comparable. [`App::private_y_from_variant_set`] gives the reason: the call set +/// of a vendor comes from a different instrument. const VCF_PRIVATE_MIN_GQ: u32 = 20; -/// Derived-allele fraction a call must reach to count as **deterministic** on a haploid chromosome. -/// chrY carries one copy, so a genuine call is essentially all-alt; a middling fraction is an -/// ambiguous locus, and an ambiguous call cannot support a private-variant claim. +/// The fraction of derived reads that a call needs to be **deterministic** on a haploid chromosome. +/// +/// A cell holds one copy of chrY. So a true call has almost no reference read. A fraction between +/// the two states marks a locus with two answers, and such a call can not support a claim about a +/// private variant. const VCF_PRIVATE_MIN_AF: f64 = 0.95; -/// Depth ceiling, as a multiple of the donor's own typical depth at good calls. +/// The maximum depth of a call, as a factor of the usual depth of that donor at a good call. /// -/// chrY carries one copy, so a locus drawing far more reads than the rest of the chromosome is -/// collecting them from somewhere else — a collapsed repeat. This was found by reviewing a candidate -/// branch whose two carriers sat at DP 413 and 504 against a median of 57, each holding a stubborn -/// ~5% reference allele: the shape of a paralogous pile-up, and it cleared every other gate. Three -/// times the median keeps ~91% of quality calls while removing that tail. +/// A cell holds one copy of chrY. A locus with many more reads than the rest of the chromosome takes +/// those reads from another place. That place is a collapsed repeat. +/// +/// A review of one candidate branch found this fault. Its two carriers had a DP of 413 and 504, +/// against a median of 57. Each one also held about 5% reference reads. That shape is a pile of +/// reads from a paralog, and the call passed each other gate. +/// +/// A limit of three times the median keeps about 91% of the good calls and removes that group. const VCF_PRIVATE_MAX_DEPTH_RATIO: u32 = 3; -/// Minimum quality-passing calls before a depth ratio is trustworthy. Below this the median is not a -/// description of the donor's coverage, and the rule abstains rather than judging against noise. +/// The count of good calls that the code needs before it can trust a depth ratio. Below this count, +/// the median does not describe the coverage of the donor. The rule then does nothing, and it makes +/// no comparison against noise. const VCF_PRIVATE_MIN_CALLS_FOR_RATIO: usize = 20; impl App { - /// The **private bucket for a variant set** — the VCF counterpart of [`Self::private_y_variants`]. + /// The **private bucket of a variant set**. It is the VCF form of + /// [`Self::private_y_variants`]. + /// + /// The key of the private-Y data was always an alignment. The code walks a BAM file or a CRAM + /// file, or its GVCF sidecar, and it caches the result under `alignment_id`. + /// + /// A subject whose Y data arrived as a VCF file from another tool has no alignment. So the app + /// never offered this option to that subject. On R1b-CTS4466Plus, about 1,600 of the 1,881 + /// members are in that group. For this reason, each cohort feature that needs private variants + /// had almost no data. + /// + /// The classification is the same as the classification of the alignment path, by design. The + /// code removes the backbone that it placed. It then removes each call outside the callable mask + /// of the cohort, and each call on the blocklist of the cohort. It then separates the known + /// off-path calls from the new ones. /// - /// Private-Y has always been keyed on an alignment: it walks a BAM/CRAM (or its GVCF sidecar) and - /// caches against `alignment_id`. A subject whose Y data arrived as an externally processed VCF - /// has no alignment, so the option was never offered — on R1b-CTS4466Plus that is ~1,600 of 1,881 - /// members, and it is why cohort features that depend on private variants had almost nothing to - /// work with. + /// Two things are different: the source of the evidence, and the test of the reliability of the + /// donor. /// - /// The classification is deliberately the same as the alignment path's: subtract the placed - /// backbone, drop anything outside the cohort callable mask or on the cohort-shared blocklist, - /// then split off-path-known from novel. What differs is where the evidence comes from and how - /// the donor's own reliability is judged: + /// - **The placement** uses [`Self::vset_base_calls`]. So the terminal comes from the genotypes + /// at each tree position, and those genotypes include the hom-ref calls. It does not come from + /// the few derived calls alone. + /// - **There is no self-callable mask**, because a VCF file holds no coverage track. The + /// evidence of each call takes its place. The code removes a call with a `FILTER` flag, a call + /// below [`VCF_PRIVATE_MIN_DP`], and each heterozygous call on chrY. A cell holds one copy of + /// chrY, so such a call comes from a paralog or from a read that the mapper placed wrongly. + /// Those calls are about two thirds of the chrY rows of a Big Y test. + /// - **The depth and the allele fraction come from the source**, so [`PublishGate`] judges these + /// calls on real read evidence. A set that the app imported before it stored the evidence has + /// a `call_schema` of 1. Such a set gives nothing that the app can publish, and that result is + /// correct. /// - /// - **Placement** uses [`Self::vset_base_calls`], so the terminal is derived from tree-position - /// genotypes (including hom-ref) rather than the handful of derived calls alone. - /// - **There is no self-callable mask** — a VCF carries no coverage track. Its place is taken by - /// the source's own per-call evidence: a `FILTER`-flagged call is dropped, as is one below - /// [`VCF_PRIVATE_MIN_DP`], and a chrY heterozygote is dropped outright — on a haploid - /// chromosome that is a paralog or mismapping artefact, and it is ~2/3 of a Big Y's chrY rows. - /// - **Depth and allele fraction are the source's**, so [`PublishGate`] judges these calls on real - /// read evidence. A set imported before evidence capture (`call_schema` 1) therefore yields - /// nothing publishable, which is the honest outcome rather than a fabricated one. + /// **These counts do not compare with the counts of the alignment path, and they are not yet + /// good enough for branch inference.** /// - /// **Not comparable to the alignment path's counts, and not yet fit for branch inference.** This - /// yields a median ~175 novel calls per donor against the GVCF path's 3–13. The gap is the - /// instrument, not a defect here: the alignment path reads GATK HaplotypeCaller at ploidy 1, - /// while a vendor export is a diploid caller emitting far more chrY calls, and only ~10% of the - /// difference is reachable by matching DP/GQ gates. Note also that `Novel` means "not - /// branch-defining in *this* tree" — the tree is FTDNA's supported branches plus splits solved - /// from the cohort, not a catalogue of known Y variation — so a real, well-known variant that - /// defines no branch classifies as novel here. Feeding these buckets to the block tree's - /// candidate detection took CTS4466 from 3 candidates to 20 (39 conflicts, 105 recurrent - /// positions dropped) on only 111 of ~1,600 sets, which is why - /// [`Self::private_y_for_biosamples`] does not union them yet. + /// This path gives a median of about 175 new calls for each donor. The GVCF path gives 3 to 13. + /// The difference is the instrument and not a fault here. The alignment path reads GATK + /// HaplotypeCaller at ploidy 1. A vendor export comes from a diploid caller, and that caller + /// writes many more chrY calls. A match of the DP gates and the GQ gates removes only about 10% + /// of the difference. + /// + /// Note also that `Novel` means "this variant defines no branch in *this* tree". The tree holds + /// the branches that FTDNA supports, and the splits that the cohort resolved. It is not a + /// catalogue of each known Y variant. So a real, well-known variant that defines no branch is + /// `Novel` here. + /// + /// The block tree read these buckets for its candidate detection. On CTS4466 the count of + /// candidates went from 3 to 20, with 39 conflicts and 105 recurrent positions removed. Only 111 + /// of about 1,600 sets took part. For this reason, + /// [`Self::private_y_for_biosamples`] does not yet join them. pub async fn private_y_from_variant_set(&self, set: &VariantSet) -> Result { use navigator_analysis::haplo; - // Without per-call evidence every quality gate below is a no-op, and the result is a list of - // whatever the vendor's caller emitted — on a real set that is 400-550 "novel" calls against - // ~70 for the same donor's evidence-bearing set. A call we cannot judge is the most - // non-deterministic kind there is, so refuse rather than publish a number that looks like a - // finding. Re-importing the source populates `CallEvidence` (migration 0042). + // With no evidence for each call, every quality gate below does nothing. The result is + // then the list that the caller of the vendor wrote. On a real set that list holds 400 to + // 550 "new" calls. The set of the same donor with evidence holds about 70. + // + // A call that the app can not judge is the least deterministic call of all. So the method + // refuses, and it does not publish a number that looks like a result. + // + // A second import of the source writes the `CallEvidence` rows. See migration 0042. if !set.has_evidence() { return Err(AppError::Import(format!( "variant set {} carries no per-call evidence (call_schema {}); re-import it to enable private-Y", @@ -966,9 +1131,12 @@ impl App { .values() .flat_map(|n| n.loci.iter().map(|l| l.position)) .collect(); - // `pv2`: the chrY structural masks are now lifted to the set's own build, so a `pv1` - // bucket was classified with **no** structural mask on anything but CHM13 and its counts - // are inflated. The version is the invalidation — `--force` cannot reach this cache. + // The `pv2` key. The code now moves the chrY structural masks to the build of the + // set. + // + // A `pv1` bucket used **no** structural mask on any build except CHM13, so its counts + // are too high. The version number removes the old value from the cache, and the + // `--force` option can not reach this cache. format!("pv2:{}", crate::haplogroup::genotype_cache_key("chrY", None, &targets)) }; if let Ok(Some(json)) = variant_set_private_y::get(self.store.pool(), set.id, &cache_key).await { @@ -992,12 +1160,14 @@ impl App { mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_CALLABLE_MASK", "chrY_callable_mask", t)); let cohort_shared = mask_token.and_then(|t| load_y_position_bed("NAVIGATOR_Y_COHORT_SHARED", "chrY_cohort_shared_sites", t)); - // The structural BEDs are CHM13-native and lifted to whatever this set is in — without that - // a GRCh38 set has no structural mask and its private counts inflate into the hundreds. + // The structural BED files are native to CHM13, and the code moves them to the build of + // this set. Without that step, a GRCh38 set has no structural mask, and its count of private + // variants grows into the hundreds. let regions = self.y_structural_regions_for(&build).await; - // Quality-passing calls first, so the depth ceiling below is measured against the donor's own - // good coverage rather than against a median dragged down by the junk we are about to drop. + // Take the calls that pass the quality gates first. The depth limit below then compares + // against the good coverage of the donor. Without this order, it compares against a median + // that the calls below the gates make smaller. let passing: Vec<&navigator_domain::variants::VariantCall> = set .calls .iter() @@ -1025,8 +1195,9 @@ impl App { position: c.position, reference, alternate, - // The source's own numbers; absent when it gave none, which the publish gate - // then (correctly) refuses rather than treating as evidence. + // The numbers of the source. The value is absent when the source gave none. + // The publish gate then refuses that call, and that decision is correct. It + // must not read an absent value as evidence. depth: c.evidence.dp.unwrap_or(0), alt_depth: c.evidence.ad_alt.unwrap_or(0), allele_fraction: c.evidence.allele_fraction().unwrap_or(0.0), @@ -1051,12 +1222,17 @@ impl App { } } -/// Whether a genotype is a single-allele (hemizygous / homozygous-alt) call. +/// Shows whether a genotype holds one allele. Such a call is hemizygous or homozygous for the +/// alternate allele. +/// +/// A cell holds one copy of chrY. So a heterozygous call there is not possible in biology. It marks +/// a paralog, or a locus where the mapper placed the reads wrongly. /// -/// chrY is haploid, so a heterozygous call there has no biological reading: it is a paralogous or -/// mismapped locus. In a real Big Y export those are ~2/3 of the chrY rows, and admitting them would -/// make the private set mostly artefact. An absent genotype is admitted — a source that reports no GT -/// is not asserting heterozygosity. +/// In a real Big Y export, those calls are about two thirds of the chrY rows. With them, most of the +/// private set is an artefact. +/// +/// The function accepts a call with no genotype. A source that writes no GT field makes no statement +/// about heterozygosity. fn is_hemizygous(gt: Option<&str>) -> bool { let Some(gt) = gt else { return true }; let alleles: Vec<&str> = gt.split(['/', '|']).filter(|a| *a != ".").collect(); @@ -1069,8 +1245,9 @@ mod vcf_private_y_tests { #[test] fn a_chr_y_heterozygote_is_rejected() { - // chrY is haploid: a het call is a paralog or a mismapping, and it is ~2/3 of a Big Y's - // chrY rows — admitting them would make the private set mostly artefact. + // A cell holds one copy of chrY. So a heterozygous call marks a paralog, or a read that + // the mapper placed wrongly. Those calls are about two thirds of the chrY rows of a Big Y + // test. With them, most of the private set is an artefact. assert!(!is_hemizygous(Some("0/1"))); assert!(!is_hemizygous(Some("1|2"))); assert!(!is_hemizygous(Some("1/2"))); @@ -1086,16 +1263,18 @@ mod vcf_private_y_tests { #[test] fn a_source_that_reports_no_genotype_is_not_treated_as_heterozygous() { - // Absence of a GT is not an assertion about ploidy; rejecting it would silently discard - // every sites-only or CSV-derived set. + // An absent GT field makes no statement about the ploidy. A rule that refused it would + // remove each set with sites only, and each set from a CSV file, with no message. assert!(is_hemizygous(None)); assert!(is_hemizygous(Some("1/.")), "a partial call still carries one allele"); } } -/// Median read depth across `calls`, or `None` when too few carry one to describe the donor's -/// coverage. Median rather than mean: the pile-ups this exists to find would drag a mean upward and -/// hide themselves behind it. +/// The median read depth across `calls`. The function returns `None` when too few calls hold a depth +/// to describe the coverage of the donor. +/// +/// The function takes the median and not the mean. The piles of reads that this code must find make +/// a mean larger, and they then hide behind that larger value. fn median_depth(calls: &[&navigator_domain::variants::VariantCall]) -> Option { let mut depths: Vec = calls.iter().filter_map(|c| c.evidence.dp).collect(); if depths.len() < VCF_PRIVATE_MIN_CALLS_FOR_RATIO { @@ -1127,7 +1306,7 @@ mod depth_ratio_tests { #[test] fn the_median_ignores_the_pile_ups_it_exists_to_find() { - // A mean would be dragged up by the outliers and hide them behind itself. + // The extreme values make a mean larger, and they then hide behind that larger value. let mut calls: Vec = (0..VCF_PRIVATE_MIN_CALLS_FOR_RATIO).map(|_| call(Some(50))).collect(); calls.push(call(Some(2584))); calls.push(call(Some(1191))); diff --git a/crates/navigator-app/src/ftdna_import.rs b/crates/navigator-app/src/ftdna_import.rs index a50e3226..7adece22 100644 --- a/crates/navigator-app/src/ftdna_import.rs +++ b/crates/navigator-app/src/ftdna_import.rs @@ -1,11 +1,17 @@ -//! FTDNA project import — the matching/dedup engine + two-phase plan/commit (design §5/§6). +//! The FTDNA project import. This module holds the engine that matches a kit to a subject and +//! finds a duplicate. It also holds the two steps of the import, which are the plan and the commit. +//! See design §5 and §6. //! -//! Phase 1 scope (roster + ancestry, the spine): parse the batch CSVs, join by kit number, match -//! each kit against the workspace, and produce a reviewable **plan** (dry-run, no writes). A separate -//! commit step applies the plan with the admin's resolutions for fuzzy candidates. +//! Phase 1 covers the roster and the ancestry, which are the base of the import. The module does +//! four steps. It parses the batch CSV files, joins them by the kit number, matches each kit +//! against the workspace, and makes a **plan** for the administrator. This phase writes nothing. //! -//! Deep per-member data (Big Y / mtDNA / Family Finder) and the wide Y-STR chart are layered on by -//! later slices; this module only wires identity + MDKA + membership. +//! A separate commit step applies the plan. That step uses the decisions of the administrator for +//! each candidate that the engine is not sure about. +//! +//! A later change adds the deep data of each member, which is Big Y, mtDNA, and Family Finder. It +//! also adds the wide Y-STR chart. This module only connects the identity, the MDKA rows, and the +//! membership. use std::collections::BTreeMap; use std::path::PathBuf; @@ -28,10 +34,11 @@ struct CatalogSample { accession: Option, } -/// Tuning for the matching engine. +/// The values that control the engine that matches a kit to a subject. #[derive(Debug, Clone)] pub struct FtdnaImportOptions { - /// Minimum fuzzy score (0..1) for a workspace Subject to be offered as a merge candidate. + /// The lowest score, from 0 to 1, that lets the engine offer a subject as a merge candidate. + /// This score is not exact, and the engine calculates it from the name. pub fuzzy_threshold: f32, } @@ -62,8 +69,9 @@ pub struct FuzzyCandidate { pub reasons: Vec, } -/// How the matcher proposes to handle a kit. Auto-merge is locked for an exact vendor-id hit; fuzzy -/// hits are queued for the admin (never auto-merged). +/// The action that the engine proposes for a kit. The engine merges without a question only for an +/// exact match on the vendor id. For a match that is not exact, it adds the kit to a list, and the +/// administrator decides. The engine never merges such a kit on its own. #[derive(Debug, Clone)] pub enum MatchKind { /// No workspace match → create a new Subject. @@ -91,8 +99,10 @@ pub struct FtdnaPlanRow { pub input: FtdnaSubjectInput, } -/// Recognized-input + scan counts for the review header — so a missing/misclassified file (e.g. no -/// roster) is immediately visible rather than silently producing all-orphan rows. +/// The counts of the input files that the code recognized, and of the rows that it read. The review +/// header shows these counts. So the administrator sees an absent file, or a file with the wrong +/// class, at once. One example is an import with no roster. Without these counts, such an import +/// gives rows with no subject and no message. #[derive(Debug, Clone, Default)] pub struct FtdnaPlanStats { /// Roster rows parsed from `Member_Information`. @@ -142,14 +152,14 @@ pub enum FtdnaResolution { Merge(SampleGuid), /// Treat as a new Subject. New, - /// Don't import this kit at all. + /// Do not import this kit at all. Skip, } /// What the commit did. #[derive(Debug, Clone, Default)] pub struct FtdnaImportSummary { - /// The project the kits were imported into (resolved/created at commit). + /// The project that received the kits. The commit step finds this project or makes it. pub project_id: i64, pub created: usize, pub merged: usize, @@ -164,8 +174,11 @@ pub struct FtdnaImportSummary { pub errors: Vec, } -/// A Subject's imported genealogy bundle: vendor ids, FTDNA member labels, and MDKA rows. PII — -/// for local display only (never federated). Empty when nothing was imported for the Subject. +/// The genealogy data that the app imported for one subject. It holds the vendor ids, the FTDNA +/// member labels, and the MDKA rows. +/// +/// This data is personal. The app shows it on this machine only, and it never sends it to the +/// network. The value is empty when the app imported nothing for the subject. #[derive(Debug, Clone, Default)] pub struct FtdnaGenealogy { pub external_ids: Vec, @@ -174,7 +187,7 @@ pub struct FtdnaGenealogy { } impl FtdnaGenealogy { - /// Nothing imported → the detail card can be skipped. + /// Shows that the app imported nothing. The UI can then skip the detail card. pub fn is_empty(&self) -> bool { self.external_ids.is_empty() && self.member.is_none() && self.mdka.is_empty() } @@ -191,10 +204,15 @@ impl App { }) } - /// Attach a vendor id (kit number) to a Subject from the subject editor. Rejects a blank - /// source/id, and refuses to bind a `(source, external_id)` that already belongs to a *different* - /// Subject (the `(source, external_id)` uniqueness is the dedup anchor — never silently re-point - /// it; the caller resolves the conflict). Idempotent for the same Subject. + /// Add a vendor id, which is a kit number, to a subject from the subject editor. + /// + /// The method refuses a blank source and a blank id. It also refuses a `(source, external_id)` + /// pair that belongs to a *different* subject. + /// + /// That pair is unique, and the app uses it to find a duplicate donor. The method must never + /// move the pair to another subject without a message. The caller resolves such a conflict. + /// + /// A second call for the same subject is safe. pub async fn add_external_id( &self, guid: SampleGuid, @@ -218,7 +236,8 @@ impl App { /// Detach a vendor id (by row id) from a Subject. pub async fn delete_external_id(&self, id: i64) -> Result<(), AppError> { - // Recover the owning subject before the row is gone, so we can refresh its published record. + // Read the subject of this row before the code deletes the row. The app then refreshes + // the published record of that subject. let guid = external_id::get(self.store.pool(), id).await?.map(|e| e.biosample_guid); external_id::delete(self.store.pool(), id).await?; if let Some(guid) = guid { @@ -227,13 +246,22 @@ impl App { Ok(()) } - /// Backfill public-catalog external ids (`IGSR`/`HGDP`/INSDC) derivable from each subject's local - /// provenance ([`navigator_domain::identity::catalog_ids_from_provenance`]) — so bulk-imported - /// public datasets publish ids that match their existing AppView catalog rows. Deterministic and - /// network-free; a friendly-name-only sample contributes nothing. Idempotent (skips ids already - /// present); a `(namespace, value)` already owned by a *different* subject is counted as a - /// conflict and left untouched (never silently re-pointed). `apply == false` is a dry run. - /// Adds via the store directly (no per-id re-publish); re-publish the affected subjects after. + /// Add the public-catalog external ids that the code can derive from the local provenance of + /// each subject. The namespaces are `IGSR`, `HGDP`, and INSDC, and + /// [`navigator_domain::identity::catalog_ids_from_provenance`] derives them. + /// + /// A public dataset that a user imported in bulk then publishes ids that match its rows in the + /// catalog of the AppView. + /// + /// The method is deterministic and uses no network. A sample with only a friendly name gives no + /// id. A second call is safe, because the method skips an id that already exists. + /// + /// The method counts a `(namespace, value)` pair that belongs to a *different* subject as a + /// conflict, and it changes nothing. It never moves such a pair without a message. + /// `apply == false` makes the method report the changes and write nothing. + /// + /// The method writes to the store directly, and it does not publish a record for each id. + /// Publish the subjects that changed after the method completes. pub async fn backfill_catalog_ids( &self, project_id: Option, @@ -274,7 +302,8 @@ impl App { if row.biosample_guid == b.guid { out.ids_added += 1; } else { - // (namespace,value) already belongs to another subject — a dup import; leave it. + // This (namespace, value) pair belongs to another subject. The user + // imported the same data twice. Change nothing. out.conflicts += 1; } } @@ -283,10 +312,14 @@ impl App { Ok(out) } - /// Fetch one public-catalog sample record from the AppView samples API (`/api/v1/samples/{alias}`, - /// public read) by its alias (= our `donor_identifier`). `Ok(None)` for a 404 (alias unknown to - /// the catalog — expected while server-side corrections are pending). The authoritative - /// `accession` it returns is the datum our local `sample_accession` lacks. + /// Read one public-catalog sample record from the samples API of the AppView. The path is + /// `/api/v1/samples/{alias}`, the read is public, and the alias is our `donor_identifier`. + /// + /// A 404 response gives `Ok(None)`, which means that the catalog does not know the alias. That + /// result is normal while a correction on the server is not complete. + /// + /// The `accession` value in the response has authority. Our local `sample_accession` field does + /// not hold it. async fn fetch_catalog_sample(&self, base: &str, alias: &str) -> Result, AppError> { let url = format!("{}/api/v1/samples/{alias}", base.trim_end_matches('/')); let resp = self @@ -309,14 +342,24 @@ impl App { Ok(Some(s)) } - /// Resolve each subject against the AppView samples API and attach, **in one pass**, the full set - /// of public-catalog ids: the catalog *name* id (IGSR/HGDP, derived from the donor id) **and** the - /// authoritative INSDC *accession* the API returns (`SAMN…` → BIOSAMPLE, `ERS…` → ENA, `SRS…` → - /// SRA) — plus correcting the local `sample_accession` placeholder. A superset of - /// [`backfill_catalog_ids`](Self::backfill_catalog_ids) (the offline name-only path); use that one - /// when the API is unavailable. By default only subjects whose `donor_identifier` looks like a - /// catalog alias (IGSR/HGDP) are queried (`all` overrides), to avoid hammering the API with - /// friendly-name 404s. `apply == false` is a dry run. `limit` caps how many are queried. + /// Look up each subject in the samples API of the AppView and add the full set of + /// public-catalog ids **in one pass**. + /// + /// The set holds two kinds of id. The first is the catalog *name* id, which is an IGSR id or an + /// HGDP id, and the code derives it from the donor id. The second is the INSDC *accession* that + /// the API returns, and that value has authority. A `SAMN` prefix gives BIOSAMPLE, an `ERS` + /// prefix gives ENA, and an `SRS` prefix gives SRA. The method also corrects the local + /// `sample_accession` field, which holds a temporary value. + /// + /// This method does more than [`backfill_catalog_ids`](Self::backfill_catalog_ids), which uses + /// the name only and needs no network. Use that method when the API is not available. + /// + /// By default the method queries only a subject whose `donor_identifier` looks like a catalog + /// alias, which is an IGSR alias or an HGDP alias. The `all` option removes that limit. The + /// default stops many 404 responses for a friendly name. + /// + /// `apply == false` makes the method write nothing. `limit` sets the maximum count of + /// queries. pub async fn backfill_accessions( &self, project_id: Option, @@ -335,7 +378,7 @@ impl App { continue; } } - // Skip samples whose name isn't a recognizable catalog alias unless `--all`. + // Skip samples whose name is not a recognizable catalog alias unless `--all`. if !all && navigator_domain::identity::catalog_ids_from_provenance(&b.donor_identifier, None).is_empty() { continue; } @@ -356,8 +399,9 @@ impl App { }; out.resolved += 1; let fetched_acc = sample.accession.as_deref().map(str::trim).filter(|a| !a.is_empty()); - // One pass: the catalog *name* id (from the donor id) + the authoritative INSDC *accession* - // (from the API, when it's a real one) — the union of both sources via the shared helper. + // One pass gives both ids. The catalog *name* id comes from the donor id. The INSDC + // *accession* comes from the API, when the API holds a real one. The shared helper + // joins the two sources. let ids = navigator_domain::identity::catalog_ids_from_provenance(&b.donor_identifier, fetched_acc); if ids.is_empty() { continue; @@ -403,11 +447,15 @@ impl App { Ok(out) } - /// Re-publish a subject's biosample anchor after its identifier set changed, so the AppView's - /// mirror (which full-replaces `external_ids`) honors the add/remove. Deterministic rkey → the - /// re-publish overwrites in place. **Only for a subject already federated** and while signed in — - /// signed out, or a never-published subject, is a no-op (we don't newly federate a donor just - /// because a local id was attached). + /// Publish the biosample anchor of a subject again, after the set of identifiers of that + /// subject changed. The mirror of the AppView replaces the full `external_ids` field, so it + /// then holds each addition and each removal. + /// + /// The method uses a fixed rkey, so the second publish replaces the record. + /// + /// The method acts **only for a subject that the app already published**, and only while an + /// account is active. With no active account, or for a subject that the app never published, it + /// does nothing. A new local id must not put a donor on the network for the first time. async fn republish_biosample_ids(&self, guid: SampleGuid) -> Result<(), AppError> { let Some(did) = self.current_account() else { return Ok(()); @@ -421,8 +469,9 @@ impl App { self.publish_biosample(guid).await } - /// Insert or update a Subject's MDKA for one lineage from the subject editor (one row per - /// lineage; stamps `updated_at`). Pass a `source` of `MANUAL` for hand-entered rows. + /// Insert or change the MDKA of a subject for one lineage, from the subject editor. There is + /// one row for each lineage, and the method sets `updated_at`. Give a `source` of `MANUAL` for + /// a row that the user typed. pub async fn upsert_mdka(&self, guid: SampleGuid, mdka: NewMdka) -> Result<(), AppError> { let now = Utc::now().to_rfc3339(); mdka::upsert(self.store.pool(), guid, &mdka, &now).await?; @@ -485,8 +534,8 @@ impl App { ystr: ystr.len(), scanned_subjects: 0, }; - // A roster was provided iff there are member rows — only then is "orphan" (data without a - // roster row) a meaningful flag. + // The import holds a roster only when it holds member rows. The "orphan" mark applies + // only in that case. An orphan is data with no roster row. let roster_provided = !members.is_empty(); // Join by kit number (BTreeMap → stable, kit-sorted plan). @@ -530,7 +579,8 @@ impl App { label: display_label(&kit, &input), kit_number: kit, y_terminal, - // Orphan only when a roster was provided but this kit isn't in it. + // Mark the kit as an orphan only when the import holds a roster and that roster + // does not name the kit. in_roster: !roster_provided || roster.contains(&input.kit_number), ystr_count: input.ystr_markers.len(), kind, @@ -545,8 +595,9 @@ impl App { }) } - /// Apply a plan. `resolutions` carries the admin's choice for each fuzzy (`NeedsConfirm`) kit; - /// an unresolved fuzzy row defaults to **New** (conservative — never silently merge). + /// Apply a plan. `resolutions` holds the decision of the administrator for each kit with the + /// `NeedsConfirm` mark. A kit with that mark and no decision becomes a **New** subject. This + /// default is the safe one, because the method must never merge a kit without a decision. pub async fn commit_ftdna_import( &self, plan: &FtdnaImportPlan, @@ -555,7 +606,7 @@ impl App { let mut summary = FtdnaImportSummary::default(); let now = Utc::now().to_rfc3339(); - // Resolve the target project, creating it now if the plan targeted a new one. + // Find the target project. Make the project now when the plan names a new one. let project_id = match plan.project_id { Some(id) => id, None => { @@ -629,7 +680,8 @@ impl App { } }; - // Vendor identity (idempotent; never steals a conflicting id). + // The vendor identity. A second call is safe, and the code never moves an id that + // belongs to another subject. external_id::add(pool, guid, IdSource::FTDNA, &input.kit_number).await?; // FTDNA-reported member labels. @@ -648,7 +700,8 @@ impl App { ) .await?; - // MDKA from paternal (Y) + maternal (Mt) ancestry, when there's anything worth storing. + // The MDKA rows from the paternal (Y) ancestry and the maternal (Mt) ancestry. The code + // writes a row only when the ancestry holds a value. let mut wrote = 0; if let Some(m) = input.paternal.as_ref().and_then(|a| mdka_from(a, Lineage::Y)) { mdka::upsert(pool, guid, &m, now).await?; @@ -667,9 +720,10 @@ impl App { .map(subgroup_role); biosample_project::add(pool, guid, project_id, role.as_deref(), now).await?; - // Y-STR profile from the wide overview (Phase 2). Attached only when CREATING a new Subject; - // on a merge the existing Subject already carries its own data sources, so we add the FTDNA - // identity/membership/MDKA metadata above but skip duplicating the Y-STR profile. + // The Y-STR profile from the wide overview, which is Phase 2. The code adds the profile + // only when it makes a new subject. On a merge, the subject already holds its own data + // sources. So the code adds the FTDNA identity, the membership, and the MDKA data above, + // and it does not add a second Y-STR profile. let wrote_str = !input.ystr_markers.is_empty() && target.is_none(); if wrote_str { str_profile::create( @@ -699,9 +753,12 @@ impl App { Ok(external_id::list_for(self.store.pool(), guid).await?) } - /// Reverse of [`external_ids`]: the Subject bound to a `(source, external_id)` vendor id, if any. - /// This is the exact-match dedup anchor (design §5.1) — e.g. resolve an FTDNA kit number to the - /// biosample it was imported under. Returns `None` when the id is unknown to the workspace. + /// The reverse of [`external_ids`]. The method returns the subject of a + /// `(source, external_id)` vendor id, when one exists. + /// + /// This lookup is the exact-match anchor that finds a duplicate donor, in design §5.1. One use + /// is to find the biosample of an FTDNA kit number. The method returns `None` when the + /// workspace does not hold the id. pub async fn find_biosample_by_external_id( &self, source: &str, @@ -724,14 +781,17 @@ impl App { Ok(mdka::list_for(self.store.pool(), guid).await?) } - /// Project ids a Subject belongs to (via the M:N membership table). + /// The ids of the projects that hold this subject. The method reads the M:N membership + /// table. pub async fn project_membership_ids(&self, guid: SampleGuid) -> Result, AppError> { Ok(biosample_project::list_projects_for(self.store.pool(), guid).await?) } - /// Autocluster a project's members by Y-STR and propagate SNP branches to STR-only members - /// (the project clustering view). Branch per member = its FTDNA-reported terminal SNP; markers = - /// the merged Y-STR profiles. The O(n²) compute runs on a blocking thread. + /// Group the members of a project by their Y-STR values, and copy an SNP branch to a member + /// that has only STR values. The project cluster view shows this result. + /// + /// The branch of a member is the terminal SNP that FTDNA reports for it. The markers are the + /// merged Y-STR profiles. The calculation costs O(n²), so it runs on its own thread. pub async fn cluster_project_ystr( &self, project_id: i64, @@ -839,10 +899,15 @@ impl App { reasons.push(format!("same Y terminal {ex}")); } } - // Y-STR genetic distance — a SAME-PERSON signal only at (near-)zero GD over many markers. - // A loose GD threshold floods inside a single-haplogroup project, where every member is - // related and within-project distances of GD 3–11 over 100 markers are normal. Only an - // exact (or off-by-one) haplotype uniquely identifies the same person, not a clade cousin. + // The genetic distance of the Y-STR values. This distance shows the SAME PERSON only + // when it is zero, or almost zero, across many markers. + // + // A high limit gives many false results in a project with one haplogroup. Each member + // of such a project is a relative, and a distance of 3 to 11 across 100 markers is + // normal there. + // + // Only an exact haplotype, or a haplotype with one difference, names the same person. A + // larger distance names a cousin in the same clade. if !input.ystr_markers.is_empty() && !e.ystr.is_empty() { let (diff, compared) = navigator_domain::strprofile::str_distance(&input.ystr_markers, &e.ystr); if compared >= 67 && diff <= 1 { @@ -882,8 +947,9 @@ impl App { struct ExistingSubject { guid: SampleGuid, donor_identifier: String, - /// Terminal SNP of the subject's computed Y consensus (may be an ISOGG long-form label that - /// doesn't reduce to an SNP — then Y-STR is the reliable signal). + /// The terminal SNP of the Y consensus that the app calculated for the subject. The value can + /// be a long ISOGG label with no SNP inside it. In that case the Y-STR values are the signal + /// that the code can trust. y_terminal: Option, /// The subject's merged Y-STR markers (across all imported profiles), for genetic-distance match. ystr: Vec, @@ -899,8 +965,9 @@ fn empty_input(kit: &str) -> FtdnaSubjectInput { } } -/// The terminal SNP token of a haplogroup label or clade path: the last segment after splitting on -/// `>` (clade) or `-` (haplogroup prefix). `"R-FGC29071"` and `"CTS4466>S1115>FGC29071"` → `FGC29071`. +/// The terminal SNP token of a haplogroup label or a clade path. The function splits the text on +/// `>` for a clade, or on `-` for a haplogroup prefix, and returns the last part. Both +/// `"R-FGC29071"` and `"CTS4466>S1115>FGC29071"` give `FGC29071`. fn terminal_snp(label: &str) -> Option { let t = label.rsplit(['>', '-']).next()?.trim(); (!t.is_empty()).then(|| t.to_string()) @@ -919,7 +986,7 @@ fn display_label(kit: &str, input: &FtdnaSubjectInput) -> String { } } -/// Drop FTDNA redaction/placeholder names so they don't pollute identifiers or matching. +/// Drop FTDNA redaction/placeholder names so they do not pollute identifiers or matching. fn clean_name(name: Option<&str>) -> Option { let n = name?.trim(); if n.is_empty() || n.eq_ignore_ascii_case("REDACTED") { @@ -929,7 +996,8 @@ fn clean_name(name: Option<&str>) -> Option { } } -/// Build an MDKA payload from an ancestry row, or `None` if it carries nothing worth storing. +/// Build an MDKA value from an ancestry row. The function returns `None` when the row holds no +/// data for the store. fn mdka_from(a: &AncestryRow, lineage: Lineage) -> Option { if a.ancestor_name.is_none() && a.origin_place.is_none() && a.country.is_none() && a.latitude.is_none() { return None; @@ -957,13 +1025,14 @@ fn panel_name_for_count(n: usize) -> String { } } -/// The clade `Sub Group` value as a membership role: keep it compact (the terminal segment), dropping -/// the leading sort number. +/// The `Sub Group` value of a clade, as a membership role. The function keeps the last part only, +/// and it removes the sort number at the start. fn subgroup_role(sub_group: &str) -> String { terminal_snp(sub_group).unwrap_or_else(|| sub_group.trim().to_string()) } -/// Jaccard overlap of lowercased word tokens (len ≥ 2) — a cheap name-similarity proxy in `0..=1`. +/// The Jaccard overlap of the word tokens, in lower case, with a length of 2 or more. The value is +/// a fast measurement of the similarity of two names, from 0 to 1. fn name_similarity(a: &str, b: &str) -> f32 { let toks = |s: &str| -> std::collections::HashSet { s.split(|c: char| !c.is_ascii_alphanumeric()) diff --git a/crates/navigator-app/src/haplogroup.rs b/crates/navigator-app/src/haplogroup.rs index 036285b9..1d619272 100644 --- a/crates/navigator-app/src/haplogroup.rs +++ b/crates/navigator-app/src/haplogroup.rs @@ -3,13 +3,13 @@ use super::*; use crate::fastpath::{chr_m_gvcf_for_alignment, chr_y_gvcf_for_alignment}; -/// Analysis-artifact `kind` for cached per-alignment tree-genotype base calls (see -/// [`App::base_calls`]). The `algorithm_version` carries the site-set hash, so distinct trees / -/// contigs / lift paths get distinct cache rows. +/// Analysis-artifact `kind` for the cached tree-genotype base calls of one alignment (see +/// [`App::base_calls`]). The `algorithm_version` carries the site-set hash, so a different tree, +/// contig, or lift path gets a different cache row. const GENOTYPE_KIND: &str = "tree-genotype"; -/// Parse a stored painting JSON into a [`PaintingResult`], tolerating the legacy form (a bare -/// `Vec` with no side labels) by wrapping it with the neutral Side A/B defaults. +/// Parse a stored painting JSON into a [`PaintingResult`]. It accepts the legacy form, which is a +/// bare `Vec` with no side labels, and gives it the neutral Side A/B defaults. fn parse_painting_json(s: &str) -> Result { match serde_json::from_str::(s) { Ok(r) => Ok(r), @@ -23,11 +23,15 @@ fn parse_painting_json(s: &str) -> Result { } } -/// Which phased side (0/1) carries the parent's transmitted alleles, by **transmission consistency**: -/// at sites where the child is heterozygous and the parent homozygous, the parent transmitted a known -/// allele — the side carrying it is the parent's. (Parent-child IBD is genome-wide IBD1, so IBD -/// overlap alone can't distinguish the sides; the phased alleles can.) `None` when there are too few -/// informative sites or the signal is ambiguous (guards against a mis-called relative). +/// Which phased side (0/1) carries the parent's transmitted alleles, by **transmission +/// consistency**. +/// +/// Take the sites where the child is heterozygous and the parent homozygous. There the parent +/// transmitted a known allele, and the side that holds it is the parent's. Parent-child IBD is +/// genome-wide IBD1, so IBD overlap alone can not separate the sides. The phased alleles can. +/// +/// `None` when there are too few informative sites, or when the signal is ambiguous. That guards +/// against a mis-called relative. fn anchor_side_to_parent(phased: &navigator_analysis::phasing::PhasedGenotypes, parent: &[SiteGenotype]) -> Option { let pd: std::collections::HashMap<(&str, i64), i32> = parent .iter() @@ -66,8 +70,8 @@ fn anchor_side_to_parent(phased: &navigator_analysis::phasing::PhasedGenotypes, } } -/// `(this-side, other-side)` labels from a parent's recorded sex — once one parent is anchored the -/// other side is definitionally the other parent. `None` if the sex isn't a clear male/female. +/// `(this-side, other-side)` labels from a parent's recorded sex. Once one parent has an anchor, +/// the other side must be the other parent. `None` if the sex is not a clear male or female. fn parent_labels_for_sex(sex: Option<&str>) -> Option<(&'static str, &'static str)> { match sex.map(|s| s.trim().to_ascii_lowercase()).as_deref() { Some("female") | Some("f") => Some(("Mother", "Father")), @@ -76,8 +80,9 @@ fn parent_labels_for_sex(sex: Option<&str>) -> Option<(&'static str, &'static st } } -/// The two side labels for a painting: Mother/Father (or "Parent: "/"Other parent") when the -/// painting is phased and a parent side was anchored; otherwise the neutral Side A/Side B. +/// The two side labels for a painting. A phased painting with an anchor on a parent side gives +/// Mother and Father, or "Parent: " and "Other parent". Any other painting gives the neutral +/// Side A and Side B. fn build_side_labels(phased: bool, anchor: Option, parent: Option<&(Option, String)>) -> [String; 2] { match (phased, anchor, parent) { (true, Some(side), Some((sex, name))) => { @@ -94,23 +99,25 @@ fn build_side_labels(phased: bool, anchor: Option, parent: Option<&(Option>> = std::sync::OnceLock::new(); fn tree_memo() -> &'static std::sync::Mutex> { TREE_MEMO.get_or_init(|| std::sync::Mutex::new(HashMap::new())) } -/// Per-source weighted genotype calls (position → base) for one build's pooling group in -/// [`App::place_y_consensus`]: one `(source type, position→base)` entry per contributing source. +/// Weighted genotype calls (position → base) from each source, for one build's pool group in +/// [`App::place_y_consensus`]. Each source that contributes gives one `(source type, +/// position→base)` entry. type YSourceCalls = Vec<(SourceType, HashMap)>; -/// A stable cache key (used as the artifact `algorithm_version`) for a tree-genotype base call: -/// the queried `contig`, the lift source build, and an FNV-1a hash of the **sorted target -/// positions** + their count. A changed tree (added/removed/moved positions) changes the hash → -/// cache miss → fresh walk; the BAM `source_sig` handles a changed alignment file separately. +/// A stable cache key for a tree-genotype base call, which the artifact uses as its +/// `algorithm_version`. It holds the queried `contig`, the lift source build, and an FNV-1a hash of +/// the **sorted target positions** with their count. A tree that adds, removes, or moves a position +/// changes the hash, which misses the cache and forces a fresh walk. The BAM `source_sig` handles a +/// changed alignment file separately. pub(crate) fn genotype_cache_key(contig: &str, source_build: Option<&str>, targets: &HashSet) -> String { let mut sorted: Vec = targets.iter().copied().collect(); sorted.sort_unstable(); @@ -128,15 +135,16 @@ pub(crate) fn genotype_cache_key(contig: &str, source_build: Option<&str>, targe for p in &sorted { feed(&p.to_le_bytes()); } - // `g3`: chrY native genotyping now also resolves indel loci (additive derived sentinels), so the - // cached result differs from the SNP-only `g1` payload — bump on any genotyping-logic change so a - // stale payload isn't reused (the site-set hash alone doesn't capture logic changes). + // `g3`: chrY native genotyping now also resolves indel loci, as additive derived sentinels. So + // the cached result is different from the SNP-only `g1` payload. Raise this on any change to + // the genotyping logic, so that no code reuses a stale payload. The site-set hash alone does + // not catch a change of logic. format!("g3:{contig}:{}:{h:016x}", sorted.len()) } -/// Callable chrY bases from a coverage result — the FTDNA Big Y generation discriminator (see -/// [`App::refine_big_y_generation`]). A Big Y has reads only on chrY, so this is its whole callable -/// footprint. Build-agnostic (`chrY`/`Y`). +/// Callable chrY bases from a coverage result. This is how the code tells the FTDNA Big Y +/// generations apart (see [`App::refine_big_y_generation`]). A Big Y has reads only on chrY, so +/// this is its whole callable footprint. It accepts either build name (`chrY` or `Y`). fn callable_chr_y_bases(cov: &Coverage) -> u64 { cov.contig_callable .iter() @@ -148,9 +156,10 @@ fn callable_chr_y_bases(cov: &Coverage) -> u64 { impl App { // ---- result exports (gap §6) ------------------------------------------- - /// Format a cached result as a shareable file body (TSV / HTML / BED). The UI writes the - /// returned string to the user-chosen path. Errors when the source result hasn't been computed - /// yet (`NotFound`). [`ExportRequest::CallableBed`] re-walks the BAM (no cached intervals). + /// Format a cached result as a file body to share (TSV / HTML / BED). The UI writes the + /// returned string to the path that the user chose. It gives an error (`NotFound`) when the app + /// has not computed the source result yet. [`ExportRequest::CallableBed`] walks the BAM again, + /// because the cache holds no intervals. pub async fn export_content(&self, req: &ExportRequest) -> Result { match req { ExportRequest::CoverageTsv(id) => Ok(export::coverage_tsv(&self.require_coverage(*id).await?)), @@ -206,9 +215,10 @@ impl App { .ok_or_else(|| AppError::Store(StoreError::NotFound(format!("ancestry for alignment {alignment_id}")))) } - /// Walk each analyzed contig for its CALLABLE intervals (BED export). Re-reads the BAM — the - /// coverage artifact stores only per-contig callable *counts*, not the intervals. Uses the - /// contig list from the cached coverage result, so coverage must have been run first. + /// Walk each analyzed contig for its CALLABLE intervals (BED export). It reads the BAM again, + /// because the coverage artifact stores only a callable *count* for each contig, and not the + /// intervals. It takes the contig list from the cached coverage result, so coverage must run + /// first. async fn callable_intervals_all(&self, alignment_id: i64) -> Result)>, AppError> { let cov = self.require_coverage(alignment_id).await?; let contigs: Vec = cov.contig_coverage_stats.iter().map(|s| s.contig.clone()).collect(); @@ -250,7 +260,7 @@ impl App { alternate: v.alternate.to_string(), rs_id: None, genotype: None, - // Derived from an rCRS diff, not a source VCF — there is no evidence to carry. + // This comes from an rCRS diff, and not from a source VCF, so it has no evidence. evidence: Default::default(), }) .collect(); @@ -266,9 +276,9 @@ impl App { Ok(variant_set::create(self.store.pool(), &new).await?) } - /// Assign an mtDNA haplogroup to a stored sequence: fetch (and cache) the FTDNA mt-DNA - /// haplotree and rank haplogroups by the Kulczynski measure over the sample's base - /// calls. RSRS-anchored and reference-free (no rCRS needed). Best first. + /// Assign an mtDNA haplogroup to a stored sequence. Fetch the FTDNA mt-DNA haplotree, cache + /// it, and rank the haplogroups by the Kulczynski measure over the sample's base calls. The + /// method is RSRS-anchored and reference-free, and needs no rCRS. The best comes first. pub async fn assign_mtdna_haplogroup(&self, mtdna_id: i64) -> Result { let tree_json = self.fetch_ftdna_mt_tree().await?; let assignment = self.assign_mtdna_haplogroup_with_tree(mtdna_id, &tree_json).await?; @@ -295,7 +305,7 @@ impl App { } /// Record (upsert) a source's haplogroup call for donor-level reconciliation. Defaults to the - /// internal `NavigatorWalk` provenance tier (the external fast path records via `record_call_fp`). + /// internal `NavigatorWalk` provenance tier. The external fast path uses `record_call_fp`. pub async fn record_haplogroup_call( &self, biosample_guid: SampleGuid, @@ -315,7 +325,7 @@ impl App { } /// Like [`record_haplogroup_call`](Self::record_haplogroup_call) but stamps the input - /// fingerprint (file + tree content hashes) so a later run can skip re-scoring. + /// fingerprint (file and tree content hashes), so a later run does not score it again. async fn record_haplogroup_call_fp( &self, biosample_guid: SampleGuid, @@ -345,7 +355,8 @@ impl App { Ok(()) } - /// Record an assignment's top candidate as a per-source call (no-op if no match). + /// Record an assignment's top candidate as a call for that source. It does nothing if there + /// is no match. async fn record_call( &self, biosample_guid: SampleGuid, @@ -395,11 +406,13 @@ impl App { Ok(()) } - /// The preferred external (sidecar fast-path) call for an alignment's DNA type — present only - /// when the "prefer external caller" policy is on **and** such a call exists. When present, - /// Navigator's internal caller must not re-walk the CRAM: returning this call instead is what - /// protects an external GATK4/1240K placement from being diluted or overwritten (the - /// PRJEB37976 ancient-DNA fix). See `documents/design/external-caller-precedence.md`. + /// The preferred external (sidecar fast-path) call for an alignment's DNA type. It is present + /// only when the "prefer external caller" policy is on **and** such a call exists. + /// + /// When it is present, Navigator's internal caller must not walk the CRAM again. To give this + /// call back instead is what protects an external GATK4/1240K placement. Without it, a walk + /// could dilute or overwrite that placement, which was the PRJEB37976 ancient-DNA fix. See + /// `documents/design/external-caller-precedence.md`. pub(crate) async fn preferred_external_call( &self, biosample_guid: SampleGuid, @@ -416,8 +429,9 @@ impl App { Ok(haplogroup_call::get_one(self.store.pool(), biosample_guid, dna_type, &key).await?) } - /// Whether an alignment already carries a preferred external call for a DNA type — the gate the - /// UI worker uses to skip enqueuing the internal Y/mt genotyping in "Full Analysis". + /// Whether an alignment already carries a preferred external call for a DNA type. This is the + /// gate that the UI worker uses. With it, "Full Analysis" does not put the internal Y/mt + /// genotyping in the queue. pub async fn has_preferred_external_call(&self, alignment_id: i64, dna_type: DnaType) -> Result { let Ok(bio) = self.biosample_of_alignment(alignment_id).await else { return Ok(false); @@ -428,12 +442,16 @@ impl App { .is_some()) } - /// "Compare callers": the trusted external caller vs Navigator's internal caller for one - /// alignment. **Forces** the internal walk regardless of the prefer-external policy — it records - /// its own `aln:{id}` / `aln:{id}:mt` (`NavigatorWalk`) rows and never touches the external `:ext` - /// row, so the comparison is non-destructive to the external call. Returns Y (for Y-bearing - /// subjects) and mtDNA, each with both terminals; a divergence is the ancient-DNA-damage signal - /// the "skip the internal walk" default is protecting against. See external-caller-precedence §6. + /// "Compare callers": the trusted external caller against Navigator's internal caller, for one + /// alignment. + /// + /// It **forces** the internal walk, whatever the prefer-external policy says. It records its + /// own `aln:{id}` and `aln:{id}:mt` (`NavigatorWalk`) rows, and never touches the external + /// `:ext` row. So the comparison does no damage to the external call. + /// + /// It returns Y, for a subject that has a Y chromosome, and mtDNA. Each one carries both + /// terminals. A difference between them is the ancient-DNA-damage signal that the "skip the + /// internal walk" default protects against. See external-caller-precedence §6. pub async fn compare_callers(&self, alignment_id: i64) -> Result, AppError> { let bio = self.biosample_of_alignment(alignment_id).await.ok(); let mut out = Vec::new(); @@ -488,7 +506,7 @@ impl App { /// The reconciled donor-level haplogroup consensus across all recorded sources. A user /// manual override, when set, replaces the computed terminal (flagged `overridden`). /// - /// See [`names_a_branch`] for why the placed-label rule below is not simply + /// See [`names_a_branch`] for why the placed-label rule below is more than /// `prefer_external && has_external`. pub async fn haplogroup_consensus( &self, @@ -498,25 +516,32 @@ impl App { let calls = haplogroup_call::list_for_with_provenance(self.store.pool(), biosample_guid, dna_type).await?; let prefer_external = prefer_external_calls(); let has_external = calls.iter().any(|(p, _)| *p == CallProvenance::External); - // Per-run label reconciliation supplies the lineage / compatibility / divergence warnings — - // honoring provenance: when the user prefers the external caller and one placed this subject, - // it wins the vote (a damaged ancient-DNA CRAM walk cannot out-score it). + // The label reconcile of each run supplies the lineage, compatibility, and divergence + // warnings, and it obeys the provenance. When the user prefers the external caller, and one + // placed this subject, that call wins the vote. A damaged ancient-DNA CRAM walk can not + // out-score it. let mut consensus = reconciliation::reconcile_with_provenance(&calls, prefer_external); - // …the genome-level PLACED call (consensus_profile.consensus_label, from build_{y,mt}_profile) - // is normally authoritative. Phase 2 makes that placement GVCF-sourced on preferred-external - // subjects (place_{y,mt}_consensus → consensus_base_calls, no CRAM walk), so a freshly built - // label already agrees with the external call. We still skip it here so a *stale* label left - // by a pre-Phase-2 (CRAM-pooled) build cannot resurface before the profile is rebuilt — the - // external reconcile is the safe authority for these subjects. + // …the genome-level PLACED call is normally the authority. It is + // consensus_profile.consensus_label, from build_{y,mt}_profile. Phase 2 takes that + // placement from the GVCF on a preferred-external subject (place_{y,mt}_consensus → + // consensus_base_calls, with no CRAM walk). So a label that the app has just built already + // agrees with the external call. + // + // The code still skips it here. A *stale* label from a pre-Phase-2 build, which pooled the + // CRAM, must not come back before a rebuild of the profile. For these subjects the external + // reconcile is the safe authority. // // …*unless* the reconcile has no branch name to offer. A call whose stored haplogroup is a - // variant string rather than a branch (see [`names_a_branch`]) is not an authority worth - // protecting: `altai363p` held one external call reading `chrY:5216846A>C [Node721]` while a - // freshly re-placed profile said `R-YP1507`, and this guard suppressed the good label in - // favour of the raw one — so a re-place appeared to do nothing to the assigned branch name - // even though it had rewritten it correctly. Skipping the placed label is only ever right + // variant string, and not a branch (see [`names_a_branch`]), is not an authority worth + // protection. + // + // `altai363p` held one external call that read `chrY:5216846A>C [Node721]`, while a freshly + // re-placed profile gave `R-YP1507`. This guard suppressed the good label in favour of the + // raw one. A re-place then looked as though it did nothing to the assigned branch name, + // even though it had rewritten the name correctly. To skip the placed label is right only // when what replaces it is better. + // // Read it as: skip the placed label only when a preferred external call offers a real // branch name to skip it *for*. let reconciled_names_a_branch = consensus.as_ref().is_some_and(|c| names_a_branch(&c.haplogroup)); @@ -571,8 +596,8 @@ impl App { /// Donor-level Y and mtDNA terminal haplogroups for **every** subject, for the subjects /// list. Reconciles each subject's recorded calls (and applies any manual override) in - /// memory from two bulk queries. `(guid → (Y terminal, mt terminal))`; either is `None` - /// when nothing is recorded. + /// memory from two bulk queries. The map is `(guid → (Y terminal, mt terminal))`. A terminal + /// is `None` when the store holds nothing for it. pub async fn haplogroup_terminals( &self, ) -> Result, Option)>, AppError> { @@ -595,9 +620,10 @@ impl App { } } } - // The genome-level placed terminal (build_{y,mt}_profile) wins over the per-run label vote, - // so the subjects table matches the detail tab — except on a preferred-external subject, where - // the CRAM-pooled placement is skipped in favor of the external call (as in haplogroup_consensus). + // The genome-level placed terminal (build_{y,mt}_profile) wins over the label vote of each + // run, so the subjects table matches the detail tab. A preferred-external subject is the + // exception: there the code skips the CRAM-pooled placement, and takes the external call, + // as haplogroup_consensus does. for (guid_s, dna_type_s, label) in navigator_store::consensus_profile::list_labels(self.store.pool()).await? { let Ok(uuid) = guid_s.parse::() else { continue; @@ -678,13 +704,15 @@ impl App { Ok(()) } - /// Load the persisted **observations** for a subject + DNA type — the raw genotype snapshot with - /// no interpretation. Cheap (no genotyping). The shared loader behind [`cached_y_profile`] / - /// [`cached_mt_profile`]; those interpret it against the current tree. `None` until a build runs. + /// Load the stored **observations** for a subject and DNA type. This is the raw genotype + /// snapshot, with no interpretation. It is low-cost, with no genotyping. It is the shared + /// loader behind [`cached_y_profile`] and [`cached_mt_profile`], which interpret it against the + /// current tree. `None` until a build runs. /// - /// Backward-compat: a payload written before the observation-first switch is a baked - /// [`ConsensusProfile`] (no `schema_version`); it is normalized to an [`ObservedProfile`] using - /// its stored per-source bases (a source with no base becomes a no-call until the next rebuild). + /// For compatibility: a payload from before the observation-first change is a complete + /// [`ConsensusProfile`], with no `schema_version`. The loader normalizes it to an + /// [`ObservedProfile`] from the bases that it stored for each source. A source with no base + /// becomes a no-call until the next rebuild. async fn load_observed_profile( &self, biosample_guid: SampleGuid, @@ -696,7 +724,7 @@ impl App { return Ok(None); }; let value: serde_json::Value = serde_json::from_str(&row.payload)?; - // New payloads carry `schema_version`; legacy baked profiles don't. + // New payloads carry `schema_version`; legacy baked profiles do not. if value.get("schema_version").is_some() { Ok(Some(serde_json::from_value(value)?)) } else { @@ -705,9 +733,10 @@ impl App { } } - /// Persist a reconciled consensus snapshot — the low-level row writer shared by every DNA type - /// (Y / mt key on [`DnaType`], autosomal keys on `"Auto"`; the payload is whatever profile shape - /// that type uses). The scalar columns mirror the summary header for quick listing. + /// Store a reconciled consensus snapshot. This is the low-level row writer that every DNA type + /// shares. Y and mt key on [`DnaType`], and the autosomal type keys on `"Auto"`. The payload is + /// the profile shape that the type uses. The scalar columns mirror the summary header, for a + /// fast listing. #[allow(clippy::too_many_arguments)] async fn persist_consensus_row( &self, @@ -738,9 +767,10 @@ impl App { Ok(()) } - /// Persist a Y/mt **observation** snapshot (the payload) plus the interpreted `summary` header for - /// quick listing. Only observations are stored; state/status are re-derived on load by - /// [`interpret_y_profile`](Self::interpret_y_profile) / [`interpret_mt_profile`]. + /// Store a Y/mt **observation** snapshot, which is the payload, plus the interpreted `summary` + /// header for a fast listing. The store keeps only the observations. + /// [`interpret_y_profile`](Self::interpret_y_profile) and [`interpret_mt_profile`] derive the + /// state and the status again at load time. async fn persist_observed_profile( &self, biosample_guid: SampleGuid, @@ -761,10 +791,11 @@ impl App { .await } - /// The Y/mt polarity map (SNP name → ancestral/derived) from the **current** tree for the - /// configured provider — the input to [`navigator_domain::consensus::interpret`]. DecodingUs uses - /// the tree's true phylogenetic polarity; FTDNA the parsed FTDNA tree's polarity. Empty when the - /// tree is unavailable (interpret then falls back to each variant's stored ref/alt). + /// The Y/mt polarity map (SNP name → ancestral/derived) from the **current** tree of the + /// configured provider. It is the input to [`navigator_domain::consensus::interpret`]. + /// DecodingUs uses the tree's true phylogenetic polarity. FTDNA uses the polarity of the parsed + /// FTDNA tree. The map is empty when the tree is not available, and interpret then falls back + /// to each variant's stored ref/alt. async fn current_y_polarity(&self) -> std::collections::BTreeMap { match y_tree_provider() { YTreeProvider::DecodingUs => self @@ -803,13 +834,14 @@ impl App { interpret_observed(observed, &pol) } - /// The Y-profile for a subject, if one has been built — cheap (no genotyping). `None` until - /// [`build_y_profile`](Self::build_y_profile) runs. + /// The Y-profile for a subject, if the app has built one. It is low-cost, with no genotyping. + /// `None` until [`build_y_profile`](Self::build_y_profile) runs. /// - /// Loads the stored **observations** and interprets them against the **current** Y tree polarity - /// on every read — so a corrected/updated tree (or a provider switch) flips the derived/ancestral - /// states with no rebuild and no BAM re-read. Legacy profiles are normalized to observations on - /// load; those persisted before bases were stored show no-calls until one rebuild. + /// It loads the stored **observations** and interprets them against the **current** Y tree + /// polarity on every read. So a corrected tree, or a change of provider, flips the derived and + /// ancestral states with no rebuild and no second BAM read. The load normalizes a legacy + /// profile to observations. A profile that the app stored before it kept bases shows no-calls + /// until one rebuild. pub async fn cached_y_profile(&self, biosample_guid: SampleGuid) -> Result, AppError> { match self.load_observed_profile(biosample_guid, DnaType::Y).await? { Some(observed) => Ok(Some(self.interpret_y_profile(observed).await)), @@ -817,15 +849,19 @@ impl App { } } - /// Build (and persist) the multi-source Y-variant profile: reconcile each Y-bearing source's - /// per-SNP calls — every alignment's haplogroup placement, the combined chip/BISDNA placement, - /// and the private-Y bucket — into one concordance view (confirmed / novel / conflict / - /// single-source per SNP, with per-source provenance + per-observation quality weighting). - /// Expensive (re-genotypes each alignment), so it's an explicit action; the result is persisted - /// so [`cached_y_profile`](Self::cached_y_profile) reloads it instantly. Sources without Y data - /// are skipped. + /// Build the multi-source Y-variant profile, and store it. + /// + /// It reconciles the SNP calls of each source that has Y data into one concordance view. Those + /// sources are every alignment's haplogroup placement, the combined chip/BISDNA placement, and + /// the private-Y bucket. Each SNP gets a status of confirmed, novel, conflict, or + /// single-source, with the provenance of each source and a quality weight for each + /// observation. + /// + /// It costs a lot, because it genotypes each alignment again, so it is an explicit action. The + /// store keeps the result, and [`cached_y_profile`](Self::cached_y_profile) reloads it + /// immediately. It skips a source with no Y data. pub async fn build_y_profile(&self, biosample_guid: SampleGuid) -> Result { - // Females have no Y chromosome — don't build or persist a Y variant profile for them. + // A female has no Y chromosome. Do not build or store a Y variant profile for her. if !self.subject_has_y_dna(biosample_guid).await? { return Ok(YProfile { variants: Vec::new(), @@ -837,18 +873,20 @@ impl App { let mut sources: Vec<(String, SourceType, Vec)> = Vec::new(); - // WGS / Y-NGS evidence: the **genome-consensus** deep placement — every alignment's chrY - // calls pooled on ONE tree+coordinate space and placed once ([`place_y_consensus`]). Its - // root→terminal lineage is the SNP set the sample carries all the way down to the deep - // terminal, so the descent report renders a populated backbone. + // WGS / Y-NGS evidence: the **genome-consensus** deep placement. Every alignment's chrY + // calls pool on ONE tree and coordinate space, and place once ([`place_y_consensus`]). The + // root→terminal lineage is the SNP set that the sample carries all the way down to the + // deep terminal. So the descent report draws a full backbone. // - // Previously this looped each alignment through `y_assignment_full`, whose *per-alignment* - // placement is shallow on lifted CHM13 Big Y data (it stops a few clades down): the profile - // then carried only the root→shallow-terminal SNPs while the profile's terminal came from - // the deeper pooled consensus — so the descent walked terminal→root over SNPs the profile - // never recorded, rendering every node below the shallow terminal as no-call (the "all - // no-call below F / reversed SNPs" bug). Pooling first keeps the variants and the terminal - // on the same deep placement. Genotypes are cached, so this reuses the Y walk already paid. + // The old code looped each alignment through `y_assignment_full`. That placement, for one + // alignment, is shallow on lifted CHM13 Big Y data, and stops a few clades down. The + // profile then carried only the root→shallow-terminal SNPs, while its terminal came from + // the deeper pooled consensus. So the descent walked terminal→root over SNPs that the + // profile never recorded, and drew every node below the shallow terminal as a no-call. + // That was the "all no-call below F / reversed SNPs" bug. + // + // To pool first keeps the variants and the terminal on the same deep placement. The cache + // holds the genotypes, so this reuses the Y walk that the app already paid for. let consensus_assignment = self.place_y_consensus(biosample_guid).await?; if let Some(asg) = &consensus_assignment { let obs = snp_obs_from_assignment(asg, true); @@ -857,9 +895,10 @@ impl App { } } - // One source *per chip/BISDNA panel* (a distinct VariantSet per import — 23andMe, - // AncestryDNA, BISDNA chromo2, …), so the profile shows which test confirmed each SNP and a - // single mistyped panel surfaces as a conflict rather than being averaged into "consumer tests". + // One source for *each chip or BISDNA panel*. Each import gives a distinct VariantSet: + // 23andMe, AncestryDNA, BISDNA chromo2, and so on. So the profile shows which test + // confirmed each SNP, and one mistyped panel shows as a conflict. An average over + // "consumer tests" would hide it. let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; let chip_sets: Vec<&VariantSet> = vsets.iter().filter(|s| s.source_type == SourceType::Chip).collect(); if !chip_sets.is_empty() { @@ -889,10 +928,11 @@ impl App { } } - // One source *per vendor Y-NGS VCF* (FTDNA Big Y / YSEQ / Full Genomes / Nebula / Dante — - // every non-chip VariantSet with chrY calls). Placed against each set's stored build so a - // GRCh38 Big Y reconciles alongside any WGS alignment; tagged with the set's real source - // type (TargetedNgs / WgsShortRead / …) so it carries the right concordance weight. + // One source for *each vendor Y-NGS VCF*: FTDNA Big Y, YSEQ, Full Genomes, Nebula, Dante, + // and every other non-chip VariantSet with chrY calls. Each one places against its own + // stored build, so a GRCh38 Big Y reconciles beside any WGS alignment. Each carries the + // set's real source type (TargetedNgs, WgsShortRead, …), which gives it the correct + // concordance weight. let ngs_sets: Vec<&VariantSet> = vsets.iter().filter(|s| s.source_type != SourceType::Chip).collect(); if !ngs_sets.is_empty() { let mut tree_cache: HashMap = HashMap::new(); @@ -929,8 +969,9 @@ impl App { PrivateClass::OffPathKnown(n) => n.clone(), PrivateClass::Novel => String::new(), // keyed by position }; - // Carry the observed base (= the called alt) so interpret re-derives Derived from - // the call's own ref/alt — no baked state, consistent with every other source. + // Carry the observed base, which is the called alt, so that interpret derives + // Derived again from the call's own ref/alt. The code stores no state here, + // and this matches every other source. let mut o = YObsInput::observed( name, v.position, @@ -939,8 +980,9 @@ impl App { Some(v.alternate), false, ); - // De-novo calls carry read depth; a structural-region (palindrome/amplicon) call - // is paralog-suspect → down-weight via the region modifier. + // A de-novo call carries a read depth. A call in a structural region, a + // palindrome or an amplicon, can be a paralog, so the region modifier + // down-weights it. o.depth = Some(v.depth); // Down-weight by the structural-region quality modifier (palindrome 0.4, // ampliconic 0.3, heterochromatin/centromere 0.1…); unique sequence = 1.0. @@ -953,13 +995,14 @@ impl App { } } - // Group the sources into an observation-only snapshot — no baked state. State/status are - // interpreted against the current tree on read (`interpret_y_profile`), so a corrected tree - // (incl. an FTDNA tree whose reference-as-ancestral polarity is inverted at some sites) flips - // the display without a rebuild. + // Group the sources into a snapshot of observations only, and store no state. The read + // path interprets the state and the status against the current tree + // (`interpret_y_profile`). So a corrected tree flips the display with no rebuild. This + // also covers an FTDNA tree that inverts its reference-as-ancestral polarity at some sites. let mut observed = yprofile::to_observed(&sources); - // Genome-level placement: the pooled call set placed once (computed above — not a vote among - // the per-run terminal labels). Falls back to the label reconciliation when nothing places. + // Genome-level placement: the pooled call set, placed once. The code above computes it, + // and it is not a vote among the terminal labels of each run. It falls back to the label + // reconcile when nothing places. observed.terminal_hint = match &consensus_assignment { Some(a) => a.ranked.first().map(|r| r.name.clone()), None => self @@ -979,9 +1022,10 @@ impl App { Ok(profile) } - /// The mtDNA consensus profile for a subject, if built — cheap (no genotyping). Loads stored - /// observations and interprets them against the current rCRS tree polarity on every read (the - /// mtDNA half of the observation-first fix — previously mt states could not re-interpret at all). + /// The mtDNA consensus profile for a subject, if the app has built one. It is low-cost, with no + /// genotyping. It loads the stored observations and interprets them against the current rCRS + /// tree polarity on every read. This is the mtDNA half of the observation-first fix. Before it, + /// an mt state could not take a new interpretation at all. pub async fn cached_mt_profile(&self, biosample_guid: SampleGuid) -> Result, AppError> { match self.load_observed_profile(biosample_guid, DnaType::Mt).await? { Some(observed) => Ok(Some(self.interpret_mt_profile(observed).await)), @@ -989,24 +1033,31 @@ impl App { } } - /// Build (and persist) the multi-source mtDNA consensus profile — the mtDNA adapter over the - /// generic [`navigator_domain::consensus`] engine. Reconciles each mt-bearing source's - /// defining-mutation calls (every alignment's chrM placement, each imported mtDNA FASTA - /// sequence's placement, and the combined chip mtDNA placement) into one concordance view, - /// keyed by phylotree **mutation name** (rCRS-coordinate, build-independent). Persisted with - /// `dna_type='Mt'` so [`cached_mt_profile`](Self::cached_mt_profile) reloads it instantly. - /// Expensive (re-places each alignment's chrM), so it's an explicit action; mt-less sources skip. + /// Build the multi-source mtDNA consensus profile, and store it. This is the mtDNA adapter + /// over the generic [`navigator_domain::consensus`] engine. + /// + /// It reconciles the mutation calls of each source that has mt data into one concordance view. + /// Those sources are every alignment's chrM placement, the placement of each imported mtDNA + /// FASTA sequence, and the combined chip mtDNA placement. The key is the phylotree **mutation + /// name**, which is an rCRS coordinate and is independent of build. + /// + /// The store keeps it with `dna_type='Mt'`, so + /// [`cached_mt_profile`](Self::cached_mt_profile) reloads it immediately. It costs a lot, + /// because it places each alignment's chrM again, so it is an explicit action. It skips a + /// source with no mt data. pub async fn build_mt_profile(&self, biosample_guid: SampleGuid) -> Result { - // One mt tree in rCRS coordinates (DecodingUs remapped from hs1, FTDNA fallback), shared by - // the per-source placements below and the pooled terminal — so the variants and the terminal - // sit on the same tree + coordinate space (the Y-profile fix, applied to mtDNA). + // One mt tree in rCRS coordinates. DecodingUs remaps it from hs1, with FTDNA as the + // fallback. The placement of each source below shares it with the pooled terminal. So the + // variants and the terminal sit on the same tree and coordinate space. This is the + // Y-profile fix, applied to mtDNA. let (tree, provider) = self.mt_tree_rcrs().await?; let source_calls = self.mt_source_calls(biosample_guid, &tree).await?; - // One source per contributing test — each alignment's chrM, each imported FASTA, the chip mt - // panel — placed individually on the shared tree so the profile shows which test confirmed - // each mutation (name-keyed reconcile across sources). Sparse sources (chip) use the robust - // assembler; dense ones (WGS/FASTA) the exact one. + // One source for each test that contributes: each alignment's chrM, each imported FASTA, + // and the chip mt panel. Each places on its own on the shared tree, so the profile shows + // which test confirmed each mutation. The reconcile across sources uses the name as its + // key. A sparse source, such as a chip, uses the robust assembler. A dense one, such as WGS + // or FASTA, uses the exact assembler. let mut sources: Vec<(String, SourceType, Vec)> = Vec::new(); for (label, st, calls) in &source_calls { let assignment = if *st == SourceType::Chip { @@ -1024,12 +1075,13 @@ impl App { let mut observed = yprofile::to_observed(&sources); // Interpret once (against the current mt polarity) for the return value + summary header. let mut profile = self.interpret_mt_profile(observed.clone()).await; - // Genome-level placement of the pooled chrM call set on the same tree. A subject with no - // derived mutations carries no real placement — an alignment with a handful of off-target - // chrM reads (a Big Y) genotypes to nothing below the root. Report no terminal rather than a - // root label, and never resurrect a stale persisted root call (the old "very few mt reads → - // RSRS" artifact); the profile is meaningful only once some mutation is derived (checked on - // the *interpreted* variants). + // Genome-level placement of the pooled chrM call set on the same tree. + // + // A subject with no derived mutations has no real placement. An alignment with a few + // off-target chrM reads, such as a Big Y, genotypes to nothing below the root. Report no + // terminal, and do not report a root label. Never bring back a stale root call from the + // store, which was the old "very few mt reads → RSRS" artifact. The profile says nothing + // until some mutation goes derived, and the check runs on the *interpreted* variants. let terminal = if profile .variants .iter() @@ -1051,7 +1103,7 @@ impl App { observed.terminal_hint = terminal.clone(); profile.terminal = terminal; - // Persist observations (keyed dna_type='Mt') with the tree provider actually used. + // Store the observations, keyed dna_type='Mt', with the tree provider that the code used. self.persist_observed_profile( biosample_guid, DnaType::Mt, @@ -1063,31 +1115,37 @@ impl App { Ok(profile) } - /// **Genome-level Y placement**: pool every source's tree-locus genotype (each alignment's - /// native-build placement calls + each chip/BISDNA panel's chrY calls) into one call set by a - /// weighted [`pool_bases`] vote — keyed by SNP **name** so sources on different builds merge — - /// then place that pooled set on one canonical tree **once** via [`assemble_assignment`]. This - /// replaces voting among the per-run terminal *labels*: a sparse run no longer drags the call - /// shallow, and a branch confirmed by any source informs the placement. `Ok(None)` when the - /// subject has no Y-bearing source. Re-genotypes each source (like [`build_y_profile`]), so it's - /// only run as part of that explicit action. + /// **Genome-level Y placement**. + /// + /// Pool the tree-locus genotype of every source into one call set, by a weighted + /// [`pool_bases`] vote. Those genotypes are each alignment's native-build placement calls and + /// each chip or BISDNA panel's chrY calls. The key is the SNP **name**, so sources on different + /// builds merge. Then place that pooled set on one canonical tree **once**, through + /// [`assemble_assignment`]. + /// + /// This replaces a vote among the terminal *labels* of each run. A sparse run no longer pulls + /// the call shallow, and a branch that any source confirms informs the placement. + /// + /// `Ok(None)` when the subject has no source with Y data. It genotypes each source again, as + /// [`build_y_profile`] does, so it runs only as part of that explicit action. pub async fn place_y_consensus(&self, biosample_guid: SampleGuid) -> Result, AppError> { - // Females have no Y chromosome — no genome consensus to place. + // A female has no Y chromosome, so there is no genome consensus to place. if !self.subject_has_y_dna(biosample_guid).await? { return Ok(None); } - // The consensus follows the user's configured tree provider (Preferences / - // NAVIGATOR_Y_TREE_PROVIDER), same as the per-alignment placement. + // The consensus follows the user's configured tree provider (Preferences or + // NAVIGATOR_Y_TREE_PROVIDER), the same as the placement of one alignment. match y_tree_provider() { YTreeProvider::DecodingUs => self.place_y_consensus_decodingus(biosample_guid).await, YTreeProvider::Ftdna => self.place_y_consensus_ftdna(biosample_guid).await, } } - /// FTDNA-provider genome consensus: pool every WGS alignment + GRCh38 vendor Y-VCF on the FTDNA - /// GRCh38 tree (`base_calls` lifts CHM13/GRCh37 sources into GRCh38) and place once. One tree + - /// one coordinate space keeps polarity/coverage consistent; chips (sparse, various builds) stay in - /// the variant profile's name-keyed reconcile. + /// FTDNA-provider genome consensus. Pool every WGS alignment and GRCh38 vendor Y-VCF on the + /// FTDNA GRCh38 tree, and place once. `base_calls` lifts a CHM13 or GRCh37 source into GRCh38. + /// One tree and one coordinate space keep the polarity and the coverage consistent. A chip is + /// sparse and can be on any build, so it stays in the name-keyed reconcile of the variant + /// profile. async fn place_y_consensus_ftdna(&self, biosample_guid: SampleGuid) -> Result, AppError> { let tree_json = self.fetch_ftdna_y_tree().await?; let tree = navigator_analysis::haplo::parse_ftdna_json(&tree_json).map_err(AppError::Import)?; @@ -1095,9 +1153,10 @@ impl App { let mut sources: Vec<(SourceType, HashMap)> = Vec::new(); let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; for a in &alignments { - // Lifted GRCh38-coordinate calls; sources lacking chrY / a reference are skipped. A - // preferred-external alignment is genotyped from its chrY GVCF (lifted native→GRCh38 by - // `gvcf_base_calls`) instead of walking the CRAM. + // Lifted GRCh38-coordinate calls. The code skips a source with no chrY or no + // reference. It genotypes a preferred-external alignment from its chrY GVCF, which + // `gvcf_base_calls` lifts from the native build to GRCh38, and it does not walk the + // CRAM. let calls = match (prefer_external_calls(), chr_y_gvcf_for_alignment(a)) { (true, Some(gvcf)) => self .gvcf_base_calls(a.id, "chrY", &gvcf, &tree, tree_build_for_contig("chrY")) @@ -1114,7 +1173,7 @@ impl App { sources.push((SourceType::WgsShortRead, calls)); } } - // Dense GRCh38 vendor Y-NGS VCFs pool alongside the WGS; non-GRCh38 sets wouldn't match. + // Dense GRCh38 vendor Y-NGS VCFs pool alongside the WGS; non-GRCh38 sets would not match. let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; for set in &vsets { if set.source_type == SourceType::Chip || !is_grch38_build(&set.reference_build) { @@ -1132,27 +1191,32 @@ impl App { Ok(Some(assemble_assignment(&tree, &pooled))) } - /// DecodingUs-provider genome consensus (the default): genotype every WGS alignment against the - /// DecodingUs Y tree in each source's *native* build, group by build, pool by position, and place - /// on the build carrying the most evidence. + /// DecodingUs-provider genome consensus, which is the default. Genotype every WGS alignment + /// against the DecodingUs Y tree in each source's *native* build. Group by build, pool by + /// position, and place on the build with the most evidence. async fn place_y_consensus_decodingus( &self, biosample_guid: SampleGuid, ) -> Result, AppError> { - // Genotype every WGS alignment against the **DecodingUs** Y tree — the workspace's configured - // provider, served from the local cache — in each source's *native* build (`hs1` for CHM13, - // `GRCh38`, `GRCh37`). No liftover and no FTDNA dependency: the per-alignment genotype is - // exactly the one the Y assignment already cached, so this reuses that walk rather than paying - // a second, FTDNA-coordinate one. Sources are grouped by build and pooled by **position** - // within one coordinate space (a single build per subject is the norm); the build carrying the - // most evidence is placed once. Pooling across builds by position would mix coordinate systems - // — cross-build merging lives in the variant profile's name-keyed reconcile, not here. + // Genotype every WGS alignment against the **DecodingUs** Y tree, which is the workspace's + // configured provider, served from the local cache. Use each source's *native* build: + // `hs1` for CHM13, `GRCh38`, or `GRCh37`. There is no liftover and no FTDNA dependency, + // because the genotype of one alignment is exactly the one that the Y assignment already + // cached. So this reuses that walk, and does not pay for a second walk in FTDNA + // coordinates. + // + // The code groups the sources by build, and pools them by **position** inside one + // coordinate space. One build for each subject is the norm. It then places the build with + // the most evidence, once. A pool across builds by position would mix coordinate systems. + // The merge across builds lives in the name-keyed reconcile of the variant profile, and + // not here. let tree_json = self.fetch_decodingus_y_tree().await?; let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; - // Parse the DecodingUs tree once per distinct build the sources use (cheap — the JSON is - // memoized). Built up front so the async genotyping loop holds only shared borrows of `trees`. + // Parse the DecodingUs tree once for each distinct build that the sources use. This is + // low-cost, because a memo holds the JSON. The code builds them first, so the async + // genotyping loop holds only shared borrows of `trees`. let mut builds: std::collections::HashSet<&'static str> = alignments .iter() .filter_map(|a| decodingus_build_key(&a.reference_build)) @@ -1182,8 +1246,9 @@ impl App { continue; }; let Some(tree) = trees.get(bk) else { continue }; - // Native build → no liftover; the cache-key matches the Y assignment's, so a CRAM walk is - // a hit — but a preferred-external alignment is genotyped from its GVCF instead (no decode). + // The build is native, so there is no liftover. The cache key matches the one of the Y + // assignment, so a CRAM walk is a hit. But the code genotypes a preferred-external + // alignment from its GVCF instead, with no decode. let Ok(calls) = self.consensus_base_calls(a, "chrY", tree, None).await else { continue; }; @@ -1192,9 +1257,9 @@ impl App { } } - // Vendor Y-NGS VCFs (FTDNA Big Y / YSEQ / Full Genomes / Nebula) are dense direct Y-SNP calls; - // fold each into its own build's group (strand-reconciled to that build's tree). Chips stay in - // the variant profile's name-keyed reconcile. + // A vendor Y-NGS VCF (FTDNA Big Y, YSEQ, Full Genomes, Nebula) holds dense, direct Y-SNP + // calls. Fold each one into the group of its own build, strand-reconciled to that build's + // tree. A chip stays in the name-keyed reconcile of the variant profile. for set in &vsets { if set.source_type == SourceType::Chip { continue; @@ -1216,7 +1281,8 @@ impl App { } } - // Place on the build carrying the most evidence (the subject's primary coordinate space). + // Place on the build with the most evidence, which is the subject's primary coordinate + // space. let Some(bk) = by_build .iter() .max_by_key(|(_, s)| s.iter().map(|(_, c)| c.len()).sum::()) @@ -1228,11 +1294,15 @@ impl App { Ok(Some(assemble_assignment(&trees[bk], &pooled))) } - /// Diagnostic: dump the Y **descent** for one subject SNP-by-SNP — the reported state + observed - /// base against the **incoming tree's** polarity in every DecodingUs build (hs1 / GRCh38 / GRCh37). - /// This is the "compare the tree vs the calls" log: a backbone SNP the sample must carry that reads - /// ancestral shows here as `state=Ancestral base=` with a build whose polarity is - /// flipped (`hs1: A>G GRCh38: G>A`), pinpointing a tree-polarity problem vs a genotyping one. + /// Diagnostic: dump the Y **descent** for one subject, SNP by SNP. It shows the reported state + /// and the observed base against the polarity of the **new tree** in every DecodingUs build + /// (hs1, GRCh38, GRCh37). + /// + /// This is the "compare the tree against the calls" log. Take a backbone SNP that the sample + /// must carry, but that reads ancestral. Its line shows `state=Ancestral base=`. + /// Beside that line, one build shows a flipped polarity (`hs1: A>G GRCh38: G>A`). That + /// separates a problem in the tree polarity from a problem in the genotyping. + /// /// Read-only. TSV: `node snp pos state base hs1 GRCh38 GRCh37`. pub async fn debug_y_descent(&self, biosample_guid: SampleGuid) -> Result { use navigator_analysis::haplo; @@ -1284,13 +1354,16 @@ impl App { Ok(out) } - /// Diagnostic: genotype a **single alignment** against the DecodingUs Y tree in its native build - /// and dump, per SNP down the placed lineage, the raw read pileup **behind** each call — the - /// reference base, the A/C/G/T passing-read tally, the consensus base, the tree's ancestral/derived - /// alleles, and the resulting state. This is the "calls generated" log: it shows whether a backbone - /// SNP that reads ancestral is (a) genuinely ancestral in the reads, (b) a coordinate/position - /// mismatch (reads a different base than the tree allele), or (c) a low-depth artifact. Read-only. - /// TSV: `node snp pos tree(anc>der) ref A C G T depth called state`. + /// Diagnostic: genotype a **single alignment** against the DecodingUs Y tree in its native + /// build. For each SNP down the placed lineage, dump the raw read pileup **behind** the call. + /// The dump holds the reference base, the A/C/G/T tally of the reads that pass, the consensus + /// base, and the two tree alleles. + /// + /// This is the "calls the code made" log. Take a backbone SNP that reads ancestral. This says + /// whether it is truly ancestral in the reads, or a coordinate mismatch, or an artifact of low + /// depth. A coordinate mismatch gives a base other than the tree allele. + /// + /// Read-only. TSV: `node snp pos tree(anc>der) ref A C G T depth called state`. pub async fn debug_y_calls(&self, alignment_id: i64) -> Result { use navigator_analysis::{caller, haplo, reader}; let aln = self.alignment_or_err(alignment_id).await?; @@ -1302,16 +1375,17 @@ impl App { }; let json = self.fetch_decodingus_y_tree().await?; let tree = haplo::parse_decodingus_json(&json, bk).map_err(AppError::Import)?; - // Native-build genotyping (no liftover) — the same walk place_y_consensus uses; the cache hit - // means `base_calls` returns the identical winning bases we're auditing here. + // Genotype in the native build, with no liftover. This is the same walk that + // place_y_consensus uses. The cache hits, so `base_calls` gives back exactly the bases that + // won, which are the bases under audit here. let calls = self.base_calls(alignment_id, "chrY", &tree, None).await?; let assignment = assemble_assignment(&tree, &calls); if assignment.lineage.is_empty() { return Ok(format!("alignment {alignment_id} ({bk}): no Y placement\n")); } - // Localize the BAM/CRAM and resolve its reference exactly as `base_calls` does, then tally the - // raw reads at the lineage positions and read the reference base there. + // Localize the BAM/CRAM and resolve its reference exactly as `base_calls` does. Then tally + // the raw reads at the lineage positions, and read the reference base there. let bam = self .localize(Path::new( &aln.bam_path.clone().ok_or(AppError::MissingPaths(alignment_id))?, @@ -1339,7 +1413,8 @@ impl App { caller::tally_at(&bam2, &contig2, &targets2, ¶ms, ref2.as_deref()) }) .await??; - // Reference base per lineage position (0-based index into the contig), best-effort. + // The reference base at each lineage position (0-based index into the contig), + // best-effort. let refseq: Option> = match reference.as_deref() { Some(r) => { let (r, c) = (r.to_path_buf(), resolved.clone()); @@ -1390,9 +1465,10 @@ impl App { Ok(out) } - /// Pick a single alignment to target for [`Self::debug_y_calls`] when only a subject is given: - /// prefer a CHM13/HiFi alignment (native tree, no liftover — the cleanest to audit), else the - /// first. Returns `None` when the subject has no alignments. + /// Pick a single alignment for [`Self::debug_y_calls`] when the caller gives only a subject. + /// Prefer a CHM13/HiFi alignment, which uses the native tree with no liftover and is the + /// easiest to audit. If there is none, take the first. Returns `None` when the subject has no + /// alignments. pub async fn pick_y_debug_alignment(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let pick = alignments @@ -1410,10 +1486,10 @@ impl App { Ok(pick.map(|a| a.id)) } - /// Pick a single alignment to genotype **mtDNA** against: skip Y-only runs (an FTDNA Big-Y - /// carries no `chrM` reads, so it would yield an all-no-call report while a usable WGS - /// alignment sat unselected), then prefer CHM13, else the first survivor. If *every* run is - /// Y-only, fall back to the first alignment rather than reporting "no alignment". + /// Pick a single alignment to genotype **mtDNA** against. Skip a Y-only run: an FTDNA Big-Y + /// carries no `chrM` reads, so it would give an all-no-call report while a usable WGS alignment + /// sat unpicked. Then prefer CHM13, else take the first one left. If *every* run is Y-only, + /// fall back to the first alignment, and do not report "no alignment". pub async fn pick_mt_alignment(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut mt_capable: Vec<&Alignment> = Vec::new(); @@ -1447,10 +1523,12 @@ impl App { } } - /// Diagnostic: trace the DecodingUs genome-consensus Y placement for one subject — the pooled - /// build, the Kulczynski top candidates + admissibility, the assembled terminal, a strict - /// root→tip descent, and the per-node derived/ancestral tally down the assembled lineage (so an - /// over-deepening tunnel through ancestral branches is visible). Read-only. + /// Diagnostic: trace the DecodingUs genome-consensus Y placement for one subject. + /// + /// It shows the pooled build, the top Kulczynski candidates with their admissibility, and the + /// assembled terminal. It then shows a strict root→tip descent, and the derived and ancestral + /// tally at each node down the assembled lineage. That last part makes a tunnel through + /// ancestral branches visible. Read-only. pub async fn debug_y_placement(&self, biosample_guid: SampleGuid) -> Result { use navigator_analysis::haplo; let tree_json = self.fetch_decodingus_y_tree().await?; @@ -1559,10 +1637,15 @@ impl App { Ok(out) } - /// Assemble a subject's lightweight [`YMatchProfile`] from **cached** data only (no re-genotyping): - /// the persisted consensus Y profile (derived/novel SNP-name sets + terminal), the terminal's - /// root→tip lineage from `tree`, and the first imported Y-STR panel's markers. `Ok(None)` when the - /// subject has neither a placed Y profile nor an STR panel — nothing to match on. + /// Assemble a subject's lightweight [`YMatchProfile`] from **cached** data only, with no + /// second genotype pass. + /// + /// It takes three things. The first is the stored consensus Y profile, which holds the derived + /// and novel SNP-name sets and the terminal. The second is the terminal's root→tip lineage from + /// `tree`. The third is the markers of the first imported Y-STR panel. + /// + /// `Ok(None)` when the subject has no placed Y profile and no STR panel, because then there is + /// nothing to match on. async fn y_match_profile( &self, b: &Biosample, @@ -1620,14 +1703,17 @@ impl App { })) } - /// Rank every other workspace subject against `query_guid` by Y relatedness (gap §2) — shared - /// derived/novel SNPs, divergence haplogroup, Y-STR genetic distance, and rough SNP/STR TMRCA. - /// One-vs-all over the workspace (or one project when `project_id` is set); local-only. Consumes - /// **cached** profiles so it's cheap over hundreds of subjects (no re-genotyping). `Ok(vec![])` - /// when the query subject has no matchable Y data. + /// Rank every other workspace subject against `query_guid` by Y relatedness (gap §2). The + /// signals are the shared derived and novel SNPs, the divergence haplogroup, the Y-STR genetic + /// distance, and a rough SNP and STR TMRCA. + /// + /// It compares one subject against all, over the workspace, or over one project when + /// `project_id` has a value. It is local only. It reads **cached** profiles, with no second + /// genotype pass, so it is low-cost over hundreds of subjects. `Ok(vec![])` when the query + /// subject has no Y data to match on. pub async fn y_matches(&self, query_guid: SampleGuid, project_id: Option) -> Result, AppError> { - // The tree only supplies the divergence haplogroup; shared-SNP and STR matching work without - // it, so a fetch failure degrades gracefully rather than failing the whole search. + // The tree supplies the divergence haplogroup alone. The shared-SNP and STR match work + // without it, so a failed fetch loses only that one field, and the search continues. let tree = match self.fetch_ftdna_y_tree().await { Ok(json) => navigator_analysis::haplo::parse_ftdna_json(&json).ok(), Err(_) => None, @@ -1638,7 +1724,7 @@ impl App { None => self.list_all_biosamples().await?, }; - // The query subject may sit outside the chosen project — load it directly if so. + // The query subject can sit outside the chosen project. Load it directly if it does. let query_bio = match candidates.iter().find(|b| b.guid == query_guid).cloned() { Some(b) => b, None => biosample::get(self.store.pool(), query_guid) @@ -1663,12 +1749,14 @@ impl App { Ok(ranked) } - /// Per-source rCRS-coordinate mtDNA calls for a subject — `(label, type, calls)` keyed by rCRS - /// position — shared by [`place_mt_consensus`] (pooled placement) and [`build_mt_profile`] - /// (per-source concordance). `tree` must be in rCRS coordinates: each alignment's `chrM` is - /// genotyped against it (cached; `base_calls` maps a CHM13 `chrM` back to rCRS, GRCh38/rCRS - /// direct), each imported FASTA is sampled at every rCRS position, and the chip mt panel is - /// strand-reconciled to it. + /// The rCRS-coordinate mtDNA calls of each source for a subject, as `(label, type, calls)` + /// keyed by rCRS position. [`place_mt_consensus`] uses it for the pooled placement, and + /// [`build_mt_profile`] for the concordance across sources. + /// + /// `tree` must be in rCRS coordinates. Each alignment's `chrM` genotypes against it, from the + /// cache, and `base_calls` maps a CHM13 `chrM` back to rCRS, while GRCh38 and rCRS are direct. + /// Each imported FASTA samples at every rCRS position. The chip mt panel strand-reconciles to + /// it. async fn mt_source_calls( &self, biosample_guid: SampleGuid, @@ -1676,8 +1764,8 @@ impl App { ) -> Result)>, AppError> { let mut sources: Vec<(String, SourceType, HashMap)> = Vec::new(); - // Each alignment's chrM genotype. `None` source-build → rCRS-direct / CHM13-chrM lift. A - // preferred-external alignment is genotyped from its chrM GVCF instead of the CRAM. + // Each alignment's chrM genotype. A `None` source-build means rCRS-direct or a CHM13-chrM + // lift. The code takes a preferred-external alignment from its chrM GVCF, not the CRAM. let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; for a in &alignments { let Ok(calls) = self.consensus_base_calls(a, "chrM", tree, None).await else { @@ -1692,7 +1780,7 @@ impl App { } } - // Each imported mtDNA FASTA — the full sequence sampled at every rCRS position. + // Each imported mtDNA FASTA: the full sequence, sampled at every rCRS position. for s in &self.list_mtdna_sequences(biosample_guid).await? { let Some(seq) = mtdna_store::get(self.store.pool(), s.id).await? else { continue; @@ -1714,10 +1802,11 @@ impl App { let sets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; - // Each imported **non-chip** variant set carrying chrM SNPs — a whole-genome VCF or a - // CompleteGenomics masterVar. These report forward-strand ref/alt on rCRS coordinates - // (GRCh37/GRCh38 chrM = rCRS), so the alt base is used raw like an alignment's chrM call - // (no TOP-strand reconciliation, unlike the chip panel below). + // Each imported **non-chip** variant set that holds chrM SNPs: a whole-genome VCF, or a + // CompleteGenomics masterVar. Such a set gives forward-strand ref/alt on rCRS coordinates, + // because GRCh37 and GRCh38 chrM are rCRS. So the code takes the alt base raw, as it does + // for an alignment's chrM call. There is no TOP-strand reconcile, unlike the chip panel + // below. for set in sets.iter().filter(|s| s.source_type != SourceType::Chip) { let calls: HashMap = set .calls @@ -1759,12 +1848,15 @@ impl App { Ok(sources) } - /// **Genome-level mtDNA placement**: the mt counterpart to [`place_y_consensus`]. Pools every - /// source's rCRS-coordinate genotype ([`mt_source_calls`]) by [`pool_votes`] vote keyed by - /// **position** (rCRS is the only mt coordinate system → no name indirection), then places the - /// pooled set on the mt tree once. The tree is the **DecodingUs** mt tree (the configured - /// provider) remapped onto rCRS, with the FTDNA mt tree as fallback ([`mt_tree_rcrs`]). `Ok(None)` - /// when the subject has no mt-bearing source. + /// **Genome-level mtDNA placement**: the mt counterpart to [`place_y_consensus`]. + /// + /// It pools the rCRS-coordinate genotype of every source ([`mt_source_calls`]) by a + /// [`pool_votes`] vote, keyed by **position**. There is no indirection through a name, because + /// rCRS is the only mt coordinate system. It then places the pooled set on the mt tree once. + /// + /// The tree is the **DecodingUs** mt tree, from the configured provider, remapped onto rCRS. + /// The FTDNA mt tree is the fallback ([`mt_tree_rcrs`]). `Ok(None)` when the subject has no + /// source with mt data. pub async fn place_mt_consensus(&self, biosample_guid: SampleGuid) -> Result, AppError> { let (tree, _provider) = self.mt_tree_rcrs().await?; let sources = self.mt_source_calls(biosample_guid, &tree).await?; @@ -1784,12 +1876,17 @@ impl App { Ok(Some(assignment)) } - /// Build a YFull-style [`DescentReport`] for a subject's Y or mtDNA lineage from the **already - /// persisted** variant profile — no re-genotyping. Reads the cached profile for its terminal + - /// per-SNP states (keyed by build-independent SNP name), then walks the FTDNA tree from the - /// terminal to the root, attaching each node's defining SNPs with the sample's call (`NoCall` for - /// an untested equivalent). `Ok(None)` when the profile isn't built yet or has no terminal — the - /// UI then offers to build it (one expensive, persisted step that also powers the variant tabs). + /// Build a YFull-style [`DescentReport`] for a subject's Y or mtDNA lineage from the variant + /// profile that the store **already holds**. There is no second genotype pass. + /// + /// It reads the cached profile for its terminal and the state of each SNP, keyed by the SNP + /// name, which is independent of build. It then walks the FTDNA tree from the terminal to the + /// root. At each node it attaches the SNPs that define the node, with the sample's call, and + /// gives `NoCall` for an equivalent that no test covered. + /// + /// `Ok(None)` when no build has run yet, or when the profile has no terminal. The UI then + /// offers to build it. That is one costly step, and the store keeps the result, which also + /// feeds the variant tabs. pub async fn descent_report( &self, biosample_guid: SampleGuid, @@ -1797,8 +1894,8 @@ impl App { ) -> Result, AppError> { use navigator_domain::consensus::ConsensusState; - // Cheap first: the persisted profile. No profile / no terminal → nothing to draw, and we - // skip the (multi-MB) tree fetch + parse entirely. + // The low-cost step first: the stored profile. With no profile, or no terminal, there is + // nothing to draw, and the code skips the multi-MB tree fetch and parse. let profile = match dna { DnaType::Y => self.cached_y_profile(biosample_guid).await?, DnaType::Mt => self.cached_mt_profile(biosample_guid).await?, @@ -1808,32 +1905,37 @@ impl App { return Ok(None); }; - // Render on the configured provider's tree so the node names + defining SNPs line up with the - // profile's placement (which followed the same provider). Y: DecodingUs in the subject's - // native build, or the FTDNA GRCh38 tree; mtDNA: DecodingUs remapped hs1→rCRS, or FTDNA — via - // `mt_tree_rcrs`, which already honors the provider. + // Draw on the tree of the configured provider. The node names and their SNPs then line up + // with the profile's placement, which followed the same provider. For Y that is DecodingUs + // in the subject's native build, or the FTDNA GRCh38 tree. For mtDNA it is DecodingUs + // remapped from hs1 to rCRS, or FTDNA, through `mt_tree_rcrs`, which already obeys the + // provider. let tree = match dna { DnaType::Y => match y_tree_provider() { YTreeProvider::DecodingUs => { let json = self.fetch_decodingus_y_tree().await?; // Parse in the tree's **native** hs1 space, not the subject's alignment build. // - // This report joins the profile to the tree by SNP *name* (`state_by_name` - // below); the loci positions it carries are for display and export only. So - // parsing under a narrower build buys nothing and costs loci: a variant with no - // coordinate in the parse build is silently dropped - // (`flatten_du_node`'s `coordinates.get(build_key)?`), and a node whose every - // defining variant is dropped survives as a real node with no SNPs — which the - // renderer then correctly hides as an empty block. + // This report joins the profile to the tree by SNP *name*, through + // `state_by_name` below. The loci positions that it carries are for display and + // export only. // - // That is not hypothetical. hs1 covers 99.8% of the tree's ~204k variants, - // GRCh38 only 86.5%: most DecodingUs-discovered (`DU`-named) SNPs exist in CHM13 - // coordinates alone, since only a few hundred were ever mapped back to the older - // references. `1087` is placed at `R-DU17762`, whose sole defining variant - // `DU17762` has an hs1 coordinate and nothing else — so parsing that subject - // under GRCh38 (which it was, its first alignment being GRCh38) emptied the - // terminal block and the descent visibly stopped one branch short, at - // `R-BY57568`, while the terminal name itself was right. + // So a parse under a narrower build gains nothing and loses loci. A variant + // with no coordinate in the parse build drops out with no message, at + // `flatten_du_node`'s `coordinates.get(build_key)?`. A node that loses every + // one of its variants stays as a real node with no SNPs. The renderer then + // correctly hides it as an empty block. + // + // That is not a theory. hs1 covers 99.8% of the tree's ~204k variants, and + // GRCh38 only 86.5%. Most DecodingUs-discovered (`DU`-named) SNPs exist in + // CHM13 coordinates alone, because only a few hundred ever mapped back to the + // older references. + // + // Subject `1087` places at `R-DU17762`. The one variant that defines that node, + // `DU17762`, has an hs1 coordinate and nothing else. A parse of that subject + // under GRCh38, which is what it got, because its first alignment is GRCh38, + // emptied the terminal block. The descent then stopped one branch short, at + // `R-BY57568`, although the terminal name itself was right. navigator_analysis::haplo::parse_decodingus_json(&json, DECODINGUS_NATIVE_BUILD) .map_err(AppError::Import)? } @@ -1860,7 +1962,8 @@ impl App { (v.name.clone(), state) }) .collect(); - // The actual consensus nucleotide per SNP, so the descent shows/exports the observed allele. + // The consensus nucleotide at each SNP, so the descent shows and exports the observed + // allele. let base_by_name: std::collections::HashMap = profile .variants .iter() @@ -1883,14 +1986,19 @@ impl App { Ok(Some(DescentReport { dna, terminal, nodes })) } - /// Build a [`BranchReport`]: the sample's genotype at every defining marker of `node_query`'s - /// descendant subtree (Y or mtDNA), with per-marker evidence — for spot-checking placement and - /// exchanging observations. `node_query` matches a haplogroup name (`R-FGC29071`) or a defining - /// marker (`FGC29071`); `max_depth` bounds descent (`None` = the whole subtree). + /// Build a [`BranchReport`]. It gives the sample's genotype at every marker that defines a + /// node in the descendant subtree of `node_query`, for Y or mtDNA. Each marker carries its + /// evidence. The report lets you spot-check a placement, and exchange observations. + /// + /// `node_query` matches a haplogroup name (`R-FGC29071`) or a marker name (`FGC29071`). + /// `max_depth` bounds the descent, and `None` means the whole subtree. /// - /// Genotypes the subtree **fresh** over the tree's loci (not the placement profile), so branches - /// the sample is *ancestral* for are reported too. Observed bases + evidence come from a per- - /// sample chrY GVCF sidecar when present (rich DP/AD/GQ, ref blocks), else the pileup caller. + /// It genotypes the subtree **fresh** over the tree's loci, and not from the placement profile. + /// So the report also covers a branch that the sample is *ancestral* for. + /// + /// The observed bases and the evidence come from the sample's own chrY GVCF sidecar when there + /// is one. That sidecar holds DP/AD/GQ and ref blocks. If there is none, they come from the + /// pileup caller. pub async fn branch_report( &self, alignment_id: i64, @@ -1905,9 +2013,10 @@ impl App { // Tree + observed base calls over ALL tree loci (covers off-path descendant branches). let (tree, calls, contig, gvcf) = match dna { DnaType::Y => { - // Probe the sidecar *before* genotyping: with one present the tree comes straight - // from JSON and the calls from the GVCF, so the per-locus pileup walk is skipped - // entirely (the point of the fast path — cf. `assign_y_from_gvcf`). + // Probe the sidecar *before* the genotype step. When one is present, the tree + // comes straight from JSON and the calls come from the GVCF. The code then skips + // the pileup walk at each locus, which is the point of the fast path. Compare + // `assign_y_from_gvcf`. match crate::fastpath::chr_y_gvcf_for_alignment(&aln) { Some(gvcf) => { let build_key = decodingus_build_key(&aln.reference_build).ok_or_else(|| { @@ -1931,8 +2040,8 @@ impl App { DnaType::Mt => { // `mt_tree_rcrs` hands back the *provider* (decodingus/ftdna), not a reference // build. `tree_source_build` must stay `None`: a non-build string there makes - // `lifted_targets` return early, skipping the rCRS↔chrM map a CHM13 alignment - // needs (its chrM is a circular permutation of rCRS). + // `lifted_targets` return early. It would then skip the rCRS↔chrM map that a + // CHM13 alignment needs, because its chrM is a circular permutation of rCRS. let (tree, _provider) = self.mt_tree_rcrs().await?; let calls = self.base_calls(alignment_id, "chrM", &tree, None).await?; (tree, calls, "chrM", None) @@ -1950,7 +2059,7 @@ impl App { let root = tree.nodes.get(&root_id).map(|n| n.name.clone()).unwrap_or_default(); let subtree = navigator_analysis::haplo::subtree_report(&tree, &calls, root_id, max_depth); - // GVCF per-marker evidence (Y with a sidecar): ungated DP/AD/GQ, off-thread. + // GVCF evidence at each marker (Y with a sidecar): DP/AD/GQ with no gate, off-thread. let evidence = match gvcf { Some(gvcf) => { let positions: HashSet = subtree.iter().map(|r| r.snp.position).collect(); @@ -1965,7 +2074,7 @@ impl App { .into_iter() .map(|r| { let pos = r.snp.position; - // Either allele being multi-base (or empty) means this isn't a clean SNV. + // Either allele being multi-base (or empty) means this is not a clean SNV. let is_indel = r.snp.derived.chars().count() != 1 || r.snp.ancestral.chars().count() != 1; let (source, dp, ad, gq) = match evidence.get(&pos).copied() { Some(e) if e.refblock => ("gvcf_refblock", None, None, e.gq), @@ -1973,8 +2082,9 @@ impl App { None if gvcf_backed => ("gvcf", None, None, None), None => ("pileup", None, None, None), }; - // The conditions are orthogonal — an uncalled indel is both — so compose the tags - // rather than report only the first. Picking one let "indel/MNV" hide a no-call. + // The conditions are orthogonal, because an indel with no call is both. So build + // up the tags, and do not report the first alone. One tag let "indel/MNV" hide a + // no-call. let mut tags: Vec<&str> = Vec::new(); if is_indel { tags.push("indel/MNV"); @@ -2030,8 +2140,9 @@ impl App { )) } - /// The persisted autosomal consensus-profile snapshot for a subject, if built — cheap (no - /// genotyping). `None` until [`build_autosomal_profile`](Self::build_autosomal_profile) runs. + /// The stored autosomal consensus-profile snapshot for a subject, if the app has built one. It + /// is low-cost, with no genotyping. `None` until + /// [`build_autosomal_profile`](Self::build_autosomal_profile) runs. pub async fn cached_autosomal_profile( &self, biosample_guid: SampleGuid, @@ -2042,24 +2153,33 @@ impl App { } } - /// Build (and persist) the multi-source **autosomal** consensus profile — the diploid (0/1/2) - /// adapter over the generic [`navigator_domain::consensus`] engine. Genotypes every WGS alignment - /// and imported chip over the canonical CHM13 **IBD panel** ([`ibd_panel_dosages`](Self::ibd_panel_dosages)) - /// and reconciles the per-site dosages into a voted genotype (confirmed where sources agree, - /// conflict where they don't), keyed by rsID. Persisted with `dna_type='Auto'`. Requires the IBD - /// panel asset (built with `panelbuild ibd-panel`); errors if it's missing. + /// Build the multi-source **autosomal** consensus profile, and store it. This is the diploid + /// (0/1/2) adapter over the generic [`navigator_domain::consensus`] engine. + /// + /// It genotypes every WGS alignment and imported chip over the canonical CHM13 **IBD panel** + /// ([`ibd_panel_dosages`](Self::ibd_panel_dosages)). It then reconciles the dosage at each site + /// into a voted genotype, keyed by rsID. A site where the sources agree counts as confirmed, + /// and a site where they disagree counts as a conflict. + /// + /// The store keeps it with `dna_type='Auto'`. It needs the IBD panel asset, which + /// `panelbuild ibd-panel` builds, and it gives an error when that asset is absent. pub async fn build_autosomal_profile(&self, biosample_guid: SampleGuid) -> Result { - // Full build: genotype any alignment whose panel dosages aren't cached yet. + // Full build: genotype any alignment whose panel dosages are not cached yet. self.build_autosomal_profile_inner(biosample_guid, false).await } - /// **Progressive refresh** of the autosomal consensus (progressive-consensus, docs §7.17): reduce - /// over the per-source dosages that are **already available** — every chip / WGS-VCF (which resolve - /// cheaply with no decode) plus any alignment whose panel dosages are *cached* - /// ([`Self::cached_alignment_panel_dosages`]) — **without** decoding an uncached alignment. Cheap - /// and safe to call after every import; alignments get their dosages populated separately by the - /// panel batch-process mode, and the next refresh folds them in. Returns the refreshed profile, or - /// `Ok(None)` when the subject has no available autosomal source yet. + /// **Progressive refresh** of the autosomal consensus (progressive-consensus, docs §7.17). + /// + /// It reduces over the dosages of each source that are **already available**. Those are every + /// chip and WGS-VCF, which resolve at low cost with no decode, plus any alignment whose panel + /// dosages are in the cache ([`Self::cached_alignment_panel_dosages`]). It **never** decodes an + /// alignment that the cache does not hold. + /// + /// It is low-cost and safe to call after every import. The panel batch-process mode fills in + /// the dosages of an alignment separately, and the next refresh folds them in. + /// + /// It returns the refreshed profile, or `Ok(None)` when the subject has no autosomal source + /// available yet. pub async fn refresh_autosomal_consensus( &self, biosample_guid: SampleGuid, @@ -2068,28 +2188,35 @@ impl App { .await .map(Some) .or_else(|e| match e { - // "no source" isn't an error for a refresh — the subject just has nothing cached yet. + // "no source" is not an error for a refresh. The cache holds nothing for this + // subject yet. AppError::Import(_) => Ok(None), other => Err(other), }) } - /// **Panel batch-process mode** (progressive-consensus, docs §7.17): genotype one alignment at - /// the full-1240k IBD panel and **cache** the dosages ([`Self::ibd_panel_dosages`]) — the - /// expensive per-source step (a whole-genome decode) that populates the consensus progressively. - /// Returns the number of panel sites genotyped. **Does not** refresh the consensus — the caller - /// refreshes **once** after a batch (reconciling millions of observations per source is wasted - /// work if repeated per alignment); use [`Self::refresh_autosomal_consensus`] at the batch - /// boundary. If the dosages are already cached this is a cheap read. + /// **Panel batch-process mode** (progressive-consensus, docs §7.17). Genotype one alignment at + /// the full-1240k IBD panel, and **cache** the dosages ([`Self::ibd_panel_dosages`]). That is + /// the costly step for each source, a whole-genome decode, and it fills the consensus one + /// source at a time. It returns the count of panel sites that it genotyped. + /// + /// It **does not** refresh the consensus. The caller refreshes **once** after a batch, with + /// [`Self::refresh_autosomal_consensus`] at the batch boundary. One reconcile handles millions + /// of observations for each source, so to repeat it for each alignment wastes work. + /// + /// If the cache already holds the dosages, this is a low-cost read. pub async fn genotype_panel_for_alignment(&self, alignment_id: i64) -> Result { Ok(self.ibd_panel_dosages(IbdSource::Alignment(alignment_id)).await?.len()) } - /// The subject's best alignment for panel genotyping, by **callable quality**: - /// `genome_territory × pct_10x × (1 − pct_exc_mapq)` — well-mapped, diploid-callable bases. Build- - /// agnostic (the IBD panel re-keys GRCh37/38 as well as CHM13), so it picks the cleanest - /// whole-genome WGS over a deep-but-targeted or too-shallow test. Requires a recorded BAM/CRAM and - /// a cached coverage artifact; `None` when the subject has neither. + /// The subject's best alignment for panel genotyping, by **callable quality**, which is + /// `genome_territory × pct_10x × (1 − pct_exc_mapq)`. That measures the bases that are + /// well-mapped and callable as a diploid. + /// + /// It works on any build, because the IBD panel re-keys GRCh37, GRCh38, and CHM13. So it picks + /// the cleanest whole-genome WGS over a test that is deep but targeted, or one that is too + /// shallow. It needs a recorded BAM/CRAM and a cached coverage artifact, and gives `None` when + /// the subject has neither. async fn best_callable_alignment(&self, biosample_guid: SampleGuid) -> Result, AppError> { let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut best: Option<(f64, i64)> = None; @@ -2108,13 +2235,17 @@ impl App { Ok(best.map(|(_, id)| id)) } - /// **Panel batch-process mode, subject-level** (progressive-consensus, docs §7.17): genotype the - /// subject's single **best-callable** alignment ([`Self::best_callable_alignment`]) at the 1240k - /// panel and refresh the autosomal consensus **once**. Chips and WGS-VCFs need no genotyping — - /// they resolve into the consensus during the refresh — so this pays at most one whole-genome - /// decode per subject (vs one per redundant same-person alignment). Returns - /// `(alignment_id, sites)`, or `None` when the subject has no callable alignment (its chips/VCFs - /// still get folded into the consensus). + /// **Panel batch-process mode, at subject level** (progressive-consensus, docs §7.17). + /// Genotype the subject's single **best-callable** alignment + /// ([`Self::best_callable_alignment`]) at the 1240k panel, and refresh the autosomal consensus + /// **once**. + /// + /// A chip or a WGS-VCF needs no genotyping, because it resolves into the consensus during the + /// refresh. So this pays for one whole-genome decode for each subject at most, and not one for + /// each redundant alignment of the same person. + /// + /// It returns `(alignment_id, sites)`, or `None` when the subject has no callable alignment. + /// The refresh still folds that subject's chips and VCFs into the consensus. pub async fn genotype_panel_for_subject( &self, biosample_guid: SampleGuid, @@ -2125,7 +2256,8 @@ impl App { } else { None }; - // Reconcile once — folds the freshly-cached alignment (if any) plus every chip / WGS-VCF. + // Reconcile once. This folds in the alignment that the code has just cached, if there is + // one, plus every chip and WGS-VCF. let _ = self.refresh_autosomal_consensus(biosample_guid).await?; Ok(picked) } @@ -2152,19 +2284,23 @@ impl App { }; let mut sources: Vec<(String, SourceType, Vec)> = Vec::new(); - // Remember the last source error: if *every* source fails (e.g. the panel asset is missing), - // surface it rather than silently returning an empty profile; a one-off per-source failure - // (a chip with no stored raw file, an alignment lacking a BAM) is just skipped. + // Remember the last source error. If *every* source fails, because the panel asset is + // absent for example, show that error. Do not return an empty profile with no message. + // + // The code skips a failure in one source on its own. That covers a chip with no stored raw + // file, or an alignment with no BAM. let mut last_err: Option = None; - // One source per WGS alignment (panel-genotyped, cached per alignment). The IBD panel carries - // every build's coordinates, so `ibd_panel_dosages` genotypes a CHM13 alignment at its native - // loci and a GRCh37/GRCh38 alignment at that build's loci, re-keying the result to canonical - // CHM13. A build the panel doesn't cover yields no genotypes and is skipped downstream. + // One source for each WGS alignment, genotyped at the panel, with a cache entry for each + // alignment. The IBD panel carries the coordinates of every build. So `ibd_panel_dosages` + // genotypes a CHM13 alignment at its native loci, and a GRCh37 or GRCh38 alignment at that + // build's loci. It then re-keys the result to canonical CHM13. A build that the panel does + // not cover gives no genotypes, and the code downstream skips it. let alignments = alignment::list_for_biosample(self.store.pool(), biosample_guid).await?; for a in &alignments { - // Progressive refresh reduces over cached dosages only — an uncached alignment is skipped - // (its dosages get populated by the panel batch mode), never decoded inline here. + // The progressive refresh reduces over cached dosages only. It skips an alignment that + // the cache does not hold, and never decodes one here. The panel batch mode fills in + // the dosages of such an alignment. let dosages = if cached_alignments_only { self.cached_alignment_panel_dosages(a.id).await? } else { @@ -2184,7 +2320,8 @@ impl App { } } - // One source per imported chip (resolved to canonical panel dosages, no alignment needed). + // One source for each imported chip, resolved to canonical panel dosages, with no + // alignment. let chips = self.list_chip_profiles(biosample_guid).await?; for c in &chips { match self.ibd_panel_dosages(IbdSource::Chip(c.id)).await { @@ -2198,11 +2335,13 @@ impl App { } } - // One source per **genome-wide** imported variant set — a WGS VCF or a CompleteGenomics - // masterVar (no alignment needed; resolved to panel dosages with unlisted sites taken as - // hom-reference). Only `WgsShortRead`/`WgsLongRead`: that hom-ref default is valid solely - // for a source that genotyped the whole genome — a targeted Big Y (`TargetedNgs`) or Sanger - // panel lists only a handful of sites and must NOT imply hom-ref everywhere else. + // One source for each **genome-wide** imported variant set: a WGS VCF, or a + // CompleteGenomics masterVar. Neither needs an alignment. Each resolves to panel dosages, + // and a site that it does not list counts as hom-reference. + // + // Accept only `WgsShortRead` and `WgsLongRead`. That hom-ref default is correct only for a + // source that genotyped the whole genome. A targeted Big Y (`TargetedNgs`), or a Sanger + // panel, lists a few sites, and must NOT mean hom-ref everywhere else. let vsets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; for set in &vsets { if !matches!(set.source_type, SourceType::WgsShortRead | SourceType::WgsLongRead) { @@ -2219,9 +2358,10 @@ impl App { } } - // One source per imported **external autosomal call set** (a trusted 1240K EIGENSTRAT set — - // GATK4 / pileupCaller). Resolved to CHM13 panel dosages at import and stored, so it pools in - // with no CRAM decode (available to both the full build and the progressive refresh). + // One source for each imported **external autosomal call set**, which is a trusted 1240K + // EIGENSTRAT set from GATK4 or pileupCaller. The import resolves it to CHM13 panel dosages + // and stores them. So it pools in with no CRAM decode, and both the full build and the + // progressive refresh can use it. for row in navigator_store::external_panel_dosage::list_for_biosample(self.store.pool(), biosample_guid).await? { match serde_json::from_str::>(&row.dosages) { @@ -2240,7 +2380,7 @@ impl App { return Err(e); // e.g. the IBD panel asset isn't built yet } if cached_alignments_only { - // A progressive refresh with nothing available yet: don't persist an empty consensus + // A progressive refresh with nothing available yet: do not persist an empty consensus // (refresh_autosomal_consensus maps this to Ok(None)). return Err(AppError::Import("no cached autosomal sources yet".into())); } @@ -2276,10 +2416,10 @@ impl App { Ok(profile) } - /// Build the `com.decodingus.atmosphere.haplogroupReconciliation` record JSON for a - /// subject + DNA type from the stored consensus, per-run calls, manual override, and - /// audit log. mtDNA heteroplasmy observations and an optional identity-verification - /// result are passed in (the caller computes them from the relevant alignments). + /// Build the `com.decodingus.atmosphere.haplogroupReconciliation` record JSON for a subject + /// and DNA type. It reads the stored consensus, the calls of each run, the manual override, and + /// the audit log. The caller supplies the mtDNA heteroplasmy observations and an optional + /// identity-verification result, and computes both from the relevant alignments. async fn reconciliation_record( &self, biosample_guid: SampleGuid, @@ -2405,7 +2545,7 @@ impl App { } /// Publish a subject's haplogroup reconciliation record to the signed-in account's PDS - /// (with refresh-on-expiry and retry/backoff via [`AsyncSync`]). + /// [`AsyncSync`] refreshes the token when it expires, and retries with a backoff. pub async fn publish_reconciliation( &self, biosample_guid: SampleGuid, @@ -2428,7 +2568,7 @@ impl App { .await } - /// All recorded per-source calls for a subject + DNA type (for display / audit). + /// Every recorded call, from every source, for a subject and DNA type, to show or to audit. pub async fn haplogroup_calls( &self, biosample_guid: SampleGuid, @@ -2437,8 +2577,8 @@ impl App { Ok(haplogroup_call::list_for(self.store.pool(), biosample_guid, dna_type).await?) } - /// Like [`assign_mtdna_haplogroup`](Self::assign_mtdna_haplogroup) but with the tree - /// JSON supplied directly (no network) — the testable core. + /// Like [`assign_mtdna_haplogroup`](Self::assign_mtdna_haplogroup), but the caller supplies the + /// tree JSON directly, with no network. This is the core that a test can drive. pub async fn assign_mtdna_haplogroup_with_tree( &self, mtdna_id: i64, @@ -2477,23 +2617,30 @@ impl App { self.fetch_tree(&url, "decodingus-ytree.json").await } - /// DecodingUs mtDNA tree-with-variants JSON from our AppView (`/api/v1/mt-tree/full`), host - /// from [`decodingus_appview_url`]. Same schema as the Y tree; coordinates are keyed by build, - /// but the mt tree currently carries only `hs1` (CHM13 `chrM`) positions — a *rotation* of rCRS - /// (~577, plus local indels), so callers must remap onto rCRS via [`mt_tree_rcrs`]. On-disk - /// cached like the other trees. + /// DecodingUs mtDNA tree-with-variants JSON from our AppView (`/api/v1/mt-tree/full`), with the + /// host from [`decodingus_appview_url`]. The schema is the same as the Y tree, and the build + /// keys the coordinates. + /// + /// But the mt tree now carries `hs1` (CHM13 `chrM`) positions alone. Those are a *rotation* of + /// rCRS, by about 577, plus local indels. So a caller must remap onto rCRS through + /// [`mt_tree_rcrs`]. The on-disk cache holds it, as it holds the other trees. pub(crate) async fn fetch_decodingus_mt_tree(&self) -> Result { let url = self.appview_url("mt-tree/full"); self.fetch_tree(&url, "decodingus-mttree.json").await } - /// The **mtDNA placement tree in rCRS coordinates**, with the provider tag. Honors the - /// configured Y-tree provider (the Preferences toggle / `NAVIGATOR_Y_TREE_PROVIDER`): when it's - /// set to FTDNA, use the FTDNA mt tree (already rCRS) directly. Otherwise prefer the DecodingUs - /// mt tree remapped from its native `hs1` (CHM13 `chrM`) positions onto rCRS — so it drops - /// straight into the existing rCRS mt pipeline (FASTA/chip sources and the `chrM` genotyper all - /// speak rCRS) — and still fall back to FTDNA when the DecodingUs tree or the CHM13 `chrM` needed - /// to build the remap is unavailable. + /// The **mtDNA placement tree in rCRS coordinates**, with the provider tag. + /// + /// It obeys the configured Y-tree provider, which is the Preferences toggle or + /// `NAVIGATOR_Y_TREE_PROVIDER`. When that names FTDNA, use the FTDNA mt tree directly, because + /// it is already rCRS. + /// + /// If not, prefer the DecodingUs mt tree, remapped from its native `hs1` (CHM13 `chrM`) + /// positions onto rCRS. It then drops straight into the rCRS mt pipeline, where the FASTA and + /// chip sources and the `chrM` genotyper all use rCRS. + /// + /// Fall back to FTDNA when the DecodingUs tree is absent, or when the CHM13 `chrM` that the + /// remap needs is absent. pub(crate) async fn mt_tree_rcrs(&self) -> Result<(navigator_analysis::haplo::HaploTree, &'static str), AppError> { if !matches!(y_tree_provider(), YTreeProvider::Ftdna) { if let Some(tree) = self.decodingus_mt_tree_rcrs().await { @@ -2505,15 +2652,17 @@ impl App { Ok((tree, "ftdna")) } - /// The DecodingUs mt tree parsed and remapped from `hs1` (CHM13 `chrM`) coordinates onto rCRS. - /// `None` (→ FTDNA fallback) when the tree can't be fetched, or the CHM13 reference isn't cached - /// to build the `hs1`↔rCRS map. Best-effort so an offline / reference-less workspace still works. + /// The DecodingUs mt tree, parsed and remapped from `hs1` (CHM13 `chrM`) coordinates onto rCRS. + /// It gives `None`, and the caller falls back to FTDNA, in two cases. The fetch fails, or the + /// cache has no CHM13 reference to build the `hs1`↔rCRS map from. It is best-effort, so a + /// workspace that is offline, or that has no reference, still works. async fn decodingus_mt_tree_rcrs(&self) -> Option { let json = self.fetch_decodingus_mt_tree().await.ok()?; let mut tree = navigator_analysis::haplo::parse_decodingus_json(&json, "hs1").ok()?; let hs1_to_rcrs = self.hs1_to_rcrs_mt_map().await?; - // Remap each defining locus from hs1 (CHM13 chrM) to rCRS; drop any that don't map (indel - // regions near the rotation wrap). An emptied node still exists in the topology. + // Remap each locus of a node from hs1 (CHM13 chrM) to rCRS. Drop any that does not map, + // which happens in the indel regions near the rotation wrap. A node that loses every locus + // still exists in the topology. for node in tree.nodes.values_mut() { node.loci.retain_mut(|l| match hs1_to_rcrs.get(&l.position) { Some(&r) => { @@ -2527,8 +2676,9 @@ impl App { } /// The `hs1` (CHM13 `chrM`, 1-based) → rCRS (1-based) position map, memoized for the process. - /// Built by aligning the bundled rCRS to the cached CHM13 reference's `chrM` (rotation-aware). - /// `None` when the CHM13 reference isn't cached (never forces a multi-GB download for this). + /// The code builds it with an alignment of the bundled rCRS to the cached CHM13 reference's + /// `chrM`, and it knows about the rotation. + /// `None` when the CHM13 reference is not cached (never forces a multi-GB download for this). async fn hs1_to_rcrs_mt_map(&self) -> Option> { static MAP: std::sync::OnceLock>> = std::sync::OnceLock::new(); if let Some(m) = MAP.get() { @@ -2560,18 +2710,19 @@ impl App { .await } - /// The AppView's full instrument→lab map (`GET /api/v1/sequencer/lab-instruments`), on-disk - /// cached like the trees (7-day TTL + offline fallback). Looked up locally so a batch import - /// makes one network call, not one per sample. + /// The AppView's full instrument→lab map (`GET /api/v1/sequencer/lab-instruments`). The on-disk + /// cache holds it, as it holds the trees, with a 7-day TTL and an offline fallback. The lookup + /// is local, so a batch import makes one network call, and not one for each sample. async fn fetch_lab_instruments(&self) -> Result, AppError> { let url = self.appview_url("sequencer/lab-instruments"); let json = self.fetch_tree(&url, "sequencer-lab-instruments.json").await?; serde_json::from_str(&json).map_err(|e| AppError::Import(format!("parsing lab-instruments: {e}"))) } - /// Resolve an instrument id to a lab display name via the AppView (cached). Normalizes the - /// returned name to the local [`labs`] catalog's canonical display name when it matches. - /// `None` if the instrument has no association or the AppView is unreachable (best-effort). + /// Resolve an instrument id to a lab display name through the AppView, from the cache. It + /// normalizes the returned name to the canonical display name of the local [`labs`] catalog + /// when the two match. It gives `None` when the instrument has no association, or when the + /// AppView is unreachable. It is best-effort. pub async fn lookup_lab_by_instrument(&self, instrument_id: &str) -> Option { let id = instrument_id.trim(); if id.is_empty() { @@ -2586,11 +2737,14 @@ impl App { ) } - /// Resolve the FTDNA Big Y **generation** for a generic Targeted-Y run from its callable chrY - /// footprint: a Big Y-500 covers ≤ ~10 Mb of callable chrY, and only the newer Big Y-700 - /// consistently exceeds it. Only acts on an FTDNA `TARGETED_Y` run — a header `@RG LB` label - /// already pins the generation at import (those are `BIG_Y_500`/`BIG_Y_700`, never `TARGETED_Y`, - /// so they're never second-guessed here), and a non-FTDNA targeted-Y stays generic. Idempotent. + /// Resolve the FTDNA Big Y **generation** for a generic Targeted-Y run, from its callable chrY + /// footprint. A Big Y-500 covers about 10 Mb of callable chrY or less, and only the newer Big + /// Y-700 goes above that consistently. + /// + /// It acts on an FTDNA `TARGETED_Y` run alone. A header `@RG LB` label already fixes the + /// generation at import time, and gives `BIG_Y_500` or `BIG_Y_700`, never `TARGETED_Y`. So this + /// never questions such a run. A targeted-Y run that is not FTDNA stays generic. The function + /// is idempotent. pub(crate) async fn refine_big_y_generation(&self, run: &SequenceRun, callable_chr_y: u64) -> Option<&'static str> { const BIG_Y_500_MAX_CALLABLE: u64 = 10_000_000; if run.test_type != "TARGETED_Y" { @@ -2614,9 +2768,10 @@ impl App { Some(code) } - /// [`Self::refine_big_y_generation`] keyed off an alignment's freshly computed (or cached) - /// coverage — the callable-chrY base count is the discriminator. Called after coverage runs. - /// Returns the new code when the generation changed (so the caller can refresh the run card). + /// [`Self::refine_big_y_generation`], keyed off an alignment's coverage, which the app has just + /// computed or read from the cache. The count of callable chrY bases is what separates the two + /// generations. The caller runs this after coverage. It returns the new code when the + /// generation changed, so that the caller can refresh the run card. pub async fn refine_big_y_generation_for_alignment( &self, alignment_id: i64, @@ -2629,12 +2784,14 @@ impl App { Ok(None) } - /// Resolve the sequencing lab for every run that has an inferred `instrument_id` but no facility - /// yet, via the AppView (one cached fetch). Best-effort; returns how many were filled. Run after - /// import and on startup so pre-existing runs pick up newly-seeded associations. + /// Resolve the sequencing lab for every run that has an inferred `instrument_id` but no + /// facility yet, through the AppView, with one cached fetch. It is best-effort, and returns how + /// many it filled. Run it after an import and at startup, so an older run takes up an + /// association that the app has just seeded. pub async fn backfill_run_labs(&self) -> Result { - // One network/cache fetch (empty when offline — the FTDNA test-type normalization below is - // local and still runs for runs whose facility was resolved earlier, e.g. subject 103589). + // One fetch, from the network or the cache. It is empty when the app is offline. The FTDNA + // test-type normalization below is local, and still runs for a run whose facility the app + // resolved earlier, such as subject 103589. let list = self.fetch_lab_instruments().await.unwrap_or_default(); let by_instrument: HashMap<&str, &str> = list .iter() @@ -2660,9 +2817,10 @@ impl App { None => None, }, }; - // A run we now know is FTDNA but typed as the generic TARGETED_Y is a Big Y — pick - // its generation (500/700) from cached coverage when it's already been analyzed, so - // pre-existing runs get corrected on startup without a re-analysis. + // A run that the app now knows is FTDNA, but that carries the generic TARGETED_Y + // type, is a Big Y. Pick its generation, 500 or 700, from the cached coverage when + // an analysis has already run. So an older run corrects itself at startup, with no + // second analysis. let _ = facility; // (resolved above; the refine reads facility off the run record) if run.test_type == "TARGETED_Y" { if let Ok(Some(cov)) = self.cached_coverage_for_run(run.id).await { @@ -2677,8 +2835,8 @@ impl App { Ok(filled) } - /// Cached coverage for a run, via its first alignment that has one (Big Y runs have a single - /// alignment). `None` when the run hasn't been analyzed yet. + /// Cached coverage for a run, through the first alignment that has one. A Big Y run has one + /// alignment. It gives `None` when no analysis has run yet. async fn cached_coverage_for_run(&self, run_id: i64) -> Result, AppError> { for aln in alignment::list_for_run(self.store.pool(), run_id).await? { if let Some(cov) = self.cached_coverage(aln.id).await? { @@ -2688,16 +2846,11 @@ impl App { Ok(None) } - /// A cached-or-downloaded haplotree JSON. The on-disk cache has a **7-day life** (see - /// [`TREE_CACHE_TTL`]): a fresh cache short-circuits the network; a stale or missing cache - /// triggers a re-download (and refresh). If the re-download fails (e.g. the AppView is - /// unreachable) but a stale copy exists, the stale copy is used rather than failing — so the - /// app keeps working offline, just on an older tree. (A server-side ETag/version would let us - /// revalidate without a full re-download; tracked as an AppView backlog item.) - /// Force a fresh pull of the haplotrees on the next placement: clear the session memo AND delete - /// the on-disk tree caches, so a corrected AppView tree (e.g. a polarity fix) is picked up without - /// an app restart. Observation-first profiles then re-interpret against the new tree on read — no - /// re-genotyping. Returns the number of cache files removed. + /// Force a fresh pull of the haplotrees on the next placement. It clears the session memo AND + /// deletes the on-disk tree caches. So the app takes up a corrected AppView tree, such as a + /// polarity fix, with no restart. An observation-first profile then interprets against the new + /// tree on read, with no second genotype pass. Returns the count of cache files that it + /// removed. pub async fn refresh_trees(&self) -> Result { tree_memo().lock().unwrap().clear(); let mut removed = 0usize; @@ -2714,16 +2867,29 @@ impl App { Ok(removed) } + /// A haplotree JSON, from the cache or from a download. The on-disk cache has a **7-day life** + /// (see [`TREE_CACHE_TTL`]). A fresh cache keeps the code off the network. A cache that is + /// stale or absent starts a download, which also refreshes the cache. + /// + /// If that download fails, but a stale copy exists, the code uses the stale copy and does not + /// fail. An unreachable AppView is one such failure. So the app still runs offline, on an older + /// tree. + /// + /// A server-side ETag or version would let the app revalidate with no full download. That is an + /// AppView backlog item. async fn fetch_tree(&self, url: &str, cache_file: &str) -> Result { - // Session memo: the Y/mt haplotrees are 4–121 MB and each placement consults them several - // times (per alignment, per vendor set, and for the polarity map). A single genome-consensus - // build alone would otherwise re-read/re-validate them repeatedly, and a *stale*-cache refresh - // blocks on the network. Resolve each tree at most once per process and serve every later call - // from memory — trees are effectively static within a session, so this is the batch's biggest - // win (a project pass was spending minutes per subject re-fetching the 121 MB FTDNA tree). - // Keyed by the *resolved* path, not the bare file name: `NAVIGATOR_TREE_DIR` can point the - // same `cache_file` at different trees, and a name-keyed memo would serve the first one for - // the rest of the process. + // Session memo. The Y and mt haplotrees are 4–121 MB. One placement consults them many + // times: for each alignment, for each vendor set, and for the polarity map. Without this + // memo, one genome-consensus build would read and check them again and again. A refresh of + // a *stale* cache would also block on the network. + // + // So resolve each tree once for each process at most, and serve every later call from + // memory. A tree does not change inside a session, so this is the biggest gain in the + // batch. A project pass took minutes for each subject to fetch the 121 MB FTDNA tree again. + // + // The key is the *resolved* path, and not the bare file name. `NAVIGATOR_TREE_DIR` can + // point the same `cache_file` at a different tree. A memo keyed by name would then serve + // the first tree for the rest of the process. let path = tree_cache_path(cache_file); let key = path.to_string_lossy().into_owned(); let memo = tree_memo(); @@ -2737,13 +2903,15 @@ impl App { let json = if fresh { cached.expect("fresh implies present") } else { - // Stale or absent → *conditional*, time-bounded refresh. When we have a cached copy and - // its stored ETag we send `If-None-Match`: an unchanged tree comes back as a tiny `304` - // (a few bytes) instead of re-streaming the full ~60–127 MB body — the curated tree - // changes only every week or so, so most refreshes are 304s. Any failure (connect/ - // timeout, a non-2xx/304 status, or a body read cut short — the whole-request timeout - // also covers streaming the body, see [`TREE_DOWNLOAD_TIMEOUT`]) falls back to the cached - // copy when present; only a first-ever fetch with no cache errors. + // A stale or absent cache starts a *conditional* refresh with a time bound. With a + // cached copy and its stored ETag, the code sends `If-None-Match`. An unchanged tree + // then comes back as a small `304`, a few bytes, instead of the full 60–127 MB body. + // The curated tree changes about once a week, so most refreshes are 304s. + // + // Any failure falls back to the cached copy when there is one. Such a failure is a + // connect or a timeout, a status that is not 2xx or 304, or a body read that stops + // short. The whole-request timeout also covers the body read, see + // [`TREE_DOWNLOAD_TIMEOUT`]. Only a first fetch with no cache gives an error. enum TreeFetch { NotModified, Modified { body: String, etag: Option }, @@ -2828,9 +2996,10 @@ impl App { pub async fn assign_mtdna_haplogroup_from_alignment(&self, alignment_id: i64) -> Result { let bio = self.biosample_of_alignment(alignment_id).await.ok(); - // Prefer an external (sidecar-GVCF) mt call over re-walking the CRAM — same rationale as the - // Y path (see `assign_y_haplogroup`); this is the guard the unguarded single-alignment - // "Full Analysis" was missing, so an internal re-run no longer overwrites the GATK4 mt call. + // Prefer an external mt call, from the sidecar GVCF, over a second walk of the CRAM. The + // reason is the same as on the Y path, see `assign_y_haplogroup`. This is the guard that + // the single-alignment "Full Analysis" did not have. So a second internal run no longer + // overwrites the GATK4 mt call. if let Some(guid) = bio { if let Some(call) = self.preferred_external_call(guid, DnaType::Mt, alignment_id).await? { return Ok(assignment_from_call(&call)); @@ -2840,10 +3009,13 @@ impl App { self.assign_mtdna_haplogroup_walk(alignment_id, bio).await } - /// The internal-caller mtDNA placement: place chrM against the FTDNA mt tree and record under the - /// walk key (`aln:{id}:mt`, `NavigatorWalk`), skipping the re-score when the fingerprint is - /// unchanged. Split out of [`assign_mtdna_haplogroup_from_alignment`] so [`compare_callers`] can - /// force the internal walk even when an external call is preferred. + /// The internal-caller mtDNA placement. Place chrM against the FTDNA mt tree, and record it + /// under the walk key (`aln:{id}:mt`, `NavigatorWalk`). It skips the score step when the + /// fingerprint stays the same. + /// + /// This is split out of [`assign_mtdna_haplogroup_from_alignment`], so that + /// [`compare_callers`] can force the internal walk even when an external call is the preferred + /// one. pub(crate) async fn assign_mtdna_haplogroup_walk( &self, alignment_id: i64, @@ -2852,7 +3024,7 @@ impl App { let source_key = format!("aln:{alignment_id}:mt"); let tree_json = self.fetch_ftdna_mt_tree().await?; - // Cache: skip re-scoring when the file and the mt tree are unchanged. + // Cache: skip the score step when the file and the mt tree stay the same. let fingerprint = self .alignment_content_hash(alignment_id) .await @@ -2888,7 +3060,8 @@ impl App { Ok(assignment) } - /// mtDNA assignment + per-SNP lineage evidence (for exact GRCh38-vs-CHM13 comparison). + /// The mtDNA assignment, plus the lineage evidence at each SNP, for an exact comparison of + /// GRCh38 against CHM13. pub async fn assign_mtdna_haplogroup_detail( &self, alignment_id: i64, @@ -2897,10 +3070,10 @@ impl App { self.assign_haplogroup_detail(alignment_id, "chrM", &tree_json).await } - /// Scan an alignment's chrM pileup for heteroplasmic positions — sites where a second - /// mitochondrial allele coexists above the noise floor. A screening pass for the - /// reconciliation view (a curator judges real heteroplasmy vs. artefacts); ascending - /// by position. Requires a chrM-bearing BAM. + /// Scan an alignment's chrM pileup for heteroplasmic positions. Those are the sites where a + /// second mitochondrial allele is present above the noise floor. This is a first pass for the + /// reconciliation view, where a curator then separates real heteroplasmy from an artefact. The + /// output is in order of position, lowest first. It needs a BAM that holds chrM. pub async fn mtdna_heteroplasmy(&self, alignment_id: i64) -> Result, AppError> { // Resolve the reference for decode (see alignment_reference_for_decode): required for a CRAM, // None for a BAM. The chrM pileup finds a second allele from reads; it needs no reference base. @@ -2914,15 +3087,13 @@ impl App { .map_err(Into::into) } - /// Estimate the donor's ancestry for an alignment by the allele-frequency likelihood: load - /// the (build-matched) AIMs panel, genotype the sample at its sites with the GL caller, and - /// score each super-population's binomial likelihood. Persists the result; returns it for - /// display. Requires a recorded BAM/CRAM and a resolvable reference (CRAM/genotyping). - /// Estimate autosomal ancestry from the subject's **consensus** — no BAM genotyping. Reads the - /// cached autosomal [`DiploidProfile`] (reconciled 0/1/2 dosages over the probe panel, pooled - /// across all WGS + chip sources), bridges it to genotypes, and runs the same estimators as the - /// per-alignment path used to. Persisted under the consensus pseudo-source - /// ([`CONSENSUS_SOURCE_ID`]). Errors if the autosomal consensus hasn't been built yet. + /// Estimate autosomal ancestry from the subject's **consensus**, with no BAM genotyping. + /// + /// It reads the cached autosomal [`DiploidProfile`], which holds reconciled 0/1/2 dosages over + /// the probe panel, pooled across every WGS and chip source. It bridges that to genotypes, and + /// runs the same estimators that the alignment path ran. The store keeps the result under + /// the consensus pseudo-source ([`CONSENSUS_SOURCE_ID`]). It gives an error when no build of + /// the autosomal consensus has run yet. pub async fn estimate_ancestry_from_consensus( &self, biosample_guid: SampleGuid, @@ -2932,11 +3103,13 @@ impl App { })?; let genotypes = consensus_genotypes(&profile); - // The consensus is canonical CHM13; the AIM freq / PCA assets are keyed by (contig,pos) there. + // The consensus uses canonical CHM13. The AIM freq and PCA assets key on (contig,pos) + // there. let build = ReferenceBuild::Chm13v2; let reference_version = "chm13v2.0".to_string(); - // Auto-download the prebuilt panels on first use (no `panelbuild`). The super-pop panel is - // required; PCA + fine frequencies are optional (best-effort — the feature degrades if absent). + // Download the prebuilt panels on first use, so nobody has to run `panelbuild`. The + // super-pop panel is necessary. The PCA and the fine frequencies are optional, and + // best-effort: the feature does less when they are absent. self.ensure_ancestry_asset(build, &ancestry_panel_path(build)).await?; let _ = self.ensure_ancestry_asset(build, &ancestry_pca_path(build)).await; let _ = self @@ -2964,12 +3137,17 @@ impl App { if let Some(pca) = pca_bytes.and_then(|b| ancestry_analysis::PcaLoadings::from_bytes(&b).ok()) { result.pca_coordinates = Some(ancestry_analysis::project_pca(&genotypes, &pca)); } - // Deep (ancient) ancestry is NOT computed here — it is the separate `estimate_deep_ancestry` - // path. NB it is *not* a heavier genotyping pass: it reads the SAME cached autosomal - // consensus these modern/fine estimators use (the consensus is the full ~1.15M-site 1240k - // IBD-panel union, not a 20k subset), and just intersects the larger qpAdm f4 panel against - // it (documents/design/ancient-ancestry-rebuild.md §7.14). Kept out of this hot path because the - // qpAdm fit is a distinct, on-demand model. + // Deep (ancient) ancestry does NOT run here. It is the separate + // `estimate_deep_ancestry` path. + // + // It is *not* a heavier genotype pass. It reads the SAME cached autosomal + // consensus that these modern and fine estimators use. That consensus is the full + // 1240k IBD-panel union, of about 1.15M sites, and not a 20k subset. Deep ancestry only + // intersects the larger qpAdm f4 panel against it + // (documents/design/ancient-ancestry-rebuild.md §7.14). + // + // It stays out of this hot path because the qpAdm fit is a distinct model, and it runs + // on demand. let ancient: Option = None; let _ = &ancient_bytes; (result, ancient, fine) @@ -2990,33 +3168,39 @@ impl App { Ok(result) } - /// **Deep (ancient) ancestry via qpAdm** (documents/design/ancient-ancestry-rebuild.md §7.14, §7.16) — - /// the validated WHG / EEF / Steppe breakdown. + /// **Deep (ancient) ancestry through qpAdm** (documents/design/ancient-ancestry-rebuild.md + /// §7.14, §7.16). This is the WHG / EEF / Steppe breakdown that passed its checks. + /// + /// It consumes the subject's **autosomal consensus** ([`Self::build_autosomal_profile`]), which + /// is the same multi-source object that modern and fine ancestry use. The IBD panel's resolver + /// builds that consensus for each build. So it already pools every source, **re-keyed to + /// canonical CHM13**. That covers WGS on any reference, GRCh37, GRCh38, and CHM13, and consumer + /// chips, and it needs no alignment. /// - /// Consumes the subject's **autosomal consensus** ([`Self::build_autosomal_profile`]) — the same - /// multi-source object modern/fine ancestry uses. The consensus is built by the IBD panel's - /// per-build resolver, so it already pools every source **re-keyed to canonical CHM13**: WGS on - /// any reference (GRCh37/38 as well as CHM13) *and* consumer chips, with no alignment required. - /// That is why deep ancestry works multi-reference and chip-only — it inherits the frontend the - /// other estimates share. It fits `target = Σ wᵢ·sourcesᵢ` by qpAdm f4 over the (now - /// CHM13-canonical, §7.16) qpAdm panel and persists the result under the consensus pseudo-source. + /// That is why deep ancestry works across references and on a chip alone: it inherits the front + /// end that the other estimates share. It fits `target = Σ wᵢ·sourcesᵢ` by qpAdm f4 over the + /// qpAdm panel, which is now CHM13-canonical (§7.16). The store keeps the result under the + /// consensus pseudo-source. /// - /// **Requires** the autosomal consensus (errors if absent — build it via the Autosomal tab, same - /// contract as modern ancestry). Given it, this is a fast fit over cached genotypes; the heavy - /// full-1240k genotyping happens once, in the shared consensus build. + /// It **needs** the autosomal consensus, and gives an error when that is absent. Build it from + /// the Autosomal tab, which is the same contract as modern ancestry. Given it, this is a fast + /// fit over cached genotypes. The heavy full-1240k genotype pass runs once, in the shared + /// consensus build. /// - /// Returns `Ok(None)` when the feature is gated off, the asset is not installed, the consensus has - /// no autosomal calls, or the deep model does not apply (non-European / model rejected / infeasible - /// weights). `None` persists nothing — keeping an inapplicable breakdown off the UI *and* out of - /// the PDS. + /// It returns `Ok(None)` in four cases. A gate holds the feature off, the asset is not + /// installed, the consensus has no autosomal calls, or the deep model does not apply. The model + /// does not apply to a non-European sample, to a fit that the test rejects, or to weights that + /// are infeasible. `None` stores nothing, which keeps an inapplicable breakdown off the UI + /// *and* out of the PDS. pub async fn estimate_deep_ancestry(&self, biosample_guid: SampleGuid) -> Result, AppError> { if !crate::ANCIENT_ANCESTRY_ENABLED { return Ok(None); } let build = ReferenceBuild::Chm13v2; let reference_version = "chm13v2.0".to_string(); - // Auto-download the prebuilt qpAdm + super-pop panels on first use (no `panelbuild`). qpAdm is - // best-effort (deep ancestry is simply unavailable if it can't be fetched); the super panel is required. + // Download the prebuilt qpAdm and super-pop panels on first use. Nobody must run + // `panelbuild`. qpAdm is best-effort: deep ancestry is unavailable when the fetch fails. + // The super panel is necessary. let _ = self.ensure_ancestry_asset(build, &ancestry_qpadm_path(build)).await; self.ensure_ancestry_asset(build, &ancestry_panel_path(build)).await?; let qpadm_path = ancestry_qpadm_path(build); @@ -3029,11 +3213,13 @@ impl App { .ok_or_else(|| AppError::AncestryPanelMissing(super_path.clone()))?; let super_panel = AncestryPanel::from_bytes(&super_bytes)?; - // The pooled autosomal consensus (all sources, any build + chips, canonical CHM13, full 0/1/2 - // dosages). **Required**, not built on demand — same contract as modern ancestry: the heavy - // build runs through the Autosomal-tab flow (with progress), and this is a fast read over it. - // Both the scope gate and the qpAdm fit read the *same* genotypes; every panel here is - // CHM13-canonical (§7.16), so no per-site re-keying is needed. + // The pooled autosomal consensus: all sources, any build, plus chips, on canonical CHM13, + // with full 0/1/2 dosages. It is **necessary**, and this code does not build it on demand. + // That is the same contract as modern ancestry. The heavy build runs through the + // Autosomal-tab flow, which shows progress, and this is a fast read over the result. + // + // The scope gate and the qpAdm fit read the *same* genotypes. Every panel here is + // CHM13-canonical (§7.16), so the code re-keys no site. let profile = self.cached_autosomal_profile(biosample_guid).await?.ok_or_else(|| { AppError::Import( "build the autosomal consensus first (Autosomal tab) before estimating deep ancestry".into(), @@ -3073,23 +3259,26 @@ impl App { Ok(result) } - /// **Deep-ancestry stability diagnostic** — the §3.4 validation gates, on a real subject. + /// **Deep-ancestry stability diagnostic**: the §3.4 check gates, on a real subject. + /// + /// It fits the ancient mixture again and again, over different *views* of the same person. The + /// views are the pooled consensus, each source that contributes on its own, and random subsets + /// of the sites. /// - /// Fits the ancient mixture repeatedly over different *views* of the same person: the pooled - /// consensus, each contributing source on its own (a 30× WGS and a consumer chip are genotyped - /// by completely different means, so agreeing across them is the strongest evidence the estimate - /// tracks the donor and not the assay), and random subsets of the sites. + /// A 30× WGS and a consumer chip genotype by completely different means. So an answer that + /// holds across the two is the strongest evidence that the estimate follows the donor, and not + /// the assay. /// - /// This is the test the previous implementation failed most spectacularly — the same person came - /// out WHG 72.6% from the consensus and WHG 7.4% from their own 28× BAM — so it is the one worth - /// being able to re-run on demand. Rows are diagnostics, never persisted or published; the - /// `reported` flag records whether the shipping estimator would have accepted that fit. + /// This is the test that the earlier implementation failed worst. The same person came out at + /// WHG 72.6% from the consensus, and WHG 7.4% from their own 28× BAM. So it is the one to be + /// able to run again on demand. A row is a diagnostic, and the app never stores or publishes + /// it. The `reported` flag records whether the released estimator would have accepted that fit. /// - /// Every row reports its dispersion even when the applicability gate rejects it, so a rejection - /// can be read as a magnitude rather than taken on faith. + /// Every row reports its dispersion, even when the applicability gate rejects it. A rejection + /// then has a magnitude, and nobody must take it on trust. pub async fn ancient_ancestry_stability(&self, biosample_guid: SampleGuid) -> Result, AppError> { - // Build the consensus on demand — this is a diagnostic, and requiring the caller to have - // clicked through the GUI first would make it useless from the CLI. + // Build the consensus on demand. This is a diagnostic. If the caller had to click through + // the GUI first, the CLI could not use it. let profile = match self.cached_autosomal_profile(biosample_guid).await? { Some(p) => p, None => self.build_autosomal_profile(biosample_guid).await?, @@ -3098,8 +3287,8 @@ impl App { let path = ancestry_freq_ancient_path(build); let bytes = read_verified_asset(build, &path)?.ok_or_else(|| AppError::AncestryPanelMissing(path.clone()))?; let panel = AncestryPanel::from_bytes(&bytes)?; - // The super-pop panel too: deep ancestry is scoped by the modern estimate, so each view has - // to be scored by both models or the diagnostic wouldn't be reproducing the shipped policy. + // The super-pop panel too. The modern estimate sets the scope of deep ancestry. So both + // models must score each view, or the diagnostic would not follow the released policy. let super_path = ancestry_panel_path(build); let super_bytes = read_verified_asset(build, &super_path)? .ok_or_else(|| AppError::AncestryPanelMissing(super_path.clone()))?; @@ -3124,7 +3313,7 @@ impl App { sites: r.snps_with_genotype, dispersion: r.fit_distance.unwrap_or(f64::NAN), european: ancestry_analysis::west_eurasian_share(&modern), - // The shipping estimator's own verdict — not a re-derivation of it. + // The verdict of the released estimator itself, and not a copy of it. reported: ancestry_analysis::estimate_ancient_admixture( genotypes, &panel, @@ -3144,12 +3333,15 @@ impl App { let consensus = consensus_genotypes(&profile); fit("consensus (pooled)".to_string(), &consensus); - // Sites a chip actually called — the intersection target for the refit below. The - // stability failure has WGS reporting ~80% Steppe where the chips report ~58%; this asks - // whether the split is *which sites* each technology reaches (WGS scores ~19.7k, a chip - // ~5–8k) or the calls themselves. If a WGS source restricted to chip-covered sites moves - // toward the chip answer, the extra WGS-only sites carry the bias; if it stays put, the - // WGS dosages do. + // The sites that a chip called, which is the intersection target for the refit below. + // + // In the stability failure, WGS gives about 80% Steppe where the chips give about 58%. + // This asks whether the split comes from *which sites* each technology reaches, or from + // the calls themselves. WGS scores about 19.7k sites, and a chip 5–8k. + // + // Take a WGS source and hold it to the chip-covered sites. If it moves toward the chip + // answer, the extra WGS-only sites carry the bias. If it stays where it was, the WGS + // dosages carry it. let chip_sites: std::collections::HashSet = profile .variants .iter() @@ -3161,10 +3353,10 @@ impl App { .map(|v| v.name.clone()) .collect(); - // Diagnostic dump (NAVIGATOR_ANCIENT_DUMP=): per consensus site, whether a chip - // covers it, the pooled dosage, and the three source frequencies. Lets us see directly - // what makes the non-chip sites favour Steppe once every intrinsic site property - // (MAF/strand/polarity/ts-tv) has been ruled out. + // Diagnostic dump (NAVIGATOR_ANCIENT_DUMP=). For each consensus site it gives + // whether a chip covers it, the pooled dosage, and the three source frequencies. That + // shows directly what makes the non-chip sites favour Steppe, once the code has ruled + // out every property of the site itself: MAF, strand, polarity, and ts-tv. if let Ok(dump_path) = std::env::var("NAVIGATOR_ANCIENT_DUMP") { let want = std::env::var("NAVIGATOR_ANCIENT_ALN").unwrap_or_else(|_| "#9".into()); let freq: std::collections::HashMap<(&str, i64), &Vec> = panel @@ -3196,10 +3388,12 @@ impl App { let _ = std::fs::write(&dump_path, out); } - // Strand-ambiguous SNPs (A/T, C/G): ref and alt are Watson–Crick complements, so which - // allele the panel counted as "alt" can't be recovered from the alleles alone. When a - // panel built from one dataset is genotyped against reads oriented by another, these are - // the sites that silently invert — the classic merge bias, and one chips routinely drop. + // Strand-ambiguous SNPs (A/T, C/G). Ref and alt are Watson–Crick complements here, so + // the alleles alone do not say which one the panel counted as "alt". + // + // Take a panel from one dataset, and genotype it against reads that another dataset + // oriented. These are the sites that invert, with no warning. That is the classic merge + // bias, and a chip usually drops these sites. let is_ambiguous = |v: &navigator_domain::consensus::DiploidVariant| -> bool { matches!( (v.reference.as_str(), v.alternate.as_str()), @@ -3238,10 +3432,10 @@ impl App { }; // Each source alone: take that source's own observed dosage at each site. For a WGS - // source, also refit it three ways to localize the stability bias: - // ∩chip — sites the chips cover (does WGS match the chip answer there?) - // ∁chip — the WGS-only complement (do those sites carry the bias?) - // ¬ambig — all sites minus strand-ambiguous A/T,C/G (does dropping them fix it?) + // source, also refit it three ways, to find where the stability bias sits. + // ∩chip the sites the chips cover. Does WGS match the chip answer there? + // ∁chip the WGS-only complement. Do those sites carry the bias? + // ¬ambig all sites, less the strand-ambiguous A/T and C/G. Does that fix it? for label in &source_labels { let is_chip = profile.variants.iter().any(|v| { v.sources @@ -3265,8 +3459,8 @@ impl App { } } - // Density: deterministic thinning of the pooled consensus. A well-conditioned fit barely - // moves when half the evidence is removed; an over-fit one lurches. + // Density: thin the pooled consensus, deterministically. A well-conditioned fit hardly + // moves when half the evidence goes. An over-fit one jumps. for (keep, label) in [(2usize, "consensus ÷2 sites"), (4, "consensus ÷4 sites")] { let thinned: Vec = consensus.iter().step_by(keep).cloned().collect(); fit(label.to_string(), &thinned); @@ -3277,25 +3471,29 @@ impl App { .map_err(AppError::from) } - /// The persisted ancestry estimate for an alignment, if one has been computed. + /// The stored ancestry estimate for an alignment, if the app has computed one. pub async fn ancestry_for_alignment(&self, alignment_id: i64) -> Result, AppError> { Ok(ancestry_result::get_for_alignment(self.store.pool(), alignment_id).await?) } - /// The persisted **fine-population** admixture estimate for an alignment, if one was computed - /// (the `ancestry_freq_global` asset was present at estimation time). Drives the super→fine - /// hierarchy rows; the super-pop donut keeps using the primary ([`ancestry_for_alignment`]). + /// The stored **fine-population** admixture estimate for an alignment, if the app computed one. + /// That needs the `ancestry_freq_global` asset to be present at estimate time. It feeds the + /// super→fine hierarchy rows. The super-pop donut continues to use the primary estimate + /// ([`ancestry_for_alignment`]). pub async fn fine_ancestry_for_alignment(&self, alignment_id: i64) -> Result, AppError> { Ok(ancestry_result::get_for_alignment_method(self.store.pool(), alignment_id, "FINE_ADMIXTURE").await?) } - /// Reference population centroids on (PC1, PC2) for the alignment's build — the backdrop - /// for the PCA scatter. `(population_code, pc1, pc2)`; empty if no PCA loadings are present. - /// Reference population centroids in the **consensus** PC frame for the PCA scatter. The donor's - /// projected coordinate (`AncestryResult::pca_coordinates`) is always computed against the CHM13 - /// PCA asset (the canonical consensus frame — see [`estimate_ancestry_from_consensus`]), so the - /// backdrop centroids must come from that same asset regardless of which source is selected. - /// Returns an empty vec when the asset isn't installed (the caller shows "reference not built"). + /// Reference population centroids on (PC1, PC2) for the alignment's build. They are the + /// backdrop for the PCA scatter, as `(population_code, pc1, pc2)`. The list is empty when no + /// PCA loadings are present. + /// + /// Reference population centroids in the **consensus** PC frame, for the PCA scatter. The code + /// always computes the donor's projected coordinate (`AncestryResult::pca_coordinates`) against + /// the CHM13 PCA asset, which is the canonical consensus frame. See + /// [`estimate_ancestry_from_consensus`]. So the backdrop centroids must come from that same + /// asset, whichever source the user selects. It returns an empty vec when the asset is not + /// installed, and the caller then shows "reference not built". pub async fn ancestry_pca_reference(&self) -> Result, AppError> { let build = ReferenceBuild::Chm13v2; let Ok(bytes) = std::fs::read(ancestry_pca_path(build)) else { @@ -3317,9 +3515,10 @@ impl App { .collect()) } - /// The cached chromosome painting for a subject, if one was painted from the **current** autosomal - /// consensus (signature = the consensus's `last_reconciled_at`). `None` if absent or stale (the - /// consensus was rebuilt since). Cheap — a cache read, no genotyping or HMM. + /// The cached chromosome painting for a subject, if the app painted it from the **current** + /// autosomal consensus. The signature is the consensus's `last_reconciled_at`. It gives `None` + /// when the cache has none, or when the entry is stale because a rebuild of the consensus came + /// after it. It is low-cost: a cache read, with no genotyping and no HMM. pub async fn cached_painting(&self, biosample_guid: SampleGuid) -> Result, AppError> { let Some(row) = consensus_profile::get(self.store.pool(), biosample_guid, "Auto").await? else { return Ok(None); @@ -3333,17 +3532,22 @@ impl App { Ok(Some(parse_painting_json(&p.payload)?)) } - /// Paint each chromosome with local ancestry from the subject's **consensus** — no BAM walk. The - /// explicit compute/refresh path: it always re-runs and refreshes the cache, keyed to the - /// consensus's `last_reconciled_at`, so a painter code change takes effect without wiping the - /// consensus. The cheap cache-read path is [`Self::cached_painting`] (used on subject load). + /// Paint each chromosome with local ancestry from the subject's **consensus**, with no BAM + /// walk. /// - /// **Parent-split**: when the phased-haplotype reference asset is present, the consensus is - /// statistically phased (Li & Stephens) and each side painted independently → two genuine - /// parental sides. If a parent is found in the workspace (a `ParentChild` IBD relationship), the - /// side carrying that parent's transmitted alleles is anchored and the sides are labelled - /// Mother/Father; otherwise Side A/Side B. Without the asset it falls back to the unphased diploid - /// painter (two arbitrary sorted copies). + /// This is the explicit compute path. It always runs again, and refreshes the cache, keyed to + /// the consensus's `last_reconciled_at`. So a change to the painter code takes effect, and + /// nobody must delete the consensus. [`Self::cached_painting`] is the low-cost cache read, and + /// the app uses it when it loads a subject. + /// + /// **Parent-split**: when the phased-haplotype reference asset is present, the code phases the + /// consensus statistically (Li & Stephens) and paints each side on its own. That gives two + /// genuine parental sides. + /// + /// The workspace can hold a parent, through a `ParentChild` IBD relationship. The side that + /// carries that parent's transmitted alleles then takes an anchor, and the labels become Mother + /// and Father. If not, they stay Side A and Side B. Without the asset, the code falls back to + /// the unphased diploid painter, which gives two sorted copies. pub async fn paint_local_ancestry_from_consensus( &self, biosample_guid: SampleGuid, @@ -3362,14 +3566,16 @@ impl App { let build = ReferenceBuild::Chm13v2; let reference_version = "chm13v2.0".to_string(); - // Super-pop AIM panel (required — the emission frequencies for both painters). + // The super-pop AIM panel. It is necessary, and gives the emission frequencies for both + // painters. self.ensure_ancestry_asset(build, &ancestry_panel_path(build)).await?; let panel_path = ancestry_panel_path(build); let panel_bytes = read_verified_asset(build, &panel_path)? .ok_or_else(|| AppError::AncestryPanelMissing(panel_path.clone()))?; let panel = AncestryPanel::from_bytes(&panel_bytes)?; - // Phased-haplotype reference (optional) — its presence switches on the parent-split path. + // The phased-haplotype reference is optional. When it is present, the parent-split path + // switches on. let haps_path = ancestry_haps_path(build); let _ = self.ensure_ancestry_asset(build, &haps_path).await; // best-effort auto-download let hap_ref = read_verified_asset(build, &haps_path) @@ -3389,13 +3595,15 @@ impl App { Some((g, sex, name)) => (Some(g), Some((sex, name))), None => (None, None), }; - // Copying-LAI knobs from settings (read here, on the async side, and moved into the blocking - // closure) so live Settings edits recalibrate the painter on the next paint. + // The copying-LAI knobs come from the settings. The code reads them here, on the async + // side, and moves them into the closure that blocks. So an edit in Settings recalibrates + // the painter on the next paint. let lai_params = copying_lai_params(); let (segments, phased, anchor_side) = tokio::task::spawn_blocking(move || { - // Genome-wide super-pop composition — the prior anchoring both painters (and the copying - // LAI's global-composition gate that suppresses spurious continents). + // The genome-wide super-pop composition. It is the earlier estimate that anchors both + // painters. The copying-LAI global-composition gate also uses it, to suppress a + // continent that is not there. let composition = ancestry_analysis::estimate_admixture(&genotypes, &panel, &reference_version); let prior: Vec<(String, f64)> = composition .components @@ -3414,10 +3622,11 @@ impl App { let pairs: Vec<(&str, i32)> = lengths.iter().map(|(k, v)| (k.as_str(), *v)).collect(); let gmap = load_genetic_map(build, &pairs); - // Statistically phase, then paint each side by haplotype copying against the - // reference (RFMix-style) — resolves fine sub-populations from haplotype structure, - // superseding the frequency-emission painter + AF fine step. The `prior` gates the - // reference to the continents the sample actually has. + // Phase statistically, then paint each side by haplotype copying against the + // reference, in the RFMix style. That resolves fine sub-populations from the + // haplotype structure, and replaces the frequency-emission painter and the AF + // fine step. The `prior` holds the reference to the continents that the sample + // has. let phaser = ReferencePhaser::new(&hap, &gmap, PhaseParams::default()); let phased_g = phaser.phase(&genotypes); let segs = navigator_analysis::lai::paint_copying_lai(&phased_g, &hap, &gmap, &prior, &lai_params); @@ -3447,7 +3656,8 @@ impl App { phased, }; - // Cache keyed to the consensus signature so it's reused until the consensus is rebuilt. + // The cache keys on the consensus signature. So the app reuses the entry until a rebuild + // of the consensus. sig_cache::PAINTING .upsert( self.store.pool(), @@ -3463,24 +3673,29 @@ impl App { /// Find a workspace subject that is this subject's parent, to anchor the painted sides, and load /// its consensus genotypes. Returns `(genotypes, recorded-sex, display-name)`, or `None`. /// - /// Scalable by construction — it must **not** scan the whole workspace (which can hold tens of - /// thousands of subjects). Candidates are restricted to the child's **home project** (families / - /// trios are imported together) and hard-capped; a workspace with no small family context simply - /// gets no auto-anchoring (the sides stay Side A/B). Each candidate is screened by a cheap - /// **Mendelian test** — a true parent shares ≥1 allele at every site, so opposite-homozygous - /// sites are ~0 — which is O(sites) and needs no IBD segment detection or genetic map. The best - /// (lowest opposite-homozygosity) candidate below the threshold is returned; the phased-side - /// assignment itself is done later by transmission consistency. + /// It scales by construction. It must **not** scan the whole workspace, which can hold tens of + /// thousands of subjects. So the candidates come from the child's **home project**, and a hard + /// cap bounds them. An import brings a family or a trio in together. A workspace with no small + /// family context gets no automatic anchor, and the sides stay Side A and Side B. + /// + /// A low-cost **Mendelian test** screens each candidate. A true parent shares one allele or + /// more at every site, so the count of opposite-homozygous sites is near 0. That test is + /// O(sites), and it needs no IBD segment detection and no genetic map. + /// + /// It returns the best candidate below the threshold, which is the one with the lowest + /// opposite-homozygosity. Transmission consistency then assigns the phased side later. async fn find_parent_for_anchor( &self, child: SampleGuid, child_genotypes: &[SiteGenotype], ) -> Option<(Vec, Option, String)> { - // Auto-anchoring is for small family/trio projects; larger sets are skipped (a research - // corpus shouldn't trigger a many-way scan, and parent detection there isn't meaningful). + // The automatic anchor is for a small family or trio project. The code skips a larger set. + // A research corpus must not start a many-way scan, and parent detection there means + // little. const MAX_CANDIDATES: usize = 32; - // Opposite-homozygous fraction below which a pair is treated as parent-child (Mendel forbids - // opposite homozygotes for a true parent-child pair; the slack absorbs genotyping error). + // The opposite-homozygous fraction below which a pair counts as parent-child. Mendel + // forbids opposite homozygotes in a true parent-child pair, and the slack here absorbs a + // genotyping error. const MAX_OPP_HOM_FRAC: f64 = 0.01; const MIN_SHARED_SITES: u32 = 500; @@ -3532,9 +3747,10 @@ impl App { Some(consensus_genotypes(&profile)) } - /// The cached ROH result for a subject, if one was computed from the **current** autosomal - /// consensus (signature = the consensus's `last_reconciled_at`). `None` if absent or stale (the - /// consensus was rebuilt since). Cheap — a cache read, no genotyping or HMM. + /// The cached ROH result for a subject, if the app computed it from the **current** autosomal + /// consensus. The signature is the consensus's `last_reconciled_at`. It gives `None` when the + /// cache has none, or when the entry is stale because a rebuild of the consensus came after it. + /// It is low-cost: a cache read, with no genotyping and no HMM. pub async fn cached_roh(&self, biosample_guid: SampleGuid) -> Result, AppError> { let Some(row) = consensus_profile::get(self.store.pool(), biosample_guid, "Auto").await? else { return Ok(None); @@ -3549,10 +3765,12 @@ impl App { } } - /// Detect runs of homozygosity from the subject's **consensus** — no BAM walk. Returns the cached - /// result when it matches the current consensus signature; otherwise runs the 2-state autozygosity - /// HMM over the consensus genotypes and caches it keyed to the consensus's `last_reconciled_at`. - /// The genome-wide F_ROH and length-class breakdown are the endogamy / consanguinity signal. + /// Detect runs of homozygosity from the subject's **consensus**, with no BAM walk. + /// + /// It returns the cached result when that matches the current consensus signature. If not, it + /// runs the 2-state autozygosity HMM over the consensus genotypes, and caches the result keyed + /// to the consensus's `last_reconciled_at`. The genome-wide F_ROH and the length-class + /// breakdown are the endogamy and consanguinity signal. pub async fn compute_roh_from_consensus(&self, biosample_guid: SampleGuid) -> Result { let row = consensus_profile::get(self.store.pool(), biosample_guid, "Auto") .await? @@ -3561,7 +3779,7 @@ impl App { })?; let sig = row.last_reconciled_at.clone(); - // Cache hit (same consensus signature) → return without recomputing. + // A cache hit, with the same consensus signature. Return, and compute nothing. if let Some(r) = sig_cache::ROH.get(self.store.pool(), biosample_guid).await? { if r.sig == sig { return Ok(serde_json::from_str(&r.payload)?); @@ -3571,7 +3789,8 @@ impl App { let profile: DiploidProfile = serde_json::from_str(&row.payload)?; let genotypes = consensus_genotypes(&profile); let result = tokio::task::spawn_blocking(move || { - // Per-contig max position → genetic-map lengths (CHM13 consensus space, uniform fallback). + // The maximum position in each contig gives the genetic-map lengths, in CHM13 + // consensus space, with a uniform fallback. let mut lengths: std::collections::BTreeMap = std::collections::BTreeMap::new(); for g in &genotypes { let e = lengths.entry(g.contig.clone()).or_insert(1); @@ -3583,7 +3802,8 @@ impl App { }) .await?; - // Cache keyed to the consensus signature so it's reused until the consensus is rebuilt. + // The cache keys on the consensus signature. The app reuses the entry until a rebuild of + // the consensus. sig_cache::ROH .upsert( self.store.pool(), @@ -3596,12 +3816,13 @@ impl App { Ok(result) } - /// The subject's archaic percentile within their inferred super-population, plus that cohort's - /// label. `None` when the distribution asset is absent or the subject's ancestry is unknown. + /// The subject's archaic percentile inside their inferred super-population, plus that cohort's + /// label. The value is `None` when the distribution asset is absent, or when nobody knows the + /// subject's ancestry. /// - /// The cohort is keyed to the subject's **super-population**, not their fine population (design - /// §9 Q3): a fine-grained cohort would be more specific, but it would also let an ancestry error - /// move the archaic headline, and a wrong percentile is worse than a coarse one. + /// The cohort keys on the subject's **super-population**, and not on their fine population + /// (design §9 Q3). A fine-grained cohort would be more specific. But it would also let an error + /// in the ancestry move the archaic figure, and a wrong percentile is worse than a coarse one. async fn archaic_percentile( &self, biosample_guid: SampleGuid, @@ -3617,8 +3838,9 @@ impl App { }; let dist = ArchaicCountDistribution::from_bytes(&bytes)?; - // The subject's dominant super-population, from the cached consensus ancestry. No estimate → - // no cohort → no percentile, rather than defaulting to one and quietly mis-ranking them. + // The subject's dominant super-population, from the cached consensus ancestry. With no + // estimate there is no cohort, and so no percentile. Do not fall back to a default + // cohort, which would give the subject a wrong rank with no warning. let Ok(ancestry) = self.estimate_ancestry_from_consensus(biosample_guid).await else { return Ok(None); }; @@ -3639,8 +3861,9 @@ impl App { /// The subject's alignment that already carries genome-wide de-novo diploid calls, if any. /// - /// Probes chr1 as the marker: the whole-genome pass writes one artifact per autosome, so a - /// cached chr1 means that alignment has been called. Cheap enough at a handful of alignments. + /// It probes chr1 as the marker. The whole-genome pass writes one artifact for each autosome, + /// so a cached chr1 means that a call ran on that alignment. The cost is small at a few + /// alignments. async fn alignment_with_diploid_calls(&self, biosample_guid: SampleGuid) -> Result, AppError> { for a in alignment::list_for_biosample(self.store.pool(), biosample_guid).await? { if a.bam_path.is_none() { @@ -3663,12 +3886,12 @@ impl App { Ok(None) } - /// The cached Tier B archaic segment result for a subject, if current for the alignment and - /// caller version it was produced from. + /// The cached Tier B archaic segment result for a subject. It must still match the alignment + /// and the caller version that produced it. /// - /// Gated by [`crate::ARCHAIC_SEGMENTS_ENABLED`] on the **read** path as well as the compute - /// path: rows persisted before the gate went in are still in the workspace, and a read-only - /// gate is the difference between withholding a result and merely declining to recompute it. + /// [`crate::ARCHAIC_SEGMENTS_ENABLED`] gates the **read** path and the compute path both. The + /// workspace still holds rows from before the gate went in. A gate on the read path withholds + /// such a result. A gate on the compute path alone would only stop a new computation. pub async fn cached_archaic_segments( &self, biosample_guid: SampleGuid, @@ -3695,21 +3918,23 @@ impl App { /// Call archaic **segments** (Tier B) from the subject's genome-wide de-novo diploid calls. /// - /// **Uses already-cached calls and refuses otherwise.** Genome-wide diploid calling is an - /// hours-long whole-genome pass; kicking one off from a UI click would look like a hang. The - /// contract mirrors "build the autosomal consensus first" — the caller runs `navigator call` - /// (or the analysis flow) and this is then a fast read over the cache. + /// **It uses calls that the cache already holds, and refuses in all other cases.** Genome-wide + /// diploid calling is a whole-genome pass that takes hours. To start one from a UI click would + /// look like a hang. + /// + /// The contract mirrors "build the autosomal consensus first". The caller runs `navigator call`, + /// or the analysis flow, and this is then a fast read over the cache. /// - /// CHM13 only, because the Tier B assets are CHM13 and segment coordinates have no per-build - /// loci to re-key through. + /// CHM13 only, because the Tier B assets are CHM13, and a segment coordinate has no loci in + /// another build to re-key through. pub async fn call_archaic_segments_for_subject( &self, biosample_guid: SampleGuid, ) -> Result { - // Withheld: the caller reproduces the cohort mean and nothing about the individual. An - // error rather than an empty result, because every caller of this asked for a computation — - // silently returning zero segments would read as "you have no archaic ancestry", which is - // a far worse claim than "we are not reporting this". + // Withheld: the caller reproduces the cohort mean, and says nothing about the individual. + // Give an error, and not an empty result. Every caller of this asked for a computation. + // A quiet return of zero segments would read as "you have no archaic ancestry". That is a + // much worse claim than "we do not report this". if !crate::ARCHAIC_SEGMENTS_ENABLED { return Err(AppError::Import( "archaic segment calling is disabled: validated against hmmix's per-individual \ @@ -3718,10 +3943,10 @@ impl App { .into(), )); } - // Prefer an alignment that already HAS genome-wide diploid calls over the - // highest-coverage one. Those calls are an hours-long per-alignment pass, so a subject with - // several CHM13 alignments (this one has four) would otherwise be told to re-run work they - // have already done, just on a different alignment. + // Prefer an alignment that already HAS genome-wide diploid calls over the one with the + // highest coverage. Those calls take hours for each alignment. Take a subject with more + // than one CHM13 alignment, and this one has four. Without this rule they would have to run + // work again that they had already done, only on a different alignment. let aln = match self.alignment_with_diploid_calls(biosample_guid).await? { Some(a) => a, None => self.best_callable_alignment(biosample_guid).await?.ok_or_else(|| { @@ -3737,8 +3962,8 @@ impl App { "archaic segments currently require a CHM13 alignment (the Tier B assets are CHM13-only)".into(), )); } - // Computed from the contigs actually cached, so a later genome-wide pass invalidates a - // partial result instead of inheriting it. + // Computed from the contigs in the cache, so a later genome-wide pass invalidates a + // partial result, and does not take it over. let sig = archaic_segment_sig(aln, &crate::called_diploid_contigs(&self.store, aln).await?); if let Some(row) = sig_cache::ARCHAIC_SEGMENTS .get(self.store.pool(), biosample_guid) @@ -3749,8 +3974,8 @@ impl App { } } - // Gather whatever de-novo diploid calls are already cached, per autosome. Never computed - // here — see the doc comment. + // Gather the de-novo diploid calls that the cache holds, for each autosome. This code + // never computes them. See the doc comment. let mut calls: Vec = Vec::new(); let mut contigs_present = 0usize; for c in 1..=22u8 { @@ -3785,12 +4010,13 @@ impl App { }; let classify = ArchaicClassify::from_bytes(&load(crate::archaic_classify_path(rb))?)?; let callable = ArchaicCallable::from_bytes(&load(crate::archaic_callable_path(rb))?)?; - // Tier A's panel carries the per-archaic-genome calls the concordance filter needs — the - // single largest quality lever measured (precision 54 % -> 90 %). + // Tier A's panel carries the calls of each archaic genome, which the concordance filter + // needs. That filter is the largest measured gain in quality: precision 54 % to 90 %. let panel = ArchaicMarkerPanel::from_bytes(&load(crate::archaic_markers_path(rb))?)?; let (_, reference) = self.alignment_bam_reference(aln).await?; - // Genetic map over the contigs actually present, so transitions are recombination-scaled. + // The genetic map over the contigs that are present, so recombination scales the + // transitions. let mut lengths: std::collections::BTreeMap = std::collections::BTreeMap::new(); for g in &calls { let e = lengths.entry(g.contig.clone()).or_insert(1); @@ -3813,8 +4039,9 @@ impl App { } let mut observations = std::collections::BTreeMap::new(); for (contig, pos_map) in &by_contig { - // One contig's reference at a time: whether a diagnostic site is informative depends - // on the reference base there, and holding all of CHM13 would cost 3.1 GB. + // One contig's reference at a time. Whether a diagnostic site tells us anything + // depends on the reference base there, and all of CHM13 would cost 3.1 GB of + // memory. let seq = navigator_analysis::reader::read_contig_sequence(&reference, contig)?; let obs = am::observations_for_contig( contig, @@ -3855,16 +4082,20 @@ impl App { /// Genotype the archaic marker panel directly from one alignment. /// - /// The consensus only carries the 1240k/IBD loci, so this is what gives a WGS subject the full - /// panel rather than the ~2.6 % that happens to intersect 1240k. Results are cached per - /// alignment under a kind salted with the panel's manifest hash, so recalibrating the panel - /// invalidates stale genotypes instead of silently mixing site sets. + /// The consensus carries the 1240k/IBD loci alone. So this is what gives a WGS subject the + /// full panel, and not the 2.6 % or so that happens to intersect 1240k. /// - /// Works on **any build the panel carries loci for**: CHM13 natively, and GRCh37/38 via the - /// panel's per-build coordinates (offline lift, oriented at build time — no runtime liftover). - /// A dosage measured on a non-CHM13 build is re-keyed to the CHM13 alleles before it is - /// returned, because that build's ref/alt may be swapped or strand-flipped relative to CHM13 - /// and feeding the raw dosage through would invert those sites silently. + /// The cache holds a result for each alignment, under a kind that the panel's manifest hash + /// salts. So a new calibration of the panel invalidates a stale genotype, and no code mixes two + /// site sets together. + /// + /// It works on **any build that the panel has loci for**: CHM13 natively, and GRCh37 and GRCh38 + /// through the panel's coordinates for each build. Those come from an offline lift, oriented at + /// build time, with no liftover at run time. + /// + /// A dosage measured on a build other than CHM13 is re-keyed to the CHM13 alleles before it + /// goes back. Such a build can swap its ref and alt against CHM13, or flip them to the other + /// strand. The raw dosage would then invert those sites, with no warning. async fn genotype_archaic_for_alignment( &self, alignment_id: i64, @@ -3909,7 +4140,8 @@ impl App { }) .await?? } else if panel.sites.iter().any(|s| s.locus(&build).is_some()) { - // Match the panel's per-build contig names to the file's naming (`chr1` vs `1`). + // Match the panel's contig names for that build to the names in the file, `chr1` + // against `1`. let (bam_h, ref_h) = (bam.clone(), reference.clone()); let file_contigs = tokio::task::spawn_blocking(move || navigator_analysis::reader::contig_names(&bam_h, ref_h.as_deref())) @@ -3993,10 +4225,10 @@ impl App { Ok(genotypes) } - /// Count archaic markers from **one specific alignment**, bypassing both the cache and the - /// best-callable pick. Used for cross-build validation: the same person's GRCh38 and CHM13 - /// alignments should agree, which is the check that the per-build loci and the dosage re-keying - /// are correct rather than merely plausible. + /// Count archaic markers from **one specific alignment**. It goes around the cache, and around + /// the best-callable pick. It is for a check across builds: the same person's GRCh38 and CHM13 + /// alignments must agree. That check shows that the loci of each build, and the re-key of the + /// dosage, are correct, and not only plausible. pub async fn archaic_for_alignment( &self, _biosample_guid: SampleGuid, @@ -4016,8 +4248,9 @@ impl App { ) } - /// The cached archaic (Tier A) marker count for a subject, if one was computed from the - /// **current** autosomal consensus. `None` if absent or stale. Cheap — a cache read. + /// The cached archaic (Tier A) marker count for a subject, if the app computed it from the + /// **current** autosomal consensus. It gives `None` when the cache has none, or when the entry + /// is stale. It is low-cost: a cache read. pub async fn cached_archaic(&self, biosample_guid: SampleGuid) -> Result, AppError> { let Some(row) = consensus_profile::get(self.store.pool(), biosample_guid, "Auto").await? else { return Ok(None); @@ -4034,17 +4267,19 @@ impl App { } } - /// Count the subject's archaic (Neanderthal / Denisovan) marker copies from the **consensus** — - /// no BAM walk. Returns the cached result when it matches the current consensus signature. + /// Count the subject's archaic (Neanderthal / Denisovan) marker copies from the **consensus**, + /// with no BAM walk. It returns the cached result when that matches the current consensus + /// signature. /// - /// The headline is a count over what was actually assayed (copies carried of copies possible), - /// so chip and WGS input both yield an honest figure without comparing across data types. + /// The reported figure is a count over the sites that the test covered: the copies carried, of + /// the copies possible. So a chip and a WGS both give an honest figure, and neither one + /// compares across data types. /// - /// The **percentile is deliberately left unset for sparse input**. A consumer chip covers only a - /// few percent of the panel, and those sites are its common tail, so ranking such a count against - /// the WGS-scored reference cohort would produce a confidently wrong number (design §10). Until - /// per-site frequencies land in the distribution asset, the percentile is only filled when the - /// subject's call rate is comparable to the cohort's. + /// The **percentile stays unset for sparse input, on purpose**. A consumer chip covers a few + /// percent of the panel, and those sites are its common tail. To rank such a count against the + /// WGS-scored reference cohort would give a confident but wrong number (design §10). The code + /// fills the percentile only when the subject's call rate is near the cohort's. That holds + /// until the distribution asset carries a frequency for each site. pub async fn estimate_archaic_from_consensus( &self, biosample_guid: SampleGuid, @@ -4054,10 +4289,10 @@ impl App { .ok_or_else(|| { AppError::Import("build the autosomal consensus first (Autosomal tab) before the archaic report".into()) })?; - // Load the panel BEFORE the cache check: the cache signature is salted with the panel's - // hash as well as the consensus signature, because rebuilding the panel changes the site - // list and the per-class split, and keying on the consensus alone would serve a stale count - // computed against a different panel. + // Load the panel BEFORE the cache check. The panel's hash salts the cache signature, and + // so does the consensus signature. A new build of the panel changes the site list and the + // split by class. A key on the consensus alone would then serve a stale count, computed + // against a different panel. let build = ReferenceBuild::Chm13v2; self.ensure_ancestry_asset(build, &crate::archaic_markers_path(build)) .await?; @@ -4074,16 +4309,18 @@ impl App { } let panel = ArchaicMarkerPanel::from_bytes(&bytes)?; - // Start from the consensus — it covers chips and every non-alignment source, but only where - // the archaic panel intersects the 1240k/IBD loci the consensus is built over (~2.6% of the - // panel). Design §5 assumed the consensus spanned the genome; it does not. + // Start from the consensus. It covers chips and every source that is not an alignment. + // + // But it reaches only the sites where the archaic panel intersects the 1240k/IBD loci of + // the consensus. That is about 2.6% of the panel. Design §5 assumed that the consensus + // spanned the genome. It does not. let profile: DiploidProfile = serde_json::from_str(&row.payload)?; let mut genotypes = consensus_genotypes(&profile); - // Then genotype the panel DIRECTLY from the subject's best-callable alignment, which is the - // only way a WGS subject reaches the other 97%. Best-effort: a subject with no callable - // alignment (chip-only) keeps the consensus-derived coverage and simply reports a lower - // call rate, which the "X of Y" headline states honestly. + // Then genotype the panel DIRECTLY from the subject's best-callable alignment. That is the + // only way a WGS subject reaches the other 97%. It is best-effort. A subject with no + // callable alignment, such as a chip-only subject, keeps the coverage from the consensus + // and reports a lower call rate. The "X of Y" figure states that honestly. if let Some(aln) = self.best_callable_alignment(biosample_guid).await? { match self.genotype_archaic_for_alignment(aln, &panel).await { Ok(direct) if !direct.is_empty() => { @@ -4103,10 +4340,11 @@ impl App { tokio::task::spawn_blocking(move || navigator_analysis::archaic::count_archaic_markers(&genotypes, &panel)) .await?; - // Percentile — valid at ANY coverage now, because the cohort is scored over exactly the - // sites this subject called rather than over the whole panel. A chip reaching ~3% of the - // panel is compared against what the cohort would score on those same 3%, so the call-rate - // artefact that used to pin every chip user near the 0th percentile is gone (design §10). + // The percentile is now valid at ANY coverage. The code scores the cohort over exactly the + // sites that this subject called, and not over the whole panel. Take a chip that reaches + // about 3% of the panel. The comparison is against what the cohort would score on those + // same 3%. So the call-rate artefact that used to hold every chip user near the 0th + // percentile is gone (design §10). if let Some((pct, cohort)) = self .archaic_percentile(biosample_guid, &result, &panel_fingerprint) .await? @@ -4127,9 +4365,10 @@ impl App { Ok(result) } - /// An alignment's content SHA-256, computed once at import. Read from the record if present, - /// else computed now (hashing the file) and stored — so batch-imported alignments are hashed - /// lazily on first analysis, then cached on the row. + /// An alignment's content SHA-256, computed once at import. The code reads it from the record + /// when that holds one. If not, it reads the file, computes the hash now, and stores it. So an + /// alignment from a batch import gets its hash at the first analysis, and the row then holds + /// it. async fn alignment_content_hash(&self, alignment_id: i64) -> Result { let aln = self.alignment_or_err(alignment_id).await?; let bam = aln.bam_path.clone(); @@ -4142,8 +4381,9 @@ impl App { h } }; - // Register the file by its content hash (gap §5-p2): stable identity across moves, and the - // dedup/accessibility registry. Idempotent — a moved file just updates its path here. + // Register the file by its content hash (gap §5-p2). That gives it a stable identity + // across a move, and it is the registry for dedup and accessibility. The call is + // idempotent: a file that moved only updates its path here. if let Some(path) = bam { let size = std::fs::metadata(&path).ok().map(|m| m.len() as i64); let now = Utc::now().to_rfc3339(); @@ -4157,13 +4397,14 @@ impl App { Ok(hash) } - /// All tracked source files (content-hash identity) — for the Data Sources view. + /// Every tracked source file, with its content-hash identity, for the Data Sources view. pub async fn list_source_files(&self) -> Result, AppError> { Ok(source_file::list(self.store.pool()).await?) } - /// Re-check each tracked file's path on disk and update its accessibility flag. Returns how many - /// are now missing (moved/deleted) — surfaced as a "file missing" marker in the UI. + /// Check each tracked file's path on disk again, and update its accessibility flag. It returns + /// how many are now absent, because a move or a delete. The UI shows those with a "file + /// missing" marker. pub async fn verify_source_files(&self) -> Result { let now = Utc::now().to_rfc3339(); let mut missing = 0; @@ -4183,10 +4424,12 @@ impl App { Ok(missing) } - /// The active Y tree's content hash (first 16 hex), the `yt:` half of a placement fingerprint. - /// Standalone — [`y_score_fingerprint`](Self::y_score_fingerprint) can only be computed for a - /// subject that owns an alignment to hash, which excludes VCF-only subjects and anyone whose - /// source file has moved. + /// The active Y tree's content hash (the first 16 hex characters), which is the `yt:` half of a + /// placement fingerprint. + /// + /// It stands alone. [`y_score_fingerprint`](Self::y_score_fingerprint) works only for a subject + /// that owns an alignment to hash. That leaves out a VCF-only subject, and anyone whose source + /// file has moved. pub async fn current_y_tree_hash(&self) -> Result { let json = match y_tree_provider() { YTreeProvider::DecodingUs => self.fetch_decodingus_y_tree().await?, @@ -4195,24 +4438,26 @@ impl App { Ok(sha256_str(&json)[..16].to_string()) } - /// The active mtDNA tree's content hash (first 16 hex) — the `mt:` half. + /// The active mtDNA tree's content hash (the first 16 hex characters), the `mt:` half. pub async fn current_mt_tree_hash(&self) -> Result { let json = self.fetch_ftdna_mt_tree().await?; Ok(sha256_str(&json)[..16].to_string()) } - /// Subjects whose Y or mtDNA calls were placed against **a different haplotree** than the one - /// now active, and so are due a re-placement. + /// Subjects whose Y or mtDNA calls sit on **a different haplotree** from the one now active. + /// They are due a second placement. /// - /// This is the selector, not the sweep: `rebuild-signatures` already re-places a set of - /// subjects, and `assign_*_haplogroup_walk` already no-ops when a subject's fingerprint still - /// matches — so feeding this list to that sweep is cheap on anything already current. + /// This is the selector, and not the sweep. `rebuild-signatures` already places a set of + /// subjects again, and `assign_*_haplogroup_walk` already does nothing when a subject's + /// fingerprint still matches. So this list costs little when it goes to that sweep and + /// everything in it is current. /// - /// `include_unknown` adds the subjects whose calls predate the fingerprint field, which is a - /// provenance backfill rather than a response to a tree change — see the store function. + /// `include_unknown` adds the subjects whose calls come from before the fingerprint field + /// existed. That is a provenance backfill, and not an answer to a change of tree. See the store + /// function. /// - /// A tree that cannot be fetched yields no subjects for that DNA type rather than an error: the - /// point of the sweep is to act on a *known* new tree, and "the network is down" is not one. + /// A tree that the app can not fetch gives no subjects for that DNA type, and no error. The + /// sweep acts on a *known* new tree. "The network is down" is not one. pub async fn subjects_placed_against_another_tree( &self, include_unknown: bool, @@ -4244,20 +4489,21 @@ impl App { Ok(out) } - /// Subjects whose **genome-level consensus label names a branch the current Y tree does not - /// carry** — the other half of tree staleness, and the one that actually bites. + /// Subjects whose **genome-level consensus label names a branch that the current Y tree does + /// not carry**. This is the other half of tree staleness, and the half that does the damage. /// - /// A consensus is *derived* from the per-source calls and persisted separately, with no tree - /// stamp of its own. So it can rot while every call beneath it is current: `GMWOF5428705` holds - /// a call placed against today's tree (`E-C116698`) under a consensus of `E-FT400514:n0` last - /// reconciled four weeks earlier. A sweep keyed on call fingerprints alone cannot see that. + /// The code *derives* a consensus from the calls of each source, and stores it on its own, with + /// no tree stamp. So it can go stale while every call below it is current. `GMWOF5428705` holds + /// a call placed against today's tree (`E-C116698`), under a consensus of `E-FT400514:n0` that + /// the app last reconciled four weeks before. A sweep keyed on call fingerprints alone can not + /// see that. /// - /// Testing the label against the tree's node names catches it directly and needs no schema - /// change: a label absent from the tree is stale by definition, whatever the cause — and it is - /// exactly the set the block tree drops into `unplaced`. + /// A test of the label against the tree's node names catches it directly, and needs no change + /// of schema. A label that the tree does not have is stale by definition, whatever the cause. + /// It is exactly the set that the block tree puts into `unplaced`. pub async fn subjects_labelled_off_tree(&self) -> Result, AppError> { - // Node *names* are build-independent, so any build key yields the same index — `hs1` is the - // DecodingUs tree's native space and needs no liftover. + // A node *name* is the same in every build, so any build key gives the same index. `hs1` + // is the DecodingUs tree's native space, and it needs no liftover. let tree = match y_tree_provider() { YTreeProvider::DecodingUs => { let json = self.fetch_decodingus_y_tree().await?; @@ -4281,9 +4527,10 @@ impl App { Ok(out) } - /// Fingerprint of the inputs to a Y-haplogroup score: the alignment's content hash + the - /// active Y-tree's content hash. Unchanged inputs → a re-score is unnecessary. Errors (e.g. - /// the tree is unreachable and uncached) disable caching for this run rather than failing. + /// Fingerprint of the inputs to a Y-haplogroup score: the alignment's content hash, plus the + /// active Y-tree's content hash. When the inputs do not change, a second score is unnecessary. + /// An error turns the cache off for this run, and does not stop the run. Such an error happens + /// when the tree is unreachable and the cache has none. async fn y_score_fingerprint(&self, alignment_id: i64) -> Result { let file_hash = self.alignment_content_hash(alignment_id).await?; let tree_json = match y_tree_provider() { @@ -4294,16 +4541,15 @@ impl App { Ok(format!("f:{}|yt:{}", &file_hash[..16], &tree_hash[..16])) } - /// Assign a Y haplogroup to an alignment: place the sample against the configured Y tree - /// (DecodingUs by default — our tree, native CHM13 coords, no liftover — falling back to - /// FTDNA if the AppView is unreachable), call the sample's base at each tree position on - /// chrY, and rank by Kulczynski. Requires a recorded BAM/CRAM path. Skips re-scoring when - /// the alignment file and tree are unchanged since the last run (see [`Self::y_score_fingerprint`]). - /// Whether a subject should be scored for Y-DNA. Females have no Y chromosome, so Y placement, - /// consensus, and the Y variant profile only produce an empty/degenerate call from mismapped - /// chrY reads — skip them. Sex comes from `biosample.sex` (user-provided, or written back by the - /// sex walker, which runs before the Y step). Male / Unknown / unrecorded → scored (`true`), so a - /// low-confidence or missing inference, or an XXY subject, is never silently dropped. + /// Whether the app must score a subject for Y-DNA. + /// + /// A female has no Y chromosome. Y placement, the consensus, and the Y variant profile would + /// then give an empty or degenerate call, from mismapped chrY reads, so skip her. The sex comes + /// from `biosample.sex`, which the user gives, or which the sex walker writes back. That walker + /// runs before the Y step. + /// + /// Male, Unknown, and a subject with no record all score (`true`). So the app never drops a + /// low-confidence inference, a missing one, or an XXY subject, with no word. pub(crate) async fn subject_has_y_dna(&self, biosample_guid: SampleGuid) -> Result { let sex = biosample::get(self.store.pool(), biosample_guid) .await? @@ -4311,10 +4557,17 @@ impl App { Ok(!matches!(sex.as_deref().map(str::trim), Some(s) if s.eq_ignore_ascii_case("female"))) } + /// Assign a Y haplogroup to an alignment. Place the sample against the configured Y tree, call + /// the sample's base at each tree position on chrY, and rank by Kulczynski. + /// + /// The default tree is DecodingUs, which is our own, in native CHM13 coordinates, with no + /// liftover. It falls back to FTDNA when the AppView is unreachable. It needs a recorded + /// BAM/CRAM path. It skips the score step when the alignment file and the tree have not changed + /// since the last run (see [`Self::y_score_fingerprint`]). pub async fn assign_y_haplogroup(&self, alignment_id: i64) -> Result { let bio = self.biosample_of_alignment(alignment_id).await.ok(); - // Females have no Y chromosome — don't genotype chrY or record a Y call for them. + // A female has no Y chromosome. Do not genotype chrY, and record no Y call, for her. if let Some(guid) = bio { if !self.subject_has_y_dna(guid).await? { return Ok(HaploAssignment { @@ -4325,9 +4578,10 @@ impl App { } } - // A trusted external caller (GATK4 GVCF) already placed this alignment via the sidecar fast - // path and the user prefers it: return that call instead of re-walking the CRAM. On damaged - // ancient DNA the walk would place a different, wrong terminal and clobber the external one. + // A trusted external caller (GATK4 GVCF) already placed this alignment, through the + // sidecar fast path, and the user prefers it. Return that call, and do not walk the CRAM + // again. On damaged ancient DNA the walk would place a different, wrong terminal, and + // overwrite the external one. if let Some(guid) = bio { if let Some(call) = self.preferred_external_call(guid, DnaType::Y, alignment_id).await? { return Ok(assignment_from_call(&call)); @@ -4337,19 +4591,21 @@ impl App { self.assign_y_haplogroup_walk(alignment_id, bio).await } - /// The internal-caller Y placement: genotype chrY against the configured tree and record the call - /// under the walk key (`aln:{id}`, `NavigatorWalk` provenance), skipping the re-score when the - /// alignment + tree fingerprint is unchanged. Split out of [`assign_y_haplogroup`] so - /// [`compare_callers`] can force the internal walk even when an external call is preferred. + /// The internal-caller Y placement. Genotype chrY against the configured tree, and record the + /// call under the walk key (`aln:{id}`, with `NavigatorWalk` provenance). It skips the score + /// step when the fingerprint of the alignment and the tree has not changed. + /// + /// This is split out of [`assign_y_haplogroup`], so that [`compare_callers`] can force the + /// internal walk even when an external call is the preferred one. pub(crate) async fn assign_y_haplogroup_walk( &self, alignment_id: i64, bio: Option, ) -> Result { let source_key = format!("aln:{alignment_id}"); - // Input fingerprint = alignment content hash + active Y-tree content hash. If it matches the - // recorded call's stamp, neither the file nor the tree changed → return the recorded call - // without re-scoring (the expensive BAM genotyping). + // The input fingerprint is the alignment content hash plus the active Y-tree content hash. + // When it matches the stamp on the recorded call, neither the file nor the tree changed. So + // return the recorded call, and do not run the costly BAM genotype step again. let fingerprint = self.y_score_fingerprint(alignment_id).await.ok(); if let (Some(bio), Some(fp)) = (bio, fingerprint.as_deref()) { if haplogroup_call::stored_fingerprint(self.store.pool(), bio, DnaType::Y, &source_key) @@ -4379,17 +4635,18 @@ impl App { Ok(assignment) } - /// Freshly place an alignment against the configured Y tree, returning the **full** assignment - /// **including per-branch SNP evidence** (the cached [`assign_y_haplogroup`] path returns only - /// the terminal). Expensive (genotypes chrY tree sites in the BAM) — used by the Y-variant - /// profile, which the user builds explicitly. + /// Place an alignment against the configured Y tree afresh, and return the **full** assignment, + /// **with the SNP evidence of each branch**. The cached [`assign_y_haplogroup`] path returns the + /// terminal alone. This costs a lot, because it genotypes the chrY tree sites in the BAM. The + /// Y-variant profile uses it, and the user builds that profile explicitly. async fn y_assignment_full(&self, alignment_id: i64) -> Result { match y_tree_provider() { YTreeProvider::DecodingUs => match self.assign_y_decodingus(alignment_id).await { Ok(a) => Ok(a), - // A gone alignment file is not a tree problem, and the fallback reads the same - // absent file — so it can only fail again, after another tree download, having - // logged that the DecodingUs tree was unavailable when it was not. Raise it. + // An alignment file that is gone is not a tree problem. The fallback reads the + // same absent file, so it can only fail again, after a second tree download. It + // would also log that the DecodingUs tree was unavailable, when it was not. Raise + // the error here. Err(e) if e.is_missing_alignment_file() => Err(e), Err(e) => { // AppView unreachable / build unsupported / parse failure → FTDNA fallback. @@ -4407,20 +4664,24 @@ impl App { } } - /// Place against the DecodingUs Y tree from our AppView, using the alignment's **native** - /// build coordinates (`hs1` for CHM13, `GRCh38`, `GRCh37`) — queried directly, **no - /// liftover**. This is the intended architecture (the AppView owns multi-build coordinates; - /// Navigator stays liftover-free). Today the AppView's `hs1` coords cover the decoding-us - /// backbone but not the FTDNA-grafted tips, so deep CHM13 placement is limited until the - /// AppView enriches `hs1` for every variant (lift GRCh38→hs1 at ingest or on the fly). + /// Place against the DecodingUs Y tree from our AppView, in the alignment's **native** build + /// coordinates: `hs1` for CHM13, `GRCh38`, or `GRCh37`. The query goes directly to those + /// coordinates, with **no liftover**. This is the architecture we want: the AppView owns the + /// coordinates of every build, and Navigator holds no liftover. + /// + /// At present the AppView's `hs1` coordinates cover the decoding-us backbone, but not the tips + /// that come from FTDNA. So a deep CHM13 placement has a limit. That limit holds until the + /// AppView gives every variant an `hs1` coordinate, by a lift from GRCh38 to hs1, at ingest + /// time or on demand. async fn assign_y_decodingus(&self, alignment_id: i64) -> Result { let (tree, calls) = self.y_decodingus_tree_calls(alignment_id).await?; Ok(assemble_assignment(&tree, &calls)) } - /// The (DecodingUs tree at the alignment's **native** build, full tree-locus base calls) for one - /// alignment — the genotype [`assign_y_decodingus`] scores. Factored so the consensus pool can - /// re-key it by SNP name and merge it with other sources. + /// A pair for one alignment: the DecodingUs tree at the alignment's **native** build, and the + /// full base calls at the tree loci. That is the genotype which [`assign_y_decodingus`] scores. + /// It is a separate function, so that the consensus pool can re-key it by SNP name and merge it + /// with other sources. pub(crate) async fn y_decodingus_tree_calls( &self, alignment_id: i64, @@ -4432,9 +4693,11 @@ impl App { aln.reference_build )) })?; - // Before the tree fetch, not after: a gone alignment file cannot be genotyped against any - // tree, so downloading one first is pure waste — and its io error surfacing from *below* the - // fetch is what let `y_assignment_full` mistake it for the tree being unavailable. + // Do this before the tree fetch, and not after. No tree can genotype an alignment file + // that is gone, so a download first is pure waste. + // + // There is a second reason. When the io error came from *below* the fetch, + // `y_assignment_full` read it as an unavailable tree. Self::alignment_file(&aln)?; let tree_json = self.fetch_decodingus_y_tree().await?; let tree = navigator_analysis::haplo::parse_decodingus_json(&tree_json, build_key).map_err(AppError::Import)?; @@ -4443,14 +4706,20 @@ impl App { Ok((tree, calls)) } - /// Assign a Y haplogroup from the subject's imported **BISDNA / Y-SNP-panel** calls — no - /// alignment required. Builds a derived-allele call map from the subject's `Chip`-sourced - /// variant sets (the panel's positive calls, each `position → derived base`) and scores it - /// against the Y tree on `build` (the subject's alignment build, else `"hs1"`). Uses the - /// DecodingUs tree at the native build (FTDNA fallback only on GRCh38, where positions - /// match), and the chip-robust terminal selection ([`assemble_assignment_robust`]). The - /// call is recorded as a reconciliation source. Only derived (positive) calls drive the - /// Kulczynski ranking, so the stored positives-only variant set is sufficient. + /// Assign a Y haplogroup from the subject's imported **BISDNA / Y-SNP-panel** calls. It needs + /// no alignment. + /// + /// It builds a derived-allele call map from the subject's `Chip`-sourced variant sets. Those + /// hold the panel's positive calls, each one a `position → derived base`. It then scores that + /// map against the Y tree on `build`, which is the subject's alignment build, or `"hs1"` when + /// there is none. + /// + /// It uses the DecodingUs tree at the native build. FTDNA is a fallback on GRCh38 alone, where + /// the positions match. Terminal selection uses the chip-robust + /// [`assemble_assignment_robust`]. The store records the call as a reconciliation source. + /// + /// Only a derived, positive call drives the Kulczynski rank. So the stored variant set, which + /// holds the positives alone, is enough. pub async fn assign_y_bisdna( &self, biosample_guid: SampleGuid, @@ -4459,8 +4728,9 @@ impl App { // Derived-allele calls from the subject's chip-sourced variant sets (BISDNA positives). let sets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; - // Placement build: explicit override, else the build stored on a chip set at import, - // else (pre-migration sets with no stored build) re-derive from the subject's alignment. + // The placement build. An explicit override wins. If there is none, take the build that + // the import stored on a chip set. A set from before the migration has no stored build, and + // the code then derives one from the subject's alignment. let build = match build { Some(b) => b.to_string(), None => match sets @@ -4509,12 +4779,14 @@ impl App { Ok(assignment) } - /// Autosomal variant sites of a set as `(bare-contig, position, a1, a2)` reference-forward - /// allele pairs — the input to the whole-genome IBD-panel resolve. Only chr1–chr22 (the panel - /// is autosomal). The genotype string is turned back into an allele pair: `1/1` → `(alt, alt)`; - /// het (`0/1`, `1/.`, or an absent genotype — one listed alt means at least one copy) → - /// `(ref, alt)`; tri-allelic `1/2` and haploid `1` (Y/mt, never autosomal) are dropped as - /// ambiguous for a diploid dosage. + /// The autosomal variant sites of a set, as `(bare-contig, position, a1, a2)` allele pairs on + /// the reference forward strand. They are the input to the whole-genome IBD-panel resolve. Only + /// chr1 to chr22 count, because the panel is autosomal. + /// + /// The code turns the genotype string back into an allele pair. `1/1` gives `(alt, alt)`. A het + /// gives `(ref, alt)`, and a het is `0/1`, `1/.`, or an absent genotype, because one listed alt + /// means one copy or more. It drops a tri-allelic `1/2`, and a haploid `1`, which is Y or mt + /// and never autosomal. Both are ambiguous for a diploid dosage. fn vset_autosomal_calls(set: &VariantSet) -> Vec<(String, i64, char, char)> { set.calls .iter() @@ -4539,10 +4811,13 @@ impl App { .collect() } - /// Resolve a **genome-wide** variant set to canonical CHM13 IBD-panel dosages (unlisted panel - /// sites ⇒ hom-reference — see [`IbdPanel::resolve_whole_genome`]). Needs the IBD panel asset. - /// Returns an empty vec for a set with no autosomal calls (e.g. a Y-only VCF). Not cached — the - /// resolve is cheap and, like the chip path, recomputed when the autosomal consensus is rebuilt. + /// Resolve a **genome-wide** variant set to canonical CHM13 IBD-panel dosages. A panel site + /// that the set does not list counts as hom-reference, see + /// [`IbdPanel::resolve_whole_genome`]. It needs the IBD panel asset. It returns an empty vec + /// for a set with no autosomal calls, such as a Y-only VCF. + /// + /// The cache does not hold the result. The resolve is low-cost, and it runs again when a + /// rebuild of the autosomal consensus happens, the same as the chip path. pub(crate) async fn variant_set_panel_dosages(&self, set: &VariantSet) -> Result, AppError> { let calls = Self::vset_autosomal_calls(set); if calls.is_empty() { @@ -4554,8 +4829,8 @@ impl App { Ok(dosages) } - /// chrY genotype calls (`position → uppercase ALT base`) from a variant set — the shared - /// extractor behind the chip / vendor-VCF Y placements. + /// chrY genotype calls (`position → uppercase ALT base`) from a variant set. This is the + /// shared extractor behind the chip and vendor-VCF Y placements. fn vset_chr_y_calls(set: &VariantSet) -> HashMap { set.calls .iter() @@ -4564,18 +4839,22 @@ impl App { .collect() } - /// Tree-position genotypes for a variant set — the VCF counterpart of [`Self::base_calls`]. + /// Tree-position genotypes for a variant set. This is the VCF counterpart of + /// [`Self::base_calls`]. /// - /// [`Self::vset_chr_y_calls`] can only report the stored rows, which are the donor's *derived* - /// calls: the workspace never recorded where he is confidently ancestral, so placement cannot - /// separate "ancestral" from "not covered" and every backbone node scores as no-call. Re-reading - /// the source VCF at the tree's positions recovers the hom-ref rows it already contains, which is - /// what the CRAM path gets for free by genotyping every target. + /// [`Self::vset_chr_y_calls`] can report the stored rows alone, and those are the donor's + /// *derived* calls. The workspace never recorded where he is confidently ancestral. So a + /// placement can not separate "ancestral" from "not covered", and every backbone node scores as + /// a no-call. /// - /// Cached in `variant_set_genotype` under the same site-set hash the alignment path uses, so a - /// changed tree misses rather than serving genotypes for sites that moved. Falls back to the - /// stored derived calls whenever the source is unavailable (never recorded, file since moved, or - /// unreadable) — strictly no worse than the previous behaviour. + /// A second read of the source VCF, at the tree's positions, recovers the hom-ref rows that it + /// already holds. The CRAM path gets those for free, because it genotypes every target. + /// + /// The cache holds the result in `variant_set_genotype`, under the same site-set hash that the + /// alignment path uses. So a changed tree misses the cache, and the code serves no genotype + /// for a site that moved. It falls back to the stored derived calls whenever the + /// source is unavailable: never recorded, moved since, or unreadable. That is no worse than the + /// earlier behaviour. pub(crate) async fn vset_base_calls( &self, set: &VariantSet, @@ -4614,14 +4893,19 @@ impl App { calls } - /// Place the subject's vendor **Y-NGS VCF** variant sets — FTDNA Big Y / Full Genomes Y Elite / - /// YSEQ / Nebula / Dante, i.e. anything imported as a non-[`Chip`](SourceType::Chip) - /// [`VariantSet`] carrying chrY calls — and record a per-source donor call for each. These are - /// direct Y-SNP genotype calls (the gold-standard placement input), placed against the configured - /// tree on each set's stored build (FTDNA Big Y is GRCh38, the FTDNA tree's native build → no - /// liftover). Best-effort per set: one that errors or lacks chrY calls is skipped. Returns the - /// number of sets placed. Called on import (so a Big Y VCF places without a manual Refresh) and - /// re-runnable. The vendor-VCF counterpart to [`assign_y_bisdna`](Self::assign_y_bisdna). + /// Place the subject's vendor **Y-NGS VCF** variant sets, and record one donor call for each + /// source. Those sets are FTDNA Big Y, Full Genomes Y Elite, YSEQ, Nebula, and Dante. In + /// general they are any [`VariantSet`] from an import that has chrY calls and that is not a + /// [`Chip`](SourceType::Chip). + /// + /// These are direct Y-SNP genotype calls, which are the best placement input. Each one places + /// against the configured tree, on the build that the set stores. FTDNA Big Y is GRCh38, which + /// is the FTDNA tree's native build, so it needs no liftover. + /// + /// It is best-effort for each set, and skips one that errors or that has no chrY calls. It + /// returns the count of sets it placed. The import calls it, so a Big Y VCF places with no + /// manual Refresh, and it is safe to run again. It is the vendor-VCF counterpart to + /// [`assign_y_bisdna`](Self::assign_y_bisdna). pub async fn assign_y_vendor_vcfs(&self, biosample_guid: SampleGuid) -> Result { let sets = variant_set::list_for_biosample(self.store.pool(), biosample_guid).await?; let mut tree_cache: HashMap = HashMap::new(); @@ -4647,10 +4931,12 @@ impl App { continue; } let assignment = Self::place_chip_panel(&tree_cache[&build], calls); - // A set with no tree-defining SNP matched carries no Y signal — e.g. an off-haplotree - // FTDNA "Private Variants" report (novel loci only), or an autosomal/mt VCF that slipped - // through. Recording its placeholder placement would conflict with a real per-source - // call and collapse the donor consensus to root, so skip it. + // A set that matched no SNP of a tree node carries no Y signal. One example is an + // FTDNA "Private Variants" report, which holds novel loci alone and sits off the + // haplotree. Another is an autosomal or mt VCF that got through. + // + // A record of its placeholder placement would conflict with a real call from another + // source, and would pull the donor consensus back to the root. So skip it. if assignment.ranked.first().map_or(true, |t| t.matched == 0) { continue; } @@ -4667,15 +4953,16 @@ impl App { Ok(placed) } - /// Fetch + parse the Y haplotree for a chip placement on `build`. DecodingUs is native multi-build - /// (no liftover); the FTDNA tree is GRCh38-only, so it's a fallback only when the calls are GRCh38. - /// Shared by the combined [`assign_y_bisdna`](Self::assign_y_bisdna) placement and the per-panel - /// Y-profile sources, so the tree is fetched once. + /// Fetch and parse the Y haplotree for a chip placement on `build`. DecodingUs is native on + /// every build, and needs no liftover. The FTDNA tree is GRCh38 only, so it is a fallback when + /// the calls are GRCh38, and at no other time. The combined + /// [`assign_y_bisdna`](Self::assign_y_bisdna) placement shares it with the Y-profile source of + /// each panel, so one fetch serves them all. pub(crate) async fn chip_y_tree(&self, build: &str) -> Result { - // Honor the configured Y-tree provider (the alignment placement path does too). With the - // FTDNA provider, place against the FTDNA tree directly — no DecodingUs call. The default - // (DecodingUs) keeps the prior behavior, with an FTDNA fallback for a GRCh38 chip when the - // DecodingUs tree is unavailable. + // Obey the configured Y-tree provider, as the alignment placement path does. With the + // FTDNA provider, place against the FTDNA tree directly, and make no DecodingUs call. The + // default is DecodingUs, which keeps the earlier behaviour. There a GRCh38 chip falls back + // to FTDNA when the DecodingUs tree is unavailable. if matches!(y_tree_provider(), YTreeProvider::Ftdna) { let json = self.fetch_ftdna_y_tree().await?; let mut tree = navigator_analysis::haplo::parse_ftdna_json(&json).map_err(AppError::Import)?; @@ -4696,8 +4983,8 @@ impl App { } } - /// Place one chip/BISDNA panel's chrY calls on `tree` (strand-reconciled), without persisting — - /// for assembling the per-panel sources of the Y-variant profile. + /// Place one chip or BISDNA panel's chrY calls on `tree`, strand-reconciled, and store nothing. + /// It builds the source of each panel for the Y-variant profile. fn place_chip_panel(tree: &navigator_analysis::haplo::HaploTree, calls: HashMap) -> HaploAssignment { let calls = strand_reconcile_to_tree(tree, calls); assemble_assignment_robust(tree, &calls) @@ -4761,26 +5048,30 @@ impl App { Ok(assemble_assignment(&tree, &calls)) } - /// Like [`assign_haplogroup_from_alignment`], but also returns the per-SNP evidence along - /// the called terminal's lineage (each defining mutation's Derived/Ancestral/NoCall state). - /// For exact comparisons (e.g. GRCh38 vs a lifted CHM13 call). - /// Full Y-haplogroup placement **report** for an alignment (gap §8): the ranked candidate - /// haplogroups (with score / matched-vs-expected) + the defining-SNP evidence along the reported - /// lineage (each SNP's derived / ancestral / no-call state). A fresh placement against the - /// configured provider tree — heavier than the cached terminal label, so it's button-driven. + /// Full Y-haplogroup placement **report** for an alignment (gap §8). It holds the ranked + /// candidate haplogroups, each with a score and a matched-against-expected count. It also holds + /// the SNP evidence along the reported lineage, with the derived, ancestral, or no-call state + /// of each SNP. + /// + /// It is a fresh placement against the configured provider tree. That costs more than the + /// cached terminal label, so a button starts it. pub async fn y_haplogroup_report( &self, alignment_id: i64, ) -> Result<(HaploAssignment, Vec), AppError> { - // Route through the provider-correct placement (DecodingUs native multi-build, FTDNA - // fallback with polarity normalization) — not a raw `parse_ftdna_json` of whatever tree the - // provider returns, which fails on the DecodingUs schema. The assignment already carries the - // root→terminal lineage evidence. + // Go through the placement that matches the provider. DecodingUs is native on every build, + // and FTDNA is the fallback, with its polarity normalized. Do not call `parse_ftdna_json` + // on whatever tree the provider returns, because that fails on the DecodingUs schema. The + // assignment already carries the root→terminal lineage evidence. let assignment = self.y_assignment_full(alignment_id).await?; let lineage = assignment.lineage.clone(); Ok((assignment, lineage)) } + /// Like [`assign_haplogroup_from_alignment`], but it also returns the evidence at each SNP + /// along the lineage of the called terminal. That is the Derived, Ancestral, or NoCall state of + /// each mutation that defines a node. It is for an exact comparison, such as GRCh38 against a + /// lifted CHM13 call. pub async fn assign_haplogroup_detail( &self, alignment_id: i64, @@ -4796,8 +5087,8 @@ impl App { Ok((assignment, lineage, calls)) } - /// Parse the tree, build the per-position base calls (lifting onto the alignment's build - /// when needed), and return both. Shared by the assignment + detail entry points. + /// Parse the tree, build the base call at each position, and return both. It lifts onto the + /// alignment's build when it must. The assignment and the detail entry points share it. pub(crate) async fn tree_base_calls( &self, alignment_id: i64, @@ -4805,10 +5096,13 @@ impl App { tree_json: &str, ) -> Result<(navigator_analysis::haplo::HaploTree, HashMap), AppError> { let mut tree = navigator_analysis::haplo::parse_ftdna_json(tree_json).map_err(AppError::Import)?; - // Harden the FTDNA tree: it records the GRCh38 *reference* base as "ancestral", so at the - // sites where the reference carries the derived allele its polarity is inverted (the - // CT-M168 amber artifact). Normalize against the DecodingUs tree's true polarity (best - // effort — offline FTDNA mode keeps the raw FTDNA polarity). Y only (no mt polarity source). + // Harden the FTDNA tree. It records the GRCh38 *reference* base as "ancestral". Take a + // site where the reference carries the derived allele: there the tree inverts the polarity. + // That is the CT-M168 amber artifact. + // + // Normalize against the DecodingUs tree's true polarity. It is best-effort: an offline + // FTDNA mode keeps the raw FTDNA polarity. This is for Y only, because there is no mt + // polarity source. if contig.eq_ignore_ascii_case("chrY") { if let Some(pol) = self.decodingus_y_polarity().await { let flipped = navigator_analysis::haplo::normalize_polarity(&mut tree, &pol); @@ -4831,10 +5125,11 @@ impl App { navigator_analysis::haplo::decodingus_polarity_map(&json).ok() } - /// Resolve a canonical contig name (`chrY`, `chrM`) to the name actually present in the - /// alignment header, tolerating naming conventions: the `chr` prefix (GRCh37/hg19 drop it) - /// and the `M`/`MT` mitochondrial spelling. Returns `None` when no equivalent contig is in - /// the header (the caller then queries the requested name and surfaces the original error). + /// Resolve a canonical contig name (`chrY`, `chrM`) to the name that the alignment header + /// holds. It accepts the usual conventions: the `chr` prefix, which GRCh37 and hg19 drop, and + /// the `M` or `MT` form for the mitochondrion. It returns `None` when the header has no + /// equivalent contig. The caller then queries the name it asked for, and shows the original + /// error. async fn resolve_header_contig( &self, bam: &Path, @@ -4857,29 +5152,24 @@ impl App { .into_iter() .find(|cand| names.iter().any(|n| n.eq_ignore_ascii_case(cand))) .and_then(|cand| { - // Return the header's exact casing/spelling so the region query matches. + // Return the exact form from the header, so the region query matches. names.iter().find(|n| n.eq_ignore_ascii_case(&cand)).cloned() })) } - /// Base-call an alignment at a parsed tree's positions on `contig`. `tree_source_build` is - /// the build the tree's positions are in: when it differs from the alignment build the - /// positions are lifted (chrY chain), queried there, and mapped back; `None` (e.g. a - /// DecodingUs tree already in the alignment's build, or mt/rCRS-direct) queries directly. + /// Tree-locus base calls for one alignment, for the **genome-consensus placement**. + /// + /// It prefers the alignment's external sidecar GVCF, which needs no CRAM decode, when the + /// "prefer external caller" policy is on and the GVCF is present. If not, it takes the cached + /// CRAM walk ([`base_calls`]). /// - /// The result (tree-position → base) is cached as a versioned analysis artifact keyed by the - /// queried **site set** (a hash of the tree's positions) + contig + lift source, and - /// invalidated by the alignment's `source_sig` (BAM/CRAM mtime:size). This is the BAM-walk - /// chokepoint for *every* genotyping path (Y/mt placement, the variant profile, genome - /// consensus), so a profile **rebuild** reuses the cached genotypes instead of re-walking the - /// reads — only a changed file or a changed tree site set forces a fresh walk. - /// Tree-locus base calls for one alignment for the **genome-consensus placement**, preferring the - /// alignment's external sidecar GVCF (no CRAM decode) when the "prefer external caller" policy is - /// on and the GVCF is present; otherwise the cached CRAM walk ([`base_calls`]). A drop-in for the - /// per-alignment genotype in `place_{y,mt}_consensus`, so a preferred-external (e.g. ancient-DNA) - /// subject's damaged CRAM is not re-walked and cannot dilute the pooled placement (Phase 2 of - /// `documents/design/external-caller-precedence.md` §4.5). `tree_source_build` matches what `base_calls` - /// receives — `None` for a native-build tree (DecodingUs Y, rCRS mt), the tree's build for a lift. + /// It replaces the alignment genotype step in `place_{y,mt}_consensus`. Take a + /// preferred-external subject, such as an ancient-DNA one. The code then does not walk that + /// subject's damaged CRAM, and the CRAM can not dilute the pooled placement. See Phase 2 of + /// `documents/design/external-caller-precedence.md` §4.5. + /// + /// `tree_source_build` matches what `base_calls` receives: `None` for a tree in the native + /// build (DecodingUs Y, rCRS mt), and the tree's build for a lift. pub(crate) async fn consensus_base_calls( &self, aln: &Alignment, @@ -4902,6 +5192,21 @@ impl App { self.base_calls(aln.id, contig, tree, tree_source_build).await } + /// Base-call an alignment at a parsed tree's positions on `contig`. + /// + /// `tree_source_build` names the build that the tree's positions are in. When that is not the + /// alignment build, the code lifts the positions through the chrY chain, queries them there, + /// and maps them back. `None` queries directly, as for a DecodingUs tree that is already in the + /// alignment's build, or for a direct mt or rCRS query. + /// + /// The cache holds the result (tree-position → base) as a versioned analysis artifact. Its key + /// is the queried **site set**, which is a hash of the tree's positions, plus the contig and + /// the lift source. The alignment's `source_sig`, its BAM/CRAM mtime and size, invalidates it. + /// + /// This is the one place where *every* genotype path walks the BAM: Y and mt placement, the + /// variant profile, and the genome consensus. So a **rebuild** of a profile reuses the cached + /// genotypes, and does not read the reads again. Only a changed file, or a changed tree site + /// set, forces a fresh walk. async fn base_calls( &self, alignment_id: i64, @@ -4929,15 +5234,16 @@ impl App { } let aln = self.alignment_or_err(alignment_id).await?; - // Copy off a slow/removable volume to local disk first — the per-locus genotyping read is a - // network round-trip per record otherwise (see App::localize). + // Copy off a slow or removable volume to a local disk first. Without that, the genotype + // read at each locus is one network round trip for each record (see App::localize). let bam = self.localize(&Self::alignment_file(&aln)?).await; let bam = bam.path().to_path_buf(); - // Resolve the reference even when none was stored at import. A CRAM can't be decoded - // without it, so resolve (download on a miss) via the gateway from the alignment's build — - // e.g. the already-cached `chm13v2.0.fa` for a CHM13 CRAM. A BAM needs no reference to - // read reads, so only adopt a *cached* FASTA (never force a multi-GB download just to - // supply the chrM liftover map). + // Resolve the reference even when the import stored none. The code can not decode a CRAM + // without it. So resolve it through the gateway, from the alignment's build, and download + // it on a miss. An example is the cached `chm13v2.0.fa` for a CHM13 CRAM. + // + // A BAM needs no reference to read its reads. So for a BAM take a *cached* FASTA only. + // Never force a multi-GB download to supply the chrM liftover map. let is_cram = bam.extension().is_some_and(|e| e.eq_ignore_ascii_case("cram")); let reference = match aln.reference_path { Some(p) => Some(PathBuf::from(p)), @@ -4959,10 +5265,13 @@ impl App { ) .await?; - // Indel loci (multi-base ancestral/derived) on the tree — genotyped separately on the native - // chrY path (VCF left-anchored; needs the reference to normalize + know deleted bases). Their - // resolved sentinel overlays the (meaningless) base call at the anchor. Liftover of indel - // coordinates isn't handled, so only the native path (no lift) contributes them. + // The indel loci on the tree, which have a multi-base ancestral and derived allele. The + // native chrY path genotypes them on its own. Each one carries a left anchor, as in a VCF. + // The code needs the reference to normalize them, and to know the deleted bases. + // + // The resolved sentinel goes over the base call at the anchor, which means nothing. The + // code does not lift an indel coordinate, so the native path, with no lift, is the only one + // that supplies them. let indel_targets: Vec<(i64, String, String)> = if contig.eq_ignore_ascii_case("chrY") { tree.nodes .values() @@ -4977,10 +5286,10 @@ impl App { let calls = match lifted { Some(lifted) => self.build_calls_from_lifted(&bam, reference.as_deref(), lifted).await?, None => { - // Match the requested contig to the header's naming convention: GRCh37/hg19 - // (still the medical-space default) use bare `Y`/`MT`; CHM13/GRCh38 use - // `chrY`/`chrM`. Without this a `chrY` query against a `Y`-named header errors - // out and the placement falls back to a worse tree. + // Match the requested contig to the convention of the header. GRCh37 and hg19, + // which are still the default in medicine, use a bare `Y` or `MT`. CHM13 and + // GRCh38 use `chrY` and `chrM`. Without this, a `chrY` query against a header that + // names it `Y` gives an error, and the placement falls back to a worse tree. let resolved = self .resolve_header_contig(&bam, reference.as_deref(), contig) .await? @@ -5011,10 +5320,14 @@ impl App { Ok(calls) } - /// Lift the haplotree's positions onto the alignment's build, or `None` to query the tree - /// positions directly. **chrY**: uses the (auto-downloaded) GRCh38→build liftover chain. - /// **chrM**: a self-generated rCRS↔`chrM` map — bundled rCRS aligned to *this* reference's - /// `chrM` (CHM13 builds only; GRCh38/rCRS `chrM` is already rCRS → direct). + /// Lift the haplotree's positions onto the alignment's build. `None` means: query the tree + /// positions directly. + /// + /// **chrY** uses the GRCh38→build liftover chain, which the app downloads on demand. + /// + /// **chrM** uses an rCRS↔`chrM` map that the app builds itself, from an alignment of the + /// bundled rCRS to *this* reference's `chrM`. That is for a CHM13 build only. On GRCh38 the + /// `chrM` is already rCRS, so the query is direct. pub(crate) async fn lifted_targets( &self, reference_build: &str, @@ -5029,10 +5342,10 @@ impl App { // chrY: downloaded nuclear chain (when the tree build differs from the alignment). if let Some(src) = tree_source_build { - // `src` must name a reference build. A tree *provider* ("decodingus"/"ftdna") would - // fall through the `differ` test to the `return Ok(None)` below, silently disabling - // liftover — for chrM that skips the rCRS↔chrM map and miscalls every marker. Refuse - // it here rather than answer with wrong coordinates. + // `src` must name a reference build. A tree *provider*, "decodingus" or "ftdna", would + // fall through the `differ` test to the `return Ok(None)` below. That turns the + // liftover off with no warning. For chrM it skips the rCRS↔chrM map, and every marker + // is then wrong. Refuse it here, and do not answer with wrong coordinates. let Some(src_build) = canonical_build(src) else { return Err(AppError::Import(format!( "lifted_targets: tree_source_build {src:?} is not a reference build \ @@ -5057,7 +5370,8 @@ impl App { if contig.eq_ignore_ascii_case("chrM") && canonical_build(reference_build) == Some(ReferenceBuild::Chm13v2) { let Some(reference) = reference else { return Ok(None) }; let reference = reference.to_path_buf(); - // Align bundled rCRS to this reference's chrM (cheap, ~16.5 kb) → (rcrs, chrM) pairs. + // Align the bundled rCRS to this reference's chrM, which is small at about 16.5 kb. + // That gives (rcrs, chrM) pairs. let map = tokio::task::spawn_blocking(move || { navigator_analysis::reader::read_contig_sequence(&reference, "chrM").map(|chrm| { let chrm = String::from_utf8_lossy(&chrm).into_owned(); @@ -5097,8 +5411,9 @@ impl App { reference: Option<&Path>, lifted: Vec, ) -> Result, AppError> { - // Group lifted positions by their target contig + a back-map (lifted → tree position, - // plus whether the lift was to the minus strand → the base needs complementing). + // Group the lifted positions by their target contig, and build a back-map from the lifted + // position to the tree position. The map also records a lift to the minus strand, where the + // code must take the complement of the base. let mut by_contig: HashMap> = HashMap::new(); let mut back: HashMap<(String, i64), (i64, bool)> = HashMap::new(); for lp in lifted { @@ -5106,7 +5421,7 @@ impl App { back.insert((lp.contig, lp.pos), (lp.tree_pos, lp.reverse)); } - // Only query contigs the alignment actually has (drop off-target lifts). + // Query only the contigs that the alignment has, and drop an off-target lift. let header_contigs: HashSet = { let bam = bam.to_path_buf(); let reference = reference.map(|p| p.to_path_buf()); @@ -5118,8 +5433,8 @@ impl App { let mut calls: HashMap = HashMap::new(); for (qcontig, set) in by_contig { - // Tolerate naming conventions between the lift target and the header (e.g. a `chrY` - // lift against a GRCh37 `Y`-named header): query the header's actual spelling. + // Accept a difference of convention between the lift target and the header, such as a + // `chrY` lift against a GRCh37 header that names it `Y`. Query the form in the header. let bare = navigator_analysis::contig::bare(&qcontig); let Some(query_contig) = header_contigs .iter() @@ -5150,9 +5465,9 @@ impl App { } } -/// Interpret an [`navigator_domain::consensus::ObservedProfile`] against a polarity map into the -/// app's display [`ConsensusProfile`] (carrying provenance + terminal). The single place observations -/// become the interpreted view. +/// Interpret an [`navigator_domain::consensus::ObservedProfile`] against a polarity map, into the +/// app's display [`ConsensusProfile`], which holds the provenance and the terminal. This is the one +/// place where an observation becomes the interpreted view. fn interpret_observed( observed: navigator_domain::consensus::ObservedProfile, polarity: &std::collections::BTreeMap, @@ -5174,10 +5489,10 @@ fn interpret_observed( } } -/// Normalize a legacy baked [`ConsensusProfile`] payload into an -/// [`navigator_domain::consensus::ObservedProfile`], preserving each source's stored observed base (a -/// base-less legacy source becomes a no-call on interpret until the next rebuild). Load-time -/// backward-compat only. +/// Normalize a legacy [`ConsensusProfile`] payload into an +/// [`navigator_domain::consensus::ObservedProfile`]. It keeps the observed base that each source +/// stored. A legacy source with no base becomes a no-call at interpret time, until the next +/// rebuild. This is for compatibility at load time, and nothing else. fn observed_from_legacy(legacy: ConsensusProfile) -> navigator_domain::consensus::ObservedProfile { use navigator_domain::consensus::{ObservedProfile, ObservedSource, ObservedVariant, SourceSummary}; ObservedProfile { @@ -5237,7 +5552,7 @@ mod lifted_targets_tests { let fa = dir.join("rotated-chrM.fa"); std::fs::write(&fa, format!(">chrM\n{chrm}\n")).unwrap(); - // name, length, offset-of-first-base, bases-per-line, bytes-per-line + // name, length, offset of the first base, bases in a line, bytes in a line std::fs::write(fa.with_extension("fa.fai"), format!("chrM\t{n}\t6\t{n}\t{}\n", n + 1)).unwrap(); fa } @@ -5280,9 +5595,9 @@ mod lifted_targets_tests { assert!(lifted.iter().any(|l| l.tree_pos == 263 && l.pos > 16_000)); } - /// The bug this guards: `mt_tree_rcrs` returns a tree *provider*, not a build. Passing it as - /// `tree_source_build` used to fall through to `Ok(None)`, silently skipping the chrM map above - /// and miscalling every marker. It must be refused instead. + /// The bug that this guards against: `mt_tree_rcrs` returns a tree *provider*, and not a + /// build. As a `tree_source_build` it used to fall through to `Ok(None)`. That skipped the chrM + /// map above with no warning, and made every marker wrong. The code must refuse it instead. #[tokio::test] async fn a_tree_provider_is_refused_as_a_reference_build() { let app = App::new(Store::open_in_memory().await.unwrap()); @@ -5302,8 +5617,8 @@ mod lifted_targets_tests { } } - /// A real build that matches the alignment's build needs no lift — still `Ok(None)`, not an - /// error. Pins that the new guard didn't narrow the chrY path. + /// A real build that matches the alignment's build needs no lift. It stays `Ok(None)`, and is + /// not an error. This pins that the new guard did not narrow the chrY path. #[tokio::test] async fn a_matching_reference_build_still_means_no_lift() { let app = App::new(Store::open_in_memory().await.unwrap()); @@ -5316,16 +5631,18 @@ mod lifted_targets_tests { } } -/// Whether `label` names a branch, as opposed to the variant string a placement falls back to when -/// the node it landed on carried no usable name. +/// Whether `label` names a branch. The other case is the variant string that a placement falls +/// back to, when the node it landed on carried no usable name. +/// +/// A branch name is alphanumeric, with hyphens and dots: `R-DU17762`, `A0-T`, `E-FT400514:n0`. The +/// fallback shows the variant of the node instead: `chrY:5216846A>C [Node721]`, or +/// `CP086569.2:27785335 G->A`. /// -/// Branch names are alphanumeric with hyphens and dots — `R-DU17762`, `A0-T`, `E-FT400514:n0`. The -/// fallback renders the defining variant instead: `chrY:5216846A>C [Node721]`, -/// `CP086569.2:27785335 G->A`. Keys on `>`, which every `ref>alt` rendering contains and no -/// haplogroup name does — a colon alone would misjudge `E-FT400514:n0`, which is a real label. +/// The test keys on `>`. Every `ref>alt` form holds one, and no haplogroup name does. A colon alone +/// would misjudge `E-FT400514:n0`, which is a real label. /// -/// Both examples are real rows from this workspace, on an external call and a navigator-walk call -/// respectively, so the fallback is not confined to one code path. +/// Both examples are real rows from this workspace. One is on an external call, and the other on a +/// navigator-walk call. So the fallback is not limited to one code path. fn names_a_branch(label: &str) -> bool { !label.contains('>') } @@ -5444,12 +5761,14 @@ mod painting_anchor_tests { #[test] fn anchors_to_the_side_carrying_the_transmitted_allele() { - // Parent is hom-alt everywhere → transmits the alt allele; the side carrying alt is theirs. + // The parent is hom-alt everywhere, so they transmit the alt allele. The side that holds + // alt is theirs. let parent: Vec = (0..60).map(|i| parent_geno(1 + i as i64 * 1000, 2)).collect(); assert_eq!(anchor_side_to_parent(&child_het(60, 0), &parent), Some(0)); assert_eq!(anchor_side_to_parent(&child_het(60, 1), &parent), Some(1)); - // Parent hom-ref everywhere → transmits ref; the side carrying ref (the OTHER side) is theirs. + // The parent is hom-ref everywhere, so they transmit ref. The side that holds ref, which + // is the OTHER side, is theirs. let parent_ref: Vec = (0..60).map(|i| parent_geno(1 + i as i64 * 1000, 0)).collect(); assert_eq!(anchor_side_to_parent(&child_het(60, 0), &parent_ref), Some(1)); } diff --git a/crates/navigator-app/src/ibd_exchange.rs b/crates/navigator-app/src/ibd_exchange.rs index 805b9bec..ad6d8c81 100644 --- a/crates/navigator-app/src/ibd_exchange.rs +++ b/crates/navigator-app/src/ibd_exchange.rs @@ -5,9 +5,10 @@ use super::*; impl App { // ---- IBD Phase 2: encrypted edge-to-edge exchange (D1 substrate) ------- // - // The AppView brokers discovery/consent + relays opaque ciphertext (never decrypts). These - // wrap the `/api/v1/exchange/*` endpoints; the crypto (X25519/X3DH-lite/AES-GCM) lives in - // `navigator_sync::exchange`. All calls are device-key-signed (no per-call OAuth). + // The AppView is the broker for discovery and consent. It also relays opaque ciphertext, and + // it never decrypts that ciphertext. These methods wrap the `/api/v1/exchange/*` endpoints. + // The cryptography, which is X25519, X3DH-lite, and AES-GCM, is in + // `navigator_sync::exchange`. The device key signs each call, and no call uses OAuth. /// The signed-in account's X25519 identity key (load-or-generate), with its public half /// published to the AppView (`POST /exchange/key`, idempotent upsert) so partners can fetch it. @@ -24,8 +25,8 @@ impl App { Ok(ik) } - /// Fetch a peer's published X25519 public key (STANDARD base64), or `None` if they haven't - /// published one. Public read — no signature. + /// Read the X25519 public key that a peer published, in STANDARD base64. The method returns + /// `None` when the peer published no key. This read is public and needs no signature. pub async fn fetch_exchange_key(&self, did: &str) -> Result, AppError> { let url = self.appview_url("exchange/key"); let resp = self @@ -46,11 +47,15 @@ impl App { Ok(v.get("x25519_pub").and_then(|x| x.as_str()).map(str::to_string)) } - /// Open an exchange request to a specific partner DID — the direct counterpart to the - /// suggestion-mediated [`ibd_introduce`] (`POST /api/v1/exchange/request`). Generates an opaque - /// request URI, signs the canonical request message, and posts it. The partner discovers it via - /// [`exchange_incoming`] (symmetric-blind) and consents; on mutual consent a session opens. Returns - /// the request URI to track. `scope` carries an optional project scope (team-ACL-gated server-side). + /// Open an exchange request to one partner DID (`POST /api/v1/exchange/request`). This method + /// is the direct form of [`ibd_introduce`], which works through a suggestion. + /// + /// The method makes an opaque request URI, signs the canonical request message, and sends it. + /// The partner finds the request with [`exchange_incoming`], which is symmetric-blind, and then + /// agrees. After both parties agree, a session opens. + /// + /// The method returns the request URI, and the caller uses it to track the request. `scope` can + /// carry a project scope, and the server gates that scope with the team ACL. pub async fn exchange_request( &self, partner_did: &str, @@ -103,7 +108,8 @@ impl App { }) } - /// Poll for inbound (symmetric-blind) exchange requests awaiting this account's consent. + /// Poll for the exchange requests that arrived and that need the consent of this account. The + /// view is symmetric-blind. pub async fn exchange_incoming(&self) -> Result, AppError> { let v = self.exchange_get_poll("exchange/incoming", &[]).await?; Ok(v.get("items") @@ -169,8 +175,9 @@ impl App { .unwrap_or_default()) } - /// Relay an opaque ciphertext `blob` to `to_did` in a session. The signed hash binds the blob to - /// its routing (the broker stores ciphertext only). Returns the broker envelope id. + /// Relay an opaque ciphertext `blob` to `to_did` in a session. The signed hash binds the blob + /// to its route. The broker stores only ciphertext. The method returns the envelope id of the + /// broker. pub async fn exchange_relay(&self, session_id: &str, to_did: &str, seq: i32, blob: &str) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let dev = self.ensure_device_key().await?; @@ -225,11 +232,15 @@ impl App { self.appview_post("exchange/ack", body).await.map(|_| ()) } - /// Establish a shared session key for a consent-ready session: publish/load our identity key, - /// fetch the partner's, exchange ephemeral keys via the relay (handshake, seq 0), and derive the - /// X3DH-lite session key. Polls the relay up to ~15s for the partner's handshake. The returned - /// [`EstablishedSession`] then seals/opens payloads. (Live-only — needs a running AppView + the - /// partner edge online to complete the handshake.) + /// Make a shared session key for a session that both parties agreed to. + /// + /// The method publishes or reads our identity key, then reads the identity key of the partner. + /// The two edges exchange short-life keys through the relay, as a handshake at seq 0. The + /// method then derives the X3DH-lite session key. It polls the relay for up to 15 seconds for + /// the handshake of the partner. + /// + /// The [`EstablishedSession`] value then seals a payload and opens a payload. This method needs + /// a live AppView and a partner edge that is online, so a test can not run it offline. pub async fn open_exchange_session(&self, info: &ExchangeSessionInfo) -> Result { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let ik = self.ensure_exchange_key().await?; @@ -241,7 +252,8 @@ impl App { let hs = exchange::Envelope::handshake(&ek).to_blob().map_err(AppError::Sync)?; self.exchange_relay(&info.session_id, &info.partner_did, 0, &hs).await?; - // Wait for the partner's handshake (seq 0 / a Handshake envelope), acking just it. + // Wait for the handshake of the partner, which is seq 0 in a Handshake envelope. + // Acknowledge only that envelope. let mut their_ek: Option = None; for _ in 0..15 { for env in self.exchange_relay_pull(&info.session_id).await? { @@ -290,8 +302,9 @@ impl App { .await } - /// Pull + decrypt + ack the data payloads waiting on an established session (returns plaintexts - /// in pull order). Non-data / undecryptable envelopes are left un-acked. + /// Read, decrypt, and acknowledge each data payload on an open session. The method returns the + /// plaintexts in the order of the read. It does not acknowledge an envelope that holds no data, + /// or an envelope that it can not decrypt. pub async fn exchange_receive(&self, session: &EstablishedSession) -> Result>, AppError> { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let mut out = Vec::new(); @@ -299,7 +312,7 @@ impl App { let Ok(parsed) = exchange::Envelope::from_blob(&env.blob) else { continue; }; - // AAD binds the sender's routing: from = the partner (sender), to = us. + // The AAD binds the route of the sender. `from` is the partner, and `to` is us. let aad = exchange::relay_aad(&session.session_id, &env.from_did, &did, env.seq); if let Ok(pt) = exchange::open(&session.key, &aad, &parsed) { out.push(pt); @@ -309,12 +322,19 @@ impl App { Ok(out) } - /// Run a **federated IBD exchange** over an established session (gap §4): send our IBD-panel - /// dosages, receive the partner's, detect IBD locally (both peers run the symmetric detector → - /// identical summary), then exchange + verify signed [`IbdAttestation`]s. `agreed` ⇒ the partner's - /// signature verified and both summary hashes match. Only panel dosages cross the wire (encrypted; - /// the broker never sees them). `my_source` supplies our dosages; the refs are opaque biosample - /// pointers carried in the attestation. Live-only — needs the partner edge online. + /// Do a **federated IBD exchange** on an open session (gap §4). + /// + /// The method sends the dosages of our IBD panel and receives the dosages of the partner. It + /// then finds the IBD segments on this machine. Both peers run the same symmetric detector, so + /// both get the same summary. The two edges then exchange signed [`IbdAttestation`] values and + /// check them. + /// + /// The `agreed` field is true when the signature of the partner is correct and the two summary + /// hashes are the same. + /// + /// Only the panel dosages cross the network. The code encrypts them, and the broker never sees + /// them. `my_source` gives our dosages. The refs are opaque pointers to a biosample, and the + /// attestation carries them. The method needs the partner edge online. pub async fn exchange_ibd( &self, session: &EstablishedSession, @@ -337,8 +357,9 @@ impl App { .await } - /// The dosage-level core of [`exchange_ibd`] — takes the panel dosages directly (e.g. from a - /// consensus profile, or synthetic vectors in tests) rather than resolving an [`IbdSource`]. + /// The core of [`exchange_ibd`] at the dosage level. This method takes the panel dosages + /// directly and does not read an [`IbdSource`]. The dosages can come from a consensus profile, + /// or from a test vector. pub async fn exchange_ibd_with_dosages( &self, session: &EstablishedSession, @@ -351,8 +372,9 @@ impl App { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; let dev = self.ensure_device_key().await?; - // Fit the relay's 1 MiB envelope: decimate a large panel (both peers apply the same - // position-based rule, so the intersection is preserved). Detect on the decimated set we send. + // Make the data fit the 1 MiB envelope of the relay. For a large panel, the code removes + // sites. Both peers use the same rule, which depends on the position, so the two sets still + // intersect. The detector uses the smaller set that the code sends. let my_sites = decimate_for_exchange(my_sites); // 1. Send our dosages (the IBD panel is on CHM13 / hs1). @@ -363,7 +385,7 @@ impl App { self.exchange_send(session, 1, &dos.to_bytes().map_err(AppError::Import)?) .await?; - // 2. Receive the partner's dosages (buffering any attestation that arrives early). + // 2. Receive the dosages of the partner. Keep an attestation that arrives too early. let mut partner_sites: Option> = None; let mut partner_att: Option = None; for _ in 0..EXCHANGE_POLL_ROUNDS { @@ -382,7 +404,8 @@ impl App { let partner_sites = partner_sites .ok_or_else(|| AppError::AppView("partner IBD dosages not received (peer offline?)".into()))?; - // 3. Detect IBD locally (symmetric — the partner computes the same summary). + // 3. Find the IBD segments on this machine. The detector is symmetric, so the partner + // calculates the same summary. let comparison = detect_ibd_sites(&my_sites, &partner_sites, ReferenceBuild::Chm13v2, config); // 4. Sign our attestation over the computed summary. @@ -421,7 +444,7 @@ impl App { let partner_att = partner_att.ok_or_else(|| AppError::AppView("partner attestation not received (peer offline?)".into()))?; - // 7. Verify the partner's signature + summary-hash agreement. + // 7. Check the signature of the partner and compare the two summary hashes. let sig_ok = du_atproto::verify_did_key( &partner_att.signing_public_key, partner_att.canonical().as_bytes(), @@ -468,8 +491,8 @@ impl App { .map(|c| IbdSource::Chip(c.id))) } - /// The subject's IBD-panel dosages from its best source (panel-restricted — only the canonical IBD - /// sites, not the whole genome, so that's all that can leave the device). + /// The IBD-panel dosages of the subject, from its best source. The set holds only the + /// canonical IBD sites and not the full genome. So no other site can leave the device. pub async fn ibd_dosages_for_subject(&self, guid: SampleGuid) -> Result, AppError> { let source = self.best_ibd_source_for_subject(guid).await?.ok_or_else(|| { AppError::Import("no IBD-capable data for this subject (need an alignment or a chip profile)".into()) @@ -507,8 +530,9 @@ impl App { ) .await?; self.record_ibd_exchange(guid, session, request_uri, &result).await?; - // Advance the ledger before either publish: the comparison is done and persisted, so the - // conversation is complete whether or not the network steps below succeed. + // Advance the ledger before the two publish steps. The comparison is complete and in the + // store. So the conversation is complete, and a failure in the network steps below does + // not change that. self.mark_matching_exchanged(guid, session, request_uri).await?; // Best-effort: publish our attestation to the PDS (skipped for did:key; never fails the exchange). let _ = self.publish_ibd_attestation(&result.my_attestation).await; @@ -557,8 +581,10 @@ impl App { Ok(navigator_store::ibd_exchange::list_for_biosample(self.store.pool(), guid).await?) } - /// Publish a signed attestation to the PDS (the AppView indexes it via Jetstream). No-op for a - /// did:key local identity (self-certifying, no repo to write). Idempotent via a session-derived rkey. + /// Publish a signed attestation to the PDS. The AppView then indexes it through Jetstream. + /// + /// The method does nothing for a local did:key identity. Such an identity certifies itself and + /// has no repository to write to. The rkey comes from the session, so a second call is safe. pub async fn publish_ibd_attestation(&self, att: &IbdAttestation) -> Result<(), AppError> { let did = self.current_account().ok_or(AppError::NotAuthenticated)?; if did.starts_with("did:key:") { @@ -578,18 +604,23 @@ impl App { Ok(()) } - /// Issue a device-key-signed `exchange-poll` GET to an `/api/v1/` endpoint, with `extra` - /// query params appended. Shared by incoming / pending / relay-pull — the exchange endpoints - /// all sign the same canonical poll string, so this is the only thing they add over - /// [`App::appview_get_signed`]. + /// Send an `exchange-poll` GET to an `/api/v1/` endpoint with a device-key signature, and + /// add the `extra` query parameters. + /// + /// Three callers use this method: the poll for requests that arrived, the poll for open + /// sessions, and the relay read. Each exchange endpoint signs the same canonical poll string. + /// That string is the only part that this method adds to [`App::appview_get_signed`]. async fn exchange_get_poll(&self, path: &str, extra: &[(&str, &str)]) -> Result { self.appview_get_signed(path, exchange::messages::poll, extra).await } - /// Enqueue the anchor records every child record references: the subject's biosample summary - /// and each of its sequence runs, at their **deterministic** rkeys so the at:// URIs resolve. - /// Idempotent — the outbox coalesces per `entity_ref` and re-publishing overwrites in place — so - /// child publishes call this freely to guarantee there's always a biosample to tie back to. + /// Put the anchor records in the queue. Each child record points to these anchors. The anchors + /// are the biosample summary of the subject and each of its sequence runs. They use **fixed** + /// rkeys, so the at:// URIs resolve. + /// + /// A second call is safe. The outbox joins the rows with the same `entity_ref`, and a second + /// publish replaces the record. So a child publish can always call this method, and a biosample + /// always exists for the child to point to. async fn ensure_subject_anchor(&self, did: &str, biosample_guid: SampleGuid) -> Result<(), AppError> { // Sequence runs first (the biosample record links to them). for run in self.list_sequence_runs(biosample_guid).await? { @@ -614,9 +645,10 @@ impl App { .await } - /// Publish the alignment's coverage summary to the signed-in account's PDS (with - /// refresh-on-expiry and retry/backoff via [`AsyncSync`]). Anchors the subject first so the - /// record's biosample/sequence-run refs resolve. + /// Publish the coverage summary of the alignment to the PDS of the active account. + /// [`AsyncSync`] refreshes an expired token and tries again after a failure, with a longer + /// delay each time. The method publishes the subject anchors first, so the biosample ref and + /// the sequence-run ref of the record resolve. pub async fn publish_coverage(&self, alignment_id: i64) -> Result<(), AppError> { let did = self.require_account()?; // auth check before touching the DB let guid = self.biosample_of_alignment(alignment_id).await?; @@ -626,25 +658,32 @@ impl App { "coverage", &format!("alignment:{alignment_id}"), NS_ALIGNMENT, - // Deterministic rkey → the idempotent put path (never a fresh create), so re-publishing - // or two concurrent drains converge on one record instead of duplicating. + // A fixed rkey selects the put path, which is safe to repeat. The code never calls + // create here. So a second publish, or two drains at the same time, give one record + // and not two. Some(&alignment_rkey(alignment_id)), value, ) .await } - /// Publish a subject's **consensus** ancestry estimate to the signed-in account's PDS — one - /// populationBreakdown record per method (ADMIXTURE / PCA_PROJECTION_GMM / FINE_ADMIXTURE / - /// G25_NMONTE), each linked to the biosample. Subject-level (the breakdown is computed from the - /// pooled autosomal consensus, not per alignment), so one authoritative record set per subject - /// rather than a conflicting set per sequencing run. The researcher opt-in act for the ancestry - /// section — anonymized population proportions only. + /// Publish the **consensus** ancestry estimate of a subject to the PDS of the active account. + /// + /// The method writes one populationBreakdown record for each method. The methods are ADMIXTURE, + /// PCA_PROJECTION_GMM, FINE_ADMIXTURE, and G25_NMONTE. Each record links to the biosample. + /// + /// The estimate belongs to the subject, not to one alignment, because the code calculates the + /// breakdown from the pooled autosomal consensus. So each subject has one record set with + /// authority. A set for each sequence run would give records that disagree. + /// + /// This is the action that a researcher opts in to for the ancestry section. Only anonymous + /// population proportions cross the network. pub async fn publish_ancestry(&self, biosample_guid: SampleGuid) -> Result<(), AppError> { let did = self.require_account()?; // auth check before touching the DB self.ensure_subject_anchor(&did, biosample_guid).await?; // the breakdown links back to it let biosample_ref = biosample_at_uri(&did, biosample_guid); - // One outbox row per method, keyed by subject+method so re-publishing coalesces per estimate. + // One outbox row for each method. The key is the subject and the method, so a second + // publish joins the rows of one estimate. for r in &self.consensus_ancestry_results(biosample_guid).await? { let value = serde_json::to_value(population_breakdown_record(r).with_biosample_ref(Some(biosample_ref.clone())))?; @@ -655,9 +694,10 @@ impl App { Ok(()) } - /// Publish the anonymized biosample summary (sex, haplogroups) **and its sequence runs** to the - /// signed-in account's PDS — the subject anchor every derived record ties back to. Deterministic - /// rkeys make it idempotent (a re-publish overwrites rather than duplicating). + /// Publish the anonymous biosample summary, which holds the sex and the haplogroups, **and its + /// sequence runs** to the PDS of the active account. These records are the subject anchor, and + /// each derived record points to them. The method uses a fixed rkey, so a second publish + /// replaces the records and does not add a copy. pub async fn publish_biosample(&self, biosample_guid: SampleGuid) -> Result<(), AppError> { let did = self.require_account()?; // auth check before touching the DB self.ensure_subject_anchor(&did, biosample_guid).await diff --git a/crates/navigator-app/src/import_profiles.rs b/crates/navigator-app/src/import_profiles.rs index 1331e0ae..ee6ce6ee 100644 --- a/crates/navigator-app/src/import_profiles.rs +++ b/crates/navigator-app/src/import_profiles.rs @@ -2,10 +2,15 @@ //! 2026-06 simplification round; `use super::*` reaches the crate-root types + free helpers. use super::*; -/// Process-wide memo of the parsed Y-SNP dictionary. Now that [`YsnpDictionary`] prefers the full -/// ~2M-row catalog, parsing it per resolve/annotate call (`y_snp_names_at` runs on every Y-SNP-table -/// view) would re-read ~200 MB each time; this parses once and reuses it. Keyed by the resolved -/// dictionary file's path + signature (mtime:size), so a refreshed dictionary is picked up. +/// One copy of the parsed Y-SNP dictionary for the full process. +/// +/// [`YsnpDictionary`] now selects the full catalog, which holds about 2 million rows. A parse of +/// that file at each call would read about 200 MB each time. The `y_snp_names_at` function runs at +/// each view of the Y-SNP table, so those calls are frequent. This value holds the result of one +/// parse. +/// +/// The key is the path of the dictionary file together with its signature, which is the mtime and +/// the size. So the code reads a new dictionary after the user replaces the file. type YsnpMemo = Mutex)>>; static YSNP_MEMO: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -47,9 +52,10 @@ impl App { ) -> Result { let text = std::fs::read_to_string(csv_path)?; let markers = strprofile::parse_csv(&text).map_err(AppError::Import)?; - // Merge into an existing same-panel profile rather than creating a duplicate — e.g. a Big Y - // CUSTOM (700/500) panel re-imported after the FTDNA project import already made one. Union - // the markers, the freshly-imported value winning on a conflict. + // Add the markers to a profile of the same panel, when one exists. Do not make a second + // profile. One example is a Big Y CUSTOM panel, of 700 or 500 markers, that the user + // imports after the FTDNA project import made the profile. The code joins the two marker + // sets. On a conflict, the value from the new import wins. if let Some(existing) = str_profile::find_by_panel(self.store.pool(), biosample_guid, panel_name).await? { let mut merged = existing.markers.clone(); for m in markers { @@ -84,10 +90,15 @@ impl App { // ---- SNP variants ------------------------------------------------------ - /// Import a subject's SNP variant calls from a file. `.vcf` is parsed as a VCF (reusing - /// the shared column parser); `.csv`/`.tsv` as a `contig,position,ref,alt[,rsid][,gt]` - /// table (a YSEQ/Sanger panel export fits this). Indels/symbolic alleles are dropped - /// (SNP-only). `source_type` sets the concordance weight (Sanger = gold standard). + /// Import the SNP variant calls of a subject from a file. + /// + /// The code parses a `.vcf` file as a VCF, with the shared column parser. It parses a `.csv` + /// file or a `.tsv` file as a `contig,position,ref,alt[,rsid][,gt]` table. A YSEQ panel export + /// and a Sanger panel export have that shape. + /// + /// The code keeps only SNPs. It removes each indel and each symbolic allele. `source_type` sets + /// the weight of the source in the concordance calculation, and a Sanger source has the highest + /// weight. pub async fn import_variants_from_file( &self, biosample_guid: SampleGuid, @@ -98,8 +109,9 @@ impl App { .file_name() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| "variants".into()); - // Match `.vcf`, plus bgzipped/gzipped `.vcf.gz` / `.vcf.bgz` (extension() alone sees only - // the trailing `.gz`, which would mis-route a compressed VCF to the CSV branch). + // Match `.vcf`, and also `.vcf.gz` and `.vcf.bgz` from bgzip or gzip. The `extension()` + // function reads only the last `.gz` part, and that value would send a compressed VCF to + // the CSV branch. let is_vcf = path .file_name() .and_then(|n| n.to_str()) @@ -107,9 +119,10 @@ impl App { .is_some_and(|n| n.ends_with(".vcf") || n.ends_with(".vcf.gz") || n.ends_with(".vcf.bgz")); let calls = if is_vcf { - // Genotype-aware: a vendor VCF (FTDNA Big Y / YSEQ) reports reference sites too, so only - // the genotype-selected ALT is kept (see parse_vcf_subject_snps). Sites-only VCFs keep - // every listed variant. Handles a bgzipped `.vcf.gz` transparently. + // The parser reads the genotype. A vendor VCF from FTDNA Big Y or YSEQ also reports + // a reference site. So the code keeps only the ALT value that the genotype selects. + // See parse_vcf_subject_snps. For a VCF with sites only, the code keeps each listed + // variant. The parser also reads a `.vcf.gz` file from bgzip. parse_vcf_subject_snps(path)? } else { let text = std::fs::read_to_string(path)?; @@ -119,10 +132,15 @@ impl App { return Err(AppError::Import("no SNP variants found in file".into())); } - // Vendor-aware tagging for VCFs: recognize FTDNA Big Y / Y Elite / YSEQ / mtFull from the - // header + filename + sibling readme, and record the vendor label, a meaningful SourceType, - // and the reference build (feeds Y/mt placement liftover). A generic VCF keeps the caller's - // label/source_type. CSV imports are unchanged. + // Find the vendor of a VCF and mark the record. The code recognizes FTDNA Big Y, Y Elite, + // YSEQ, and mtFull. It reads the header, the file name, and a readme file in the same + // directory. + // + // The code then records the vendor label, a correct SourceType, and the reference build. + // The Y placement and the mt placement use that build for the liftover. + // + // A VCF with no vendor keeps the label and the source_type of the caller. A CSV import does + // not change. let (source_label, source_type, reference_build) = if is_vcf { let (meta, contigs) = peek_vcf_header(path); let vendor = @@ -147,15 +165,20 @@ impl App { source_type, reference_build, calls, - // Recorded so the VCF can be re-read to genotype at tree positions (the role - // `alignment.bam_path` plays for a CRAM) — see `App::vset_base_calls`. + // The code records this path, so it can read the VCF again and genotype at the + // positions of the tree. The `alignment.bam_path` field has the same role for a CRAM. + // See `App::vset_base_calls`. source_path: Some(path.to_string_lossy().into_owned()), }; let set = variant_set::create(self.store.pool(), &new).await?; - // Place a vendor Y-NGS VCF (FTDNA Big Y / YSEQ / Full Genomes / …) on import so it lands a - // Y haplogroup without a manual Refresh — the VCF *is* the called Y-SNP set. Best-effort: an - // offline tree or an autosomal/mt-only VCF just leaves the calls (no chrY → no-op). + // Place a vendor Y-NGS VCF at the import, so the subject gets a Y haplogroup and the user + // does not press Refresh. Such a VCF comes from FTDNA Big Y, YSEQ, Full Genomes, or a + // similar test, and the file *is* the set of Y-SNP calls. + // + // The step is optional. The cache can hold no tree, and a VCF can hold only autosomal + // data or mt data. In each case the code writes the calls and does nothing more. A file + // with no chrY data gives no placement. let has_chr_y = set .calls .iter() @@ -168,13 +191,21 @@ impl App { Ok(set) } - /// Import a CompleteGenomics **masterVar** whole-genome variant table (`var-*-ASM.tsv[.bz2]`, - /// the old CG sequencing service's `cgatools` output). The file is streamed and decompressed - /// off-thread ([`navigator_analysis::mastervar`]) into SNP calls — each diploid het becomes a - /// `0/1`, a homozygous/haploid call a `1/1` / `1`, indels and `ref`/`no-call` spans dropped - /// (SNP-only, matching the VCF/CSV importer). Stored as a `WgsShortRead` set on GRCh37 (CG's - /// only build; chrM = rCRS), then Y-placed on import like a vendor Y-NGS VCF. mtDNA falls out - /// via the multi-source mt consensus (a non-chip set's chrM feeds `mt_source_calls`). + /// Import a CompleteGenomics **masterVar** whole-genome variant table. The file name is + /// `var-*-ASM.tsv` or `var-*-ASM.tsv.bz2`, and the `cgatools` program of the old CG sequencing + /// service wrote it. + /// + /// [`navigator_analysis::mastervar`] reads and decompresses the file on another thread, and it + /// makes SNP calls. A diploid heterozygous call becomes `0/1`. A homozygous call becomes `1/1`, + /// and a haploid call becomes `1`. The code removes each indel, each `ref` span, and each + /// `no-call` span. It keeps only SNPs, as the VCF importer and the CSV importer do. + /// + /// The code stores the result as a `WgsShortRead` set on GRCh37, which is the only build of CG. + /// The chrM contig uses rCRS. The code then places the Y haplogroup at the import, as it does + /// for a vendor Y-NGS VCF. + /// + /// The mtDNA result comes from the mt consensus of many sources. The chrM data of a set that is + /// not a chip feeds `mt_source_calls`. pub async fn import_mastervar_from_file( &self, biosample_guid: SampleGuid, @@ -219,12 +250,17 @@ impl App { Ok(set) } - /// Import an FTDNA Big Y CSV variant report (Named or Private Variants) — the data a project - /// admin gets when their access tier exposes the browser CSVs but not the BAM/CRAM/VCF. The - /// rows are GRCh38 chrY derived-allele calls, so they're stored as a `TargetedNgs` variant set - /// on GRCh38 (FTDNA's native Y-tree build) and placed via the vendor path on import — the Named - /// report lands a Y haplogroup directly (positions match the tree, no liftover). Private - /// Variants are stored too (novel loci, off-tree) for the record. + /// Import a Big Y CSV variant report from FTDNA. The report is the Named report or the Private + /// Variants report. A project administrator receives these files when the access level gives + /// the browser CSV files but no BAM file, CRAM file, or VCF file. + /// + /// Each row is a derived-allele call on chrY in GRCh38. So the code stores the rows as a + /// `TargetedNgs` variant set on GRCh38, which is the native build of the Y tree of FTDNA. The + /// code then places the subject with the vendor path at the import. + /// + /// The Named report gives a Y haplogroup directly, because its positions match the tree and + /// need no liftover. The code also stores the Private Variants. Those loci are new and are not + /// on the tree, and the store keeps them as a record. pub async fn import_ftdna_csv_variants( &self, biosample_guid: SampleGuid, @@ -241,15 +277,17 @@ impl App { source_path: Some(path.to_string_lossy().into_owned()), }; let set = variant_set::create(self.store.pool(), &new).await?; - // Place Y from the vendor (non-Chip) sets — the Named report carries the tree-defining SNPs. + // Place the Y haplogroup from the vendor sets, which are the sets that are not a chip. + // The Named report holds the SNPs that define a node of the tree. if let Err(e) = self.assign_y_vendor_vcfs(biosample_guid).await { eprintln!("FTDNA CSV Y placement deferred ({e})"); } Ok(set) } - /// Add a manually-entered variant set — paste `contig,position,ref,alt` rows (e.g. - /// Sanger/YSEQ confirmations). `source_type` sets the weight (Sanger = 1.0). + /// Add a variant set that the user typed. The user pastes `contig,position,ref,alt` rows. One + /// example is a set of confirmations from Sanger or YSEQ. `source_type` sets the weight, and a + /// Sanger source has the weight 1.0. pub async fn add_variants( &self, biosample_guid: SampleGuid, @@ -269,8 +307,9 @@ impl App { Ok(variant_set::create(self.store.pool(), &new).await?) } - /// The build to emit a subject's BISDNA calls on: the first of its alignments whose - /// reference build maps to a dictionary key, else `"hs1"` (the project default). + /// The build for the BISDNA calls of a subject. The method takes the first alignment whose + /// reference build has a dictionary key. If there is none, it returns `"hs1"`, which is the + /// default of the project. pub(crate) async fn bisdna_target_build(&self, biosample_guid: SampleGuid) -> String { if let Ok(aligns) = alignment::list_for_biosample(self.store.pool(), biosample_guid).await { for a in &aligns { @@ -282,13 +321,19 @@ impl App { "hs1".to_string() } - /// Annotate position-only Y variants with the catalogued Y-SNP **name** at that site, for the two - /// Y-SNP tables (multi-source variant profile + private-Y union). Resolves the subject's Y build - /// key (CHM13→`hs1`, else GRCh38/GRCh37 — same rule as the BISDNA importer), loads the Y-SNP - /// dictionary (the full catalog, memoized), and returns `position → canonical name` for the - /// requested positions only. Best-effort: a missing dictionary yields an empty map (not an error), - /// so the tables simply show no extra names. Looking a position up against the wrong build just - /// misses — there are no false labels, only possibly-absent ones. + /// Add the catalogued Y-SNP **name** to each Y variant that has a position and no name. Two + /// tables use this map: the variant profile with many sources, and the union of the private-Y + /// sets. + /// + /// The method finds the Y build key of the subject. A CHM13 build gives `hs1`, and the other + /// builds give GRCh38 or GRCh37. The BISDNA importer uses the same rule. + /// + /// The method then reads the Y-SNP dictionary, which is the full catalog and stays in memory. + /// It returns a map from a position to a canonical name, for the requested positions only. + /// + /// The step is optional. An absent dictionary gives an empty map and no error, and the tables + /// then show no extra name. A lookup against the wrong build finds nothing. So the table can + /// hold an absent name, but it never holds a wrong name. pub async fn y_snp_names_at( &self, biosample_guid: SampleGuid, @@ -309,14 +354,23 @@ impl App { Ok(names) } - /// Ensure a Y-SNP dictionary is present, downloading the full catalog (`dictionary.tsv`, - /// ~208 MB) from the asset release on first use — it's too big and too volatile (~weekly YBrowse - /// refresh) to bundle in the installer. No-op when a dictionary (the chromo2 panel or the full - /// catalog) is already installed, or the user pointed `NAVIGATOR_YSNP_DIR` at one. The download - /// is verified against a small published manifest (`ysnp_manifest.json`, the ancestry - /// [`AssetManifest`](navigator_analysis::manifest::AssetManifest) shape) so a rebuild is a - /// re-publish, not a client change. Best-effort — the caller then loads, degrading clearly if the - /// dictionary is still absent. Publish with `packaging/publish-assets.sh ysnp`. + /// Make sure that a Y-SNP dictionary is on the machine. At the first use, the method downloads + /// the full catalog, `dictionary.tsv`, which is about 208 MB. + /// + /// The installer does not hold that file. The file is too large, and YBrowse refreshes it about + /// once each week. + /// + /// The method does nothing when the machine already holds a dictionary. That dictionary can be + /// the chromo2 panel or the full catalog. It also does nothing when `NAVIGATOR_YSNP_DIR` points + /// to one. + /// + /// The method checks the download against a small published manifest, + /// `ysnp_manifest.json`. That file has the shape of the ancestry + /// [`AssetManifest`](navigator_analysis::manifest::AssetManifest). So a rebuild of the catalog + /// is a new publish and not a change to the client. + /// + /// The step is optional. The caller then reads the dictionary, and it reports the state clearly + /// when the file is still absent. Publish the file with `packaging/publish-assets.sh ysnp`. pub async fn ensure_ysnp_dictionary(&self) -> Result<(), AppError> { const YSNP_ASSET_BASE: &str = "https://github.com/JamesKane/decodingus-navigator/releases/download/assets-ysnp"; @@ -349,8 +403,9 @@ impl App { &mut noop, ) .await?; - // Verify the streamed digest against the manifest (no 208 MB re-read). A manifest without an - // entry passes through advisory, matching `AssetManifest::verify`. + // Check the digest from the stream against the manifest. The code does not read the 208 MB + // file again. A manifest with no entry for the file gives a warning only, as + // `AssetManifest::verify` does. if let Some(entry) = manifest.assets.get("dictionary.tsv") { if !got.eq_ignore_ascii_case(&entry.sha256) { let _ = std::fs::remove_file(&dest); @@ -363,12 +418,19 @@ impl App { Ok(()) } - /// Import a BISDNA chromo2 Y-SNP export. Each named marker is resolved to a locus via the - /// Y-SNP dictionary on `build` (when `None`, the subject's alignment build, else `"hs1"`). - /// Only **positive** (derived) calls become variant calls: a negative is not a variant. - /// `no_call`, back-mutated, and dictionary-unresolved markers are tallied but not emitted. - /// The genotype is a QC cross-check only — the file's verdict (independent of the Illumina - /// TOP strand) decides derived/ancestral. Stored as a `Chip`-weighted [`VariantSet`]. + /// Import a chromo2 Y-SNP export from BISDNA. + /// + /// The Y-SNP dictionary changes each marker name into a locus on `build`. When `build` is + /// `None`, the method uses the alignment build of the subject, and then `"hs1"`. + /// + /// Only a **positive**, or derived, call becomes a variant call. A negative call is not a + /// variant. The method counts a `no_call` marker, a back-mutated marker, and a marker that the + /// dictionary does not hold. It writes none of those three. + /// + /// The genotype is a quality cross-check only. The verdict in the file decides between derived + /// and ancestral, and that verdict does not depend on the Illumina TOP strand. + /// + /// The method stores the result as a [`VariantSet`] with the `Chip` weight. pub async fn import_bisdna_from_file( &self, biosample_guid: SampleGuid, @@ -404,10 +466,13 @@ impl App { .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| "BISDNA".into()); - // Also record an array QC summary so the chromo2 chip appears under Data Sources → - // Chip / Array Profiles (the placeable per-SNP calls live in the variant set below; a - // genotyping array legitimately has both a QC/provenance summary and its calls). BISDNA - // is a Y-only haploid panel: every called marker is a Y marker, heterozygosity is n/a. + // Also write a quality summary for the array. The chromo2 chip then appears under + // Data Sources, in the Chip and Array Profiles list. The variant set below holds the SNP + // calls that the code can place. An array correctly has both a quality summary with its + // provenance and a set of calls. + // + // BISDNA is a haploid Y panel. Each called marker is a Y marker, and heterozygosity does + // not apply. let total = calls.len() as i64; let called = total - outcome.no_call as i64; let chip = NewChipProfile { @@ -442,10 +507,13 @@ impl App { }; let variant_set = variant_set::create(self.store.pool(), &new).await?; - // Compute the Y haplogroup on import (best-effort; an offline tree just leaves the calls), - // mirroring the array path in `import_chip_profile_from_csv`. Without this a chromo2/BISDNA - // panel imports its calls but never auto-places — it has no cached alignment genotypes, so - // `rebuild-signatures` can't place it later either, leaving the subject's Y at . + // Calculate the Y haplogroup at the import. The step is optional, and with no tree the + // code writes the calls only. The array path in `import_chip_profile_from_csv` does the + // same. + // + // Without this step, a chromo2 panel from BISDNA imports its calls and never gets a + // placement. Such a panel has no alignment genotypes in the cache, so `rebuild-signatures` + // can not place it later either. The Y value of the subject then stays at . if derived_calls > 0 { if let Err(e) = self.assign_y_bisdna(biosample_guid, Some(&build)).await { eprintln!("BISDNA Y placement deferred ({e})"); @@ -473,17 +541,26 @@ impl App { // ---- chip / array profiles --------------------------------------------- - /// Import a genotyping-array raw-data export (CSV/TSV) and store its QC summary. - /// `provider` overrides vendor detection when given; `chip_version` is optional. - /// Import a genotyping-array raw-data export and (1) store its QC summary as a [`ChipProfile`], - /// (2) store the haploid Y/MT genotype rows as a `Chip`-source [`VariantSet`], and (3) - /// best-effort place the Y (and, where present, mtDNA) haplogroup on import — the consumer-array - /// counterpart to BISDNA's chromo2 path. 23andMe carries both Y and MT rows; AncestryDNA carries - /// Y but no usable mtDNA. The stored observed bases flow through the same - /// [`assign_y_bisdna`](Self::assign_y_bisdna) / [`assign_mt_chip`](Self::assign_mt_chip) + - /// `assemble_assignment_robust` placement as BISDNA, with plus-strand reconciliation to the tree. - /// Placement is best-effort: an unreachable tree (offline) leaves the calls stored for a later - /// manual "Assign … (panel)" — it does not fail the import. + /// Import the raw-data export of a genotyping array, as a CSV file or a TSV file. + /// + /// The method does three things. It writes the quality summary as a [`ChipProfile`]. It writes + /// the haploid Y rows and MT rows as a [`VariantSet`] with the `Chip` source. It then tries to + /// place the Y haplogroup, and the mtDNA haplogroup when the file holds one. + /// + /// This method is the consumer-array form of the chromo2 path of BISDNA. A 23andMe file holds + /// both Y rows and MT rows. An AncestryDNA file holds Y rows and no mtDNA rows that the app can + /// use. + /// + /// The stored bases go through the same placement as BISDNA. That path is + /// [`assign_y_bisdna`](Self::assign_y_bisdna) or [`assign_mt_chip`](Self::assign_mt_chip), + /// followed by `assemble_assignment_robust`, and it reconciles each call to the plus strand of + /// the tree. + /// + /// The placement is optional. With no network, the code stores the calls, and the user can + /// press "Assign … (panel)" later. A failed placement does not fail the import. + /// + /// `provider` replaces the vendor that the code finds, when the caller gives it. `chip_version` + /// is optional. pub async fn import_chip_profile_from_csv( &self, biosample_guid: SampleGuid, @@ -514,10 +591,12 @@ impl App { }; let profile = chip_profile::create(self.store.pool(), &new).await?; - // Pull the haploid Y/MT genotype rows and store them as Chip-source variant calls so the - // haplogroup placement (and later re-placement) has them without re-reading the file. The - // observed allele goes in both `reference` and `alternate` (we don't know the ancestral); - // the placement reads `alternate`. + // Read the haploid Y rows and MT rows, and store them as variant calls with the Chip + // source. The haplogroup placement then has them, and a later placement also has them, + // with no second read of the file. + // + // The observed allele goes in `reference` and in `alternate`, because the app does not know + // the ancestral allele. The placement reads `alternate`. let haplo = chipprofile::haplo_calls(&text); if !haplo.is_empty() { let build = chipprofile::detect_build(&text); @@ -556,8 +635,9 @@ impl App { eprintln!("chip Y placement deferred ({e})"); } } - // AncestryDNA's stray MT rows aren't a usable mtDNA panel — only place mtDNA when the - // array carries a real MT marker set (23andMe has thousands; the threshold filters noise). + // The few MT rows of an AncestryDNA file are not an mtDNA panel that the app can use. + // Place mtDNA only when the array holds a true MT marker set. A 23andMe file holds + // some thousands of such markers, and the limit below removes the noise. const MIN_MT_CALLS: usize = 20; if mt_count >= MIN_MT_CALLS { if let Err(e) = self.assign_mt_chip(biosample_guid).await { @@ -595,8 +675,9 @@ impl App { }; let seq = mtdna_store::create(self.store.pool(), &new).await?; - // Derive rCRS-relative variants and persist them, so an mtDNA FASTA yields a variant set on - // import (not only on the on-demand "show mutations" view) — like a chip/VCF import does. + // Derive the variants against rCRS and write them to the store. An mtDNA FASTA then gives + // a variant set at the import. Before this step, the set appeared only in the "show + // mutations" view. A chip import and a VCF import behave in the same way. let derived = navigator_analysis::mtvariants::derive(navigator_analysis::mtvariants::rcrs(), &seq.sequence); if !derived.is_empty() { let label = mt_vendor_label(seq.source_file_name.as_deref(), seq.defline.as_deref()); @@ -609,7 +690,8 @@ impl App { alternate: v.alternate.to_string(), rs_id: None, genotype: None, - // Derived from an rCRS diff, not a source VCF — no evidence to carry. + // These calls come from a comparison with rCRS and not from a source VCF. So + // there is no evidence to store. evidence: Default::default(), }) .collect(); @@ -626,9 +708,10 @@ impl App { let _ = variant_set::create(self.store.pool(), &set).await; } - // Haplogroup placement is intentionally NOT run here: it needs the mt haplotree (network), - // and coupling a deterministic import to a network fetch is what the alignment import - // deliberately avoids too. The mtDNA tab's "Assign mtDNA haplogroup" places it on demand. + // This method does NOT place the haplogroup, by design. A placement needs the mt + // haplotree, and a read of that tree needs the network. An import must stay deterministic, + // so it must not depend on the network. The alignment import follows the same rule. The + // user presses "Assign mtDNA haplogroup" on the mtDNA tab to place the subject. Ok(seq) } @@ -637,12 +720,17 @@ impl App { Ok(mtdna_store::list_for_biosample(self.store.pool(), biosample_guid).await?) } - /// Derive mtDNA variants for a stored sequence by comparing it to an rCRS reference - /// FASTA, and save them as a variant set (contig `rCRS`) so they appear alongside the - /// subject's other variants. The reference is validated as an mtDNA FASTA. - /// The mtDNA mutation list for a stored sequence: variants relative to the **bundled** rCRS - /// (NC_012920.1), via banded alignment — substitutions, insertions, and deletions in standard - /// mtDNA notation. On-demand (one ~16.5 kb alignment), not stored. The classic mtDNA result. + /// Derive the mtDNA variants of a stored sequence. The method compares that sequence with an + /// rCRS reference FASTA and writes the result as a variant set on the contig `rCRS`. The + /// variants then appear with the other variants of the subject. The method checks that the + /// reference file is an mtDNA FASTA. + /// + /// The mutation list holds the variants against the **bundled** rCRS sequence, NC_012920.1. A + /// banded alignment gives them. The list holds substitutions, insertions, and deletions, in the + /// standard mtDNA notation. + /// + /// The method runs at the request of the user, and it does one alignment of about 16.5 kb. It + /// stores nothing. This list is the classic mtDNA result. pub async fn mtdna_variants(&self, mtdna_id: i64) -> Result, AppError> { let seq = mtdna_store::get(self.store.pool(), mtdna_id) .await? diff --git a/crates/navigator-app/src/import_unified.rs b/crates/navigator-app/src/import_unified.rs index 783f53bf..0453962d 100644 --- a/crates/navigator-app/src/import_unified.rs +++ b/crates/navigator-app/src/import_unified.rs @@ -5,20 +5,31 @@ use super::*; impl App { // ---- unified import ---------------------------------------------------- - /// Detect a file's type and route it to the right subject importer (STR / variants / - /// chip / mtDNA), using sensible defaults. Returns the detected type. Alignment files - /// are rejected here — they attach to a sequencing test, not directly to a subject. - /// Probe a BAM/CRAM header for the build/aligner/platform/test-type (best-effort). + /// Find the type of a file and send it to the correct importer for a subject. The importers + /// cover STR data, variants, a chip export, and mtDNA data. The method uses a default value + /// where it needs one, and it returns the type that it found. + /// + /// This method refuses an alignment file. Such a file belongs to a sequence test, and it does + /// not attach to a subject directly. + /// + /// The method also reads the header of a BAM file or a CRAM file. From that header it finds + /// the build, the aligner, the platform, and the test type. That step is optional. pub async fn probe_alignment(&self, path: PathBuf) -> Result { tokio::task::spawn_blocking(move || navigator_analysis::probe::probe_alignment(&path)) .await? .map_err(AppError::from) } - /// Scan a bounded prefix of an alignment's reads to infer the instrument/library identity — - /// the `@RG SM/LB/PU` tags plus the most-frequent instrument/flowcell/platform from read names - /// (the crowd-source input for resolving the lab). Off-thread (blocking IO + CRAM decode); - /// `reference` is required for CRAM. Best-effort — callers tolerate an error. + /// Read a limited count of records from the start of an alignment, and find the identity of the + /// instrument and the library. + /// + /// That identity is the `@RG` tags `SM`, `LB`, and `PU`. It also holds the most frequent + /// instrument, flowcell, and platform in the read names. The AppView uses those values to find + /// the laboratory. + /// + /// The method runs on another thread, because it blocks on I/O and decodes a CRAM file. A CRAM + /// file also needs the `reference` value. The step is optional, and each caller continues after + /// an error. pub async fn library_stats( &self, path: PathBuf, @@ -35,20 +46,30 @@ impl App { .map_err(AppError::from) } - /// Auto-import an alignment file by probing its header: create the sequencing run (test type, - /// platform, instrument) and the alignment (reference build + aligner) with no questions - /// asked. The reference FASTA is **not** required — it's resolved from the build on demand; - /// if already cached it's stored so every analysis step has it immediately. + /// Import an alignment file with no question to the user. The method reads the header of that + /// file. + /// + /// It then makes the sequence run, with the test type, the platform, and the instrument. It also + /// makes the alignment, with the reference build and the aligner. + /// + /// The method does **not** need the reference FASTA file. It finds that file from the build when + /// a step needs it. When the cache already holds the file, the method stores its path, and each + /// analysis step then has it at once. async fn import_alignment_file( &self, biosample_guid: SampleGuid, path: &Path, test_type_override: Option<&str>, ) -> Result<(), AppError> { - // Idempotent per subject: skip only if *this* subject already has the alignment. Dedup used - // to be global (any subject), which silently skipped importing a file into a new subject when - // another subject already had it — leaving an empty subject and a misleading "imported" toast - // (e.g. re-importing a file after deleting its old subject, when a sibling subject also has it). + // A second import is safe for one subject. The code skips the file only when *this* + // subject already holds the alignment. + // + // An earlier version compared across each subject. So the code skipped a file for a new + // subject when another subject already held it, and it gave no message. The new subject + // stayed empty, and the app showed an "imported" message that was not true. + // + // One case is a second import of a file after the user deleted its earlier subject, when + // another subject also holds that file. let path_str = path.to_string_lossy().into_owned(); if alignment::list_for_biosample(self.store.pool(), biosample_guid) .await? @@ -57,25 +78,31 @@ impl App { { return Ok(()); } - // Best-effort: a probe failure falls back to filename/defaults rather than aborting. + // The step is optional. After a failed read of the header, the code uses the file name and + // its default values. It does not stop the import. let probe = self.probe_alignment(path.to_path_buf()).await.unwrap_or_default(); - // Resolve the reference first — the read-name scan needs it to decode a CRAM. + // Find the reference first. The scan of the read names needs it to decode a CRAM file. let reference_build = probe .reference_build .clone() .unwrap_or_else(|| reference_build_for(path)); - // Store the cached reference path if we have it; otherwise leave it unset (resolved on - // demand) — never block import on a download. + // Store the path of the reference when the cache holds that file. If not, leave the field + // empty, and the code finds the file when a step needs it. An import must never wait for a + // download. let reference_path = self .gateway .cached_reference(&reference_build) .map(|p| p.to_string_lossy().into_owned()); - // Read-name scan → instrument/library identity (the lab crowd-source input). Best-effort: - // it fills the platform/model the header `@RG` left blank, and the instrument/flowcell that - // never live in the header. Skipped silently if the file can't be read (e.g. CRAM with no - // resolved reference yet). + // The scan of the read names gives the identity of the instrument and the library, and the + // AppView uses those values to find the laboratory. + // + // The step is optional. It fills the platform and the model when the `@RG` header holds + // neither. It also fills the instrument and the flowcell, which no header holds. + // + // The code skips this step when it can not read the file. One case is a CRAM file with no + // reference yet. let stats = self .library_stats(path.to_path_buf(), reference_path.as_deref().map(PathBuf::from)) .await @@ -97,13 +124,19 @@ impl App { .clone() .or_else(|| stats.as_ref().and_then(|s| s.instrument_model.clone())); - // Test type: refine the header/platform guess with coverage *shape* from the BAI index — - // a targeted-Y pile-up (autosomes empty) → Big Y / Y Elite / YSEQ; an mtDNA pile-up → - // mtFull. Best-effort and cheap (O(contigs), no read scan); CRAM / unindexed BAMs have no - // profile and keep the platform-based guess. - // An explicit override (e.g. a Big_Y-700/500 directory the caller recognized) wins over - // inference — CRAMs ship no `.bai`, so the coverage-shape detector below can't see the - // targeted-Y pile-up and would otherwise fall back to the platform default (WGS). + // The test type. The code reads the *shape* of the coverage from the BAI index, and that + // shape corrects the value from the header and the platform. + // + // Many reads on chrY, with no read on an autosome, mark a Big Y test, a Y Elite test, or a + // YSEQ test. Many reads on chrM mark an mtFull test. + // + // The step is optional and fast. It costs O(contigs) and reads no record. A CRAM file and a + // BAM file with no index hold no such profile, and they keep the value from the platform. + // + // A value from the caller wins over each value above. One example is a Big_Y-700 directory + // or a Big_Y-500 directory that the caller recognized. A CRAM file has no `.bai` file, so + // the detector below can not see the reads on chrY. Without the value from the caller, the + // code would use the default of the platform, which is WGS. let test_type = match test_type_override { Some(t) => t.to_string(), None => { @@ -134,9 +167,11 @@ impl App { }) .await?; - // Persist the inferred lab/instrument identity block (the crowd-source key). The lab - // (`sequencing_facility`) stays unset — set manually, or resolved from `instrument_id` - // once the AppView lookup ships (roadmap D8). + // Write the identity of the laboratory and the instrument that the code found. The AppView + // uses those values as its key. + // + // The `sequencing_facility` field stays empty. The user sets it, or the AppView lookup + // gives it from `instrument_id` after that feature ships. See roadmap D8. if let Some(s) = &stats { let _ = sequence_run::set_library_stats( self.store.pool(), @@ -149,10 +184,15 @@ impl App { s.read_type.as_deref(), ) .await; - // Resolve the lab from the instrument id via the AppView (best-effort, cached). The - // FTDNA Big Y generation comes from the header `@RG LB` label (already in `test_type` - // above) or, on older headers that omit it, from the callable-chrY footprint after - // analysis ([`Self::refine_big_y_generation`]) — not guessed from the lab here. + // Find the laboratory from the instrument id, through the AppView. The step is + // optional, and the result goes into the cache. + // + // The generation of an FTDNA Big Y test comes from the `@RG LB` label of the header, + // and the step above already put it in `test_type`. An older header holds no such + // label. For such a file, the callable area of chrY gives the generation after the + // analysis, in [`Self::refine_big_y_generation`]. + // + // This code does not estimate the generation from the laboratory. if let Some(inst) = s.instrument_id.as_deref() { if let Some(lab) = self.lookup_lab_by_instrument(inst).await { let _ = sequence_run::set_facility(self.store.pool(), run.id, &lab).await; @@ -160,10 +200,13 @@ impl App { } } - // Defer the content hash (the file's identity, used to invalidate cached analyses): a - // whole-file SHA-256 of a multi-GB alignment would block this import for minutes with no - // feedback. Like the batch path, leave it `None` — `alignment_content_hash` computes and - // caches it lazily on the first analysis that needs it. + // Do not calculate the content hash here. That hash is the identity of the file, and the + // app uses it to find an old cache entry. + // + // A SHA-256 hash of a full alignment of many GB stops this import for some minutes, and the + // user sees nothing. The batch path also leaves the field `None`. The function + // `alignment_content_hash` calculates the hash at the first analysis that needs it, and it + // writes the value to the cache. self.record_alignment(NewAlignment { sequence_run_id: run.id, reference_build, @@ -172,8 +215,8 @@ impl App { bam_path: Some(path.to_string_lossy().into_owned()), reference_path, content_sha256: None, - // An imported alignment is an original — nothing derived it. Only realignment sets - // these, and it registers its own row. + // An imported alignment is an original alignment, and no other row made it. Only a + // realignment writes these fields, and it adds its own row. derived_from_alignment_id: None, derivation: None, }) @@ -185,9 +228,12 @@ impl App { self.add_data_with_test_type(biosample_guid, path, None).await } - /// Like [`add_data`], but forces the sequencing-run `test_type` for an alignment file instead - /// of inferring it (e.g. a bulk Big Y import where the directory layout names the test). The - /// override is ignored for non-alignment inputs (their type is intrinsic to the file). + /// The same work as [`add_data`], but the caller gives the `test_type` of the sequence run for + /// an alignment file. The code does not find that value itself. One case is a bulk Big Y import, + /// where the layout of the directories names the test. + /// + /// The method ignores that value for each other kind of file, because the file itself gives the + /// type. pub async fn add_data_with_test_type( &self, biosample_guid: SampleGuid, @@ -199,9 +245,12 @@ impl App { .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_default(); let lower = name.to_ascii_lowercase(); - // Binary/structured formats are detected by extension; only text needs a sniff. A VCF is - // sniffed too now — an all-sites (genotyped) VCF is a 1240K call set, a variant-only VCF is a - // plain variant set (see `filetype::looks_like_genotyped_callset_vcf`) — so `.vcf*` is NOT here. + // The extension of a file gives its type for each binary format and each structured + // format. Only a text file needs a look at its content. + // + // The code now also looks inside a VCF file. A VCF with each site is a 1240K call set. A VCF + // with the variants only is a plain variant set. See + // `filetype::looks_like_genotyped_callset_vcf`. So this list holds no `.vcf` pattern. let by_ext = lower.ends_with(".bam") || lower.ends_with(".cram") || lower.ends_with(".geno") @@ -255,12 +304,21 @@ impl App { Ok(detected) } - /// Batch [`add_data`]: expand any directories among `paths` into their recognized data files, - /// then auto-detect + import each into the subject, collecting a [`BatchImportSummary`]. A - /// failed/unrecognized file is recorded (not propagated) so one bad file doesn't abort the - /// batch. `progress(done, total)` ticks per file. The unified multi-file / folder importer - /// behind the GUI's Add Data button + drag-and-drop. (Distinct from [`import_project_dir`], - /// which builds a *new* multi-subject project from a NAS layout; this adds to *this* subject.) + /// The batch form of [`add_data`]. + /// + /// The method expands each directory in `paths` into the data files that it recognizes. It then + /// finds the type of each file and imports it into the subject. It collects the results in a + /// [`BatchImportSummary`] value. + /// + /// The summary records a file that failed, and a file that the code does not recognize. The + /// method does not return an error for such a file, so one bad file does not stop the batch. + /// + /// The method calls `progress(done, total)` after each file. + /// + /// The Add Data button of the GUI calls this method, and a drag-and-drop action also calls it. + /// + /// This method is not [`import_project_dir`]. That method makes a *new* project with many + /// subjects from a NAS layout. This method adds files to *this* subject. pub async fn add_data_batch( &self, biosample_guid: SampleGuid, @@ -269,9 +327,12 @@ impl App { ) -> Result { let mut files = Vec::new(); for p in &paths { - // Guard against a single picked folder that's really a *parent* of several per-sample - // folders (e.g. an FTDNA download root): recursing it would silently merge sibling - // samples into this one subject. Refuse with guidance rather than import the wrong data. + // Guard against one folder that the user picked and that is the *parent* of the + // folders of many samples. An FTDNA download root is one example. + // + // A read of each folder below it would add the samples of each folder to this one + // subject, with no message. The method refuses and gives the user a hint. It must not + // import the wrong data. if p.is_dir() { let mut these = Vec::new(); collect_data_files(p, &mut these, 0); @@ -308,18 +369,25 @@ impl App { Ok(summary) } - /// Ingest one staged **sample directory** onto an existing subject (the CLI `ingest` fast path - /// for the D2C bulk side-load). Scans `dir` into a single sample, records its alignment(s) onto - /// `biosample_guid` from the **header only** (no read decode / library scan), imports any variant - /// files, then — when `fast_path` and a haplogroup GVCF is present — runs [`Self::ingest_sidecars`] - /// to place Y + mt from the BGZF GVCFs and fill sex / read-metrics / lite-coverage from the text - /// sidecars, still **without decoding the CRAM**. Per-file [`Self::add_data`] can't do this: it - /// can't group `*.callable.bed` / `coverage.txt` / `stats.txt` to their alignment, and it would - /// route a `*.g.vcf.gz` through the plain-VCF importer instead of the GVCF haplogroup fast path. + /// Import one **sample directory** onto a subject that exists. The CLI `ingest` command uses + /// this fast path for the D2C bulk load. + /// + /// The method reads `dir` as one sample. It records each alignment onto `biosample_guid` from + /// the **header only**. It decodes no read and scans no library. It then imports each variant + /// file. + /// + /// When the caller sets `fast_path` and the directory holds a haplogroup GVCF file, the method + /// calls [`Self::ingest_sidecars`]. That method places the Y haplogroup and the mt haplogroup + /// from the BGZF GVCF files. It also fills the sex, the read metrics, and a small coverage + /// result from the text sidecar files. It decodes **no CRAM file**. /// - /// A directory that holds no alignment, variant, or haplogroup GVCF falls back to a best-effort - /// per-file [`Self::add_data`] of its contents — so a plain folder of chip/STR/mtDNA exports - /// still imports as before. + /// A call of [`Self::add_data`] for each file can not do this work. That method can not join a + /// `*.callable.bed` file, a `coverage.txt` file, or a `stats.txt` file to its alignment. It also + /// sends a `*.g.vcf.gz` file to the plain-VCF importer, and not to the GVCF fast path. + /// + /// A directory with no alignment, no variant file, and no haplogroup GVCF takes another path. + /// The method then calls [`Self::add_data`] for each file, and each call is optional. So a plain + /// folder of chip exports, STR exports, and mtDNA exports imports as it did before. pub async fn add_sample_dir( &self, biosample_guid: SampleGuid, @@ -330,8 +398,9 @@ impl App { let sample = tokio::task::spawn_blocking(move || navigator_analysis::scan::scan_sample(&scan_dir)).await?; let mut summary = SampleDirSummary::default(); - // No primary sequencing data (no alignment, no variant, no haplogroup GVCF): treat the - // directory as a loose bundle of subject files and import each as add_data would. + // The directory holds no primary sequence data: no alignment, no variant file, and no + // haplogroup GVCF file. So the code reads it as a set of separate subject files, and it + // imports each file as add_data does. let has_primary = !sample.alignment_files.is_empty() || !sample.variant_files.is_empty() || sample.sidecars.has_haplogroup_gvcf(); @@ -367,8 +436,8 @@ impl App { } }; - // Record each alignment from the header only — cheap, no read decode (the whole point of the - // fast path). Idempotent on the alignment's stored path. + // Record each alignment from the header only. That read is fast and decodes no record, + // which is the purpose of the fast path. A second call with the same stored path is safe. let existing = alignment::list_for_run(self.store.pool(), run.id).await?; for aln_path in &sample.alignment_files { let path_str = aln_path.to_string_lossy().into_owned(); @@ -401,13 +470,21 @@ impl App { summary.alignments_created += 1; } - // Import bundled variant files ONLY when there is no haplogroup GVCF. When a GVCF is present - // the fast path below is the authoritative Y/mt source, so a called `chrY.vcf.gz` sitting - // beside it (the GATK repo layout ships both) is redundant — importing it would fire a second - // Y placement and, because variant-set import isn't content-idempotent, would duplicate the - // set on a resumable re-run. Non-GVCF tiers (e.g. the b38 aengine `variants.vcf.gz`) still - // import here: there the VCF *is* the Y source. GVCFs themselves are `.g.vcf.gz`, which `scan` - // also lists as variant files — the guard keeps them out of this loop too. + // Import the variant files of this directory ONLY when it holds no haplogroup GVCF file. + // + // With a GVCF file, the fast path below is the source of the Y value and the mt value. A + // called `chrY.vcf.gz` file beside it holds the same data, and the GATK layout ships both + // files. + // + // An import of that file starts a second Y placement. A second import of a variant set also + // adds a second copy, because that import does not compare the content. So a run that + // continues an earlier run would duplicate the set. + // + // A directory with no GVCF file still imports its variant files here. In the b38 aengine + // layout, for example, the `variants.vcf.gz` file *is* the Y source. + // + // A GVCF file has the name `*.g.vcf.gz`, and `scan` also lists it as a variant file. This + // guard keeps it out of this loop. if !sample.sidecars.has_haplogroup_gvcf() { for vcf in &sample.variant_files { let name = vcf @@ -429,9 +506,10 @@ impl App { } } - // Fast path: place Y + mt from the GVCFs and fill sex / read-metrics / lite-coverage from the - // text sidecars onto the build-matching alignment — no CRAM walk. Best-effort (mirrors the - // project-import chooser at import_project_sample). + // The fast path. It places the Y haplogroup and the mt haplogroup from the GVCF files. It + // fills the sex, the read metrics, and a small coverage result from the text sidecar files, + // onto the alignment with the same build. It walks no CRAM file. The step is optional, and + // the chooser in import_project_sample works in the same way. if fast_path && sample.sidecars.has_haplogroup_gvcf() { let alns = alignment::list_for_run(self.store.pool(), run.id).await?; let chosen = sample @@ -460,10 +538,14 @@ impl App { } } - // Progressive consensus (docs §7.17): fold whatever autosomal dosages are now available into - // the subject's consensus. Cheap — chips/WGS-VCFs resolve without a decode, and a freshly- - // imported WGS alignment (dosages not yet cached) is simply skipped until the panel batch mode - // genotypes it. Best-effort: a consensus hiccup must not fail the import. + // The progressive consensus, in docs §7.17. The code adds each autosomal dosage that the + // store now holds to the consensus of the subject. + // + // The step is fast. A chip and a WGS VCF resolve with no decode. A WGS alignment from a + // recent import has no dosage in the cache. The code skips such an alignment until the + // batch mode of the panel genotypes it. + // + // The step is optional. A fault in the consensus must not fail the import. if let Err(e) = self.refresh_autosomal_consensus(biosample_guid).await { summary.errors.push(format!("consensus refresh: {e}")); } @@ -471,14 +553,22 @@ impl App { Ok(summary) } - /// Batch-import a NAS project directory: scan `{dir}/{sample}/…` and create the Project - /// plus its Biosample → SequenceRun → Alignment rows. The reference is resolved per - /// alignment: pass `Some(fasta)` to use a specific FASTA (validated with its `.fai`) for - /// every alignment, or `None` to let the gateway resolve each file's inferred build from - /// the cache. If a needed build isn't cached, returns [`AppError::ReferenceNeeded`] - /// **before any DB writes** so the UI can prompt + download, then retry. Idempotent: an - /// existing project (by name), biosample (by donor id), or alignment (by path) is reused. - /// Coverage is NOT computed here — run it per alignment or via the project report. + /// Import a NAS project directory as a batch. The method reads `{dir}/{sample}/…` and makes + /// the project with its Biosample, SequenceRun, and Alignment rows. + /// + /// The method finds the reference of each alignment. With `Some(fasta)`, it uses that one FASTA + /// file for each alignment, and it checks the `.fai` file. With `None`, the gateway finds the + /// build of each file in the cache. + /// + /// When the cache holds no file for a build that the import needs, the method returns + /// [`AppError::ReferenceNeeded`] **before it writes to the database**. The UI can then ask the + /// user, download the file, and call the method again. + /// + /// A second call is safe. The method uses a project with the same name, a biosample with the + /// same donor id, and an alignment with the same path. + /// + /// The method does NOT calculate the coverage. Run that step for one alignment, or from the + /// project report. pub async fn import_project_dir( &self, dir: &Path, @@ -490,12 +580,19 @@ impl App { .await } - /// Re-run the sidecar fast path for every alignment of a subject whose source directory still - /// carries the pipeline GVCFs — restoring external (GATK4) Y/mt calls that an older build's - /// internal walk had overwritten before provenance existed. Cheap: reads the small GVCFs, never - /// the CRAM. The external calls land on their own `:ext` keys (they cannot clobber, and with the - /// "prefer external caller" policy they win the consensus). Returns `(y_placed, mt_placed)`. - /// This is the operational fix for a workspace imported before external-caller precedence. + /// Run the sidecar fast path again, for each alignment of a subject whose source directory + /// still holds the GVCF files of the pipeline. + /// + /// The method returns the external Y calls and mt calls from GATK4. An older build ran its + /// internal walk and replaced those calls, before the app recorded a provenance. + /// + /// The method is fast. It reads the small GVCF files and never the CRAM file. + /// + /// Each external call goes to its own `:ext` key. So it can replace no other call, and the + /// "prefer external caller" policy makes it win the consensus. + /// + /// The method returns `(y_placed, mt_placed)`. It is the correction for a workspace that a user + /// imported before the app had external-caller precedence. pub async fn reingest_external_for_biosample( &self, biosample_guid: SampleGuid, @@ -521,10 +618,14 @@ impl App { Ok((y_placed, mt_placed)) } - /// [`Self::import_project_dir`] with a per-sample progress callback `progress(done, total, - /// sample_id)`, invoked before each sample so a large NAS import (thousands of samples) can - /// stream a status bar instead of appearing frozen. `done` is the 0-based index about to - /// process; the first call fires only after the (potentially slow) header-probe pre-flight. + /// The work of [`Self::import_project_dir`], with a progress callback for each sample. The + /// method calls `progress(done, total, sample_id)` before each sample. + /// + /// So a large NAS import of some thousands of samples can move a status bar. Without it, the app + /// looks stopped. + /// + /// The `done` value is the 0-based index of the next sample. The first call comes after the + /// header probe of the preflight, and that step can be slow. pub async fn import_project_dir_with_progress( &self, dir: &Path, @@ -533,7 +634,8 @@ impl App { fast_path: bool, mut progress: impl FnMut(usize, usize, &str), ) -> Result { - // An explicit FASTA must exist and be indexed; it applies to every alignment. + // A FASTA file from the caller must exist and must have an index. It applies to each + // alignment. if let Some(path) = &reference { if !path.exists() { return Err(AppError::Import(format!( @@ -553,10 +655,12 @@ impl App { let scan_dir = dir.to_path_buf(); let discovered = tokio::task::spawn_blocking(move || navigator_analysis::scan::scan(&scan_dir)).await??; - // Detect each alignment's reference build from its **header** (only the header, so it's - // cheap and needs no reference FASTA). The filename is an unreliable signal — most NAS - // project layouts don't put the build in the name — so probe first, fall back to the - // filename, and record how each build was decided for the import diagnostics. + // Find the reference build of each alignment from its **header**. The code reads the + // header only, so this step is fast and needs no reference FASTA file. + // + // The file name is not a reliable source, because most NAS layouts do not put the build in + // that name. So the code reads the header first and uses the file name second. It also + // records the source of each build, for the import report. let all_paths: Vec = discovered .samples .iter() @@ -573,17 +677,22 @@ impl App { }) .await?; - // Resolve each *distinct* detected build to a reference path. A build the gateway can't - // canonicalize falls back to the CHM13v2.0 default rather than aborting the whole batch; - // a known build that isn't cached is surfaced as a recoverable download need. `effective_of` - // maps a detected build to the one actually stored on the alignment (after any fallback). + // Find a reference path for each *distinct* build that the code detected. + // + // A build that the gateway does not recognize takes the CHM13v2.0 default, so the batch + // continues. A known build with no file in the cache becomes a download that the UI can + // start. + // + // The `effective_of` map takes a build that the code detected and gives the build that the + // alignment row holds, after each default above. let explicit = reference.as_ref().map(|p| p.to_string_lossy().into_owned()); let mut resolved: HashMap = HashMap::new(); // effective build -> FASTA path let mut effective_of: HashMap = HashMap::new(); // detected build -> effective build let mut needs: Vec = Vec::new(); let mut reference_notes: Vec = Vec::new(); - // Alignment count + a representative detection source, per distinct detected build. + // The count of alignments, and one example of the detection source, for each distinct + // build. let mut per_build: BTreeMap = BTreeMap::new(); for (build, source) in detected.values() { let e = per_build.entry(build.clone()).or_insert((0, *source)); @@ -592,9 +701,10 @@ impl App { for (detected_build, (count, source)) in &per_build { let count = *count; - // Effective build: keep the detected one when the gateway recognizes it (or an explicit - // FASTA overrides everything); otherwise fall back to the default so unlabeled files - // still import instead of killing the batch. + // The build that the row holds. Keep the build that the code detected when the gateway + // recognizes it. A FASTA file from the caller replaces each such value. If neither + // applies, use the default build. A file with no label then still imports, and the + // batch continues. let (effective, defaulted) = if explicit.is_some() || !matches!(self.gateway.reference_status(detected_build), RefStatus::Unknown) { (detected_build.clone(), false) @@ -603,9 +713,12 @@ impl App { }; effective_of.insert(detected_build.clone(), effective.clone()); - // Resolve the effective build to a FASTA once (explicit > already-resolved > cache > - // gateway status). A download need is collected; an unresolvable build is recorded - // without a FASTA (resolved on demand at analysis time) rather than aborting. + // Find one FASTA file for that build. The order is the file from the caller, a file + // that the code already found, the cache, and then the status of the gateway. + // + // The method collects each download that the import needs. It records a build with no + // file and no FASTA path, and the analysis finds that file later. It does not stop the + // import. let path: Option = if let Some(ref p) = explicit { Some(p.clone()) } else if let Some(p) = resolved.get(&effective) { @@ -680,9 +793,10 @@ impl App { fast_path: FastPathSummary::default(), }; - // Import each sample independently: a single sample's failure (unreadable file, DB hiccup) - // is logged + tallied into `sample_errors` and the batch continues with the rest, rather - // than one bad sample aborting the whole import. + // Import each sample on its own. A failure in one sample goes into the log and into the + // `sample_errors` count, and the batch continues with the other samples. The causes are a + // file that the code can not read, and a fault in the database. One bad sample must not + // stop the full import. let total = discovered.samples.len(); for (i, sample) in discovered.samples.iter().enumerate() { progress(i, total, &sample.sample_id); @@ -708,10 +822,13 @@ impl App { Ok(summary) } - /// Import one sample's subject, run, alignments, and fast-path sidecars. Extracted so a failure - /// here bubbles up as this sample's error (caught by [`Self::import_project_dir`]) instead of - /// aborting the whole batch. `detected`/`effective_of`/`resolved` are the pre-flight reference - /// maps from the caller; `summary` is updated in place with what this sample contributed. + /// Import the subject, the run, the alignments, and the fast-path sidecar files of one sample. + /// + /// This method is separate, so a failure here becomes the error of this sample. + /// [`Self::import_project_dir`] catches that error, and the batch continues. + /// + /// The `detected`, `effective_of`, and `resolved` maps come from the preflight of the caller. + /// The method writes what this sample added to `summary`. #[allow(clippy::too_many_arguments)] async fn import_project_sample( &self, @@ -723,10 +840,14 @@ impl App { resolved: &HashMap, summary: &mut ProjectImportSummary, ) -> Result<(), AppError> { - // Biosample: reuse an existing subject with this donor identifier **anywhere in the - // workspace** — a person is one subject across projects. Scoping the lookup to the target - // project duplicated everyone when the same folder was re-imported under a different - // project name (a person then existed once per project). Create only when truly new. + // The biosample. Use a subject with this donor identifier from **any place in the + // workspace**. One person is one subject in each project. + // + // An earlier version looked in the target project only. So a second import of the same + // folder, under another project name, made a second subject for each person. A person then + // had one subject in each project. + // + // Make a subject only when the workspace holds none. let biosample = match biosample::find_by_donor(self.store.pool(), &sample.sample_id).await? { Some(b) => b, None => { @@ -740,8 +861,9 @@ impl App { .await? } }; - // Ensure the subject is a member of this project (idempotent on the (guid, project) PK). - // A reused subject whose *home* project is another one still joins this project's roster. + // Make sure that the subject is a member of this project. A second call is safe, because + // the primary key is the pair (guid, project). A subject whose *home* project is another + // project also joins the list of this project. biosample_project::add( self.store.pool(), biosample.guid, @@ -792,8 +914,8 @@ impl App { variant_caller: None, bam_path: Some(path_str), reference_path, - // Batch import: hash lazily on first analysis (don't stall a bulk NAS import - // hashing every multi-GB file up front). + // This is a batch import. Calculate the hash at the first analysis. A bulk NAS + // import must not stop while it hashes each file of many GB. content_sha256: None, // An imported alignment is an original; see above. derived_from_alignment_id: None, @@ -803,9 +925,14 @@ impl App { summary.alignments_created += 1; } - // Fast path: ingest the pipeline sidecars onto the build-matching alignment — - // places Y + mt from the GVCFs and fills sex/metrics/lite-coverage from the text - // sidecars, no CRAM walk. Best-effort; a failure is tallied and import continues. + // The fast path. It reads the pipeline sidecar files onto the alignment with the same + // build. + // + // It places the Y haplogroup and the mt haplogroup from the GVCF files. It also fills the + // sex, the metrics, and a small coverage result from the text sidecar files. It walks no + // CRAM file. + // + // The step is optional. A failure goes into a count, and the import continues. if fast_path && sample.sidecars.has_haplogroup_gvcf() { let alns = alignment::list_for_run(self.store.pool(), run.id).await?; let chosen = sample @@ -840,8 +967,9 @@ impl App { self.gateway.reference_status(build) } - /// Resolve a reference build to a cached, indexed `.fa`, downloading on a miss. - /// `progress(received, total)` is invoked as bytes arrive. + /// Find the indexed `.fa` file of a reference build in the cache. The method downloads that + /// file when the cache holds none. It calls `progress(received, total)` as each part of the file + /// arrives. pub async fn resolve_reference( &self, build: &str, @@ -850,8 +978,9 @@ impl App { Ok(self.gateway.resolve_reference(build, progress).await?) } - /// Resolve (and cache) a liftover chain for a build pair, downloading on a miss. The - /// cached `.chain` is then available for the haplogroup/liftover path. + /// Find the liftover chain of a build pair, and write it to the cache. The method downloads + /// that file when the cache holds none. The haplogroup path and the liftover path then read the + /// `.chain` file from the cache. pub async fn resolve_chain( &self, from: &str, @@ -861,19 +990,26 @@ impl App { Ok(self.gateway.resolve_chain(from, to, progress).await?) } - /// Re-hash a cached reference against its integrity sidecar (gap §7) — detects on-disk - /// corruption of the cached `.fa`. Runs on a blocking thread (re-reads the whole FASTA), so it's - /// an explicit, user-triggered check (Settings), not the hot path. + /// Calculate the hash of a reference in the cache again, and compare it with the sidecar file + /// that holds the correct value. See gap §7. The method finds a `.fa` file that the disk + /// damaged. + /// + /// The method reads the full FASTA file, so it runs on its own thread. The user starts it from + /// the Settings screen, and no analysis calls it. pub async fn verify_reference(&self, build: &str) -> Result { let gw = self.gateway.clone(); let build = build.to_string(); Ok(tokio::task::spawn_blocking(move || gw.verify_reference(&build)).await??) } - /// Lift a whole VCF from `source` build to `target` build (gap §7 — the GATK `LiftoverVcf` - /// replacement). Ensures the source→target chain and the target reference are resolved - /// (downloading on a miss, with progress), then runs the line-level lift on a blocking thread. - /// Returns lift/drop counts. + /// Move a full VCF file from the `source` build to the `target` build. See gap §7. This method + /// takes the place of the GATK `LiftoverVcf` tool. + /// + /// The method first finds the chain from the source to the target, and the reference of the + /// target. It downloads each file that the cache does not hold, and it reports the progress. + /// + /// It then moves each line on its own thread. It returns the count of the lines that it moved + /// and the count of the lines that it removed. pub async fn lift_vcf( &self, source: &str, @@ -883,12 +1019,14 @@ impl App { opts: navigator_refgenome::VcfLiftOpts, progress: &mut (dyn FnMut(u64, Option) + Send), ) -> Result { - // Resolve the inputs (chain + target FASTA), downloading on a miss. + // Find the two input files, which are the chain and the FASTA file of the target. Download + // each file that the cache does not hold. self.gateway.resolve_chain(source, target, progress).await?; let target_fa = self.gateway.resolve_reference(target, progress).await?; let lo = self.gateway.load_liftover(source, target)?; - // Target chrY PAR intervals (only needed when filtering them out). + // The PAR intervals of chrY on the target build. The code needs them only when it removes + // those intervals. let target_par: Vec<(i64, i64)> = if opts.filter_par { let regions = self.gateway.genome_regions(target, progress).await?; regions @@ -918,31 +1056,37 @@ impl App { Ok(stats) } - /// See [`asset_action`] for the present/stale/absent decision this drives. + /// Make sure that the ancestry asset or IBD asset at `path` is present **and current**. + /// [`asset_action`] makes the decision that this method acts on. + /// + /// The method downloads the asset, and the manifest that it checks the asset against, from the + /// published GitHub release. A user receives each panel in this way, and no user runs the + /// offline `panelbuild` tool. /// - /// Ensure a prebuilt ancestry/IBD asset at `path` is present **and current**, downloading it — - /// and the asset manifest it's verified against — from the published GitHub release. End users - /// get the panels this way instead of running the offline `panelbuild` tool. + /// There are three cases, and the manifest decides each one. /// - /// Three cases, all manifest-driven: + /// * The asset is **absent**. Download it when the manifest lists it. An optional asset that + /// the team did not publish stays absent, and its feature gives less data. + /// * The **manifest does not list the asset**. Read the manifest again one time, then test + /// again. The code refreshes a cached manifest at no other time. So an installation from + /// before the publication of an asset would never learn that the asset exists. That fault + /// occurred with `ancestry_haps`. + /// * The asset is **present with the wrong size**. The team published a new version, so replace + /// the file. A test for the file alone can not see a new version. So without this test, an + /// installation with the asset keeps the old file for all time. /// - /// * **Absent** → download it, provided the manifest lists it (an unpublished optional asset - /// simply stays absent and its feature degrades). - /// * **Manifest doesn't list it** → re-fetch the manifest once, then re-check. The cached - /// manifest is otherwise never refreshed, so an install that predates an asset's publication - /// would never learn the asset exists — which is exactly what happened to `ancestry_haps`. - /// * **Present but the wrong size** → the published asset was revised; replace it. Without this - /// an install that already has an asset keeps the stale one forever, because a revision is - /// invisible to a plain existence check. The stale file is moved aside, not deleted, and put - /// back if the download fails — a stale asset beats no asset. + /// The code moves the old file to another name and does not delete it. It puts that file back + /// when the download fails, because an old asset is better than no asset. /// - /// An explicit `$NAVIGATOR_*` path override is never fetched over or repaired: that file is the - /// user's own. Best-effort throughout — network failures leave on-disk state alone. + /// The method never downloads over a path from a `$NAVIGATOR_*` variable, and it never repairs + /// such a file. That file belongs to the user. Each step is optional, and a network failure + /// changes nothing on the disk. pub(crate) async fn ensure_ancestry_asset(&self, build: ReferenceBuild, path: &Path) -> Result<(), AppError> { let Some(name) = path.file_name().and_then(|n| n.to_str()).map(str::to_string) else { return Ok(()); }; - // Only auto-fetch to the default cache location — an explicit override is the user's own file. + // Download only to the default place in the cache. A path from a variable names a file of + // the user. let default = refgenome_cache::base_dir().join("ancestry").join(&name); if path != default { return Ok(()); @@ -951,9 +1095,12 @@ impl App { let manifest_name = format!("ancestry_manifest_{}.json", build.as_str()); let manifest_path = default.with_file_name(&manifest_name); - // (1) The manifest: fetch when absent, and re-fetch when it doesn't list this asset (a - // manifest cached before the asset was published). Keep the old copy in memory so a - // failed refresh doesn't cost us the integrity data we already had. + // (1) The manifest. Download it when the cache holds none. Download it again when it does + // not list this asset, because the cache can hold a manifest from before the team + // published that asset. + // + // Keep the old copy in memory. A failed download then does not remove the check values + // that the app already has. let listed = |m: &Option| { m.as_ref().is_some_and(|m| m.assets.contains_key(&name)) }; @@ -984,8 +1131,8 @@ impl App { return Ok(()); }; - // (2) What to do with what's on disk. Content is verified at read time by - // `read_verified_asset`; hashing every asset here would cost seconds per paint. + // (2) The decision about the file on disk. `read_verified_asset` checks the content at + // each read. A hash of each asset here costs some seconds at each paint. let on_disk = std::fs::metadata(&default).ok().map(|m| m.len()); let action = asset_action(Some(&entry), on_disk); if action == AssetAction::Ready { @@ -1035,7 +1182,8 @@ impl App { Ok(()) } Err(e) => { - // Put the old asset back: analysing against a superseded panel beats not analysing. + // Put the old asset back. An analysis against an old panel is better than no + // analysis. if let Some(aside) = stale { let _ = std::fs::rename(&aside, &default); eprintln!("ancestry assets: {name} download failed ({e}) — kept the existing copy"); @@ -1046,9 +1194,12 @@ impl App { } } - /// Load the CHM13 IBD panel — downloading the prebuilt asset from the release on first use (no - /// `panelbuild` needed). The single entry point for the panel: replaces the five call sites that - /// each errored "build it with `panelbuild ibd-panel`" when the asset was absent. + /// Read the CHM13 IBD panel. At the first use, the method downloads the asset from the release, + /// and no user runs `panelbuild`. + /// + /// This method is the one entry point for that panel. It takes the place of five call sites. + /// Each of those sites gave the error "build it with `panelbuild ibd-panel`" when the asset was + /// absent. pub(crate) async fn load_ibd_panel(&self) -> Result { let build = ReferenceBuild::Chm13v2; let path = ibd_panel_path(build); @@ -1063,10 +1214,16 @@ impl App { Ok(navigator_analysis::ibd_panel::IbdPanel::from_bytes(&bytes)?) } - /// Resolve an imported chip's genotypes to canonical CHM13 **IBD-panel** dosages — the chip→IBD - /// path (no alignment, no runtime liftover: the multi-build panel pre-computes coordinates). The - /// output [`SiteGenotype`]s are over the same CHM13 sites a WGS caller would hit, so a chip and a - /// WGS sample compare uniformly. Errors if the IBD panel asset isn't built yet. + /// Change the genotypes of an imported chip into dosages at the canonical CHM13 **IBD panel** + /// sites. This method is the path from a chip to the IBD data. + /// + /// It needs no alignment and does no liftover at run time, because the panel holds the + /// coordinates of each build. + /// + /// The [`SiteGenotype`] values that the method returns cover the same CHM13 sites that a WGS + /// caller reaches. So a chip sample and a WGS sample compare in the same way. + /// + /// The method fails when the IBD panel asset does not exist. pub async fn chip_ibd_dosages(&self, chip_profile_id: i64) -> Result, AppError> { let chip = chip_profile::get(self.store.pool(), chip_profile_id) .await? @@ -1085,14 +1242,24 @@ impl App { Ok(panel.resolve_chip(&from_build, &tuples)) } - /// Import a trusted external caller's autosomal **1240K EIGENSTRAT call set** - /// (`.geno`/`.snp`/`.ind`) for a subject — the autosomal counterpart to the Y/mt GVCF sidecar - /// fast path. Resolves the target individual's genotypes to canonical CHM13 panel dosages (no - /// CRAM decode; `resolve_chip` self-orients against the CHM13 alleles), persists them as an - /// `external` source, and refreshes the autosomal consensus so modern/fine/deep ancestry and IBD - /// pick them up. `path` may point at any member of the triplet (siblings resolved by basename). - /// The `.snp` build is GRCh37 (AADR 1240K) unless `NAVIGATOR_CALLSET_BUILD` overrides it. Returns - /// the number of resolved panel sites. + /// Import the autosomal **1240K EIGENSTRAT call set** of a trusted external caller for a + /// subject. The files are `.geno`, `.snp`, and `.ind`. This method is the autosomal form of the + /// Y and mt sidecar fast path. + /// + /// The method changes the genotypes of the target individual into dosages at the canonical CHM13 + /// panel sites. It decodes no CRAM file, and `resolve_chip` orients each call against the CHM13 + /// alleles itself. + /// + /// It stores those dosages as an `external` source, and it builds the autosomal consensus again. + /// The modern ancestry, the fine ancestry, the deep ancestry, and the IBD steps then read them. + /// + /// The `path` value can name any of the three files, because the code finds the other two from + /// the base name. + /// + /// The build of the `.snp` file is GRCh37, which the AADR 1240K set uses. The + /// `NAVIGATOR_CALLSET_BUILD` variable replaces that value. + /// + /// The method returns the count of the panel sites that it resolved. pub async fn import_callset_from_file(&self, biosample_guid: SampleGuid, path: &Path) -> Result { // Resolve the triplet from any member by shared basename. let stem = path @@ -1113,7 +1280,8 @@ impl App { } } - // AADR 1240K is GRCh37/hg19; allow a GRCh38-built call set via the env override. + // The AADR 1240K set uses GRCh37, which is also hg19. The variable lets a user import a + // call set on GRCh38. let build = std::env::var("NAVIGATOR_CALLSET_BUILD").unwrap_or_else(|_| "GRCh37".to_string()); let (g, s, i, b) = (geno.clone(), snp.clone(), ind.clone(), build.clone()); let callset = @@ -1138,13 +1306,21 @@ impl App { .await } - /// Import a trusted external caller's autosomal genotypes for a subject from a **diploid VCF/gVCF** - /// — the VCF path of the autosomal fast path (Phases 4/5). Handles a GATK4 gVCF (variant records + - /// hom-ref ref blocks) **and** a genotyped all-sites VCF (e.g. `bcftools mpileup`/`call` over the - /// 1240K sites, where every site carries an explicit `GT`). Genotypes the panel loci directly with - /// **no CRAM decode**, re-keys to canonical CHM13 (`resolve_chip`), stores the dosages as an - /// `external` source, and refreshes the autosomal consensus. Build is auto-detected from the VCF - /// header (`NAVIGATOR_CALLSET_BUILD` overrides). Returns the number of resolved panel sites. + /// Import the autosomal genotypes of a trusted external caller for a subject, from a **diploid + /// VCF file or gVCF file**. This method is the VCF path of the autosomal fast path, in phases 4 + /// and 5. + /// + /// The method reads a GATK4 gVCF file, which holds variant records and hom-ref blocks. It also + /// reads a VCF file with each site genotyped, such as the output of `bcftools mpileup` and + /// `bcftools call` across the 1240K sites. In that second file, each site holds a `GT` value. + /// + /// The method genotypes the panel loci directly and decodes **no CRAM file**. It then re-keys + /// each call to the canonical CHM13 sites with `resolve_chip`, stores the dosages as an + /// `external` source, and builds the autosomal consensus again. + /// + /// The code finds the build in the header of the VCF file, and the `NAVIGATOR_CALLSET_BUILD` + /// variable replaces that value. The method returns the count of the panel sites that it + /// resolved. pub async fn import_gvcf_callset_from_file( &self, biosample_guid: SampleGuid, @@ -1154,8 +1330,9 @@ impl App { let panel = self.load_ibd_panel().await?; - // Panel loci in the gVCF's build, grouped + sorted per contig; keep each site's reference - // allele so a hom-ref block resolves to (ref, ref). + // The panel loci in the build of the gVCF file. The code groups them by contig and sorts + // each group. It keeps the reference allele of each site, so a hom-ref block gives the pair + // (ref, ref). let mut targets_by_contig: std::collections::HashMap> = std::collections::HashMap::new(); let mut ref_allele: std::collections::HashMap<(String, i64), char> = std::collections::HashMap::new(); for site in &panel.sites { @@ -1238,7 +1415,8 @@ impl App { created_at: chrono::Utc::now().to_rfc3339(), }; navigator_store::external_panel_dosage::upsert(self.store.pool(), &row).await?; - // Fold into the autosomal consensus immediately — cheap, no decode (best-effort). + // Add the dosages to the autosomal consensus at once. The step is fast, it decodes + // nothing, and it is optional. let _ = self.refresh_autosomal_consensus(biosample_guid).await; Ok(site_count) } @@ -1259,11 +1437,16 @@ impl App { Ok(detect_ibd(&ga, &gb, ReferenceBuild::Chm13v2, config)) } - /// IBD comparison between two **subjects** from their autosomal consensuses — each subject's - /// pooled best genotype per site (across all its WGS + chip sources), no per-source genotyping. - /// This is the subject-level IBD path (consensus-driven); both subjects must have a built - /// autosomal consensus. A near-complete genome-wide match is the cross-subject identity (dedup) - /// signal — read it off the returned [`MatchSummary`]'s relationship estimate. + /// Compare two **subjects** for IBD segments, from their autosomal consensus values. + /// + /// The consensus of a subject holds the best genotype at each site, from each of its WGS sources + /// and chip sources. The method genotypes no source again. + /// + /// This path works at the level of a subject, and the consensus drives it. Both subjects must + /// have an autosomal consensus. + /// + /// A match across almost the full genome shows that the two subjects are the same person. Read + /// that result from the relationship estimate of the [`MatchSummary`] value. pub async fn compare_ibd_consensus( &self, a: SampleGuid, @@ -1283,9 +1466,11 @@ impl App { Ok(detect_ibd(&ga, &gb, ReferenceBuild::Chm13v2, config)) } - /// Dosages over the canonical CHM13 IBD-panel sites for a comparison source. A chip resolves - /// directly ([`Self::chip_ibd_dosages`]); an alignment genotypes the panel's CHM13 sites from - /// its BAM (cached per alignment, ploidy-2 autosomal). + /// The dosages at the canonical CHM13 IBD-panel sites, for one comparison source. + /// + /// A chip resolves directly, in [`Self::chip_ibd_dosages`]. An alignment genotypes the CHM13 + /// sites of the panel from its BAM file. The code caches that result for each alignment, and it + /// uses ploidy 2 on an autosome. pub async fn ibd_panel_dosages(&self, source: IbdSource) -> Result, AppError> { match source { IbdSource::Chip(id) => self.chip_ibd_dosages(id).await, @@ -1296,16 +1481,19 @@ impl App { self.variant_set_panel_dosages(&set).await } IbdSource::Alignment(id) => { - // Salt the cache key with the panel asset's manifest hash, so regenerating the panel - // (e.g. the probe superset) auto-invalidates stale per-alignment genotypes instead of - // silently serving genotypes taken over an older site set. + // Add the manifest hash of the panel asset to the cache key. A new panel, such as + // one with more probes, then makes each stored genotype of an alignment invalid. So + // the app does not return a genotype from an older set of sites with no message. let kind = ibd_panel_cache_kind(); if let Some(g) = self.load_analysis(id, &kind, caller::GENOTYPE_VERSION).await? { return Ok(g); } - // Resolve the reference for decode (see alignment_reference_for_decode): required for - // a CRAM, None for a BAM. Panel genotyping tallies SNP sites (ref/alt come from the - // panel), so a BAM consults no reference bases — don't force a download for it. + // Find the reference for the decoder. See alignment_reference_for_decode. A CRAM + // file needs it, and a BAM file uses None. + // + // The panel genotype step counts the reads at each SNP site, and the panel gives the + // reference allele and the alternate allele. So the code reads no reference base for + // a BAM file, and it must not start a download for one. let build = self.alignment_or_err(id).await?.reference_build; let (bam, reference) = self.alignment_reference_for_decode(id).await?; let panel = self.load_ibd_panel().await?; @@ -1315,8 +1503,9 @@ impl App { ); let genotypes = if is_chm13 { - // Native CHM13: genotype directly at the panel's canonical CHM13 loci — the dosage - // is already CHM13-oriented, no re-keying. + // A native CHM13 alignment. Genotype it directly at the canonical CHM13 loci + // of the panel. Each dosage is then already in the CHM13 space, and the code + // changes no key. let sites: Vec = panel .sites .iter() @@ -1341,10 +1530,15 @@ impl App { }) .await?? } else if panel.sites.iter().any(|s| s.locus(&build).is_some()) { - // GRCh37/GRCh38: the panel already carries this build's coordinates (offline - // allele-aware liftover). Genotype at the build's loci, then re-key the dosages to - // canonical CHM13 ([`IbdPanel::resolve_alignment`]) — no runtime liftover needed. - // Match the panel's per-build contig names to the file's naming (`chr1` vs `1`). + // A GRCh37 alignment or a GRCh38 alignment. The panel already holds the + // coordinates of that build, from an offline liftover that read the alleles. + // + // Genotype the sample at the loci of that build. Then re-key each dosage to the + // canonical CHM13 sites, in [`IbdPanel::resolve_alignment`]. The code does no + // liftover at run time. + // + // Match the contig names of the panel for that build to the names in the file. + // One file uses `chr1`, and another file uses `1`. let (bam_h, ref_h) = (bam.clone(), reference.clone()); let file_contigs = tokio::task::spawn_blocking(move || { navigator_analysis::reader::contig_names(&bam_h, ref_h.as_deref()) @@ -1384,8 +1578,9 @@ impl App { .await??; panel.resolve_alignment(&build, &raw) } else { - // A build the panel doesn't carry — nothing to genotype (degrade gracefully rather - // than probe the wrong loci). + // The panel holds no coordinates for this build, so there is nothing to + // genotype. The code gives a smaller result. It must not read the wrong + // loci. Vec::new() }; self.save_analysis(id, &kind, caller::GENOTYPE_VERSION, &genotypes) @@ -1395,10 +1590,13 @@ impl App { } } - /// Cached IBD-panel dosages for an alignment, **without genotyping** — `Ok(None)` when they - /// haven't been computed yet (so callers can reduce over what's available progressively rather - /// than triggering a whole-genome decode). [`Self::ibd_panel_dosages`] is the compute-and-cache - /// path; this is the read-only companion used by the progressive-consensus refresh. + /// The IBD-panel dosages of an alignment from the cache. The method genotypes **nothing**. + /// + /// It returns `Ok(None)` when no earlier run calculated those dosages. So a caller can work with + /// the data that the store holds, and it does not start a decode of the full genome. + /// + /// [`Self::ibd_panel_dosages`] calculates the dosages and writes them to the cache. This method + /// only reads them, and the progressive-consensus refresh calls it. pub async fn cached_alignment_panel_dosages( &self, alignment_id: i64, @@ -1407,9 +1605,13 @@ impl App { .await } - /// Subject-level identity verification (gap §8) — "are these two subjects the same individual?" - /// (duplicate detection). Pooled autosomal consensus genotype concordance (no panel selection), - /// corroborated by Y-STR distance. Both subjects need a built autosomal consensus. + /// A test of identity at the level of a subject, in gap §8. It answers the question "are these + /// two subjects the same person?", and the app uses it to find a duplicate. + /// + /// The method compares the genotypes of the two pooled autosomal consensus values, and it + /// selects no panel. The distance between the Y-STR values supports that comparison. + /// + /// Both subjects need an autosomal consensus. pub async fn verify_identity_consensus( &self, a: SampleGuid, @@ -1441,22 +1643,30 @@ impl App { } } -/// What [`App::ensure_ancestry_asset`] must do for one asset, from the manifest entry (`None` when -/// the manifest doesn't list it) and the on-disk size (`None` when the file is absent). +/// The action that [`App::ensure_ancestry_asset`] must take for one asset. +/// +/// The decision reads two values. The first is the manifest entry, which is `None` when the manifest +/// does not list the asset. The second is the size of the file on disk, which is `None` when the +/// file is absent. +/// +/// The decision uses the size and not a hash. The team makes a new version of an asset when it +/// builds that asset again, and the new file has a different length. The read step already checks +/// the content, and that check has authority. /// -/// Size, not hash: a published asset is revised by rebuilding it, which changes its length, and the -/// authoritative content check already happens at read time. Hashing a 133 MB panel on every call -/// would cost seconds per paint to catch a case (same size, different bytes) that read-time -/// verification catches anyway. +/// A hash of a panel of 133 MB at each call costs some seconds at each paint. It finds only one more +/// case, where two files have the same size and different bytes, and the read step finds that case +/// also. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum AssetAction { - /// Present at the published size — use it. + /// The file is present, and its size equals the published size. Use it. Ready, - /// Absent — fetch it. + /// The file is absent. Download it. Download, - /// Present but superseded (or truncated) — move aside and fetch. + /// The file is present, but a newer version exists, or the file is not complete. Move it to + /// another name and download the new file. Replace, - /// Not published for this build — leave it absent and let the feature degrade. + /// The team published no file of this asset for this build. Leave it absent, and let its + /// feature give a smaller result. Skip, } diff --git a/crates/navigator-app/src/lib.rs b/crates/navigator-app/src/lib.rs index 9cbb657e..9d6ad8d5 100644 --- a/crates/navigator-app/src/lib.rs +++ b/crates/navigator-app/src/lib.rs @@ -1,8 +1,12 @@ -//! Navigator application/command layer — the single API the UI dispatches to, and the -//! antidote to the `WorkbenchViewModel` god object. Orchestrates `navigator-store` (and -//! later analysis/sync) behind commands and queries; holds policy the old dialogs -//! embedded (identity assignment, existence checks, result (de)serialization). The UI -//! holds only view-state and dispatch — no DB calls or domain decisions in widgets. +//! The application layer of Navigator, and its command layer. This crate is the one API that the UI +//! calls. It takes the place of the `WorkbenchViewModel` type, which held too much. +//! +//! The crate controls `navigator-store`, and later the analysis code and the sync code, behind its +//! commands and queries. It also holds each policy that an old dialog held. Those policies assign an +//! identity, test that a record exists, and read and write a result. +//! +//! The UI holds the state of its views and sends commands. No widget calls the database, and no +//! widget makes a decision about the domain. use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -33,9 +37,12 @@ pub use navigator_analysis::preflight::{ }; pub use navigator_analysis::CancelToken; -/// Diagnose a BAM/CRAM **path** with no workspace record behind it — the case that matters when a -/// user is reporting a file the app refuses to read and we need the answer before deciding whether -/// importing it is even possible. Blocking; call it off the async runtime. +/// Report the state of a BAM **path** or CRAM **path** that has no record in the workspace. +/// +/// This case matters when a user reports a file that the app refuses to read. The team needs the +/// answer before it decides whether an import of that file is possible. +/// +/// The function blocks, so call it away from the async runtime. pub fn diagnose_alignment_file(alignment: &std::path::Path, reference: Option<&std::path::Path>) -> PreflightReport { navigator_analysis::preflight::diagnose(alignment, reference) } @@ -56,107 +63,148 @@ pub use navigator_domain::ancestry::{ side_label_default, AncestryResult, AncestrySegment, ConfidenceInterval, PaintingResult, PopulationComponent, SuperPopulationSummary, }; -// The ancestry panel format, re-exported so panel tooling/tests depend only on navigator-app. +// The format of the ancestry panel. This crate exports it again, so a panel tool and a test depend +// on navigator-app alone. pub use navigator_analysis::ancestry::{AncestryPanel, PanelSite as AncestryPanelSite}; -/// A haplogroup assignment: the ranked candidates plus, for the reported terminal, the -/// child branches with per-SNP evidence (why descent stopped — unsupported splits show -/// ancestral SNPs, unresolved ones show no-calls). +/// One haplogroup assignment. It holds the candidates in their order. For the terminal node that it +/// reports, it also holds each child branch with the evidence of each SNP. +/// +/// That evidence shows the reason that the descent stopped. A split with no support shows an +/// ancestral SNP. A split with no answer shows a no-call. #[derive(Debug, Clone)] pub struct HaploAssignment { pub ranked: Vec, pub branches: Vec, - /// Per-SNP evidence along the placed lineage (root→terminal): every defining mutation the - /// sample carries (or doesn't), Derived/Ancestral/NoCall. This is the set the multi-source - /// variant/mutation **profile** reconciles — distinct from `branches`, which is the *untaken* - /// child branches (explaining why descent stopped, hence largely ancestral/no-call). + /// The evidence of each SNP along the lineage, from the root to the terminal node. The list + /// holds each mutation that defines a node, and the state of the sample at that mutation. A + /// state has one of three values: Derived, Ancestral, or NoCall. + /// + /// The variant **profile**, which pools many sources, reconciles this set. + /// + /// This list is not `branches`. That field holds the child branches that the descent did not + /// take. It shows the reason that the descent stopped, so most of its states are Ancestral or + /// NoCall. pub lineage: Vec, } -/// A YFull-YReport-style descent report for one lineage (Y or mtDNA): the root→terminal path, each -/// node carrying its defining SNPs with the subject's per-SNP call state. Generic over [`DnaType`] -/// so the Y-DNA and mtDNA tabs share one model + renderer. Built by [`App::descent_report`]. +/// A descent report for one lineage, which is the Y lineage or the mtDNA lineage. The report has the +/// shape of a YFull YReport. +/// +/// It holds the path from the root to the terminal node. Each node holds the SNPs that define it, +/// and the call state of the subject at each SNP. +/// +/// The type takes a [`DnaType`] parameter, so the Y-DNA tab and the mtDNA tab share one model and +/// one renderer. [`App::descent_report`] builds it. #[derive(Debug, Clone)] pub struct DescentReport { pub dna: DnaType, /// The reported terminal haplogroup name (e.g. "R-FGC29071", "U5a1b1g"). pub terminal: String, - /// Nodes root→terminal, each with its defining SNPs + the sample's state (`NodeEvidence`). + /// The nodes from the root to the terminal node. Each node holds the SNPs that define it and + /// the state of the sample, in a `NodeEvidence` value. pub nodes: Vec, } -/// A cohort **block tree** for one project: the induced subtree of the haplotree spanning the -/// members' terminal haplogroups, each node a *block* of phylogenetically equivalent SNPs, with the -/// members hanging off their own terminal. The group-project counterpart to [`DescentReport`] — -/// where that draws one subject's root→terminal path, this draws where a whole cohort sits relative -/// to each other. Built by [`App::project_block_tree`]; see -/// `documents/design/project-block-tree.md`. +/// The **block tree** of the cohort of one project. +/// +/// The tree is the part of the haplotree that covers the terminal haplogroups of the members. Each +/// node is a *block* of SNPs that the tree treats as equivalent. Each member appears below its own +/// terminal node. +/// +/// This type is the group-project form of [`DescentReport`]. That report draws the path of one +/// subject from the root to its terminal node. This tree draws the place of each member of a cohort +/// against the other members. +/// +/// [`App::project_block_tree`] builds it. See `documents/design/project-block-tree.md`. /// -/// This view **reads** placements and never re-places, so it cannot introduce a placement error. +/// This view **reads** a placement and never makes one. So it can add no placement error. #[derive(Debug, Clone)] pub struct ProjectBlockTree { pub dna: DnaType, /// Induced-subtree blocks in pre-order (a parent always precedes its children). pub blocks: Vec, - /// Members with no placement, or whose terminal is absent from this tree. Reported rather than - /// dropped: on a multi-lab cohort provider/build skew is expected, and hiding it would - /// misrepresent how much of the project the tree actually accounts for. + /// The members with no placement, and the members whose terminal node this tree does not hold. + /// + /// The report names them and does not remove them. In a cohort from many laboratories, a + /// difference between providers and builds is normal. Without those members, the reader can not + /// see how much of the project the tree covers. pub unplaced: Vec, - /// The tree the view was drawn on (`"decodingus"` / `"ftdna"`) — `Block::loci` belong to it. + /// The tree of this view, which is `"decodingus"` or `"ftdna"`. The `Block::loci` values belong + /// to that tree. pub provider: String, - /// The coordinate space `Block::loci` positions are in. Node names and topology are - /// build-independent; only the positions are, so the view is labelled with the one build key it - /// was parsed under (the cohort's modal build). + /// The coordinate space of each position in `Block::loci`. + /// + /// The node names and the shape of the tree do not depend on the build. Only the positions + /// depend on it. So the view carries the one build key that the code parsed it under, which is + /// the most frequent build of the cohort. pub build_key: String, - /// Shared-private groupings that were **dropped** because they conflicted: their member sets - /// overlapped an accepted group without nesting inside it, so keeping both would not be a tree. - /// Surfaced rather than silently discarded — a non-zero count means recurrent calls or genuine - /// phylogenetic conflict in the cohort, which is worth knowing about. + /// The count of groups of shared private variants that the code **removed** for a conflict. + /// + /// The member set of such a group shares some members with an accepted group, and the accepted + /// group does not hold it. Two such sets can not both be a branch of one tree. + /// + /// The report gives this count and does not hide it. A count above zero shows recurrent calls, + /// or a real conflict in the phylogeny of the cohort. The reader needs that fact. pub candidate_conflicts: usize, - /// Positions rejected as **recurrent** — each would have defined a candidate branch under more - /// than one parent block, so it arose more than once and cannot mark a new branch. Counted - /// rather than hidden: a high number says the cohort's private calls carry systematic noise. + /// The count of positions that the code refused as **recurrent**. Each one would define a + /// candidate branch below more than one parent block. So it occurred more than one time, and it + /// can not mark a new branch. + /// + /// The report gives this count and does not hide it. A high value shows that the private calls + /// of the cohort hold noise from the same cause. pub candidate_recurrent: usize, } -/// One block of a [`ProjectBlockTree`]: a branch plus the run of defining SNPs that are -/// phylogenetically equivalent on it — every member below carries all of them, and nothing observed -/// in this cohort separates them. +/// One block of a [`ProjectBlockTree`]. It holds a branch and the run of SNPs that define that +/// branch. The tree treats those SNPs as equivalent. Each member below the branch carries each of +/// them, and no observation in this cohort separates them. #[derive(Debug, Clone)] pub struct Block { pub node_id: i64, pub name: String, /// Parent within the induced subtree (`None` at a root). pub parent: Option, - /// Depth within the induced subtree, root = 0 — the layout's x coordinate. + /// The depth of this block in the subtree. The root has the value 0. The layout uses this value + /// as its x coordinate. pub depth: usize, - /// The equivalent SNPs defining this block. After a collapse this is the concatenation of the - /// absorbed branches' loci, root-most first: within *this cohort* they are one undivided block. + /// The equivalent SNPs of this block. After a collapse, the list holds the loci of each absorbed + /// branch, with the loci nearest the root first. Inside *this cohort*, those branches are one + /// block that nothing divides. pub loci: Vec, /// Members whose terminal *is* this block. pub members: Vec, - /// Members at or below this block — the count to badge on a collapsed branch. + /// The count of members at this block and below it. A collapsed branch shows that count. pub subtree_members: usize, /// Names of the member-less branches this block absorbed when collapsed (root-most first). /// Empty for an ordinary block. Kept so the UI can still name what it folded away. pub collapsed: Vec, - /// True when this is a **candidate branch** — not a node in the published tree, but a grouping - /// inferred from private (unnamed) variants that two or more members share. `node_id` is - /// synthetic and negative for these; `name` is empty, because the label is the view's to - /// localize. This is the thing a published tree cannot tell you and we can: a branch that is - /// real in the data but has not been named yet. + /// True when this block is a **candidate branch**. The published tree holds no such node. The + /// code makes the group from the private variants that two members or more share, and those + /// variants have no name. + /// + /// For a candidate, `node_id` is a value that the code made, and it is negative. The `name` + /// field is empty, because the view supplies the label in the language of the user. + /// + /// A published tree can not give this answer, and the app can. The branch is real in the data, + /// and nobody has named it yet. pub candidate: bool, - /// For a candidate branch: every carrier's evidence at each shared position, so it can be - /// reviewed. Empty on a named block, whose SNPs are the tree's assertion rather than ours. + /// For a candidate branch, the evidence of each carrier at each shared position. A reader can + /// then judge the branch. The list is empty on a named block, because the tree states those SNPs + /// and this app does not. pub evidence: Vec, } -/// One carrier's evidence at one of a candidate branch's shared positions. +/// The evidence of one carrier at one shared position of a candidate branch. +/// +/// A candidate is a deduction, and the statement "three men share 1 SNP" is not enough to judge it. /// -/// A candidate is an inference, and "1 SNP shared by three men" is not enough to judge it. What -/// decides whether it is a branch or a mapping artefact is the read evidence behind each carrier's -/// call — depth, and how cleanly the derived allele dominates on a chromosome that carries one copy. -/// Carried on the aggregate so the branch can be reviewed rather than taken on trust. +/// The read evidence behind the call of each carrier decides between a branch and a mapping +/// artefact. That evidence is the depth, and the share of the reads that hold the derived allele. A +/// cell holds one copy of that chromosome, so a true call has almost no other allele. +/// +/// The aggregate carries this evidence, so a reader can judge the branch and does not trust it +/// without data. #[derive(Debug, Clone)] pub struct CandidateEvidence { pub guid: SampleGuid, @@ -167,9 +215,10 @@ pub struct CandidateEvidence { pub alternate: char, /// Read depth at the site; `0` when the source reported none. pub depth: u32, - /// Reads supporting the derived allele. + /// The count of reads that hold the derived allele. pub alt_depth: u32, - /// Derived-allele fraction — on haploid chrY a real call is essentially 1.0. + /// The share of the reads that hold the derived allele. A cell holds one copy of chrY, so a + /// true call gives a value near 1.0. pub allele_fraction: f64, /// Whether this call clears the federation publish gate. pub publishable: bool, @@ -179,16 +228,25 @@ pub struct CandidateEvidence { #[derive(Debug, Clone)] pub struct BlockMember { pub guid: SampleGuid, - /// Display name (donor identifier, else the guid) — what the tree leaf is labelled with. + /// The name for the display. The value is the donor identifier, and then the guid. The leaf of + /// the tree shows it. pub name: String, - /// Unnamed (private) variants below this member's terminal. `None` until private-Y has been - /// computed for the subject, which is distinct from `Some(0)` ("computed, none found"). - /// Populated in phase 3 (`documents/design/project-block-tree.md` §9); `None` before that. + /// The count of private variants below the terminal node of this member. Those variants have no + /// name. + /// + /// The value is `None` until the app calculates the private-Y data of the subject. That state is + /// not the same as `Some(0)`, which means that the app calculated the data and found no variant. + /// + /// Phase 3 writes this value. See `documents/design/project-block-tree.md` §9. Before that + /// phase, the value is `None`. pub private_novel: Option, - /// The **publishable** subset of the above: novel, unique-sequence, near-homozygous, with enough - /// supporting reads ([`PublishGate`]). This is the count we would stake a branch claim on, and - /// the one the block tree averages — the permissive `private_novel` is a working figure, not a - /// finding. + /// The part of the count above that the app can **publish**. Each such variant is new, is in + /// unique sequence, has almost no other allele, and has enough reads. [`PublishGate`] holds + /// those rules. + /// + /// The app makes a claim about a branch only from this count, and the block tree takes the mean + /// of it. The `private_novel` count above uses weaker rules. It is a value for work in progress, + /// and it is not a result. pub private_publishable: Option, pub private_total: Option, } @@ -202,24 +260,32 @@ pub struct UnplacedMember { pub terminal: Option, } -/// A per-marker branch report: the sample's genotype at every defining marker of a chosen tree -/// node's **descendant subtree** (Y or mtDNA), for spot-checking placement accuracy and exchanging -/// observations with other researchers. Built by [`App::branch_report`]; exported by -/// [`crate::export::branch_report_tsv`]. Unlike [`DescentReport`] (which walks the placement's -/// root→terminal ancestors from the persisted profile) this genotypes the subtree fresh, so -/// off-path branches the sample is *ancestral* for are reported too. +/// A branch report with one row for each marker. It holds the genotype of the sample at each marker +/// that defines a node in the **subtree below** a tree node that the user chose. The lineage is the Y +/// lineage or the mtDNA lineage. +/// +/// A researcher uses this report to check a placement, and to send observations to another +/// researcher. +/// +/// [`App::branch_report`] builds it, and [`crate::export::branch_report_tsv`] writes it to a file. +/// +/// This report is not a [`DescentReport`]. That report reads the ancestors of the placement from the +/// stored profile, from the root to the terminal node. This report genotypes the subtree again. So +/// it also holds each branch off the path where the sample is *ancestral*. #[derive(Debug, Clone)] pub struct BranchReport { pub dna: DnaType, /// The queried root node's haplogroup name (e.g. `R-FGC29071`). pub root: String, pub contig: String, - /// True when the observed bases + evidence came from a per-sample GVCF sidecar (else pileup). + /// True when the bases and the evidence came from the GVCF sidecar file of the sample. False + /// when they came from the pileup. pub gvcf_backed: bool, pub rows: Vec, } -/// One defining marker of a branch in a [`BranchReport`], with the sample's call + evidence. +/// One marker that defines a branch in a [`BranchReport`]. It holds the call of the sample and the +/// evidence for that call. #[derive(Debug, Clone)] pub struct BranchRow { pub node: String, @@ -230,7 +296,8 @@ pub struct BranchRow { pub derived: String, pub observed_base: Option, pub state: CallState, - /// `(ref, alt)` allele depths — `None` on ref blocks / the pileup path. + /// The depth of the reference allele and the depth of the alternate allele, as a pair. The value + /// is `None` on a reference block, and on the pileup path. pub ad: Option<(u32, u32)>, pub dp: Option, pub gq: Option, @@ -256,7 +323,8 @@ impl BranchReport { } impl DescentReport { - /// Total defining SNPs across the path the sample carries (derived). + /// The count of SNPs on the path that define a node and that the sample carries. Each one has a + /// derived state. pub fn derived(&self) -> usize { self.nodes .iter() @@ -265,7 +333,7 @@ impl DescentReport { .count() } - /// Total defining SNPs across the path (all states). + /// The count of SNPs on the path that define a node, in each state. pub fn total(&self) -> usize { self.nodes.iter().map(|n| n.snps.len()).sum() } @@ -274,29 +342,37 @@ impl DescentReport { /// How a private (off-backbone) variant relates to the tree. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum PrivateClass { - /// A known tree SNP off the assigned path — supports a finer/sibling branch. + /// A known SNP of the tree that is not on the path that the code assigned. It supports a finer + /// branch, or a branch beside the assigned one. OffPathKnown(String), - /// Not in the tree at all — a candidate for proposing a new branch. + /// The tree does not hold this SNP. It is a candidate for a new branch. Novel, } -/// A derived variant the sample carries that the haplogroup placement doesn't explain. +/// A derived variant the sample carries that the haplogroup placement does not explain. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct PrivateVariant { pub position: i64, pub reference: char, pub alternate: char, pub depth: u32, - /// Reads supporting the derived (alternate) allele. `alt_depth / depth` ≈ `allele_fraction`; - /// carried explicitly so the publish gate can require a minimum supporting-read count. Old - /// cached buckets predate this field → `serde(default)` reads them as 0 (they recompute). + /// The count of reads that hold the derived allele, which is the alternate allele. The value + /// `alt_depth / depth` is about equal to `allele_fraction`. + /// + /// The field is explicit, so the publish gate can set a minimum count of such reads. An older + /// cached bucket holds no such field, and `serde(default)` reads it as 0. The app then + /// calculates that bucket again. #[serde(default)] pub alt_depth: u32, pub allele_fraction: f64, pub class: PrivateClass, - /// Curated CHM13 chrY structural class at this position (palindrome / amplicon / AZF-DYZ), - /// if any — a paralog-prone zone where short-read mapping is unreliable, so the call is - /// suspect (annotation only; not dropped). `None` = unique sequence, or a non-CHM13 build. + /// The structural class of this position on chrY in CHM13, from the curated list. The classes + /// are a palindrome, an amplicon, and an AZF-DYZ region. + /// + /// Such a region holds paralogs, and a short read maps there without reliability. So the call is + /// doubtful. This value is an annotation only, and the code removes no call for it. + /// + /// A value of `None` means unique sequence, or a build that is not CHM13. #[serde(default)] pub region: Option, } @@ -319,13 +395,15 @@ impl PrivateBucket { .filter(|v| matches!(v.class, PrivateClass::OffPathKnown(_))) .count() } - /// Calls that fall in a curated chrY structural (paralog-prone) region — suspect, to be - /// down-weighted in reports rather than treated as confident new variants. + /// The calls in a curated structural region of chrY, which holds paralogs. Such a call is + /// doubtful. A report must give it less weight, and it must not show it as a confident new + /// variant. pub fn in_structural_region(&self) -> usize { self.variants.iter().filter(|v| v.region.is_some()).count() } - /// Novel calls in *unique* sequence (no structural-region flag) — the high-confidence - /// new-branch candidates, separated from the paralog-zone noise. + /// The new calls in *unique* sequence, with no structural-region mark. These calls are the + /// candidates for a new branch with high confidence, and they are separate from the noise of a + /// paralog region. pub fn novel_in_unique_sequence(&self) -> usize { self.variants .iter() @@ -333,10 +411,14 @@ impl PrivateBucket { .count() } - /// The **publishable** subset: novel, unique-sequence calls that also clear the strict - /// novel-marker `gate` (near-homozygous allele fraction + a supporting-read floor). This is the - /// set we federate to the AppView as unverified singleton candidates — far stricter than the - /// caller's placement gates, so a paralog/contamination/low-evidence call never becomes a claim. + /// The part that the app can **publish**. Each such call is new, is in unique sequence, and also + /// passes the strict `gate` for a new marker. That gate needs a high share of derived reads and + /// a minimum count of reads. + /// + /// The app sends this set to the AppView as a set of single candidates that nobody verified. + /// + /// The rules are much stricter than the placement rules of the caller. So a call from a paralog, + /// from contamination, or with little evidence never becomes a claim. pub fn publishable(&self, gate: PublishGate) -> Vec<&PrivateVariant> { self.variants.iter().filter(|v| gate.admits(v)).collect() } @@ -353,20 +435,25 @@ impl PrivateBucket { } } -/// Thresholds gating which private variants are confident enough to **publish** to the AppView as -/// novel-branch candidates. Haploid chrY should be effectively homozygous, so a mixed/paralog -/// allele fraction (0.5–0.9, which the placement caller still accepts) is rejected here, as is a -/// call with too few supporting reads to trust as a real singleton. +/// The limits that decide which private variants the app can **publish** to the AppView as +/// candidates for a new branch. +/// +/// A cell holds one copy of chrY, so a call there has almost no second allele. The code refuses an +/// allele fraction between 0.5 and 0.9, which marks a mixture or a paralog. The placement caller +/// still accepts such a fraction. +/// +/// The code also refuses a call with too few reads to trust as a real single variant. #[derive(Debug, Clone, Copy, PartialEq)] pub struct PublishGate { /// Minimum derived-allele fraction (haploid → expect ≈1.0). pub min_allele_fraction: f64, - /// Minimum reads supporting the derived allele. + /// The minimum count of reads that hold the derived allele. pub min_alt_depth: u32, } impl Default for PublishGate { - /// Short-read WGS defaults: near-homozygous and ≥10 supporting reads. + /// The default values for a short-read WGS sample. The call needs almost no second allele, and + /// it needs 10 reads or more. fn default() -> Self { Self { min_allele_fraction: 0.9, @@ -376,9 +463,11 @@ impl Default for PublishGate { } impl PublishGate { - /// Gate adapted to the sample's mean read length: HiFi/long reads make a confident haploid - /// observation from far fewer reads (same rationale as [`adaptive_min_depth`]), so the - /// supporting-read floor drops to 3. The allele-fraction requirement is unchanged. + /// The gate for the mean read length of the sample. + /// + /// A HiFi read, and each other long read, gives a confident haploid observation from many fewer + /// reads. [`adaptive_min_depth`] uses the same reason. So the minimum count of reads becomes 3. + /// The rule for the allele fraction does not change. pub fn for_read_len(read_len: f64) -> Self { let mut g = Self::default(); if read_len > 1000.0 { @@ -387,8 +476,8 @@ impl PublishGate { g } - /// Whether a variant clears the gate: it must be an unnamed novel in unique sequence, with a - /// near-homozygous derived fraction and enough supporting reads. + /// Shows whether a variant passes the gate. Such a variant must be new and have no name. It + /// must also be in unique sequence, have almost no second allele, and have enough reads. pub fn admits(&self, v: &PrivateVariant) -> bool { v.class == PrivateClass::Novel && v.region.is_none() @@ -425,9 +514,10 @@ mod publish_gate_tests { assert!(!g.admits(&var(PrivateClass::OffPathKnown("M269".into()), None, 30, 1.0))); // Paralog-prone structural region → rejected even when deep/homozygous. assert!(!g.admits(&var(PrivateClass::Novel, Some(YRegionClass::Palindrome), 30, 1.0))); - // Mixed allele fraction (the placement caller accepts 0.5, publishing must not). + // This site has mixed alleles. The placement caller accepts a fraction of 0.5, and a + // publish must not. assert!(!g.admits(&var(PrivateClass::Novel, None, 30, 0.6))); - // Too few supporting reads for short-read. + // A short-read sample needs more reads than this call holds. assert!(!g.admits(&var(PrivateClass::Novel, None, 4, 1.0))); } @@ -509,15 +599,21 @@ const KEYCHAIN_SERVICE: &str = "decodingus-navigator"; pub struct IbdComparison { pub summary: MatchSummary, pub segments: Vec, - /// Sites called in **both** samples — the effective comparison size. Sparse overlap (a - /// chip↔chip pair, or chip↔WGS limited to the chip's sites) weakens short-segment calls, so - /// it's surfaced rather than hidden. + /// The count of sites with a call in **both** samples. That count is the true size of the + /// comparison. + /// + /// A small overlap makes a call on a short segment weak. Two chips give such an overlap, and a + /// chip against a WGS sample also gives one, because the chip sites limit it. The report gives + /// this count and does not hide it. pub overlapping_sites: usize, } -/// A sample for an IBD comparison — either a WGS/CRAM **alignment** (genotyped at the IBD-panel -/// sites) or an imported **chip** profile (resolved to the same CHM13 sites). Both yield dosages -/// over the canonical IBD panel, so the comparison is data-type-agnostic. +/// One sample of an IBD comparison. It is a WGS **alignment** in a CRAM file, which the code +/// genotypes at the IBD-panel sites. It can also be a **chip** profile that a user imported, which +/// the code re-keys to the same CHM13 sites. +/// +/// Both forms give a dosage at each site of the canonical IBD panel. So the comparison does not +/// depend on the kind of data. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IbdSource { Alignment(i64), @@ -549,15 +645,20 @@ pub struct AssetStatus { pub verified: bool, } -/// A pseudonymous federated-IBD candidate from the AppView's match engine. The -/// `suggested_sample_guid` is the AppView's opaque handle for the counterpart (not a DID, -/// not PII) — used to request an introduction. `signals` names the sources that contributed -/// (e.g. `POPULATION_OVERLAP`, `HAPLOGROUP`, `SHARED_MATCH`) behind the composite `score`. +/// A federated-IBD candidate from the match engine of the AppView. The candidate has no name. /// -/// `target_sample_guid` is the AppView's handle for **our own** sample the candidate was ranked -/// against. We already own it, so it discloses nothing — but a self-publishing client has no other -/// way to learn its server-side sample handle, and [`App::ibd_attest`] cannot report a completed -/// comparison without it. `None` when talking to an AppView that predates that field. +/// The `suggested_sample_guid` value is the opaque handle of the AppView for the other person. It is +/// not a DID, and it holds no personal data. The app uses it to ask for an introduction. +/// +/// The `signals` field names each source behind the `score` value, such as `POPULATION_OVERLAP`, +/// `HAPLOGROUP`, and `SHARED_MATCH`. +/// +/// The `target_sample_guid` value is the handle of the AppView for **our own** sample, and the +/// engine ranked the candidate against that sample. We already own it, so it gives away nothing. +/// +/// But a client that publishes its own records has no other way to learn its handle on the server. +/// [`App::ibd_attest`] can not report a complete comparison without it. The value is `None` on an +/// AppView from before that field. #[derive(Debug, Clone, PartialEq)] pub struct IbdSuggestion { pub target_sample_guid: Option, @@ -567,30 +668,33 @@ pub struct IbdSuggestion { pub signals: Vec, } -/// How much a federated-IBD candidate's composite score is worth believing, as the Simple-mode -/// "Genetic relatives" card frames it. +/// The level of trust in the score of a federated-IBD candidate. The "Genetic relatives" card of +/// Simple mode uses these words. /// -/// This is a reading of the evidence, not a rendering of it: where the line falls between "strong" -/// and "merely possible" decides which of three claims the app makes about a stranger's relatedness -/// to the user. It lives beside [`IbdSuggestion`] rather than in the card that draws it so the rule -/// has one home — and so tuning it later is a change to the interpretation, not to a widget. +/// This value reads the evidence, and it does not only draw it. The line between "strong" and +/// "possible" decides which of three statements the app makes about the relationship between a +/// stranger and the user. +/// +/// This code is beside [`IbdSuggestion`] and not in the card that draws it. So the rule has one +/// home. A later change to it changes the reading of the evidence, and not a widget. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MatchStrength { /// The signals agree strongly; presented as a likely relative. Strong, - /// Enough agreement to be worth pursuing. + /// The two samples agree enough for the user to act. Likely, /// Weak or single-signal evidence; presented as a possibility only. Possible, } impl IbdSuggestion { - /// Classify this candidate's composite `score` into the tier the UI names. + /// Change the `score` of this candidate into the level that the UI names. + /// + /// The AppView gives a score from 0 to 1, and that score joins each of the `signals` values. So + /// the limits here are careful. The method names a candidate strong only when the evidence is + /// far above the middle of that range. /// - /// The AppView's score is a 0–1 composite over the contributing `signals`, so the cutoffs are - /// deliberately conservative: a candidate is only called strong when the evidence is well clear - /// of the middle of the range, because overstating a match invites someone to contact a stranger - /// on the strength of it. + /// A statement that is too strong makes a user write to a stranger on weak evidence. pub fn strength(&self) -> MatchStrength { if self.score >= 0.8 { MatchStrength::Strong @@ -602,12 +706,15 @@ impl IbdSuggestion { } } -/// Result of requesting an introduction to a candidate: the AppView's request URI and its -/// status (initially `PENDING`, awaiting the consent round-trip). +/// The result of a request for an introduction to a candidate. It holds the request URI of the +/// AppView and the status of that request. The first status is `PENDING`, and the two parties then +/// exchange their consent. /// -/// `purpose` is chosen server-side from the suggestion's dominant signal (`IBD_AUTOSOMAL` / `IBD_Y` -/// / `IBD_MT`) — it decides which genomic region a later attestation is filed under, so it is worth -/// recording at introduction rather than waiting for the session to reveal it. +/// The server chooses the `purpose` value from the strongest signal of the suggestion. The values +/// are `IBD_AUTOSOMAL`, `IBD_Y`, and `IBD_MT`. +/// +/// That value decides the genomic region of a later attestation. So the app records it at the +/// introduction, and it does not wait for the session to give it. #[derive(Debug, Clone, PartialEq)] pub struct IbdIntroResult { pub request_uri: String, @@ -615,8 +722,9 @@ pub struct IbdIntroResult { pub purpose: String, } -/// An inbound, **symmetric-blind** exchange request awaiting this account's consent (the initiator -/// is hidden until both parties consent). From `GET /api/v1/exchange/incoming`. +/// An exchange request that arrived and that needs the consent of this account. The view is +/// **symmetric-blind**: the app does not see the sender until both parties agree. The value comes +/// from `GET /api/v1/exchange/incoming`. #[derive(Debug, Clone, PartialEq)] pub struct IncomingRequest { pub request_uri: String, @@ -635,8 +743,9 @@ pub struct ExchangeSessionInfo { pub partner_key_uri: Option, } -/// Outcome of `POST /api/v1/exchange/consent`: `CONSENTED` (with the opened `session_id`), -/// `DECLINED`, or `PENDING` (recorded, awaiting the counterpart). +/// The result of `POST /api/v1/exchange/consent`. The value is `CONSENTED`, with the `session_id` +/// of the new session. It can also be `DECLINED`. It can also be `PENDING`, which means that the +/// server recorded our answer and waits for the other party. #[derive(Debug, Clone, PartialEq)] pub struct ConsentOutcome { pub status: String, @@ -646,9 +755,9 @@ pub struct ConsentOutcome { /// Who opened a matching conversation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MatchingDirection { - /// We asked to be introduced. + /// This account asked for the introduction. Outbound, - /// Someone asked to be introduced to us. + /// Another person asked for an introduction to this account. Inbound, } @@ -668,23 +777,24 @@ impl MatchingDirection { } } -/// Where a matching conversation stands. Deliberately records only what this edge can *know*: -/// the broker is symmetric-blind, so a partner declining is indistinguishable from a partner who -/// has not answered yet — both stay [`MatchingStatus::Requested`], and [`MatchingStatus::Declined`] -/// means **we** declined. +/// The state of a matching conversation. The value records only what this device can *know*. +/// +/// The broker is symmetric-blind. So a partner who declined looks the same as a partner who did not +/// answer, and both stay at [`MatchingStatus::Requested`]. The value +/// [`MatchingStatus::Declined`] means that **this account** declined. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MatchingStatus { /// We asked; the counterpart has not consented (or has not answered). Requested, - /// Inbound and awaiting our decision. + /// The request arrived, and this account must decide. AwaitingConsent, /// We declined. Declined, - /// Both consented — a session is open and the encrypted exchange can run. + /// Both parties agreed. A session is open, and the encrypted exchange can run. Ready, - /// The exchange ran and a result is stored. + /// The exchange ran, and the store holds a result. Exchanged, - /// The exchange was attempted and failed; `last_error` says why. + /// The exchange ran and failed. The `last_error` field gives the reason. Failed, } @@ -710,8 +820,8 @@ impl MatchingStatus { _ => MatchingStatus::Requested, } } - /// True once the conversation has nothing further to do — it either produced a result or we - /// turned it down. The UI files these away from the actionable list. + /// True when the conversation has no more work. It gave a result, or this account declined it. + /// The UI keeps such a conversation out of the list of actions. pub fn is_terminal(self) -> bool { matches!(self, MatchingStatus::Exchanged | MatchingStatus::Declined) } @@ -725,12 +835,14 @@ pub struct MatchingEntry { pub direction: MatchingDirection, pub purpose: String, pub status: MatchingStatus, - /// Revealed only after mutual consent — `None` while the request is still blind. + /// The server gives this value only after both parties agree. It is `None` while the request + /// stays blind. pub partner_did: Option, pub session_id: Option, /// The local subject whose dosages this conversation exchanges. pub biosample_guid: Option, - /// AppView sample handles (ours / theirs) — what an attestation is filed under. + /// The two sample handles of the AppView, ours and theirs. An attestation uses them as its + /// key. pub my_sample_ref: Option, pub partner_sample_ref: Option, /// Our own consent decision; `None` until we make one. @@ -744,8 +856,8 @@ pub struct MatchingEntry { pub result: Option, } -/// A pulled relay envelope: the opaque ciphertext `blob` plus its routing (`from_did`/`seq`) and the -/// broker `id` to ack. From `GET /api/v1/exchange/relay/pull`. +/// A pulled relay envelope: the opaque ciphertext `blob`, its route fields (`from_did`/`seq`), and +/// the broker `id` to ack. From `GET /api/v1/exchange/relay/pull`. #[derive(Debug, Clone, PartialEq)] pub struct RelayEnvelope { pub id: i64, @@ -754,8 +866,8 @@ pub struct RelayEnvelope { pub blob: String, } -/// A live exchange session with a derived shared key, ready to seal/open payloads. Holds key -/// material, so it is deliberately not `Debug`/`Serialize` and should be kept in memory only. +/// A live exchange session with a derived shared key, ready to seal/open payloads. It holds key +/// material. So it is deliberately not `Debug`/`Serialize`, and you must keep it in memory only. #[derive(Clone)] pub struct EstablishedSession { pub session_id: String, @@ -763,16 +875,17 @@ pub struct EstablishedSession { key: [u8; 32], } -/// Jetstream-ingest retry budget for a freshly-published device key: a 403 right after -/// publishing means the AppView hasn't ingested our `deviceKey` record yet. Exponential -/// backoff 1+2+4+8 s ≈ 15 s total before giving up. +/// Jetstream-ingest retry budget for a device key that the app published a moment ago. A 403 +/// immediately after the publish means the AppView has not ingested our `deviceKey` record yet. +/// Exponential backoff of 1+2+4+8 s, about 15 s in total, before the app stops. const DEVICE_KEY_INGEST_RETRIES: u32 = 4; /// Poll rounds (≈1s each) an IBD exchange waits for the partner's dosages / attestation. const EXCHANGE_POLL_ROUNDS: u32 = 30; -/// The fed-record collections this client publishes — a PULL reconcile scans each (mirrors the -/// `publish_*` NSIDs). Derived-summary collections are tracked but not overwritten locally. +/// The fed-record collections this client publishes. A PULL reconcile scans each one, and the set +/// mirrors the `publish_*` NSIDs. The app tracks the derived-summary collections, but it does not +/// overwrite them locally. const PUBLISHED_COLLECTIONS: &[&str] = &[ NS_BIOSAMPLE, NS_ALIGNMENT, @@ -782,14 +895,17 @@ const PUBLISHED_COLLECTIONS: &[&str] = &[ HAPLOGROUP_RECONCILIATION_COLLECTION, ]; -/// PDS collection NSID for a published IBD match attestation (the AppView indexes these via Jetstream). +/// PDS collection NSID for a published IBD match attestation (the AppView indexes these through +/// Jetstream). const IBD_ATTESTATION_COLLECTION: &str = "com.decodingus.atmosphere.ibdAttestation"; -/// Above this many sites, the exchanged dosage vector is decimated to fit the relay's 1 MiB envelope. +/// Above this many sites, the app decimates the exchanged dosage vector to fit the relay's 1 MiB +/// envelope. const EXCHANGE_SITE_BUDGET: usize = 100_000; -/// Decimation stride when over budget: keep sites at `position % N == 0`. A **position-based** rule -/// (not index) so both peers keep the *same physical sites* — preserving the IBD intersection — even -/// when their panels differ in size (WGS vs chip). Yields ~1/N of the canonical panel. +/// Decimation stride when over budget: keep sites at `position % N == 0`. The rule is +/// **position-based** and not index-based, so both peers keep the *same physical sites*. This keeps +/// the IBD intersection even when their panels differ in size (WGS against chip). The result is +/// about 1/N of the canonical panel. const EXCHANGE_DECIMATE: i64 = 16; /// Downsample a dosage vector to fit the relay envelope, deterministically + cross-peer-aligned. @@ -804,9 +920,9 @@ fn decimate_for_exchange(sites: Vec) -> Vec { .collect() } -/// Parse the AppView's `/api/v1/ibd/suggestions` body into [`IbdSuggestion`]s. Lenient on -/// field casing (camel/snake) and on the `signals` shape (object map or array) so a minor -/// contract drift degrades gracefully rather than dropping every candidate. +/// Parse the AppView's `/api/v1/ibd/suggestions` body into [`IbdSuggestion`]s. The parser accepts +/// both field casings (camel and snake) and both `signals` shapes (an object map or an array). A +/// small change to the contract then loses only some fields, and it does not drop every candidate. fn parse_ibd_suggestions(body: &serde_json::Value) -> Vec { let Some(items) = body .get("items") @@ -831,8 +947,8 @@ fn parse_ibd_suggestions(body: &serde_json::Value) -> Vec { .and_then(|v| v.as_str()) .unwrap_or("unknown") .to_string(); - // Optional: older AppViews omit it, and the row is still usable for everything but - // attesting, so a missing value must not drop the candidate. + // Optional: older AppViews omit it, and the row is still usable for everything except + // an attestation. So a missing value must not drop the candidate. let target_sample_guid = it .get("targetSampleGuid") .or_else(|| it.get("target_sample_guid")) @@ -856,9 +972,9 @@ fn parse_ibd_suggestions(body: &serde_json::Value) -> Vec { .collect() } -/// Signal names. The AppView emits an array of plain strings -/// (`["POPULATION_OVERLAP", "HAPLOGROUP"]`); also tolerate an array of `{name|source}` -/// objects or an object map (keys) so a contract tweak degrades gracefully. +/// Signal names. The AppView emits an array of plain strings, as in +/// `["POPULATION_OVERLAP", "HAPLOGROUP"]`. The parser also accepts an array of `{name|source}` +/// objects, or an object map whose keys are the names. A change to the contract then loses less. fn parse_ibd_signals(v: &serde_json::Value) -> Vec { if let Some(arr) = v.as_array() { arr.iter() @@ -922,7 +1038,7 @@ pub mod sync_reconcile; pub use settings::AppSettings; pub use update::UpdateInfo; -/// Artifact kind for de-novo calls, keyed per contig so different contigs don't +/// Artifact kind for de-novo calls, keyed by contig so different contigs do not /// overwrite each other in the cache. fn denovo_kind(contig: &str) -> String { format!("denovo_snps:{contig}") @@ -938,31 +1054,33 @@ fn tree_cache_path(file: &str) -> PathBuf { dir.join(file) } -/// Sidecar path holding the HTTP `ETag` of a cached haplotree (`.etag`). [`App::fetch_tree`] -/// sends it back as `If-None-Match` on a refresh, so an unchanged tree returns a tiny `304` instead -/// of re-streaming the full ~60–127 MB body. +/// Sidecar path that holds the HTTP `ETag` of a cached haplotree (`.etag`). +/// [`App::fetch_tree`] sends it back as `If-None-Match` on a refresh. An unchanged tree then +/// returns a small `304` instead of the full 60 to 127 MB body. fn tree_etag_path(cache_path: &Path) -> PathBuf { let mut p = cache_path.as_os_str().to_owned(); p.push(".etag"); PathBuf::from(p) } -/// How long a cached haplotree is trusted before [`App::fetch_tree`] re-downloads it. The -/// AppView's curated tree changes slowly (curator review, periodic builds), so a weekly -/// refresh keeps placements current without hitting the network on every run. Override with +/// How long the app trusts a cached haplotree before [`App::fetch_tree`] downloads it again. The +/// AppView's curated tree changes slowly (curator review, periodic builds). So a weekly refresh +/// keeps placements current, and it does not touch the network on every run. Override with /// `NAVIGATOR_TREE_TTL_DAYS` (0 = always refetch). const TREE_CACHE_TTL_DAYS_DEFAULT: u64 = 7; /// Whole-request timeout for a haplotree download. reqwest's `.timeout()` bounds the *entire* -/// request, streaming the body included — and the trees are large (the DecodingUs Y tree is ~60 MB, -/// the FTDNA Y tree ~127 MB), so a short cap aborts the body read partway (surfacing as reqwest's -/// "error decoding response body") and a refresh can then *never* complete, leaving the cache -/// permanently stale. Generous on purpose: a present cache makes any failure fall back instantly -/// (see [`App::fetch_tree`]), so only a first-ever fetch with no cache can wait this long. +/// request, and it includes the body read. The trees are large: the DecodingUs Y tree is about +/// 60 MB, and the FTDNA Y tree about 127 MB. A short cap stops the body read part of the way +/// through. reqwest reports that as "error decoding response body". A refresh can then never +/// complete, and the cache stays stale for ever. +/// +/// The value is large on purpose. A cache that is present makes any failure fall back immediately +/// (see [`App::fetch_tree`]). So only a first fetch with no cache can wait this long. const TREE_DOWNLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); /// Is the cached tree at `path` still within its TTL (default 7 days; `NAVIGATOR_TREE_TTL_DAYS` -/// overrides)? Unknown mtime / unreadable metadata → not fresh (forces a refresh attempt). +/// overrides)? Unknown mtime or unreadable metadata → not fresh, which forces a refresh. fn tree_cache_is_fresh(path: &Path) -> bool { let days = std::env::var("NAVIGATOR_TREE_TTL_DAYS") .ok() @@ -972,23 +1090,17 @@ fn tree_cache_is_fresh(path: &Path) -> bool { brief::cache_is_fresh(path, days) } -/// Score a tree against the sample calls and attach the terminal's child-branch evidence. +/// Pool every source's vote into one consensus map by a `SourceType`-weighted majority. +/// +/// The key `K` is the SNP **name** for Y, which is portable across builds. For mt it is the rCRS +/// **position**. The value `V` is a **state** for each Y SNP, which is independent of strand and +/// build. CHM13 and GRCh38 can flip a base, but neither changes whether the sample carries the +/// derived allele. For mt the value is a **base**, because mt has one coordinate system. /// -/// The Kulczynski `score` ranks the candidates by proportional similarity (and supplies the -/// alternatives list), but the *reported terminal* is chosen in two steps: (1) the best-ranked -/// candidate the path-supported parsimony guard admits — i.e. whose lineage doesn't tunnel -/// through a branch the sample contradicts (the distal-Y paralog artifact); then (2) -/// [`haplo::deepen_terminal`] descends further into any child the sample clearly entered, -/// correcting under-calls at **unsplit tree nodes** (a half-ancestral SNP block scores below -/// its parent). The chosen node is moved to the front so every `ranked.first()` consumer -/// transparently gets it. See `documents/design/PangenomeExpansion.md`. -/// Pool every source's vote into one consensus map by a `SourceType`-weighted majority, keyed by -/// `K` (SNP **name** for Y — build-portable; rCRS **position** for mt) over value `V` (a per-SNP -/// **state** for Y — strand-/build-independent, since CHM13 vs GRCh38 can flip a base but not -/// "carries the derived allele"; a **base** for mt, which has one coordinate system). The weight -/// matches the variant reconcile's [`navigator_domain::consensus::obs_weight`] `SourceType` term; -/// the highest-weight value wins per key. The pooled set is placed on the tree **once** (genome- -/// level placement) instead of voting among per-run terminal labels. +/// The weight matches the `SourceType` term of the variant reconcile, in +/// [`navigator_domain::consensus::obs_weight`]. The value with the highest weight wins for each +/// key. The function places the pooled set on the tree **once**, at the genome level. It does not +/// vote among the terminal labels of each run. fn pool_votes(sources: &[(SourceType, HashMap)]) -> HashMap where K: std::hash::Hash + Eq + Clone, @@ -1006,9 +1118,10 @@ where .filter_map(|(k, votes)| { votes .into_iter() - // Highest weight wins; on a tie break by the allele itself so the pooled call is - // deterministic (a `HashMap` iteration order otherwise picked the winner at random, - // which flipped the placed terminal between runs over identical genotypes). + // The highest weight wins. On a tie, break by the allele itself, so the pooled + // call is deterministic. Without that tie-break the `HashMap` iteration order + // picked the winner at random, which flipped the placed terminal between runs + // over identical genotypes. .max_by(|a, b| { a.1.partial_cmp(&b.1) .unwrap_or(std::cmp::Ordering::Equal) @@ -1019,6 +1132,21 @@ where .collect() } +/// Score a tree against the sample calls and attach the terminal's child-branch evidence. +/// +/// The Kulczynski `score` ranks the candidates by proportional similarity, and it supplies the +/// alternatives list. But two steps choose the *reported terminal*. +/// +/// The first step takes the best-ranked candidate that the path-supported parsimony guard admits. +/// The guard rejects a candidate whose lineage tunnels through a branch that the sample +/// contradicts, which is the distal-Y paralog artifact. +/// +/// The second step calls [`haplo::deepen_terminal`], which descends into any child that the sample +/// clearly entered. This corrects an under-call at an **unsplit tree node**, where a half-ancestral +/// SNP block scores below its parent. +/// +/// The function moves the chosen node to the front, so every `ranked.first()` consumer gets it. +/// See `documents/design/PangenomeExpansion.md`. fn assemble_assignment(tree: &navigator_analysis::haplo::HaploTree, calls: &HashMap) -> HaploAssignment { use navigator_analysis::haplo; let mut ranked = haplo::score(tree, calls); @@ -1046,15 +1174,20 @@ fn assemble_assignment(tree: &navigator_analysis::haplo::HaploTree, calls: &Hash } } -/// Terminal selection for **named Y-SNP panel** data (BISDNA chip), as opposed to the -/// alignment-tuned [`assemble_assignment`]. Such panels give confident but sparse genotype -/// calls: a handful of recurrent or mis-probed ancestral calls on backbone nodes can make the -/// strict `path_admissible` guard (designed to kill distal tunnel artifacts in *coverage- -/// limited* alignment data) veto the genuine deep lineage, dropping the call to a shallow node -/// (e.g. A1). With confident chip calls that failure mode dominates, so here we trust the -/// proportional Kulczynski top — robust to a few stray calls — then [`deepen_terminal`] into -/// clearly-entered children. (Validated: this kit's chromo2 export → R-S1121 on both the -/// DecodingUs/hs1 and FTDNA/GRCh38 trees, on the lineage to its WGS-confirmed R-FGC29071.) +/// Terminal selection for **named Y-SNP panel** data (BISDNA chip), against the alignment-tuned +/// [`assemble_assignment`]. +/// +/// Such panels give confident but sparse genotype calls. A few recurrent or mis-probed ancestral +/// calls on backbone nodes can make the strict `path_admissible` guard veto the genuine deep +/// lineage. The call then drops to a shallow node such as A1. That guard exists to kill distal +/// tunnel artifacts in alignment data with limited coverage. +/// +/// With confident chip calls that failure mode dominates. So here the code trusts the top of the +/// proportional Kulczynski rank, which survives a few stray calls. It then calls +/// [`deepen_terminal`] on the children that the sample clearly entered. +/// +/// Checked: this kit's chromo2 export gives R-S1121 on both the DecodingUs/hs1 tree and the +/// FTDNA/GRCh38 tree, on the lineage to its WGS-confirmed R-FGC29071. fn assemble_assignment_robust( tree: &navigator_analysis::haplo::HaploTree, calls: &HashMap, @@ -1063,9 +1196,10 @@ fn assemble_assignment_robust( let mut ranked = haplo::score(tree, calls); if let Some(top_id) = ranked.first().map(|r| r.id) { let terminal_id = haplo::deepen_terminal(tree, calls, top_id); - // Parsimony back-off: don't report a deeper terminal than the evidence supports. Trim any - // net-contradicted tail of the lineage (sparse-panel / damaged-aDNA over-deepening) while - // a lone contradiction outweighed by deeper derived support still reaches the deep terminal. + // Parsimony back-off: do not report a terminal deeper than the evidence supports. Trim any + // net-contradicted tail of the lineage, which a sparse panel or damaged aDNA can make too + // deep. A lone contradiction that deeper derived support outweighs still reaches the deep + // terminal. let chosen_id = support_backoff_terminal(tree, calls, terminal_id); if let Some(idx) = ranked.iter().position(|r| r.id == chosen_id) { if idx != 0 { @@ -1086,7 +1220,7 @@ fn assemble_assignment_robust( } } -/// The root→`target` path of node ids (inclusive), or empty if `target` isn't reachable. +/// The root→`target` path of node ids (inclusive), or empty if `target` is not reachable. fn lineage_ids(tree: &navigator_analysis::haplo::HaploTree, target: i64) -> Vec { fn dfs(tree: &navigator_analysis::haplo::HaploTree, id: i64, target: i64, acc: &mut Vec) -> bool { let Some(node) = tree.nodes.get(&id) else { return false }; @@ -1113,7 +1247,7 @@ fn lineage_ids(tree: &navigator_analysis::haplo::HaploTree, target: i64) -> Vec< Vec::new() } -/// Root→`name` lineage of haplogroup names from the tree (empty if the name isn't found). Used to +/// Root→`name` lineage of haplogroup names from the tree (empty if the name is not found). Used to /// derive a placed terminal's lineage path for cross-subject divergence/LCA without re-genotyping. fn lineage_names(tree: &navigator_analysis::haplo::HaploTree, name: &str) -> Vec { let Some(id) = tree.nodes.values().find(|n| n.name == name).map(|n| n.id) else { @@ -1125,14 +1259,19 @@ fn lineage_names(tree: &navigator_analysis::haplo::HaploTree, name: &str) -> Vec .collect() } -/// Back off an over-deepened terminal to the node that maximizes running support along its -/// lineage. Walking root→terminal, each node contributes `(covered derived − covered ancestral)` -/// over its defining SNPs the sample has a call for; the chosen terminal is the deepest node at -/// which that running balance peaks. A net-contradicted tail (more ancestral than derived calls — -/// a sparse chip or degraded aDNA sample tunnelling into a wrong sub-clade) is trimmed, but a tail -/// whose deeper derived calls outweigh a shallow contradiction is kept (ties favour the deeper -/// node, preserving the robust "survive a lone backbone contradiction" behaviour). Returns -/// `terminal_id` unchanged when its lineage can't be traced. +/// Back off an over-deep terminal to the node with the maximum cumulative support along its +/// lineage. +/// +/// The walk goes from the root to the terminal. Each node contributes +/// `(covered derived − covered ancestral)` over the SNPs that define it and that the sample has a +/// call for. The chosen terminal is the deepest node at which that cumulative balance peaks. +/// +/// The function trims a net-contradicted tail, which has more ancestral than derived calls. A +/// sparse chip or a degraded aDNA sample makes such a tail when it tunnels into a wrong sub-clade. +/// But the function keeps a tail whose deeper derived calls outweigh a shallow contradiction. A +/// tie favours the deeper node, which keeps the "survive a lone backbone contradiction" behaviour. +/// +/// Returns `terminal_id` unchanged when the code can not trace its lineage. fn support_backoff_terminal( tree: &navigator_analysis::haplo::HaploTree, calls: &HashMap, @@ -1162,9 +1301,10 @@ fn support_backoff_terminal( } } // Deepen on strictly more support, or on a tie *only* when this node is itself - // derived-supported. So a contradiction recovered by a deeper derived call still reaches - // the deep terminal, while a net-negative tail or a flat run of marker-less nodes (the - // sparse-panel / aDNA tunnel) is trimmed back to the last positively-supported node. + // derived-supported. A contradiction that a deeper derived call recovers then still + // reaches the deep terminal. A net-negative tail, or a flat run of nodes with no marker, + // goes back to the last node with positive support. That flat run is the sparse-panel or + // aDNA tunnel. if balance > best_balance || (balance == best_balance && node_derived) { best_balance = balance; best_id = id; @@ -1173,14 +1313,18 @@ fn support_backoff_terminal( best_id } -/// Reconcile chip genotype calls to a haplotree's strand. Consumer arrays report alleles on the -/// reference plus strand, but a subset of sites sit on the opposite strand from the tree's -/// ancestral/derived convention. For each call at a tree position: keep the observed base if it -/// already equals the ancestral or derived allele; else substitute its complement when *that* -/// matches; else keep it (a genuine no-match the scorer will count against the branch). Positions -/// absent from the tree pass through unchanged (they don't affect scoring). This is a no-op for -/// dictionary-reconciled BISDNA calls (their base is always the derived allele), so it's safe to -/// apply on the shared chip-placement path. +/// Reconcile chip genotype calls to a haplotree's strand. +/// +/// Consumer arrays report alleles on the reference plus strand. But some sites sit on the opposite +/// strand from the ancestral/derived convention of the tree. +/// +/// For each call at a tree position, keep the observed base when it already equals the ancestral +/// or the derived allele. If not, use its complement when *that* base matches. If neither matches, +/// keep the observed base, and the score counts it against the branch. A position that the tree +/// does not have passes through unchanged, and it changes no score. +/// +/// This does nothing for BISDNA calls that the dictionary reconciled, because their base is always +/// the derived allele. So it is safe on the shared chip-placement path. fn strand_reconcile_to_tree( tree: &navigator_analysis::haplo::HaploTree, calls: HashMap, @@ -1211,11 +1355,15 @@ fn strand_reconcile_to_tree( .collect() } -/// Map GVCF-decoded bases at *lifted* positions back to tree positions (the GVCF-sourced -/// analogue of [`App::build_calls_from_lifted`]). A variant base wins; otherwise a callable -/// hom-ref lifted site takes the **reference base** at that lifted position — both reverse- -/// complemented for a minus-strand lift; otherwise the position is a no-call. `ref_base` is -/// keyed by lifted position (the GVCF/reference coordinate), not the tree position. +/// Map GVCF-decoded bases at *lifted* positions back to tree positions. This is the GVCF form of +/// [`App::build_calls_from_lifted`]. +/// +/// A variant base wins. If there is none, a callable hom-ref lifted site takes the **reference +/// base** at that lifted position. A minus-strand lift reverse-complements both. Any other +/// position is a no-call. +/// +/// `ref_base` uses the lifted position as its key, which is the GVCF or reference coordinate, and +/// not the tree position. fn assemble_calls_lifted( called: &gvcf::CalledBases, lifted: &[LiftedPos], @@ -1237,12 +1385,15 @@ fn assemble_calls_lifted( calls } -/// Minimum callable/calling depth adapted to read technology. The default (4) is a -/// short-read assumption — ~4 reads to call a base confidently. Long, accurate reads (HiFi, -/// mean read length > 1 kb) make a confident haploid observation from a *single* read, so a -/// ~4× HiFi sample is callable at 1×; clamping the floor at 2 needlessly threw away half its -/// already-shallow coverage. (ONT long reads are less accurate — revisit if we ever adapt by -/// platform rather than read length.) +/// Minimum callable depth, adapted to the read technology. The default of 4 is a short-read +/// assumption: about 4 reads to call a base with confidence. +/// +/// Long, accurate reads (HiFi, with a mean read length above 1 kb) give a confident haploid +/// observation from a *single* read. So a HiFi sample at about 4x is callable at 1x. A +/// floor clamped at 2 threw away half of its already shallow coverage for no gain. +/// +/// ONT long reads are less accurate. Look at this again if the code ever adapts by platform and +/// not by read length. fn adaptive_min_depth(base: u32, read_len: f64) -> u32 { if read_len > 1000.0 { 1 @@ -1251,9 +1402,9 @@ fn adaptive_min_depth(base: u32, read_len: f64) -> u32 { } } -/// Haploid-caller params adapted to the sample's read tech (see [`adaptive_min_depth`]). -/// Sampled from the BAM head; falls back to defaults on any error. Blocking (reads the BAM) -/// — call inside `spawn_blocking`. +/// Haploid-caller params adapted to the sample's read technology (see [`adaptive_min_depth`]). +/// The function samples the head of the BAM, and falls back to the defaults on any error. It +/// blocks, because it reads the BAM, so call it inside `spawn_blocking`. fn adaptive_haploid_params(bam_path: &Path, reference: Option<&Path>) -> HaploidCallerParams { let mut params = HaploidCallerParams::default(); if let Ok((read_len, _)) = coverage::estimate_molecule_lengths(bam_path, reference) { @@ -1263,7 +1414,7 @@ fn adaptive_haploid_params(bam_path: &Path, reference: Option<&Path>) -> Haploid } /// Minimum genotyped sites for a reliable AIMs ancestry estimate (Scala `minSnpsAims`). -/// Overridable via `$NAVIGATOR_ANCESTRY_MIN_SNPS` (tests use a small panel). +/// `$NAVIGATOR_ANCESTRY_MIN_SNPS` overrides it (tests use a small panel). fn ancestry_min_snps() -> usize { std::env::var("NAVIGATOR_ANCESTRY_MIN_SNPS") .ok() @@ -1272,8 +1423,8 @@ fn ancestry_min_snps() -> usize { } /// Resolve an ancestry/IBD asset path under `/ancestry/`: an `$` override -/// (when non-empty) wins, else `/ancestry/_.`. The per-asset wrappers below -/// delegate here so the override+join+format pattern lives in one place. +/// (when non-empty) wins, else `/ancestry/_.`. The wrapper for each asset +/// below delegates here, so the override, join, and format pattern lives in one place. fn ancestry_asset_path(env_var: &str, stem: &str, build: ReferenceBuild, ext: &str) -> PathBuf { if !env_var.is_empty() { if let Ok(p) = std::env::var(env_var) { @@ -1293,47 +1444,49 @@ fn ancestry_panel_path(build: ReferenceBuild) -> PathBuf { } /// Where the PCA loadings for `build` live: `$NAVIGATOR_ANCESTRY_PCA` (override), else -/// `/ancestry/ancestry_pca_.bin`. Optional — absent means the -/// AF-likelihood estimate runs without PCA coordinates. +/// `/ancestry/ancestry_pca_.bin`. Optional. When it is absent, the +/// AF-likelihood estimate runs with no PCA coordinates. fn ancestry_pca_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_PCA", "ancestry_pca", build, "bin") } /// The fine-population frequency asset path (`$NAVIGATOR_ANCESTRY_FREQ` override, else -/// `/ancestry/ancestry_freq_global_.bin`). Optional — fine admixture is skipped if absent. +/// `/ancestry/ancestry_freq_global_.bin`). Optional. Without it, the app skips fine +/// admixture. fn ancestry_freq_global_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_FREQ", "ancestry_freq_global", build, "bin") } /// The phased-haplotype reference asset (`$NAVIGATOR_ANCESTRY_HAPS` override, else /// `/ancestry/ancestry_haps_.bin`): the phased 1000G haplotypes the statistical phaser -/// copies from, for the parent-split chromosome painter. Optional — when absent, the painter falls -/// back to the unphased diploid path (two arbitrary sorted copies rather than parental sides). +/// copies from, for the parent-split chromosome painter. Optional. When it is absent, the painter +/// falls back to the unphased diploid path, which gives two sorted copies and not parental sides. fn ancestry_haps_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_HAPS", "ancestry_haps", build, "bin") } /// The **ancient** deep-source frequency asset (`$NAVIGATOR_ANCESTRY_FREQ_ANCIENT` override, else -/// `/ancestry/ancestry_freq_ancient_.bin`): per-site WHG/ANF/Steppe alt-allele -/// frequencies, built by `panelbuild ancient-panel` from the AADR. Optional — deep ancestry is -/// skipped if absent. Built over the AIM panel's own sites, so the single genotyping pass covers it. +/// `/ancestry/ancestry_freq_ancient_.bin`): WHG/ANF/Steppe alt-allele frequencies for +/// each site, which `panelbuild ancient-panel` builds from the AADR. Optional. Without it, the app +/// skips deep ancestry. It covers the AIM panel's own sites, so one genotyping pass supplies it. /// -/// This supersedes the old `ancestry_pca_ancient_.bin`, which is no longer read by anything: -/// PCA-projected ancient centroids collapse onto the modern cloud and carry no ancient signal. +/// This replaces the old `ancestry_pca_ancient_.bin`, which nothing reads now. Ancient +/// centroids that a PCA projects collapse onto the modern cloud and carry no ancient signal. fn ancestry_freq_ancient_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_FREQ_ANCIENT", "ancestry_freq_ancient", build, "bin") } /// The qpAdm deep-ancestry panel asset path (`$NAVIGATOR_ANCESTRY_QPADM` override, else -/// `/ancestry/ancestry_qpadm_.bin`). The full-1240k Patterson-2022 config (WHG/EEF/Steppe -/// sources + sister outgroups) — see documents/design/ancient-ancestry-rebuild.md §7.14. +/// `/ancestry/ancestry_qpadm_.bin`). The full-1240k Patterson-2022 config, with +/// WHG/EEF/Steppe sources and sister outgroups. See +/// documents/design/ancient-ancestry-rebuild.md §7.14. fn ancestry_qpadm_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ANCESTRY_QPADM", "ancestry_qpadm", build, "bin") } /// The archaic (Neanderthal / Denisovan) marker panel asset path /// (`$NAVIGATOR_ARCHAIC_MARKERS` override, else `/ancestry/archaic_markers_.bin`). -/// Built by `panelbuild archaic-panel` — see documents/design/ArchaicAncestry_Design.md §4. +/// `panelbuild archaic-panel` builds it. See documents/design/ArchaicAncestry_Design.md §4. fn archaic_markers_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_MARKERS", "archaic_markers", build, "bin") } @@ -1344,20 +1497,19 @@ fn archaic_marker_dist_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_DIST", "archaic_marker_dist", build, "bin") } -/// Tier B: positions variable in the African outgroup, for stripping shared variants. -/// Cache signature for a Tier B segment result: the alignment it came from plus the caller's -/// genotype version, so re-calling with a newer caller invalidates it. +/// Cache signature for a Tier B segment result: the alignment it came from, plus the genotype +/// version of the caller. So a newer caller invalidates the result. pub(crate) fn archaic_segment_sig(alignment_id: i64, called_contigs: &[String]) -> String { // Three things make a result stale, and all three are in the key. // - // The METHOD version, because Tier B was rebuilt from a private-variant density model to - // archaic-genome matching; without it a workspace carrying output from the withdrawn caller - // would keep serving it. + // The METHOD version, because Tier B changed from a private-variant density model to a match + // against the archaic genomes. Without it, a workspace that holds output from the withdrawn + // caller would continue to serve that output. // - // The CONTIGS ACTUALLY CALLED, because the result covers only those. A subject called on chr21 - // alone and later called genome-wide would otherwise keep the two-chromosome answer forever — - // observed doing exactly that during genome-wide validation, reporting 1.94 Mb over 2 contigs - // while 22 sat cached and ready. + // The CONTIGS THE CODE CALLED, because the result covers only those. Take a subject called on + // chr21 alone, and later called genome-wide. Without this term the subject would keep the + // two-chromosome answer for ever. A genome-wide check showed exactly that: a report of 1.94 Mb + // over 2 contigs, while 22 more sat in the cache and ready. let mut contigs: Vec<&str> = called_contigs.iter().map(String::as_str).collect(); contigs.sort_unstable(); format!( @@ -1381,16 +1533,18 @@ pub(crate) async fn called_diploid_contigs( .collect()) } +/// Tier B: positions variable in the African outgroup, which let the code remove shared variants. fn archaic_outgroup_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_OUTGROUP", "archaic_outgroup_af", build, "bin") } -/// Tier B: genome-wide archaic diagnostic sites, for labelling called segments. +/// Tier B: genome-wide archaic diagnostic sites, which give a label to each called segment. fn archaic_classify_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_CLASSIFY", "archaic_classify", build, "bin") } -/// Tier B: callable bases per window. Without this the segment HMM calls mapping artifacts. +/// Tier B: the count of callable bases in each window. Without this the segment HMM calls mapping +/// artifacts. fn archaic_callable_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_ARCHAIC_CALLABLE", "archaic_callable", build, "bin") } @@ -1401,8 +1555,9 @@ fn ibd_panel_path(build: ReferenceBuild) -> PathBuf { ancestry_asset_path("NAVIGATOR_IBD_PANEL", "ibd_panel", build, "bin") } -/// The ancestry/IBD reference assets for the analysis build (CHM13), each with presence + manifest -/// verification — the "data sources" transparency line. Pure filesystem inspection (no analysis). +/// The ancestry/IBD reference assets for the analysis build (CHM13). Each one carries its presence +/// and its manifest check, which is the "data sources" transparency line. This looks only at the +/// file system, and does no analysis. pub fn ancestry_asset_status() -> Vec { let build = ReferenceBuild::Chm13v2; let manifest = load_asset_manifest(build); @@ -1442,10 +1597,11 @@ pub struct SeedSummary { pub skipped: usize, } -/// Copy every regular file in `src_dir` into `dest_dir` that isn't already present there. Never -/// overwrites an existing file — a CDN-refreshed asset must win over the bundled one. Creates -/// `dest_dir`. A missing/unreadable `src_dir` is a no-op (returns the empty summary). Pure over the -/// two directories (no globals) so it's unit-testable. +/// Copy every regular file in `src_dir` into `dest_dir` that is not already present there. It never +/// overwrites a file that exists, because an asset refreshed from the CDN must win over the bundled +/// one. It creates `dest_dir`. A `src_dir` that is missing or unreadable does nothing, and returns +/// the empty summary. The function is pure over the two directories, with no globals, so a unit +/// test can drive it. pub fn seed_assets_from(src_dir: &Path, dest_dir: &Path) -> std::io::Result { let mut summary = SeedSummary::default(); let Ok(entries) = std::fs::read_dir(src_dir) else { @@ -1458,8 +1614,8 @@ pub fn seed_assets_from(src_dir: &Path, dest_dir: &Path) -> std::io::Result