diff --git a/rust/crates/du-db/src/grid/digest.rs b/rust/crates/du-db/src/grid/digest.rs index 3dbcdbb..2c12836 100644 --- a/rust/crates/du-db/src/grid/digest.rs +++ b/rust/crates/du-db/src/grid/digest.rs @@ -84,6 +84,30 @@ impl Comparable { } } +impl Comparable { + /// Whether this digest carries **no** comparable call at all. + /// + /// Two such digests agree with each other, because every field is absent on both sides. That is + /// correct as a comparison and disastrous as a quorum: two nodes whose analysis failed would + /// agree on nothing, canonicalize a unit with no content, and be paid for it. + /// + /// So validation refuses to canonicalize on an empty digest. The node is expected to fail its + /// unit rather than submit one, but the AppView cannot rely on that — a node is untrusted by + /// construction, and that is the whole premise of adaptive replication. + pub fn is_empty(&self) -> bool { + self.sex.is_none() + && self.y_terminal.is_none() + && self.ancestry_superpop_argmax.is_none() + && self.coverage_bucket.is_none() + && self.callable_bucket.is_none() + } +} + +/// Whether a digest carries anything a quorum could be about. +pub fn has_content(digest: &Value) -> bool { + !Comparable::from_digest(digest).is_empty() +} + /// The major component of a semver-ish stack version: `"1.7.0"` → `"1"`. /// /// Only submissions from a compatible major are compared. A minor release that refactors a walker @@ -203,6 +227,19 @@ mod tests { ); } + /// …and "equally uninformative" is exactly why agreement is not enough on its own. Two nodes + /// whose analysis failed submit two empty digests, which agree. Without this check they would + /// canonicalize a unit with no content and be credited for it. + #[test] + fn an_empty_digest_has_no_content_to_agree_about() { + assert!(!has_content(&json!({"unexpected": true}))); + assert!(!has_content(&json!(null))); + assert!(!has_content(&json!({"calls": {}}))); + assert!(has_content(&d("XY", "R-A", "EUR", 30.0, 0.94))); + // One discrete call is enough to be about something. + assert!(has_content(&json!({"calls": {"sex": "XX"}}))); + } + /// The cross-repo byte contract: key order in the source JSON must not change the hash, or a /// node and the AppView would disagree about what was signed. #[test] diff --git a/rust/crates/du-db/src/grid/mod.rs b/rust/crates/du-db/src/grid/mod.rs index 5860bf1..6a61888 100644 --- a/rust/crates/du-db/src/grid/mod.rs +++ b/rust/crates/du-db/src/grid/mod.rs @@ -419,6 +419,12 @@ pub struct CurationCandidate { /// With `only_new`, samples that already have a work unit are skipped. That is the nightly path. /// Passing `false` re-projects everything, which refreshes manifests after a re-crawl. /// +/// **The manifest carries the instrument**, because the node can not choose a mapper without it. +/// A read set with no mate can be a long read or a single-end short read, and those two need +/// different presets. A long read mapped under a short-read preset does not fail — it gives +/// alignments that look correct and are wrong. `crawl_project` already stores the instrument model +/// of ENA on the library, so this only carries a value that we hold. +/// /// **`est_bases` prefers the measured `base_count`** that ENA publishes on `read_run`, and falls /// back to `reads × read_length` where a row predates that column. The fallback is only ever a /// mean-length approximation and is wrong outright for variable-length long reads, so it is a @@ -460,7 +466,8 @@ pub async fn curation_candidates( 'index_url', sf.http_locations->0->>'file_index_url', \ 'md5', sf.checksums->0->>'checksum', \ 'bytes', sf.file_size_bytes, \ - 'format', sf.file_format \ + 'format', sf.file_format, \ + 'instrument', sl.instrument \ )) ORDER BY sl.id, sf.id) AS manifest, \ ( SELECT SUM(COALESCE(l2.base_count, l2.reads::bigint * l2.read_length::bigint))::bigint \ FROM genomics.sequence_library l2 WHERE l2.sample_guid = s.sample_guid ) AS est_bases, \ diff --git a/rust/crates/du-db/tests/grid.rs b/rust/crates/du-db/tests/grid.rs index 7011c43..cd2dff7 100644 --- a/rust/crates/du-db/tests/grid.rs +++ b/rust/crates/du-db/tests/grid.rs @@ -570,6 +570,10 @@ async fn curation_projects_crawled_samples_into_work_units() { "ftp.sra.ebi.ac.uk/vol1/run/ERR200/s1.cram.crai" ); assert_eq!(cram.manifest[0]["md5"], "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + assert_eq!( + cram.manifest[0]["instrument"], "Illumina NovaSeq 6000", + "the node needs the instrument to choose a mapper preset" + ); assert_eq!(cram.total_bytes, Some(12_000_000_000)); assert_eq!( cram.est_bases, diff --git a/rust/crates/du-jobs/src/grid_validate.rs b/rust/crates/du-jobs/src/grid_validate.rs index 894cfe9..7e86616 100644 --- a/rust/crates/du-jobs/src/grid_validate.rs +++ b/rust/crates/du-jobs/src/grid_validate.rs @@ -141,6 +141,20 @@ pub async fn validate(pool: &PgPool, spot_check_rate: f64) -> anyhow::Result = winner.iter().map(|&i| subs[i].did.as_str()).collect();