From 10ad32d5472f0f7254a34e7786f72bafaf229113 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:18:18 +0100 Subject: [PATCH 1/4] refactor(report): build the five chart specs and their SVG in og-render og report charts --summary IN --charts-json OUT writes what bin/render_report.py --charts-json writes, byte for byte, and prints the same Sections line, so both run with the same arguments. The specs are built once (charts::build) for the HTML report (charts::figure) and the live page (charts::json). Bead: D1 --- rust/crates/og-render/src/charts/build.rs | 792 ++++++++++++++++++++++ rust/crates/og-render/src/charts/mod.rs | 417 +++++++++++- rust/crates/og-render/src/charts/svg.rs | 582 ++++++++++++++++ 3 files changed, 1783 insertions(+), 8 deletions(-) create mode 100644 rust/crates/og-render/src/charts/build.rs create mode 100644 rust/crates/og-render/src/charts/svg.rs diff --git a/rust/crates/og-render/src/charts/build.rs b/rust/crates/og-render/src/charts/build.rs new file mode 100644 index 000000000..faf3a8480 --- /dev/null +++ b/rust/crates/og-render/src/charts/build.rs @@ -0,0 +1,792 @@ +//! The five chart specs from a summary, read with Python's rules (an int +//! stays an int, `str()`, `int()` and `float()` as Python applies them), so +//! the specs are what `bin/report_charts.py` built. + +use indexmap::IndexMap; +use pycompat::json::Value; +use pycompat::num::{parse_float, parse_int}; +use pycompat::value::py_repr; + +use super::{Kind, Mark, MarkKind, NotDrawn, ROH_MIN_MB, Spec, Tone, chrom_len, draw}; +use crate::model::{Malformed, Summary, get, need, py, py_eq, truthy}; +use crate::notes; +use crate::sections::clinvar::{HITS_ANCHOR, HITS_SHOWN}; +use crate::sections::{ancestry_fine, prs}; + +/// The specs of the five charts, in report order (ClinVar, genome, mito, +/// fine-grained ancestry, PRS), for the sections that ran, each drawn. +pub fn build(s: &Summary) -> Result, Malformed> { + let all = [ + clinvar(s)?, + genome(s)?, + mito(s)?, + ancestry(s)?, + prs_chart(s)?, + ]; + let mut out = Vec::new(); + for mut sp in all.into_iter().flatten() { + sp.svg = draw(&sp)?; + out.push(sp); + } + Ok(out) +} + +// --- Python's rules --------------------------------------------------------------------- + +fn bad(path: &str, why: &str) -> Malformed { + Malformed(format!("{path}: {why}")) +} + +/// `int(v)`: an int, a float cut toward zero, a bool, or a text of digits. +fn py_int(v: &Value, path: &str) -> Result { + match v { + Value::Int(i) => Ok(*i), + Value::Bool(b) => Ok(i128::from(*b)), + Value::Float(x) if x.is_finite() => Ok(x.trunc() as i128), + Value::Str(t) => parse_int(t).map_err(|_| bad(path, "not an int")), + _ => Err(bad(path, "not an int")), + } +} + +/// `float(v)`; `Ok(None)` for a text that is not a number (Python's +/// ValueError), an error for a value of another kind (TypeError). +fn py_float(v: &Value, path: &str) -> Result, Malformed> { + match v { + Value::Int(i) => Ok(Some(*i as f64)), + Value::Float(x) => Ok(Some(*x)), + Value::Bool(b) => Ok(Some(f64::from(u8::from(*b)))), + Value::Str(t) => Ok(parse_float(t).ok()), + _ => Err(bad(path, "not a number")), + } +} + +/// `float(v)` that must be a number. +fn need_float(v: &Value, path: &str) -> Result { + py_float(v, path)?.ok_or_else(|| bad(path, "not a number")) +} + +/// `100 * v`, an int for an int as in Python. +fn times_100(v: &Value, path: &str) -> Result { + match v { + Value::Int(i) => Ok(Value::Int(i * 100)), + Value::Bool(b) => Ok(Value::Int(i128::from(*b) * 100)), + Value::Float(x) => Ok(Value::Float(100.0 * x)), + _ => Err(bad(path, "not a number")), + } +} + +/// `str(d.get(key, default))`. +fn s_or(d: &Value, key: &str, default: &str, path: &str) -> Result { + match get(d, key, path)? { + Some(v) => py(v), + None => Ok(default.to_owned()), + } +} + +/// `str(d.get(key) or default)`. +fn s_truthy(d: &Value, key: &str, default: &str, path: &str) -> Result { + match get(d, key, path)?.filter(|v| truthy(v)) { + Some(v) => py(v), + None => Ok(default.to_owned()), + } +} + +/// `d.get(key)`, `null` when it is not there. +fn raw(d: &Value, key: &str, path: &str) -> Result { + Ok(get(d, key, path)?.cloned().unwrap_or(Value::Null)) +} + +/// `d.get(key) or []` as a list. +fn list<'a>(d: &'a Value, key: &str, path: &str) -> Result<&'a [Value], Malformed> { + match get(d, key, path)?.filter(|v| truthy(v)) { + None => Ok(&[]), + Some(Value::Array(a)) => Ok(a), + Some(_) => Err(bad(&format!("{path}.{key}"), "not a list")), + } +} + +/// The section's data, when the section ran (`ok` or `stale`). +fn ran<'a>(s: &'a Summary, key: &'a str) -> Option<&'a Value> { + let sec = s.section(key); + sec.state.ran().then_some(sec.data) +} + +fn caveat(key: &'static str) -> Result<&'static str, Malformed> { + notes::text(key).ok_or_else(|| Malformed(format!("no caveat called {key}"))) +} + +fn spec( + id: &'static str, + section: &'static str, + kind: Kind, + title: &'static str, + caveat_key: &'static str, + marks: Vec, +) -> Result { + Ok(Spec { + id, + section, + kind, + title, + caveat_key, + caveat: caveat(caveat_key)?, + axis: IndexMap::new(), + marks, + not_drawn: Vec::new(), + text: None, + svg: String::new(), + }) +} + +fn axis(items: &[(&str, Value)]) -> IndexMap { + items + .iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect() +} + +fn plural(n: i128) -> &'static str { + if n == 1 { "" } else { "s" } +} + +/// The report's hits table for a hit it lists, the ClinVar section for one +/// past it. +fn hit_anchor(i: usize) -> &'static str { + if i < HITS_SHOWN { + HITS_ANCHOR + } else { + "clinvar" + } +} + +/// `GENE chrom:pos REF>ALT`. +fn hit_label(h: &Value, path: &str) -> Result { + Ok(format!( + "{} {}:{} {}>{}", + s_truthy(h, "gene", ".", path)?, + py(need(h, "chrom", path)?)?, + py(need(h, "pos", path)?)?, + s_or(h, "ref", "", path)?, + s_or(h, "alt", "", path)? + )) +} + +/// `^(chr[0-9XYM]+):([0-9]+)-([0-9]+)$`, where `$` also matches before a +/// final newline as in Python. +fn region(text: &str) -> Option<(String, i128, i128)> { + let t = text.strip_suffix('\n').unwrap_or(text); + let (chrom, rest) = t.split_once(':')?; + let name = chrom.strip_prefix("chr")?; + if name.is_empty() + || !name + .bytes() + .all(|b| b.is_ascii_digit() || b"XYM".contains(&b)) + { + return None; + } + let (a, b) = rest.split_once('-')?; + let digits = |x: &str| !x.is_empty() && x.bytes().all(|c| c.is_ascii_digit()); + if !digits(a) || !digits(b) { + return None; + } + Some((chrom.to_owned(), a.parse().ok()?, b.parse().ok()?)) +} + +/// Counts by what is not drawn, in the order first seen. +#[derive(Default)] +struct Tally(IndexMap); + +impl Tally { + fn add(&mut self, what: &str) { + *self.0.entry(what.to_owned()).or_insert(0) += 1; + } + + fn list(self) -> Vec { + self.0 + .into_iter() + .map(|(what, count)| NotDrawn { what, count }) + .collect() + } +} + +// --- the five charts -------------------------------------------------------------------- + +fn clinvar(s: &Summary) -> Result, Malformed> { + let Some(c) = ran(s, "clinvar") else { + return Ok(None); + }; + let path = "sections.clinvar.data"; + let empty = Value::Object(IndexMap::new()); + let by = get(c, "by_stars", path)? + .filter(|v| truthy(v)) + .unwrap_or(&empty); + let hits = list(c, "hits", path)?; + let hp = format!("{path}.hits[]"); + let low_conf = |h: &Value| -> Result { + Ok(get(h, "low_confidence", &hp)?.is_some_and(truthy)) + }; + let mut marks = Vec::new(); + for k in ["4", "3", "2", "1", "0"] { + let n = match get(by, k, &format!("{path}.by_stars"))? { + Some(v) => py_int(v, &format!("{path}.by_stars.{k}"))?, + None => 0, + }; + let mut idx = Vec::new(); + for (i, h) in hits.iter().enumerate() { + let stars = match get(h, "stars", &hp)? { + Some(v) => py(v)?, + None => "None".to_owned(), + }; + if stars == k && !low_conf(h)? { + idx.push(i); + } + } + let shown: Vec = idx + .iter() + .take(10) + .map(|&i| hit_label(&hits[i], &hp)) + .collect::>()?; + let names = shown.join(", ") + if idx.len() > 10 { " and more" } else { "" }; + let one = if k == "1" { "" } else { "s" }; + let mut m = Mark::new( + format!("stars-{k}"), + format!("{k} star{one}"), + Value::Int(n), + ); + m.tone = if n != 0 { Tone::Attention } else { Tone::Muted }; + m.anchor = Some(idx.first().map_or("clinvar", |&i| hit_anchor(i)).to_owned()); + m.detail = format!("{n} hit{} with {k} review star{one}", plural(n)) + + &if names.is_empty() { + String::new() + } else { + format!(": {names}") + }; + marks.push(m); + } + if let Some(v) = get(c, "low_confidence", path)? { + let n = py_int(v, &format!("{path}.low_confidence"))?; + let mut first = None; + for (i, h) in hits.iter().enumerate() { + if low_conf(h)? { + first = Some(i); + break; + } + } + let f = get(c, "call_floor", path)? + .filter(|v| truthy(v)) + .unwrap_or(&empty); + let fp = format!("{path}.call_floor"); + let floor = |k: &str| -> Result { + get(f, k, &fp)?.map_or(Ok("None".to_owned()), py) + }; + let mut m = Mark::new( + "low-confidence".into(), + "Low-confidence, not counted".into(), + Value::Int(n), + ); + m.tone = Tone::Muted; + m.anchor = Some(first.map_or("clinvar", hit_anchor).to_owned()); + m.detail = format!( + "{n} call{} with GQ below {} or DP below {}: listed in the hits, not counted in the stars", + plural(n), + floor("GQ")?, + floor("DP")? + ); + marks.push(m); + } + let mut sp = spec( + "clinvar-stars", + "clinvar", + Kind::Bars, + "ClinVar hits by review stars", + "CLINVAR_FREQ_NOTE", + marks, + )?; + sp.axis = axis(&[("unit", Value::Str("hits".into()))]); + Ok(Some(sp)) +} + +fn ancestry(s: &Summary) -> Result, Malformed> { + let Some(d) = ran(s, "ancestry_fine") else { + return Ok(None); + }; + let mut marks: Vec = Vec::new(); + for (level, title) in ancestry_fine::LEVELS { + let at = format!("sections.ancestry_fine.data.{level}[]"); + for r in ancestry_fine::rows(d, level)? { + let v = times_100(need(&r, "fraction", &at)?, &format!("{at}.fraction"))?; + let bound = |k: &str| -> Result { + match get(&r, k, &at)? { + None | Some(Value::Null) => Ok(Value::Null), + Some(x) => times_100(x, &format!("{at}.{k}")), + } + }; + let (low, high) = (bound("low")?, bound("high")?); + let pct = + |x: &Value| pycompat::fmt::fixed(crate::model::num(Some(x)).unwrap_or(0.0), 0); + let code = match get(&r, "code", &at)?.filter(|c| truthy(c)) { + Some(c) => py(c)?, + None => marks.len().to_string(), + }; + let mut m = Mark::new( + format!("{level}-{code}"), + py(need(&r, "group", &at)?)?, + v.clone(), + ); + m.detail = format!("{title}: {}%", pct(&v)); + if !matches!(low, Value::Null) && !matches!(high, Value::Null) { + m.detail += &format!(" (bootstrap range {}% to {}%)", pct(&low), pct(&high)); + } + m.low = low; + m.high = high; + m.anchor = Some("ancestry-fine".into()); + m.group = Some(title.to_owned()); + marks.push(m); + } + } + if marks.is_empty() { + return Ok(None); + } + let mut sp = spec( + "ancestry-fine", + "ancestry_fine", + Kind::Ranges, + "Fine-grained ancestry, with bootstrap ranges", + "ANCESTRY_FINE_NOTE", + marks, + )?; + sp.axis = axis(&[ + ("min", Value::Int(0)), + ("max", Value::Int(100)), + ("unit", Value::Str("%".into())), + ]); + Ok(Some(sp)) +} + +fn prs_chart(s: &Summary) -> Result, Malformed> { + let Some(d) = ran(s, "prs") else { + return Ok(None); + }; + let path = "sections.prs.data"; + if !get(d, "adjusted", path)?.is_some_and(truthy) { + let mut sp = spec( + "prs", + "prs", + Kind::Text, + "Polygenic risk scores", + "PRS_ANCESTRY_NOTE", + Vec::new(), + )?; + sp.text = Some(prs::note(d)?); + return Ok(Some(sp)); + } + let at = format!("{path}.scores[]"); + let mut marks = Vec::new(); + for (i, r) in list(d, "scores", path)?.iter().enumerate() { + let other = + matches!(get(r, "sex_check", &at)?, Some(Value::Str(x)) if x == prs::NOT_APPLICABLE); + let mut value = None; + if !other && let Some(p) = get(r, "percentile", &at)?.filter(|v| truthy(v)) { + value = py_float(p, &format!("{at}.percentile"))?.filter(|v| (0.0..=100.0).contains(v)); + } + let id = s_truthy(r, "pgs_id", &i.to_string(), &at)?; + let label = format!( + "{} ({})", + s_truthy(r, "condition", ".", &at)?, + s_truthy(r, "pgs_id", ".", &at)? + ); + let text = prs::percentile(r)?; + let mut m = Mark::new( + format!("pgs-{id}"), + label, + value.map_or(Value::Null, Value::Float), + ); + m.tone = if value.is_some() { + Tone::Info + } else { + Tone::Muted + }; + m.anchor = Some("prs".into()); + m.detail = match (value, text.is_empty()) { + (Some(_), false) => text, + (Some(_), true) => "no percentile".into(), + (None, false) => format!("no percentile: {text}"), + (None, true) => "no percentile".into(), + }; + marks.push(m); + } + let mut sp = spec( + "prs", + "prs", + Kind::Strip, + "Polygenic risk score percentiles", + "PRS_ANCESTRY_NOTE", + marks, + )?; + sp.axis = axis(&[ + ("min", Value::Int(0)), + ("max", Value::Int(100)), + ("unit", Value::Str("percentile".into())), + ]); + Ok(Some(sp)) +} + +/// The genome chart sits in the ROH section, so it is drawn only when ROH +/// ran; the ClinVar ticks, AnnotSV spans and MoChA bars come with it when +/// their steps ran too. +fn genome(s: &Summary) -> Result, Malformed> { + let Some(roh) = ran(s, "roh") else { + return Ok(None); + }; + let mut marks = Vec::new(); + let mut not_drawn = Tally::default(); + let at = "sections.roh.data.autosomal_over_5mb[]"; + let mut short = 0; + for (i, r) in list(roh, "autosomal_over_5mb", "sections.roh.data")? + .iter() + .enumerate() + { + let reg = s_or(r, "region", "", at)?; + let shown = || -> Result { + match get(r, "region", at)? { + Some(v) => py_repr(v).map_err(|e| Malformed(e.to_string())), + None => Ok("None".into()), + } + }; + let Some((chrom, start, end)) = region(®) else { + return Err(Malformed(format!( + "run of homozygosity {}: not a chrom:start-end region", + shown()? + ))); + }; + if end < start { + return Err(Malformed(format!( + "run of homozygosity {}: the end is before the start", + shown()? + ))); + } + let mb = match get(r, "mb", at)? { + Some(v) => need_float(v, &format!("{at}.mb"))?, + None => 0.0, + }; + if mb < ROH_MIN_MB { + short += 1; + continue; + } + if chrom_len(&chrom).is_none() { + not_drawn.add(&format!("runs of homozygosity on {chrom}")); + continue; + } + let mb_text = py(need(r, "mb", at)?)?; + let mut m = Mark::new( + format!("roh-{i}"), + format!("Run of homozygosity {reg}"), + Value::Float(mb), + ); + m.locus = Some(format!("{chrom}:{start}-{end}")); + m.chrom = Some(chrom); + m.pos = Some(start); + m.end = Some(end); + m.anchor = Some("roh".into()); + m.kind = Some(MarkKind::Roh); + m.detail = format!("{mb_text} MB run of homozygosity at {reg}"); + marks.push(m); + } + if short > 0 { + not_drawn + .0 + .insert("runs of homozygosity under 5 MB".into(), short); + } + if let Some(c) = ran(s, "clinvar") { + let hp = "sections.clinvar.data.hits[]"; + for (i, h) in list(c, "hits", "sections.clinvar.data")?.iter().enumerate() { + let chrom = match get(h, "chrom", hp)? { + Some(Value::Str(x)) => Some(x.as_str()), + _ => None, + }; + if chrom == Some("chrM") { + not_drawn.add("ClinVar hits on chrM"); + continue; + } + let Some(chrom) = chrom.filter(|c| chrom_len(c).is_some()) else { + not_drawn.add("ClinVar hits on other contigs (alt, unplaced)"); + continue; + }; + let low = get(h, "low_confidence", hp)?.is_some_and(truthy); + let pos = need(h, "pos", hp)?; + let stars = raw(h, "stars", hp)?; + let one = if py_eq(&stars, &Value::Int(1)) { + "" + } else { + "s" + }; + let mut m = Mark::new(format!("hit-{i}"), hit_label(h, hp)?, stars.clone()); + m.chrom = Some(chrom.to_owned()); + m.pos = Some(py_int(pos, &format!("{hp}.pos"))?); + m.tone = if low { Tone::Muted } else { Tone::Attention }; + m.anchor = Some(hit_anchor(i).to_owned()); + m.locus = Some(format!("{chrom}:{}", py(pos)?)); + m.kind = Some(MarkKind::Hit); + m.caveat = Some(caveat("CLINVAR_FREQ_NOTE")?.to_owned()); + m.detail = format!( + "{}, {} star{one}, {}{}", + s_truthy(h, "significance", ".", hp)?, + py(&stars)?, + s_truthy(h, "genotype", ".", hp)?, + if low { ", low-confidence call" } else { "" } + ); + marks.push(m); + } + } + if let Some(a) = ran(s, "annotsv") { + let rp = "sections.annotsv.data.rows[]"; + for (i, r) in list(a, "rows", "sections.annotsv.data")?.iter().enumerate() { + let sv = s_or(r, "sv", "", rp)?; + // AnnotSV writes SV_chrom without the chr prefix: put it back, as the collector does. + let full = if sv.starts_with("chr") { + sv.clone() + } else { + format!("chr{sv}") + }; + let Some((chrom, start, end)) = + region(&full).filter(|(c, _, _)| chrom_len(c).is_some()) + else { + not_drawn.add("AnnotSV class 4 and 5 SVs on other contigs"); + continue; + }; + let genes = match get(r, "genes", rp)?.filter(|v| truthy(v)) { + None => String::new(), + Some(Value::Array(g)) => g + .iter() + .take(5) + .map(py) + .collect::, _>>()? + .join(", "), + Some(Value::Str(g)) => g + .chars() + .take(5) + .map(String::from) + .collect::>() + .join(", "), + Some(_) => return Err(bad(&format!("{rp}.genes"), "not a list")), + }; + let genes = if genes.is_empty() { + "no gene".to_owned() + } else { + genes + }; + let class = raw(r, "acmg_class", rp)?; + let kind = s_truthy(r, "type", "SV", rp)?; + let mut m = Mark::new( + format!("sv-{i}"), + format!( + "{kind} {} (AnnotSV class {})", + py(need(r, "sv", rp)?)?, + py(&class)? + ), + class.clone(), + ); + m.locus = Some(format!("{chrom}:{start}-{end}")); + m.chrom = Some(chrom); + m.pos = Some(start); + m.end = Some(end.max(start)); + m.tone = Tone::Attention; + m.anchor = Some("structural-variants".into()); + m.kind = Some(MarkKind::Sv); + m.caveat = Some(caveat("ANNOTSV_NOTE")?.to_owned()); + m.detail = format!( + "AnnotSV class {} {kind}, {} bp, genotype {}, genes: {genes}", + py(&class)?, + s_truthy(r, "length", ".", rp)?, + s_truthy(r, "gt", ".", rp)? + ); + marks.push(m); + } + } + if let Some(mo) = ran(s, "mosaic") { + let cp = "sections.mosaic.data.calls[]"; + for (i, c) in list(mo, "calls", "sections.mosaic.data")? + .iter() + .enumerate() + { + let mut chrom = s_truthy(c, "chrom", "", cp)?; + if !chrom.is_empty() && !chrom.starts_with("chr") { + chrom = format!("chr{chrom}"); + } + // int() of start and end; a value that is not one leaves the call unplaced. + let as_int = |k: &str| -> Option { + let v = get(c, k, cp).ok()??; + py_int(v, k).ok() + }; + let span = as_int("start").zip(as_int("end")); + let Some((start, end)) = span.filter(|(a, b)| chrom_len(&chrom).is_some() && b >= a) + else { + not_drawn.add("MoChA calls on other contigs or with no position"); + continue; + }; + let fraction = raw(c, "cell_fraction", cp)?; + let mut m = Mark::new( + format!("mca-{i}"), + format!( + "MoChA {} {chrom}:{start}-{end}", + s_truthy(c, "type", "call", cp)? + ), + fraction, + ); + m.locus = Some(format!("{chrom}:{start}-{end}")); + m.chrom = Some(chrom); + m.pos = Some(start); + m.end = Some(end); + m.tone = Tone::Attention; + m.anchor = Some("mosaic-mca".into()); + m.kind = Some(MarkKind::Mca); + m.caveat = Some(caveat("MCA_NOTE")?.to_owned()); + m.detail = format!( + "mosaic {} over {} Mb, cell fraction {}", + s_truthy(c, "type", "alteration", cp)?, + s_truthy(c, "length_mb", ".", cp)?, + s_truthy(c, "cell_fraction", ".", cp)? + ); + marks.push(m); + } + } + // A position past the end of its chromosome (a file of another assembly) + // is not drawn; an end past it is cut at the end, so every mark stays + // inside its chromosome. + let mut kept = Vec::new(); + for mut m in marks { + let n = m.chrom.as_deref().and_then(chrom_len).unwrap_or(0); + if m.pos.is_some_and(|p| p > n) { + not_drawn.add("marks past the end of their GRCh38 chromosome"); + continue; + } + if m.end.is_some_and(|e| e > n) { + m.end = Some(n); + } + kept.push(m); + } + let mut sp = spec( + "genome", + "roh", + Kind::Ideogram, + "Runs of homozygosity, ClinVar hits, structural variants and mosaic alterations on the \ + chromosomes", + "ROH_NOTE", + kept, + )?; + sp.axis = axis(&[("assembly", Value::Str("GRCh38".into()))]); + sp.not_drawn = not_drawn.list(); + Ok(Some(sp)) +} + +/// `^chrM:([0-9]+) `: the position of a poly-C call. +fn polyc_pos(variant: &str) -> Option { + let rest = variant.strip_prefix("chrM:")?; + let n = rest.bytes().take_while(u8::is_ascii_digit).count(); + (n > 0 && rest.as_bytes().get(n) == Some(&b' ')).then(|| rest[..n].parse().ok())? +} + +fn mito(s: &Summary) -> Result, Malformed> { + let Some(d) = ran(s, "mito") else { + return Ok(None); + }; + let path = "sections.mito.data"; + let sp_at = format!("{path}.heteroplasmic_sites[]"); + let mut marks = Vec::new(); + let mut not_drawn = Vec::new(); + let sites = get(d, "heteroplasmic_sites", path)?.filter(|v| !matches!(v, Value::Null)); + let site_list: &[Value] = match sites { + None => &[], + Some(v) if !truthy(v) => &[], + Some(Value::Array(a)) => a, + Some(_) => return Err(bad(&format!("{path}.heteroplasmic_sites"), "not a list")), + }; + for x in site_list { + let pos = need(x, "pos", &sp_at)?; + let (p, alt, af) = ( + py(pos)?, + py(need(x, "alt", &sp_at)?)?, + need(x, "af", &sp_at)?, + ); + let mut m = Mark::new( + format!("m{p}-{alt}"), + format!("m.{p}{}>{alt}", py(need(x, "ref", &sp_at)?)?), + Value::Float(need_float(af, &format!("{sp_at}.af"))?), + ); + m.chrom = Some("chrM".into()); + m.pos = Some(py_int(pos, &format!("{sp_at}.pos"))?); + m.tone = Tone::Attention; + m.anchor = Some("mito".into()); + m.locus = Some(format!("chrM:{p}")); + m.kind = Some(MarkKind::Site); + m.detail = format!("allele fraction {}", py(af)?); + marks.push(m); + } + let missing = match (sites, get(d, "heteroplasmic", path)?.filter(|v| truthy(v))) { + (None, Some(h)) => py_int(h, &format!("{path}.heteroplasmic"))?, + _ => 0, + }; + if missing != 0 { + not_drawn.push(NotDrawn { + what: "heteroplasmic sites this summary does not list (made before it listed them)" + .into(), + count: missing, + }); + } + let pc_at = format!("{path}.polyc_heteroplasmic[]"); + for x in list(d, "polyc_heteroplasmic", path)? { + let variant = match get(x, "variant", &pc_at)? { + None => String::new(), + Some(Value::Str(v)) => v.clone(), + Some(_) => return Err(bad(&format!("{pc_at}.variant"), "not a text")), + }; + let Some(pos) = polyc_pos(&variant) else { + let shown = match get(x, "variant", &pc_at)? { + Some(v) => py_repr(v).map_err(|e| Malformed(e.to_string()))?, + None => "None".into(), + }; + not_drawn.push(NotDrawn { + what: format!("poly-C call {shown} with no position"), + count: 1, + }); + continue; + }; + let af = need(x, "af", &pc_at)?; + let mut m = Mark::new( + format!("polyc-{pos}"), + format!("{variant} (D-loop poly-C tract)"), + Value::Float(need_float(af, &format!("{pc_at}.af"))?), + ); + m.chrom = Some("chrM".into()); + m.pos = Some(pos); + m.tone = Tone::Muted; + m.anchor = Some("mito".into()); + m.locus = Some(format!("chrM:{pos}")); + m.kind = Some(MarkKind::Polyc); + m.detail = format!( + "length heteroplasmy in a D-loop poly-C tract, allele fraction {}; not counted as heteroplasmic", + py(af)? + ); + marks.push(m); + } + let floor = get(d, "heteroplasmy_floor", path)? + .cloned() + .unwrap_or(Value::Float(0.05)); + let mut sp = spec( + "mito", + "mito", + Kind::Scatter, + "Heteroplasmic chrM sites", + "MITO_NOTE", + marks, + )?; + sp.axis = axis(&[ + ("min", Value::Int(0)), + ("max", Value::Int(1)), + ("unit", Value::Str("allele fraction".into())), + ("floor", floor), + ("sites_missing", Value::Int(missing)), + ]); + sp.not_drawn = not_drawn; + Ok(Some(sp)) +} diff --git a/rust/crates/og-render/src/charts/mod.rs b/rust/crates/og-render/src/charts/mod.rs index 814b431b6..1bf958ea1 100644 --- a/rust/crates/og-render/src/charts/mod.rs +++ b/rust/crates/og-render/src/charts/mod.rs @@ -1,13 +1,414 @@ -//! `og report charts`: the chart specs the HTML report embeds and the live -//! site serves, which `bin/report_charts.py` builds today. +//! `og report charts`: the report's five charts, as data and as inline SVG, +//! built once from summary.json for both places a chart is shown: the HTML +//! report puts each SVG in its section, under that section's caveat +//! ([`figure`]), and the live page's charts plugin reads every spec with its +//! SVG from `_charts.json` ([`json`]). //! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! ```text +//! og report charts --summary IN.json [--charts-json OUT.json] +//! ``` +//! +//! It writes what `bin/render_report.py --summary IN.json --charts-json +//! OUT.json` writes, and prints the same last line (the summary's sections +//! counted by state), so the two run with the same arguments. +//! +//! The five charts, each drawn only when its section ran: +//! +//! - `clinvar-stars` (bars): ClinVar hits per review star, 4 to 0, and a +//! grey bar for the low-confidence calls, which the star counts leave out. +//! Yellow and grey only, never green; with no hit a grey frame says that +//! no hit is not an all-clear. +//! - `genome` (ideogram), drawn in the ROH section so only when ROH ran: +//! chromosomes 1 to 22, X and Y of GRCh38 with the runs of homozygosity of +//! 5 MB or more, a tick for each ClinVar hit, an outlined span for each +//! AnnotSV class 4 or 5 SV and a bar under the chromosome for each MoChA +//! call. What has no place on them is counted in `not_drawn`. +//! - `mito` (scatter): the heteroplasmic chrM sites by position and allele +//! fraction, the D-loop poly-C length variants marked apart. +//! - `ancestry-fine` (ranges): each fine-grained ancestry group the report +//! lists, its share as a dot and its bootstrap range as a line. +//! - `prs` (strip): PRS percentiles on a 0 to 100 strip, only when pgsc_calc +//! had an ancestry panel. Without one a raw score has no scale, so the +//! chart is the report's own note (`text`); a score of the other sex, or +//! one with no percentile, says "no percentile" and is never placed. +//! +//! Every caveat comes from [`crate::notes`] by its name, so the chart and the +//! report say the same thing. The SVG has no script, no style attribute and +//! no outside reference: colours are presentation attributes, links are +//! `` inside the page and each mark carries a `` and +//! `data-mark="<id>"`. + +mod build; +mod svg; use std::io::Write; -/// `og report charts ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og report charts: not ported yet"); - 2 +use indexmap::IndexMap; +use pycompat::json::{Dump, Value, dumps}; + +use crate::model::{Malformed, Summary, need, py}; + +pub use build::build; +pub use svg::draw; + +/// The `schema_version` of `<sample>_charts.json`. +pub const SCHEMA_VERSION: i128 = 1; + +/// GRCh38 primary assembly lengths (UCSC hg38 chrom.sizes), in the order the +/// genome chart draws them. The live site's family page uses them too. +pub const GRCH38: [(&str, i128); 24] = [ + ("chr1", 248_956_422), + ("chr2", 242_193_529), + ("chr3", 198_295_559), + ("chr4", 190_214_555), + ("chr5", 181_538_259), + ("chr6", 170_805_979), + ("chr7", 159_345_973), + ("chr8", 145_138_636), + ("chr9", 138_394_717), + ("chr10", 133_797_422), + ("chr11", 135_086_622), + ("chr12", 133_275_309), + ("chr13", 114_364_328), + ("chr14", 107_043_718), + ("chr15", 101_991_189), + ("chr16", 90_338_345), + ("chr17", 83_257_441), + ("chr18", 80_373_285), + ("chr19", 58_617_616), + ("chr20", 64_444_167), + ("chr21", 46_709_983), + ("chr22", 50_818_468), + ("chrX", 156_040_895), + ("chrY", 57_227_415), +]; + +/// The length of a GRCh38 primary chromosome, `None` for any other name. +pub fn chrom_len(chrom: &str) -> Option<i128> { + GRCH38.iter().find(|(c, _)| *c == chrom).map(|(_, n)| *n) +} + +/// The length of chrM. +pub const MITO_LEN: i128 = 16569; + +/// The shortest run of homozygosity the genome chart draws, in MB: the runs +/// F_ROH counts (the roh section's own floor). +pub const ROH_MIN_MB: f64 = og_summary::collect::group_c::roh::ROH_MIN_BP / 1e6; + +/// What a mark's colour means: yellow for something to look at, grey for +/// what is not counted, blue for plain information. No green anywhere: a +/// chart is not an all-clear. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Tone { + Attention, + Muted, + Info, +} + +impl Tone { + pub fn name(self) -> &'static str { + match self { + Tone::Attention => "attention", + Tone::Muted => "muted", + Tone::Info => "info", + } + } + + pub fn colour(self) -> &'static str { + match self { + Tone::Attention => "#e0b94a", + Tone::Muted => "#8a909b", + Tone::Info => "#5b9ee6", + } + } +} + +/// How a chart is drawn. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Bars, + Ranges, + Strip, + Text, + Ideogram, + Scatter, +} + +impl Kind { + pub fn name(self) -> &'static str { + match self { + Kind::Bars => "bars", + Kind::Ranges => "ranges", + Kind::Strip => "strip", + Kind::Text => "text", + Kind::Ideogram => "ideogram", + Kind::Scatter => "scatter", + } + } +} + +/// What a mark of the genome or mito chart stands for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MarkKind { + Roh, + Hit, + Sv, + Mca, + Site, + Polyc, +} + +impl MarkKind { + pub fn name(self) -> &'static str { + match self { + MarkKind::Roh => "roh", + MarkKind::Hit => "hit", + MarkKind::Sv => "sv", + MarkKind::Mca => "mca", + MarkKind::Site => "site", + MarkKind::Polyc => "polyc", + } + } +} + +/// One mark of a chart. `value`, `low` and `high` keep the summary's own +/// type (an int stays an int); `null` where they do not apply. +#[derive(Debug, Clone, PartialEq)] +pub struct Mark { + pub id: String, + pub label: String, + pub value: Value, + pub low: Value, + pub high: Value, + pub chrom: Option<String>, + pub pos: Option<i128>, + pub end: Option<i128>, + pub tone: Tone, + /// The id of the report element the mark links to. + pub anchor: Option<String>, + /// `chrom:pos` or `chrom:start-end`, for the genome viewer. + pub locus: Option<String>, + pub detail: String, + /// The heading of the ancestry level the mark belongs to. + pub group: Option<String>, + pub kind: Option<MarkKind>, + /// A genome chart mark of another section carries that section's caveat. + pub caveat: Option<String>, +} + +impl Mark { + /// A mark with nothing but its id, label and value; tone `info`. + pub fn new(id: String, label: String, value: Value) -> Mark { + Mark { + id, + label, + value, + low: Value::Null, + high: Value::Null, + chrom: None, + pos: None, + end: None, + tone: Tone::Info, + anchor: None, + locus: None, + detail: String::new(), + group: None, + kind: None, + caveat: None, + } + } + + fn to_value(&self) -> Value { + let s = |x: &Option<String>| x.clone().map_or(Value::Null, Value::Str); + let i = |x: Option<i128>| x.map_or(Value::Null, Value::Int); + let mut m = IndexMap::new(); + m.insert("id".into(), Value::Str(self.id.clone())); + m.insert("label".into(), Value::Str(self.label.clone())); + m.insert("value".into(), self.value.clone()); + m.insert("low".into(), self.low.clone()); + m.insert("high".into(), self.high.clone()); + m.insert("chrom".into(), s(&self.chrom)); + m.insert("pos".into(), i(self.pos)); + m.insert("end".into(), i(self.end)); + m.insert("tone".into(), Value::Str(self.tone.name().into())); + m.insert("anchor".into(), s(&self.anchor)); + m.insert("locus".into(), s(&self.locus)); + m.insert("detail".into(), Value::Str(self.detail.clone())); + if let Some(g) = &self.group { + m.insert("group".into(), Value::Str(g.clone())); + } + if let Some(k) = self.kind { + m.insert("kind".into(), Value::Str(k.name().into())); + } + if let Some(c) = &self.caveat { + m.insert("caveat".into(), Value::Str(c.clone())); + } + Value::Object(m) + } +} + +/// Something a chart has no place for, and how many. +#[derive(Debug, Clone, PartialEq)] +pub struct NotDrawn { + pub what: String, + pub count: i128, +} + +/// One chart: its data and its SVG. +#[derive(Debug, Clone, PartialEq)] +pub struct Spec { + pub id: &'static str, + /// The summary section whose data it shows. + pub section: &'static str, + pub kind: Kind, + pub title: &'static str, + /// The name of its caveat in [`crate::notes`], and that caveat's text. + pub caveat_key: &'static str, + pub caveat: &'static str, + pub axis: IndexMap<String, Value>, + pub marks: Vec<Mark>, + pub not_drawn: Vec<NotDrawn>, + /// The text a `text` chart shows. + pub text: Option<String>, + /// The chart drawn ([`draw`]). + pub svg: String, +} + +impl Spec { + /// The spec as `<sample>_charts.json` holds it. + pub fn to_value(&self) -> Value { + let mut m = IndexMap::new(); + m.insert("id".into(), Value::Str(self.id.into())); + m.insert("section".into(), Value::Str(self.section.into())); + m.insert("kind".into(), Value::Str(self.kind.name().into())); + m.insert("title".into(), Value::Str(self.title.into())); + m.insert("caveat_key".into(), Value::Str(self.caveat_key.into())); + m.insert("caveat".into(), Value::Str(self.caveat.into())); + m.insert("axis".into(), Value::Object(self.axis.clone())); + m.insert( + "marks".into(), + Value::Array(self.marks.iter().map(Mark::to_value).collect()), + ); + let nd = self.not_drawn.iter().map(|x| { + let mut o = IndexMap::new(); + o.insert("what".into(), Value::Str(x.what.clone())); + o.insert("count".into(), Value::Int(x.count)); + Value::Object(o) + }); + m.insert("not_drawn".into(), Value::Array(nd.collect())); + if let Some(t) = &self.text { + m.insert("text".into(), Value::Str(t.clone())); + } + m.insert("svg".into(), Value::Str(self.svg.clone())); + Value::Object(m) + } +} + +/// The HTML report's line for a chart, which goes into its section right +/// under its caveat; `None` for a `text` chart, which is the note the card +/// already prints (the live page shows it). +pub fn figure(spec: &Spec) -> Option<String> { + (spec.kind != Kind::Text).then(|| format!(" <figure class=\"chart\">{}</figure>", spec.svg)) +} + +/// `<sample>_charts.json`: the schema version, the sample, when the summary +/// was made and the charts. +pub fn json(s: &Summary) -> Result<Value, Malformed> { + let mut m = IndexMap::new(); + m.insert("schema_version".into(), Value::Int(SCHEMA_VERSION)); + m.insert("sample".into(), s.need("sample")?.clone()); + m.insert("generated_utc".into(), s.need("generated_utc")?.clone()); + let charts = build(s)?.iter().map(Spec::to_value).collect(); + m.insert("charts".into(), Value::Array(charts)); + Ok(Value::Object(m)) +} + +const USAGE: &str = "usage: og report charts --summary IN.json [--charts-json OUT.json]"; + +/// The arguments: the summary, and where to write the charts. +fn parse(args: &[String]) -> Result<(String, Option<String>), String> { + let (mut summary, mut out) = (None, None); + let mut it = args.iter(); + while let Some(arg) = it.next() { + let (name, inline) = match arg.split_once('=') { + Some((n, v)) if n.starts_with("--") => (n, Some(v.to_owned())), + _ => (arg.as_str(), None), + }; + let mut value = || { + inline + .clone() + .or_else(|| it.next().cloned()) + .ok_or_else(|| format!("argument {name}: expected one argument")) + }; + match name { + "--summary" => summary = Some(value()?), + "--charts-json" => out = Some(value()?), + n => return Err(format!("unrecognized arguments: {n}")), + } + } + Ok((summary.ok_or("give --summary")?, out)) +} + +/// `Sections: missing 28, ok 20`: the sections counted by state, as `og +/// report render` and `bin/render_report.py` end. +fn state_counts(s: &Summary) -> Result<String, Malformed> { + let Some(Value::Object(secs)) = s.get("sections") else { + return Err(Malformed("summary.sections is not an object".into())); + }; + let mut counts: Vec<(String, usize)> = Vec::new(); + for (key, sec) in secs { + let state = py(need(sec, "state", &format!("sections.{key}"))?)?; + match counts.iter_mut().find(|(k, _)| *k == state) { + Some((_, n)) => *n += 1, + None => counts.push((state, 1)), + } + } + counts.sort(); + let parts: Vec<String> = counts.iter().map(|(k, n)| format!("{k} {n}")).collect(); + Ok(format!("Sections: {}", parts.join(", "))) +} + +fn write(path: &str, body: &str) -> std::io::Result<()> { + let mut f = og_io::AtomicFile::create(path)?; + f.write_all(body.as_bytes())?; + f.commit() +} + +/// `og report charts ARGS`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + if args.iter().any(|a| a == "-h" || a == "--help") { + let _ = writeln!(out, "{USAGE}"); + return 0; + } + let (path, dest) = match parse(args) { + Ok(a) => a, + Err(why) => { + let _ = writeln!(err, "{USAGE}\nog report charts: error: {why}"); + return 2; + } + }; + let run = || -> Result<String, String> { + let text = std::fs::read_to_string(&path).map_err(|e| format!("{path}: {e}"))?; + let s = Summary::parse(&text).map_err(|e| format!("{path}: {e}"))?; + if let Some(dest) = &dest { + let v = json(&s).map_err(|e| format!("{path}: {e}"))?; + let opts = Dump { + indent: Some(1), + ..Dump::default() + }; + let body = dumps(&v, &opts).map_err(|e| format!("{path}: {e:?}"))? + "\n"; + write(dest, &body).map_err(|e| format!("{dest}: {e}"))?; + } + state_counts(&s).map_err(|e| format!("{path}: {e}")) + }; + match run() { + Ok(line) => { + let _ = writeln!(out, "{line}"); + 0 + } + Err(why) => { + let _ = writeln!(err, "og report charts: {why}"); + 1 + } + } } diff --git a/rust/crates/og-render/src/charts/svg.rs b/rust/crates/og-render/src/charts/svg.rs new file mode 100644 index 000000000..1a7bdf19c --- /dev/null +++ b/rust/crates/og-render/src/charts/svg.rs @@ -0,0 +1,582 @@ +//! Each chart as inline SVG, the same bytes `bin/report_charts.py` drew: a +//! coordinate with one decimal at most, the arithmetic in Python's order, and +//! `min` and `max` with Python's rules (a NaN passed second never wins). +//! +//! Text uses `currentColor`, so it follows the page's colour on screen and +//! in print. + +use pycompat::fmt::{fixed, general}; +use pycompat::json::Value; +use pycompat::text::html_escape; + +use super::{GRCH38, Kind, MITO_LEN, Mark, MarkKind, Spec, Tone, chrom_len}; +use crate::model::{Malformed, num, py}; + +/// The SVG's width; its height follows the content. +const W: f64 = 640.0; +const FRAME: &str = "#8a909b"; +const LINE: &str = "#4a505b"; + +fn e(s: &str) -> String { + html_escape(s).into_owned() +} + +/// Python's `min(a, b)`: `b` only when `b < a`. +fn pmin(a: f64, b: f64) -> f64 { + if b < a { b } else { a } +} + +/// Python's `max(a, b)`: `b` only when `b > a`. +fn pmax(a: f64, b: f64) -> f64 { + if b > a { b } else { a } +} + +/// A coordinate: one decimal, trailing zeros and point dropped. +fn f(x: f64) -> Result<String, Malformed> { + if !x.is_finite() { + return Err(Malformed(format!( + "a chart coordinate is {}, not a number", + pycompat::fmt::repr(x) + ))); + } + let s = fixed(x, 1); + Ok(s.trim_end_matches('0').trim_end_matches('.').to_owned()) +} + +struct Text<'a> { + size: u32, + anchor: &'a str, + extra: &'a str, +} + +const PLAIN: Text<'static> = Text { + size: 12, + anchor: "start", + extra: "", +}; + +fn text(x: f64, y: f64, s: &str, t: &Text<'_>) -> Result<String, Malformed> { + Ok(format!( + "<text x=\"{}\" y=\"{}\" font-size=\"{}\" text-anchor=\"{}\" fill=\"currentColor\"{}>{}</text>", + f(x)?, + f(y)?, + t.size, + t.anchor, + t.extra, + e(s) + )) +} + +fn sized(size: u32, anchor: &str) -> Text<'_> { + Text { + size, + anchor, + extra: "", + } +} + +/// A mark: a link to its anchor in the report, with its title and id. +fn a(m: &Mark, inner: &str) -> String { + let title = if m.detail.is_empty() { + m.label.clone() + } else { + format!("{}: {}", m.label, m.detail) + }; + format!( + "<a href=\"#{}\" data-mark=\"{}\"><title>{}{inner}", + e(m.anchor.as_deref().unwrap_or("")), + e(&m.id), + e(&title) + ) +} + +fn open(sp: &Spec, h: f64) -> Result { + Ok(format!( + "{}", + e(sp.id), + f(h)?, + e(sp.title), + e(sp.title) + )) +} + +/// Words of `s` in lines of at most `n` characters (a longer word alone). +fn wrap(s: &str, n: usize) -> Vec { + let mut lines = Vec::new(); + let mut cur = String::new(); + for w in pycompat::text::split(s, None, None) { + if !cur.is_empty() && cur.chars().count() + 1 + w.chars().count() > n { + lines.push(std::mem::replace(&mut cur, w.to_owned())); + } else { + cur = if cur.is_empty() { + w.to_owned() + } else { + format!("{cur} {w}") + }; + } + } + if !cur.is_empty() { + lines.push(cur); + } + lines +} + +/// The "Not drawn:" lines under a chart, from `y`, and the y after them. +fn footer(sp: &Spec, y: f64) -> Result<(String, f64), Malformed> { + if sp.not_drawn.is_empty() { + return Ok((String::new(), y)); + } + let parts: Vec = sp + .not_drawn + .iter() + .map(|x| format!("{} {}", x.count, x.what)) + .collect(); + let lines = wrap(&format!("Not drawn: {}.", parts.join("; ")), 100); + let t = Text { + size: 11, + anchor: "start", + extra: " opacity=\"0.75\"", + }; + let mut out = String::new(); + for (i, s) in lines.iter().enumerate() { + out += &text(8.0, y + 14.0 * i as f64, s, &t)?; + } + Ok((out, y + 14.0 * lines.len() as f64)) +} + +/// A mark's value as a number, 0 for `null` (Python's `value or 0`). +fn value(m: &Mark) -> Result { + match &m.value { + Value::Null => Ok(0.0), + v => num(Some(v)) + .ok_or_else(|| Malformed(format!("mark {}: the value is not a number", m.id))), + } +} + +/// A label cut to 40 characters. +fn short(label: &str) -> String { + if label.chars().count() <= 40 { + label.to_owned() + } else { + label.chars().take(38).collect::() + "..." + } +} + +fn bars(sp: &Spec) -> Result { + let vals: Vec = sp.marks.iter().map(value).collect::>()?; + if vals.iter().sum::() == 0.0 { + let h = 96.0; + return Ok(open(sp, h)? + + &format!( + "", + W - 2.0, + h - 2.0 + ) + + &text( + W / 2.0, + 40.0, + "No ClinVar hit in this sample.", + &sized(14, "middle"), + )? + + &text( + W / 2.0, + 64.0, + "No hit is not an all-clear: the screen reads only the variants ClinVar lists.", + &sized(12, "middle"), + )? + + ""); + } + let top = vals.iter().copied().fold(vals[0], pmax); + if top == 0.0 { + return Err(Malformed("ClinVar chart: no bar above 0".into())); + } + let (left, right, row) = (190.0, 60.0, 26.0); + let h = 16.0 + row * sp.marks.len() as f64 + 8.0; + let mut body = String::new(); + for (i, (m, v)) in sp.marks.iter().zip(&vals).enumerate() { + let y = 12.0 + i as f64 * row; + let wbar = (W - left - right) * v / top; + let bar = if *v != 0.0 { + format!( + "", + f(y)?, + f(pmax(wbar, 0.0))?, + m.tone.colour() + ) + } else { + format!( + "", + f(y)? + ) + }; + let inner = text(left - 8.0, y + 13.0, &m.label, &sized(12, "end"))? + + &bar + + &text( + left + pmax(wbar, 2.0) + 6.0, + y + 13.0, + &py(&m.value)?, + &PLAIN, + )?; + body += &a(m, &inner); + } + let (foot, h2) = footer(sp, h + 4.0)?; + Ok(open(sp, h2 + 4.0)? + &body + &foot + "") +} + +/// The ticks of a 0 to 100 axis, at `x(t)`. +fn ticks(x: impl Fn(f64) -> f64, label: impl Fn(u32) -> String) -> Result { + let mut out = String::new(); + for t in [0, 25, 50, 75, 100] { + out += &text(x(f64::from(t)), 10.0, &label(t), &sized(10, "middle"))?; + } + Ok(out) +} + +fn ranges(sp: &Spec) -> Result { + let (left, right, row) = (260.0, 50.0, 22.0); + let (x0, x1) = (left, W - right); + let sx = |v: f64| x0 + (x1 - x0) * pmax(0.0, pmin(100.0, v)) / 100.0; + let info = Tone::Info.colour(); + let mut y = 22.0; + let mut body = String::new(); + let mut group: Option<&str> = None; + let axis = format!("") + + &ticks(sx, |t| format!("{t}%"))?; + for m in &sp.marks { + if m.group.as_deref() != group { + group = m.group.as_deref(); + y += 6.0; + let bold = Text { + size: 12, + anchor: "start", + extra: " font-weight=\"600\"", + }; + body += &text(8.0, y + 10.0, group.unwrap_or("None"), &bold)?; + y += row; + } + let cy = y + 6.0; + let v = value(m)?; + let mut inner = text(left - 8.0, cy + 4.0, &short(&m.label), &sized(12, "end"))?; + if let (Some(lo), Some(hi)) = (num(Some(&m.low)), num(Some(&m.high))) { + inner += &format!( + "", + f(sx(lo))?, + f(cy)?, + f(sx(hi))?, + f(cy)? + ); + } + inner += &format!( + "", + f(sx(v))?, + f(cy)? + ); + inner += &text( + x1 + 6.0, + cy + 4.0, + &format!("{}%", fixed(v, 0)), + &sized(11, "start"), + )?; + body += &a(m, &inner); + y += row; + } + Ok(open(sp, y + 8.0)? + &axis + &body + "") +} + +fn strip(sp: &Spec) -> Result { + let (left, right, row) = (260.0, 120.0, 24.0); + let (x0, x1) = (left, W - right); + let sx = |v: f64| x0 + (x1 - x0) * v / 100.0; + let mut body = format!("") + + &ticks(sx, |t| t.to_string())?; + let mut y = 26.0; + for m in &sp.marks { + let cy = y + 6.0; + let mut inner = text(left - 8.0, cy + 4.0, &short(&m.label), &sized(12, "end"))?; + inner += &format!( + "", + f(cy)?, + f(cy)? + ); + if matches!(m.value, Value::Null) { + let faint = Text { + size: 11, + anchor: "start", + extra: " opacity=\"0.8\"", + }; + inner += &text(x0 + 4.0, cy + 4.0, "no percentile", &faint)?; + } else { + let v = value(m)?; + inner += &format!( + "", + f(sx(v))?, + f(cy)?, + m.tone.colour() + ); + inner += &text(x1 + 8.0, cy + 4.0, &general(v, 6), &sized(11, "start"))?; + } + body += &a(m, &inner); + y += row; + } + Ok(open(sp, y + 8.0)? + &body + "") +} + +fn text_chart(sp: &Spec) -> Result { + let lines = wrap(sp.text.as_deref().unwrap_or(""), 95); + let h = 20.0 + 16.0 * lines.len() as f64; + let mut out = open(sp, h)? + + &format!( + "", + W - 2.0, + f(h - 2.0)? + ); + for (i, s) in lines.iter().enumerate() { + out += &text(12.0, 20.0 + 16.0 * i as f64, s, &PLAIN)?; + } + Ok(out + "") +} + +fn ideogram(sp: &Spec) -> Result { + let (left, right, row) = (46.0, 16.0, 18.0); + let (x0, x1) = (left, W - right); + let top = GRCH38.iter().map(|(_, n)| *n).max().unwrap_or(1) as f64; + let sx = |bp: f64| x0 + (x1 - x0) * bp / top; + let row_y = |c: &str| { + GRCH38 + .iter() + .position(|(n, _)| *n == c) + .map(|i| 10 + i as i64 * 18) + }; + let (attention, info) = (Tone::Attention.colour(), Tone::Info.colour()); + let mut body = String::new(); + for (i, (c, n)) in GRCH38.iter().enumerate() { + let y = 10 + i as i64 * row as i64; + body += &text(left - 6.0, (y + 9) as f64, &c[3..], &sized(10, "end"))?; + body += &format!( + "", + y + 2, + f(sx(*n as f64) - x0)? + ); + } + for m in &sp.marks { + let chrom = m.chrom.as_deref().unwrap_or(""); + let (Some(y), Some(len)) = (row_y(chrom), chrom_len(chrom)) else { + return Err(Malformed(format!( + "mark {}: {chrom} is not a GRCh38 chromosome", + m.id + ))); + }; + let (lo, hi) = (x0, sx(len as f64)); + // The x of a mark w px wide, moved inside its chromosome's outline. + let inside = |a: f64, w: f64| pmax(lo, pmin(a, hi - w)); + let pos = m.pos.unwrap_or(0) as f64; + let span = || { + let a = sx(pos); + let b = sx(m.end.unwrap_or(0) as f64); + (a, b) + }; + let inner = match m.kind { + Some(MarkKind::Roh) => { + let (a, b) = span(); + let w = pmax(b - a, 1.5); + format!( + "", + f(inside(a, w))?, + y + 2, + f(w)? + ) + } + Some(MarkKind::Sv) => { + // An outlined span over the chromosome, at least 5 px so a small deletion is seen. + let (a, b) = span(); + let w = pmax(b - a, 5.0); + format!( + "", + f(inside(a - (w - (b - a)) / 2.0, w))?, + y - 1, + f(w)? + ) + } + Some(MarkKind::Mca) => { + // A bar under the chromosome: a share of the cells, not the whole genome. + let (a, b) = span(); + let w = pmax(b - a, 3.0); + format!( + "", + f(inside(a, w))?, + y + 12, + f(w)? + ) + } + _ => format!( + "", + f(inside(sx(pos) - 1.5, 3.0))?, + y - 1, + m.tone.colour() + ), + }; + body += &a(m, &inner); + } + let mut y = 10 + GRCH38.len() as i64 * row as i64 + 6; + let small = sized(11, "start"); + let mut legend = + format!("") + + &text( + x0 + 20.0, + (y + 8) as f64, + "run of homozygosity, 5 MB or more", + &small, + )? + + &format!( + "", + x0 + 250.0, + y - 2 + ) + + &text(x0 + 258.0, (y + 8) as f64, "ClinVar hit", &small)? + + &format!( + "", + x0 + 340.0, + y - 2, + Tone::Muted.colour() + ) + + &text( + x0 + 348.0, + (y + 8) as f64, + "low-confidence ClinVar call", + &small, + )?; + let has = |k: MarkKind| sp.marks.iter().any(|m| m.kind == Some(k)); + if has(MarkKind::Sv) || has(MarkKind::Mca) { + // A second legend row, drawn only when the chart has these marks. + y += 18; + if has(MarkKind::Sv) { + legend += &format!( + "", + y - 2 + ); + legend += &text(x0 + 20.0, (y + 8) as f64, "AnnotSV class 4 or 5 SV", &small)?; + } + if has(MarkKind::Mca) { + legend += &format!( + "", + x0 + 250.0, + y + 4 + ); + legend += &text( + x0 + 270.0, + (y + 8) as f64, + "mosaic alteration (MoChA), a share of cells", + &small, + )?; + } + } + let (foot, h) = footer(sp, (y + 26) as f64)?; + Ok(open(sp, h + 4.0)? + &body + &legend + &foot + "") +} + +fn scatter(sp: &Spec) -> Result { + let (left, right, tp, bottom) = (44.0, 16.0, 12.0, 30.0); + let h = 200.0; + let (x0, x1, y0, y1) = (left, W - right, tp, h - bottom); + let sx = |p: f64| x0 + (x1 - x0) * (p - 1.0) / (MITO_LEN - 1) as f64; + let sy = |af: f64| y1 - (y1 - y0) * pmax(0.0, pmin(1.0, af)); + let muted = Tone::Muted.colour(); + let mut body = format!( + "", + x1 - x0, + y1 - y0 + ); + for t in [0.0, 0.5, 1.0] { + body += &text(x0 - 6.0, sy(t) + 4.0, &general(t, 6), &sized(10, "end"))?; + } + for (t, anchor) in [ + (1, "start"), + (4000, "middle"), + (8000, "middle"), + (12000, "middle"), + (MITO_LEN, "end"), + ] { + body += &text( + sx(t as f64), + y1 + 14.0, + &format!("m.{t}"), + &sized(10, anchor), + )?; + } + let faint = Text { + size: 10, + anchor: "middle", + extra: " opacity=\"0.8\"", + }; + body += &text( + (x0 + x1) / 2.0, + h - 2.0, + "position on chrM; height: allele fraction", + &faint, + )?; + for (lo, hi) in [(303.0, 315.0), (16184.0, 16193.0)] { + body += &format!( + "", + f(sx(lo) - 1.0)?, + f(pmax(sx(hi) - sx(lo), 2.0))?, + y1 - y0 + ); + } + let missing = sp + .axis + .get("sites_missing") + .is_some_and(crate::model::truthy); + let mid = ((x0 + x1) / 2.0, (y0 + y1) / 2.0); + if missing { + body += &text( + mid.0, + mid.1, + "The positions of the heteroplasmic sites are not in this summary", + &sized(12, "middle"), + )?; + } else if sp.marks.is_empty() { + body += &text( + mid.0, + mid.1, + "No heteroplasmic site between allele fraction 0.05 and 0.95", + &sized(12, "middle"), + )?; + } + for m in &sp.marks { + let (x, y) = (sx(m.pos.unwrap_or(0) as f64), sy(value(m)?)); + let inner = if m.kind == Some(MarkKind::Polyc) { + format!( + "", + f(x - 4.0)?, + f(y - 4.0)? + ) + } else { + format!( + "", + f(x)?, + f(y)?, + Tone::Attention.colour() + ) + }; + body += &a(m, &inner); + } + let (foot, h2) = footer(sp, h + 10.0)?; + Ok(open(sp, h2 + 4.0)? + &body + &foot + "") +} + +/// The chart drawn as SVG, by its kind. +pub fn draw(sp: &Spec) -> Result { + match sp.kind { + Kind::Bars => bars(sp), + Kind::Ranges => ranges(sp), + Kind::Strip => strip(sp), + Kind::Text => text_chart(sp), + Kind::Ideogram => ideogram(sp), + Kind::Scatter => scatter(sp), + } +} From 29e8d4486c9c2155f1e77318fd34233b0d2dfa1d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:18:18 +0100 Subject: [PATCH 2/4] test(report): move the chart checks to cargo Every non-browser check of tests/test_report_charts.py now runs on og report charts in rust/crates/og-render/tests/charts.rs, each with its red control, plus frozen hashes of the oracle's charts JSON on the fixture and five changed copies. The Python file keeps only the live page's browser check, which needs the Python live server. Bead: D1 --- rust/crates/og-render/tests/charts.rs | 849 ++++++++++++++++++++++++++ tests/test_report_charts.py | 409 +------------ 2 files changed, 856 insertions(+), 402 deletions(-) create mode 100644 rust/crates/og-render/tests/charts.rs diff --git a/rust/crates/og-render/tests/charts.rs b/rust/crates/og-render/tests/charts.rs new file mode 100644 index 000000000..aaacc6bf8 --- /dev/null +++ b/rust/crates/og-render/tests/charts.rs @@ -0,0 +1,849 @@ +//! The report's charts (og report charts), on the invented sections of +//! tests/fixtures/report_charts/sections.json and on changed copies of them. +//! Moved here from tests/test_report_charts.py, check for check; each check +//! runs on a broken input too and must catch it. +//! +//! 1. ClinVar: one bar per star 4 to 0 with the by_stars count, then a grey +//! bar with the low-confidence count; no green; with no hit a grey frame +//! says that no hit is not an all-clear. A hit past the 50-row table links +//! to the section. +//! 2. PRS: no strip without a panel, only the report's note; with a panel a +//! score with no percentile, or one of the other sex, says "no +//! percentile" and gets no dot. +//! 3. Fine-grained ancestry: the groups are the ones the report lists, and +//! each dot lies inside its range line. +//! 4. Genome: an unreadable run of homozygosity stops the chart; a run under +//! 5 MB is not drawn; hits on chrM and on alt contigs, SVs and MoChA calls +//! with no place are counted in not_drawn; every mark is drawn inside its +//! chromosome's outline. +//! 5. Mito: one dot per heteroplasmic site, the poly-C call marked apart. +//! 6. Every caveat is the report's own, and no caveat sentence is written in +//! the charts' source. +//! 7. The charts JSON is what bin/report_charts.py wrote, byte for byte, on +//! the fixture and its changed copies (frozen as hashes of the oracle's +//! files), the same on every run, and the report paths write it. + +use std::path::{Path, PathBuf}; + +use og_render::charts::{self, GRCH38, Kind, MarkKind, Spec, build, chrom_len, draw, figure, json}; +use og_render::model::Summary; +use og_render::notes; +use og_render::sections::{ancestry_fine, prs}; +use pycompat::json::{Dump, Value, dumps, loads}; + +const GREENS: [&str; 5] = ["#d5f5e3", "#1d3b2a", "#8ee0ae", "#196f3d", "green"]; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// The fixture as a summary value: sample S, a fixed clock and its sections. +fn fixture() -> Value { + let text = + std::fs::read_to_string(repo().join("tests/fixtures/report_charts/sections.json")).unwrap(); + let f = loads(&text).unwrap(); + let Value::Object(o) = f else { panic!() }; + let mut s = indexmap::IndexMap::new(); + s.insert("sample".to_owned(), Value::Str("S".into())); + s.insert( + "generated_utc".to_owned(), + Value::Str("2026-01-01 00:00 UTC".into()), + ); + s.insert("sections".to_owned(), o["sections"].clone()); + Value::Object(s) +} + +/// `v[a][b]...` for changing a copy. +fn at<'a>(v: &'a mut Value, path: &[&str]) -> &'a mut Value { + path.iter().fold(v, |v, k| match v { + Value::Object(o) => o.get_mut(*k).unwrap_or_else(|| panic!("no {k}")), + Value::Array(a) => &mut a[k.parse::().unwrap()], + _ => panic!("{k}: not a container"), + }) +} + +fn set(v: &mut Value, path: &[&str], x: &str) { + let (last, head) = path.split_last().unwrap(); + let Value::Object(o) = at(v, head) else { + panic!() + }; + o.insert((*last).to_owned(), loads(x).unwrap()); +} + +fn summary(v: &Value) -> Summary { + Summary::parse(&dumps(v, &Dump::default()).unwrap()).unwrap() +} + +fn charts_of(v: &Value) -> Vec { + build(&summary(v)).expect("the charts") +} + +fn chart(v: &Value, id: &str) -> Spec { + charts_of(v) + .into_iter() + .find(|c| c.id == id) + .unwrap_or_else(|| panic!("no chart {id}")) +} + +fn data<'a>(v: &'a Value, sec: &str) -> &'a Value { + let Value::Object(o) = v else { panic!() }; + let Value::Object(s) = &o["sections"] else { + panic!() + }; + let Value::Object(x) = &s[sec] else { panic!() }; + &x["data"] +} + +fn int(v: &Value) -> f64 { + match v { + Value::Int(i) => *i as f64, + Value::Float(x) => *x, + Value::Str(s) => s.parse().unwrap(), + _ => panic!("{v:?}"), + } +} + +fn num(v: &Value) -> Option { + match v { + Value::Null => None, + v => Some(int(v)), + } +} + +/// The number in attribute `name` of an SVG element's text `piece`. +fn attr(piece: &str, name: &str) -> Option { + let k = format!(" {name}=\""); + let piece = format!(" {piece}"); + let i = piece.find(&k)? + k.len(); + piece[i..].split('"').next()?.parse().ok() +} + +fn redraw(sp: &mut Spec) { + sp.svg = draw(sp).unwrap(); +} + +// --- the detectors, each also run on a broken input --------------------------------------- + +fn clinvar_problems(sp: &Spec, d: &Value) -> Vec { + let mut out = Vec::new(); + let Value::Object(o) = d else { panic!() }; + let Value::Object(by) = &o["by_stars"] else { + panic!() + }; + let mut want: Vec = ["4", "3", "2", "1", "0"] + .iter() + .map(|k| int(&by[*k])) + .collect(); + want.push(int(&o["low_confidence"])); + let got: Vec = sp.marks.iter().map(|m| int(&m.value)).collect(); + if got != want { + out.push(format!( + "bars {got:?} != by_stars and low_confidence {want:?}" + )); + } + if sp.marks.last().unwrap().tone != charts::Tone::Muted { + out.push("the low-confidence bar is not grey".into()); + } + let low = sp.svg.to_lowercase(); + for g in GREENS { + if low.contains(g) { + out.push(format!("green {g} in the chart")); + } + } + out +} + +/// A dot per score with a percentile of the declared sex, "no percentile" +/// for the others. +fn prs_problems(sp: &Spec, scores: &[Value]) -> Vec { + let mut out = Vec::new(); + for (m, r) in sp.marks.iter().zip(scores) { + let Value::Object(r) = r else { panic!() }; + let pct = &r["percentile"]; + let placeable = !matches!(pct, Value::Str(s) if s.is_empty()) + && !matches!(&r["sex_check"], Value::Str(s) if s == prs::NOT_APPLICABLE); + if placeable && num(&m.value) != Some(int(pct)) { + out.push(format!( + "{}: value {:?} is not the percentile", + m.id, m.value + )); + } + if !placeable && num(&m.value).is_some() { + out.push(format!( + "{}: placed with no percentile of the declared sex", + m.id + )); + } + } + let dots = sp.svg.matches("no percentile").count() != sp.marks.len() - want { + out.push("not every score without a percentile says 'no percentile'".into()); + } + out +} + +/// Each mark with a range: its dot between the two ends of its line. +fn whisker_problems(svg: &str) -> Vec { + let mut out = Vec::new(); + for g in svg.split("").next().unwrap(); + let (Some(line), Some(dot)) = (g.find("class=\"whisker\""), g.find(" String { + if c.starts_with("chr") { + c.to_owned() + } else { + format!("chr{c}") + } +} + +/// What the genome chart misses of the AnnotSV rows and MoChA calls on the +/// primary chromosomes. +fn sv_mca_problems(sp: &Spec, svs: &[(&str, bool)], mcas: &[bool]) -> Vec { + let mut out = Vec::new(); + let mark = |id: &str| sp.marks.iter().find(|m| m.id == id); + for (i, (sv, _)) in svs.iter().enumerate() { + if chrom_len(&with_chr(sv.split(':').next().unwrap())).is_none() { + continue; + } + let id = format!("sv-{i}"); + if mark(&id).is_none_or(|m| m.anchor.as_deref() != Some("structural-variants")) + || !sp.svg.contains(&format!("data-mark=\"{id}\"")) + { + out.push(format!("AnnotSV row {sv} not drawn")); + } + } + for (i, placeable) in mcas.iter().enumerate() { + let id = format!("mca-{i}"); + if !placeable { + if mark(&id).is_some() { + out.push(format!( + "MoChA call {i} drawn, but it has no place on the chart" + )); + } + continue; + } + if mark(&id).is_none_or(|m| m.anchor.as_deref() != Some("mosaic-mca")) + || !sp.svg.contains(&format!("data-mark=\"{id}\"")) + { + out.push(format!("MoChA call {i} not drawn")); + } + } + if sp.svg.matches("class=\"sv\"").count() != svs.iter().filter(|(_, p)| *p).count() { + out.push("the outlined spans do not match the rows".into()); + } + if sp.svg.matches("class=\"mca\"").count() != mcas.iter().filter(|p| **p).count() { + out.push("the bars do not match the calls".into()); + } + out +} + +/// The marks of an ideogram drawn past either end of their chromosome's +/// outline: each mark's y is its row's plus a fixed offset per kind. +fn outside_marks(svg: &str) -> Vec { + let mut rows = Vec::new(); + for p in svg.split("").next().unwrap(); + if p.contains("height=\"8\" rx=\"4\" fill=\"none\"") { + let (x, y, w) = ( + attr(p, "x").unwrap(), + attr(p, "y").unwrap(), + attr(p, "width").unwrap(), + ); + rows.push((y - 2.0, x, x + w)); + } + } + let mut out = Vec::new(); + for p in svg.split(" 2.0, + "sv" | "tick" => -1.0, + "mca" => 12.0, + _ => continue, + }; + let p = p.split("/>").next().unwrap(); + let (x, y, w) = ( + attr(p, "x").unwrap(), + attr(p, "y").unwrap(), + attr(p, "width").unwrap(), + ); + match rows.iter().find(|(ry, _, _)| (ry - (y - off)).abs() < 0.01) { + Some((_, lo, hi)) if x >= lo - 0.01 && x + w <= hi + 0.01 => {} + _ => out.push(format!("{kind} at x={x} y={y} width={w}")), + } + } + out +} + +/// Caveat sentences written into a source file instead of taken from notes. +fn note_literals(src: &str) -> Vec { + let mut found: Vec = notes::NOTES + .iter() + .filter(|(_, t)| t.len() > 30 && src.contains(&t[..30])) + .map(|(k, _)| (*k).to_owned()) + .collect(); + if src.lines().any(|l| l.contains("_NOTE: &str =")) { + found.push("a *_NOTE constant".into()); + } + found +} + +// --- the checks --------------------------------------------------------------------------- + +#[test] +fn clinvar_bars_count_the_stars_in_yellow_and_grey() { + let s = fixture(); + let cv = chart(&s, "clinvar-stars"); + let d = data(&s, "clinvar"); + assert_eq!(clinvar_problems(&cv, d), Vec::::new()); + assert!( + cv.marks + .iter() + .all(|m| matches!(m.tone, charts::Tone::Attention | charts::Tone::Muted)) + ); + // Red controls: a green bar, and a bar off by one. + let mut green = cv.clone(); + green.svg = green + .svg + .replace(charts::Tone::Attention.colour(), "#d5f5e3"); + assert!(!clinvar_problems(&green, d).is_empty()); + let mut off = cv.clone(); + off.marks[1].value = Value::Int(int(&off.marks[1].value) as i128 + 1); + assert!(!clinvar_problems(&off, d).is_empty()); +} + +#[test] +fn no_clinvar_hit_is_a_grey_frame_that_is_not_an_all_clear() { + let mut z = fixture(); + set( + &mut z, + &["sections", "clinvar", "data", "by_stars"], + r#"{"4":0,"3":0,"2":0,"1":0,"0":0}"#, + ); + set(&mut z, &["sections", "clinvar", "data", "hits"], "[]"); + set(&mut z, &["sections", "clinvar", "data", "count"], "0"); + set( + &mut z, + &["sections", "clinvar", "data", "low_confidence"], + "0", + ); + let svg = chart(&z, "clinvar-stars").svg; + assert!(svg.contains("No hit is not an all-clear"), "{svg}"); + assert!(!svg.contains(" = g + .marks + .iter() + .filter(|m| m.kind == Some(MarkKind::Hit)) + .collect(); + let want: Vec<&str> = ticks + .iter() + .map(|m| { + if row(m.locus.as_deref().unwrap()) < 50 { + "clinvar-hits" + } else { + "clinvar" + } + }) + .collect(); + let got: Vec<&str> = ticks.iter().map(|m| m.anchor.as_deref().unwrap()).collect(); + assert_eq!(got, want); + assert!(want.contains(&"clinvar")); +} + +#[test] +fn prs_strip_places_only_percentiles_of_the_declared_sex() { + let s = fixture(); + let pr = chart(&s, "prs"); + let Value::Object(d) = data(&s, "prs") else { + panic!() + }; + let Value::Array(scores) = &d["scores"] else { + panic!() + }; + assert_eq!(pr.kind, Kind::Strip); + assert_eq!(prs_problems(&pr, scores), Vec::::new()); + // Red controls: the raw Score_SUM drawn as a dot, and the other sex's percentile. + let field = |i: usize, k: &str| { + let Value::Object(r) = &scores[i] else { + panic!() + }; + int(&r[k]) + }; + let mut raw = pr.clone(); + raw.marks[1].value = Value::Float(field(1, "score")); + redraw(&mut raw); + assert!(!prs_problems(&raw, scores).is_empty()); + let mut other = pr.clone(); + other.marks[2].value = Value::Float(field(2, "percentile")); + redraw(&mut other); + assert!(!prs_problems(&other, scores).is_empty()); +} + +#[test] +fn prs_without_a_panel_is_the_reports_note() { + let mut na = fixture(); + set(&mut na, &["sections", "prs", "data", "adjusted"], "false"); + let nc = chart(&na, "prs"); + assert_eq!(nc.kind, Kind::Text); + assert!(nc.marks.is_empty() && !nc.svg.contains(" = an.marks.iter().map(|m| m.label.clone()).collect(); + assert_eq!(got, want); + assert!(want.iter().any(|g| g == "Invented region B")); + assert!(!want.iter().any(|g| g == "Invented group C")); + assert!(an.svg.contains("whisker")); + assert_eq!(whisker_problems(&an.svg), Vec::::new()); + // Red control: a dot moved outside its range. + let mut off = an.clone(); + off.marks[0].value = Value::Float(99.0); + redraw(&mut off); + assert!(!whisker_problems(&off.svg).is_empty()); +} + +#[test] +fn an_ancestry_range_with_one_bound_is_left_out_of_the_detail() { + let mut half = fixture(); + set( + &mut half, + &["sections", "ancestry_fine", "data"], + r#"{"continent": [{"group": "Invented group D", "fraction": 0.3, "low": 0.2}]}"#, + ); + let m = &chart(&half, "ancestry-fine").marks[0]; + assert!(!m.detail.contains("bootstrap range"), "{}", m.detail); + assert_eq!(m.high, Value::Null); +} + +#[test] +fn genome_draws_runs_of_5_mb_and_counts_what_has_no_place() { + let s = fixture(); + let g = chart(&s, "genome"); + let roh: Vec<&str> = g + .marks + .iter() + .filter(|m| m.kind == Some(MarkKind::Roh)) + .map(|m| m.locus.as_deref().unwrap()) + .collect(); + assert_eq!(roh, ["chr3:10000000-22300000", "chr15:40000000-46100000"]); + let nd = |what: &str| g.not_drawn.iter().find(|x| x.what == what).map(|x| x.count); + assert_eq!(nd("runs of homozygosity under 5 MB"), Some(1)); + assert_eq!(nd("ClinVar hits on chrM"), Some(1)); + assert_eq!(nd("ClinVar hits on other contigs (alt, unplaced)"), Some(1)); + assert!(g.svg.contains("Not drawn:")); + assert!( + !g.marks + .iter() + .any(|m| matches!(m.chrom.as_deref(), Some("chrM" | "chr22_KI270879v1_alt"))) + ); + assert!(!g.svg.contains("AnnotSV class 4 or 5 SV") && !g.svg.contains("MoChA")); + // The floor is the roh section's, the 5 MB the docs promise. + assert_eq!(charts::ROH_MIN_MB, 5.0); + // No genome chart when ROH did not run, even with ClinVar hits. + let mut nr = s.clone(); + set(&mut nr, &["sections", "roh", "state"], "\"missing\""); + assert!(!charts_of(&nr).iter().any(|c| c.id == "genome")); +} + +#[test] +fn an_unreadable_run_of_homozygosity_stops_the_chart() { + for region in ["chr3:10000000_22300000", "3:1-2", "chr3:20-10"] { + let mut bad = fixture(); + set( + &mut bad, + &[ + "sections", + "roh", + "data", + "autosomal_over_5mb", + "0", + "region", + ], + &format!("\"{region}\""), + ); + let e = build(&summary(&bad)).expect_err(region); + assert!( + e.0.starts_with(&format!("run of homozygosity '{region}'")), + "{e}" + ); + } + // Control: the readable regions do not stop it. + assert!(build(&summary(&fixture())).is_ok()); +} + +const SV_ROWS: &str = r#"[ + {"sv": "chr7:117480000-117530000", "type": "DEL", "gt": "0/1", "length": "50000", "acmg_class": "4", "genes": ["CFTR"]}, + {"sv": "chr22_KI270879v1_alt:100-900", "type": "DUP", "gt": "0/1", "length": "800", "acmg_class": "5", "genes": []}, + {"sv": "17:7660000-7690000", "type": "DEL", "gt": "0/1", "length": "30000", "acmg_class": "5", "genes": ["TP53"]}]"#; +const MCA_CALLS: &str = r#"[ + {"chrom": "chr20", "start": "30000000", "end": "42000000", "length_mb": "12.0", "type": "Del", "cell_fraction": "0.12"}, + {"chrom": "chrM", "start": "100", "end": "900", "length_mb": ".", "type": "Del", "cell_fraction": "."}, + {"chrom": "chr3", "start": ".", "end": ".", "length_mb": ".", "type": "Dup", "cell_fraction": "."}, + {"chrom": "20", "start": "50000000", "end": "52000000", "length_mb": "2.0", "type": "Dup", "cell_fraction": "0.05"}]"#; + +fn with_sv_mca(rows: &str, calls: &str) -> Value { + let mut s = fixture(); + set( + &mut s, + &["sections", "annotsv"], + &format!(r#"{{"state": "ok", "data": {{"count": 3, "rows": {rows}}}}}"#), + ); + set( + &mut s, + &["sections", "mosaic"], + &format!(r#"{{"state": "ok", "data": {{"status": "assessed", "calls": {calls}}}}}"#), + ); + s +} + +#[test] +fn genome_draws_annotsv_spans_and_mocha_bars() { + let gm = chart(&with_sv_mca(SV_ROWS, MCA_CALLS), "genome"); + let svs = [ + ("chr7:117480000-117530000", true), + ("chr22_KI270879v1_alt:100-900", false), + ("17:7660000-7690000", true), + ]; + let mcas = [true, false, false, true]; + assert_eq!(sv_mca_problems(&gm, &svs, &mcas), Vec::::new()); + let nd = |what: &str| { + gm.not_drawn + .iter() + .find(|x| x.what == what) + .map(|x| x.count) + }; + assert_eq!(nd("AnnotSV class 4 and 5 SVs on other contigs"), Some(1)); + assert_eq!( + nd("MoChA calls on other contigs or with no position"), + Some(2) + ); + assert!( + gm.svg.contains("AnnotSV class 4 or 5 SV") && gm.svg.contains("mosaic alteration (MoChA)") + ); + // Red control: a genome chart that leaves out the MoChA calls. + let mut lost = gm.clone(); + lost.marks.retain(|m| m.kind != Some(MarkKind::Mca)); + redraw(&mut lost); + assert!(!sv_mca_problems(&lost, &svs, &mcas).is_empty()); +} + +#[test] +fn every_genome_mark_stays_inside_its_chromosome() { + let edge = with_sv_mca( + // past the end of chr21 (46,709,983 bp), and a 10 bp one at the very start of chr1 + r#"[{"sv": "chr21:46700000-46800000", "type": "DUP", "gt": "0/1", "length": "100000", "acmg_class": "4", "genes": []}, + {"sv": "chr1:1-10", "type": "DEL", "gt": "0/1", "length": "10", "acmg_class": "4", "genes": []}]"#, + r#"[{"chrom": "chr22", "start": "50000000", "end": "51000000", "length_mb": "1.0", "type": "Dup", "cell_fraction": "0.1"}, + {"chrom": "chr22", "start": "52000000", "end": "53000000", "length_mb": "1.0", "type": "Del", "cell_fraction": "0.1"}]"#, + ); + let ge = chart(&edge, "genome"); + assert_eq!(outside_marks(&ge.svg), Vec::::new()); + let nd = ge + .not_drawn + .iter() + .find(|x| x.what == "marks past the end of their GRCh38 chromosome"); + assert_eq!(nd.map(|x| x.count), Some(1)); + assert!(ge.marks.iter().all(|m| { + m.end + .is_none_or(|e| e <= chrom_len(m.chrom.as_deref().unwrap()).unwrap()) + })); + assert_eq!(ge.svg.matches("class=\"sv\"").count(), 2); + assert_eq!(ge.svg.matches("class=\"mca\"").count(), 1); + let cut: Vec<(&str, i128)> = ge + .marks + .iter() + .filter(|m| { + matches!(m.kind, Some(MarkKind::Sv | MarkKind::Mca)) && m.pos.unwrap() > 40_000_000 + }) + .map(|m| (m.chrom.as_deref().unwrap(), m.end.unwrap())) + .collect(); + assert_eq!(cut, [("chr21", 46_709_983), ("chr22", 50_818_468)]); + // Red control: a span drawn past its chromosome's end. + let i = ge.svg.find("::new()); + // Red control: a copied caveat is caught. + let copied = format!("{src}\nconst X: &str = \"{}\";\n", notes::ROH_NOTE); + assert_eq!(note_literals(&copied), ["ROH_NOTE"]); +} + +#[test] +fn the_grch38_lengths_are_ucscs() { + assert_eq!(GRCH38.len(), 24); + assert_eq!(chrom_len("chr1"), Some(248_956_422)); + assert_eq!(chrom_len("chrY"), Some(57_227_415)); + assert_eq!(chrom_len("chrM"), None); +} + +/// FNV-1a of a file's bytes: a frozen vector of what the oracle wrote. +fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, b| { + (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3) + }) +} + +fn charts_file(v: &Value) -> String { + let opts = Dump { + indent: Some(1), + ..Dump::default() + }; + dumps(&json(&summary(v)).unwrap(), &opts).unwrap() + "\n" +} + +/// The summaries whose charts JSON is frozen below. With OG_CHARTS_DUMP set +/// to a folder, each is written there, to render it with the oracle +/// (`bin/render_report.py --summary F --charts-json OUT`). +fn frozen() -> Vec<(&'static str, Value)> { + let mut z = fixture(); + set( + &mut z, + &["sections", "clinvar", "data", "by_stars"], + r#"{"4":0,"3":0,"2":0,"1":0,"0":0}"#, + ); + set(&mut z, &["sections", "clinvar", "data", "hits"], "[]"); + set( + &mut z, + &["sections", "clinvar", "data", "low_confidence"], + "0", + ); + let mut na = fixture(); + set(&mut na, &["sections", "prs", "data", "adjusted"], "false"); + let mut old = fixture(); + if let Value::Object(d) = at(&mut old, &["sections", "mito", "data"]) { + d.shift_remove("heteroplasmic_sites"); + } + let edge = with_sv_mca( + r#"[{"sv": "chr21:46700000-46800000", "type": "DUP", "gt": "0/1", "length": "100000", "acmg_class": "4", "genes": []}, + {"sv": "chr1:1-10", "type": "DEL", "gt": "0/1", "length": "10", "acmg_class": "4", "genes": []}]"#, + r#"[{"chrom": "chr22", "start": "50000000", "end": "51000000", "length_mb": "1.0", "type": "Dup", "cell_fraction": "0.1"}, + {"chrom": "chr22", "start": "52000000", "end": "53000000", "length_mb": "1.0", "type": "Del", "cell_fraction": "0.1"}]"#, + ); + let all = vec![ + ("fixture", fixture()), + ("no-hit", z), + ("no-panel", na), + ("no-sites", old), + ("sv-mca", with_sv_mca(SV_ROWS, MCA_CALLS)), + ("edge", edge), + ]; + if let Some(dir) = std::env::var_os("OG_CHARTS_DUMP") { + for (name, v) in &all { + let p = Path::new(&dir).join(format!("{name}.json")); + std::fs::write(p, dumps(v, &Dump::default()).unwrap()).unwrap(); + } + } + all +} + +/// The FNV-1a of the charts JSON `bin/report_charts.py` wrote at commit +/// 65095e97 for each frozen summary (render_report.py --charts-json). A +/// deliberate change to a chart changes its vector: say so in the PR. +const ORACLE: [(&str, u64); 6] = [ + ("fixture", 0x0fb8_e455_9f1d_5688), + ("no-hit", 0x97e4_dae4_149c_3c39), + ("no-panel", 0xd855_b5d4_785b_2c3a), + ("no-sites", 0xa570_9eae_3ca9_1850), + ("sv-mca", 0x003a_8f9c_2441_ba65), + ("edge", 0xca4a_fec8_1f14_13d8), +]; + +#[test] +fn the_charts_json_is_what_the_python_wrote() { + let got: Vec<(&str, u64)> = frozen() + .iter() + .map(|(n, v)| (*n, fnv(charts_file(v).as_bytes()))) + .collect(); + let shown: Vec = got + .iter() + .map(|(n, h)| format!("(\"{n}\", {h:#018x})")) + .collect(); + assert_eq!(got, ORACLE, "now: {}", shown.join(", ")); +} + +#[test] +fn the_charts_json_is_the_same_on_every_run() { + let s = fixture(); + let (a, b) = (charts_file(&s), charts_file(&s)); + assert_eq!(a, b); + let v = loads(&a).unwrap(); + let Value::Object(o) = &v else { panic!() }; + assert_eq!(o["schema_version"], Value::Int(1)); + let Value::Array(c) = &o["charts"] else { + panic!() + }; + let ids: Vec = c + .iter() + .map(|x| { + let Value::Object(x) = x else { panic!() }; + pycompat::value::py_str(&x["id"]).unwrap() + }) + .collect(); + assert_eq!( + ids, + ["clinvar-stars", "genome", "mito", "ancestry-fine", "prs"] + ); + // Red control: one changed value is a different file. + let mut moved = s.clone(); + set( + &mut moved, + &["sections", "mito", "data", "heteroplasmic_sites", "0", "af"], + "0.31", + ); + assert_ne!(charts_file(&moved), a); +} + +#[test] +fn the_verb_writes_the_file_and_counts_the_sections() { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("charts-verb"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let inp = dir.join("s.json"); + std::fs::write(&inp, dumps(&fixture(), &Dump::default()).unwrap()).unwrap(); + let outp = dir.join("S_charts.json"); + let args: Vec = [ + "--summary", + inp.to_str().unwrap(), + "--charts-json", + outp.to_str().unwrap(), + ] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!( + charts::cli(&args, &mut out, &mut err), + 0, + "{}", + String::from_utf8_lossy(&err) + ); + assert_eq!(String::from_utf8(out).unwrap(), "Sections: ok 5\n"); + assert_eq!( + std::fs::read_to_string(&outp).unwrap(), + charts_file(&fixture()) + ); + // No summary is a usage error; an unreadable one fails. + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!( + charts::cli( + &["--charts-json".into(), "x.json".into()], + &mut out, + &mut err + ), + 2 + ); + let missing = dir.join("none.json").to_str().unwrap().to_owned(); + assert_eq!( + charts::cli(&["--summary".into(), missing], &mut out, &mut err), + 1 + ); +} + +#[test] +fn both_report_paths_write_the_charts_beside_the_report() { + let nf = std::fs::read_to_string(repo().join("modules/local/html_report/main.nf")).unwrap(); + let sh = std::fs::read_to_string(repo().join("scripts/24-html-report.sh")).unwrap(); + assert!(nf.contains("--charts-json ${id}_charts.json")); + assert!(nf.contains("path(\"${meta.id}_charts.json\")")); + assert!( + nf.split("stub:") + .nth(1) + .unwrap() + .contains("${meta.id}_charts.json") + ); + assert!(sh.contains("--charts-json \"/genome/${SAMPLE}/${SAMPLE}_charts.json\"")); +} diff --git a/tests/test_report_charts.py b/tests/test_report_charts.py index 23dc05b2d..58e6b4856 100755 --- a/tests/test_report_charts.py +++ b/tests/test_report_charts.py @@ -1,37 +1,10 @@ #!/usr/bin/env python3 -"""The report's charts (bin/report_charts.py) and where bin/render_report.py -puts them, on the invented sections of tests/fixtures/report_charts/ and on -DEMO-001, the invented demo sample. +"""The report's charts on the live page, on DEMO-001, the invented demo sample. + +The charts' own checks (bin/report_charts.py, og report charts) are cargo +tests now: rust/crates/og-render/tests/charts.rs. What stays here is the +browser check of the live page, which needs the live site's Python server: -Checks, each with a red control that feeds it a broken input: - 1. ClinVar: one bar per star 4 to 0 with the by_stars count, and one grey - bar with the low-confidence count; no green in the chart; with no hit a - grey frame says that no hit is not an all-clear; - 2. PRS: no strip without a panel (adjusted false), only the report's note; - with a panel a score with no percentile, or one of the other sex, says - "no percentile" and gets no dot; - 3. fine-grained ancestry: the groups are render_report.ancestry_fine_rows, - and each dot lies inside its range line; - 4. genome: a run of homozygosity whose region cannot be read stops the - chart; a run under 5 MB is not drawn; ClinVar hits on chrM and on alt - contigs are counted in not_drawn; ROH_MIN_MB is collect_summary's; - each AnnotSV class 4 or 5 row is an outlined span and each reported MoChA - call a bar under its chromosome, linked to their sections, and one on - another contig is counted in not_drawn; every mark is drawn inside its - chromosome's outline (a span past its end is cut there, a mark that - starts past it is counted in not_drawn); - 5. mito: one dot per heteroplasmic site, the poly-C call marked apart; - 6. no caveat sentence is written in report_charts.py: they come from - render_report's *_NOTE constants; - 7. the HTML report: no