From 5b54778ec3a7078d919d688c42334721ae711649 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:25:13 +0100 Subject: [PATCH 1/3] refactor(og-pedigree): lay out and draw the pedigree in rust The layout and the SVG drawing that bin/pedigree_svg.py did, ported to og-pedigree's layout module: the same generations, couples, sibships and descent lines, and an SVG byte for byte the same. Bead: D1 --- bin/family_report.py | 207 --------- bin/pedigree_svg.py | 275 ----------- rust/crates/og-pedigree/src/layout.rs | 637 +++++++++++++++++++++++++- tests/test_live_family_depth.py | 389 ---------------- tests/test_pedigree_svg.py | 197 -------- 5 files changed, 635 insertions(+), 1070 deletions(-) delete mode 100755 bin/family_report.py delete mode 100755 bin/pedigree_svg.py delete mode 100644 tests/test_live_family_depth.py delete mode 100755 tests/test_pedigree_svg.py diff --git a/bin/family_report.py b/bin/family_report.py deleted file mode 100755 index fc7390b59..000000000 --- a/bin/family_report.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -# remnant: kept for pedigree_svg; deleted in lane R4 -"""family_report.py: what is left of the family analysis of step 38 once it -moved to `og family` (rust/crates/og-steps/src/family). Only the pedigree -and relatedness rules that other Python tools still import are kept here; -`og dev remnants` (tests/parity/remnants.tsv) checks that nothing else comes -back. Standard library only. -""" -import re - -COLUMNS = ("family", "sample", "sex", "father", "mother", "partner") -MISSING = {"", ".", "0", "-9"} -SEX = {"male": "male", "m": "male", "1": "male", - "female": "female", "f": "female", "2": "female", - "unknown": None, "": None, ".": None, "0": None, "-9": None} -NAME = re.compile(r"^[A-Za-z0-9._-]+$") - -# Relatedness (twice the kinship coefficient) bins, as KING draws them: the -# geometric middle between two degrees. -BINS = ((0.707, "same person"), (0.354, "first degree"), - (0.177, "second degree"), (0.0884, "third degree")) - - -def missing(v): - return v is None or v.strip() in MISSING - - -def read_pedigree(path): - """Rows of the pedigree TSV as dicts, and the problems found. The first - non-blank line is the header (a leading '#' is dropped); later lines - starting with '#' are comments.""" - rows, errors, header = [], [], None - with open(path) as f: - for n, raw in enumerate(f, 1): - line = raw.rstrip("\r\n") - if not line.strip(): - continue - if header is None: - header = [c.strip().lower() for c in line.lstrip("#").split("\t")] - if "sample" not in header: - return [], ["%s: the first line must be a header with a 'sample' column " - "(columns: %s, tab separated); it is '%s'" % (path, ", ".join(COLUMNS), line)] - bad = [c for c in header if c not in COLUMNS] - if bad: - errors.append("%s: unknown column%s %s; the columns are %s" - % (path, "s" if len(bad) > 1 else "", ", ".join(bad), ", ".join(COLUMNS))) - continue - if line.startswith("#"): - continue - vals = [v.strip() for v in line.split("\t")] - if len(vals) > len(header): - errors.append("%s line %d: %d fields, the header has %d (fields are tab separated)" - % (path, n, len(vals), len(header))) - continue - vals += [""] * (len(header) - len(vals)) - rec = dict(zip(header, vals)) - rec["line"] = n - rows.append(rec) - if header is None: - errors.append("%s is empty" % path) - return rows, errors - - -def build_members(rows, declared, path, source="the samplesheet"): - """Every person the pedigree names: its rows, and the parents and - partners it names without a row of their own (not sequenced, no - parents known). Returns (members, errors); a member is a dict with - sex, father, mother, partner (None when not given) and row (bool).""" - errors, members = [], {} - - for r in rows: - where = "%s line %d" % (path, r["line"]) - s = r.get("sample", "").strip() - if missing(s): - errors.append("%s: no sample name" % where) - continue - if bad_name(s): - errors.append("%s: sample name '%s' may hold only letters, digits, '.', '_' and '-' " - "and may not start with '-'" % (where, s)) - continue - if s in members: - errors.append("%s: sample '%s' has a second row; each person has one" % (where, s)) - continue - sex_raw = r.get("sex", "").strip().lower() - if sex_raw not in SEX: - errors.append("%s: sex '%s' of '%s' is not male, female or unknown" % (where, r.get("sex"), s)) - sex_raw = "" - m = {"sex": SEX[sex_raw], "row": True} - for k in ("father", "mother", "partner"): - v = r.get(k, "").strip() - if missing(v): - m[k] = None - elif bad_name(v): - errors.append("%s: %s '%s' of '%s' is not a valid sample name" % (where, k, v, s)) - m[k] = None - elif v == s: - errors.append("%s: '%s' is named as its own %s" % (where, s, k)) - m[k] = None - else: - m[k] = v - if m["father"] and m["father"] == m["mother"]: - errors.append("%s: '%s' has the same person, '%s', as father and mother" % (where, s, m["father"])) - d = declared.get(s) - if d: - if m["sex"] and m["sex"] != d: - errors.append("%s: the pedigree says '%s' is %s, %s says %s" - % (where, s, m["sex"], source, d)) - m["sex"] = m["sex"] or d - members[s] = m - - # People named but without a row: parents and partners not in the file. - for s, m in list(members.items()): - for k, sex in (("father", "male"), ("mother", "female"), ("partner", None)): - p = m.get(k) - if p and p not in members: - members[p] = {"sex": sex, "father": None, "mother": None, "partner": None, "row": False} - - for s, m in members.items(): - if m["father"] and members[m["father"]]["sex"] == "female": - errors.append("'%s' is the father of '%s' but its sex is female" % (m["father"], s)) - if m["mother"] and members[m["mother"]]["sex"] == "male": - errors.append("'%s' is the mother of '%s' but its sex is male" % (m["mother"], s)) - fathers = {m["father"] for m in members.values() if m["father"]} - mothers = {m["mother"] for m in members.values() if m["mother"]} - for p in sorted(fathers & mothers): - errors.append("'%s' is named as a father and as a mother" % p) - - for s, m in members.items(): - p = m["partner"] - if p and members[p]["row"] and members[p]["partner"] not in (None, s): - errors.append("'%s' names '%s' as partner, but '%s' names '%s'" - % (s, p, p, members[p]["partner"])) - - # No one is their own ancestor. - for s in members: - seen, todo = set(), [s] - while todo: - x = todo.pop() - for k in ("father", "mother"): - p = members.get(x, {}).get(k) - if not p: - continue - if p == s: - errors.append("'%s' is among its own ancestors: the parent columns form a loop" % s) - todo = [] - break - if p not in seen: - seen.add(p) - todo.append(p) - return members, errors - - -def bad_name(v): - return not NAME.match(v) or v.startswith("-") - - -def kinship_table(members): - """phi(a, b): the kinship coefficient of two members, from the pedigree. - Founders (no parent known) are unrelated; an unknown parent is a founder.""" - depth_memo, phi_memo = {}, {} - - def depth(x): - if x not in depth_memo: - m = members[x] - ps = [p for p in (m["father"], m["mother"]) if p] - depth_memo[x] = 1 + max(depth(p) for p in ps) if ps else 0 - return depth_memo[x] - - def phi(a, b): - if a == b: - m = members[a] - if m["father"] and m["mother"]: - return 0.5 * (1 + phi(m["father"], m["mother"])) - return 0.5 - key = (a, b) if a < b else (b, a) - if key not in phi_memo: - if depth(a) < depth(b): - a, b = b, a - m = members[a] - if not m["father"] and not m["mother"]: - phi_memo[key] = 0.0 - else: - phi_memo[key] = 0.5 * sum(phi(p, b) for p in (m["father"], m["mother"]) if p) - return phi_memo[key] - - return phi - - -def declared_relationship(a, b, members, expected): - ma, mb = members[a], members[b] - if a in (mb["father"], mb["mother"]) or b in (ma["father"], ma["mother"]): - return "parent-child" - shared = {p for p in (ma["father"], ma["mother"]) if p} & {p for p in (mb["father"], mb["mother"]) if p} - if len(shared) == 2: - return "full siblings" - if len(shared) == 1: - return "half siblings" - if ma["partner"] == b or mb["partner"] == a: - return "partners" if expected == 0 else "partners, %s" % degree(expected) - return degree(expected) if expected > 0 else "unrelated" - - -def degree(r): - for lo, name in BINS: - if r >= lo: - return name - return "unrelated" diff --git a/bin/pedigree_svg.py b/bin/pedigree_svg.py deleted file mode 100755 index 1999c3ff6..000000000 --- a/bin/pedigree_svg.py +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env python3 -"""pedigree_svg.py: draw a family's pedigree as an SVG. - - pedigree_svg.py FAMILY_DIR_OR_FILE [--title TEXT] > pedigree.svg - -Reads members.tsv (step 38's table of every person the pedigree names, with -in_pedigree and sequenced) or, when there is none, pedigree.tsv (the rows a -person wrote; parents and partners named without a row are added, as step 38 -adds them). A folder argument reads FAMILY_DIR/members.tsv, else -FAMILY_DIR/pedigree.tsv. - -The drawing uses the usual pedigree symbols: a square for a male, a circle for -a female, a diamond when the sex is not known. A filled symbol is a sequenced -person (members.tsv's sequenced column; from pedigree.tsv, a person with a row -of their own). A horizontal line joins two partners, or the two parents of a -child; a line drops from it to a bar over their children. Each generation is -one row, parents above their children. - -It only draws what the file says. It reads no genotype and checks no -relationship: step 38's relationship check does that. - -Every symbol is a , every partner line a , -and every line of descent a , so a test can read the drawing back. -""" -import argparse -import os -import sys - -HERE = os.path.dirname(os.path.abspath(__file__)) - -SIZE = 36 # a symbol's width and height -STEP_X = 120 # from one person to the next in a row -STEP_Y = 130 # from one generation to the next -MARGIN = 40 -TOP = 50 # room for the title -BG, FG, LINE, FILL = "#16181c", "#e6e6e6", "#c0c4cc", "#7aa2f7" -SEXES = ("male", "female") - - -def esc(s): - return str(s).replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) - - -def blank(v): - return v is None or v.strip() in ("", ".", "-9", "0", "NA", "na") - - -def read_members_tsv(path): - """(order, members) from members.tsv; a member is {sex, father, mother, - partner, sequenced}.""" - order, members = [], {} - with open(path) as f: - head = f.readline().rstrip("\r\n").lstrip("#").split("\t") - if "sample" not in head: - raise ValueError("%s: the first line must be a header with a 'sample' column" % path) - for line in f: - line = line.rstrip("\r\n") - if not line.strip() or line.startswith("#"): - continue - vals = line.split("\t") - r = dict(zip(head, vals + [""] * (len(head) - len(vals)))) - s = r["sample"].strip() - if blank(s) or s in members: - continue - sex = r.get("sex", "").strip().lower() - members[s] = {"sex": sex if sex in SEXES else "unknown", - "father": None if blank(r.get("father")) else r["father"].strip(), - "mother": None if blank(r.get("mother")) else r["mother"].strip(), - "partner": None if blank(r.get("partner")) else r["partner"].strip(), - "sequenced": r.get("sequenced", "").strip() == "yes"} - order.append(s) - return order, members - - -def read_pedigree_tsv(path): - """(order, members) from a pedigree file, with step 38's own reader.""" - if HERE not in sys.path: - sys.path.insert(0, HERE) - import family_report - rows, errors = family_report.read_pedigree(path) - members, more = family_report.build_members(rows, {}, path) if not errors else ({}, []) - if errors or more: - raise ValueError("; ".join(errors + more)) - out = {} - for s, m in members.items(): - out[s] = {"sex": m.get("sex") if m.get("sex") in SEXES else "unknown", "father": m.get("father"), - "mother": m.get("mother"), "partner": m.get("partner"), "sequenced": bool(m.get("row"))} - return list(members), out - - -def load(path): - """(order, members, source file) of a family folder or a members/pedigree file.""" - if os.path.isdir(path): - for name in ("members.tsv", "pedigree.tsv"): - p = os.path.join(path, name) - if os.path.isfile(p): - return load(p) - raise ValueError("%s holds no members.tsv or pedigree.tsv" % path) - with open(path) as f: - head = f.readline().rstrip("\r\n").lstrip("#").split("\t") - if "sequenced" in head: - order, members = read_members_tsv(path) - else: - order, members = read_pedigree_tsv(path) - return order, members, path - - -def couples(order, members): - """Pairs drawn joined: partners, and the two parents of a child, in file order.""" - seen, out = set(), [] - for s in order: - m = members[s] - for a, b in ((s, m["partner"]), (m["father"], m["mother"])): - if a and b and a in members and b in members: - key = frozenset((a, b)) - if key not in seen: - seen.add(key) - out.append((a, b)) - return out - - -def generations(order, members, pairs): - """Generation of each person: 0 with no parent named, else one below the - lower parent. A partner with no parent named moves down to the other - partner's row; two partners who both have parents keep their own rows - (a line then joins them across rows).""" - gen = {s: 0 for s in order} - - def rooted(s): - return any(members[s][k] in gen for k in ("father", "mother")) - - for _ in range(len(order) + 2): - changed = False - for s in order: - ps = [members[s][k] for k in ("father", "mother") if members[s][k] in gen] - g = max([gen[p] + 1 for p in ps] + [gen[s]]) - if g != gen[s]: - gen[s], changed = g, True - for a, b in pairs: - for x, y in ((a, b), (b, a)): - if not rooted(x) and gen[x] < gen[y]: - gen[x], changed = gen[y], True - if not changed: - break - return gen - - -def layout(order, members): - """{person: (x, y)} of each symbol's centre, plus the partner pairs.""" - pairs = couples(order, members) - gen = generations(order, members, pairs) - partners = {} - for a, b in pairs: - partners.setdefault(a, []).append(b) - partners.setdefault(b, []).append(a) - rank = {s: i for i, s in enumerate(order)} - rows, x_of = {}, {} - for g in sorted(set(gen.values())): - people = [s for s in order if gen[s] == g] - - def key(s): - ps = [x_of[members[s][k]] for k in ("father", "mother") if members[s][k] in x_of] - return (sum(ps) / len(ps) if ps else None, rank[s]) - - # People with a parent drawn go under their parents; one without (a partner who married in, - # or a whole first generation) goes beside the partner already placed, else in file order. - anchored = sorted([s for s in people if key(s)[0] is not None], key=key) - row, placed = [], set() - - def put(s): - if s in placed: - return - placed.add(s) - row.append(s) - for p in partners.get(s, []): - if gen.get(p) == g and key(p)[0] is None: - put(p) - - for s in anchored + [s for s in people if key(s)[0] is None]: - put(s) - rows[g] = row - for i, s in enumerate(row): - x_of[s] = i - widest = max(len(r) for r in rows.values()) - pos = {} - for g, row in rows.items(): - shift = (widest - len(row)) * STEP_X / 2 - for i, s in enumerate(row): - pos[s] = (MARGIN + SIZE / 2 + shift + i * STEP_X, TOP + SIZE / 2 + g * STEP_Y) - return pos, gen, pairs, widest, max(rows) + 1 - - -def symbol(s, m, x, y, g): - h = SIZE / 2 - fill = FILL if m["sequenced"] else BG - if m["sex"] == "male": - shape = '' - '%s fill="%s" stroke="%s" stroke-width="2"/>' - '%s' - '%s: %s, %s' - % (esc(s), m["sex"], g, "yes" if m["sequenced"] else "no", shape, fill, LINE, - x, y + h + 16, FG, esc(s), esc(s), m["sex"], "sequenced" if m["sequenced"] else "not sequenced")) - - -def draw(order, members, title="Pedigree"): - if not order: - raise ValueError("no person to draw") - pos, gen, pairs, widest, ngen = layout(order, members) - width = int(2 * MARGIN + SIZE + (widest - 1) * STEP_X) - legend_y = TOP + SIZE + (ngen - 1) * STEP_Y + 50 - height = legend_y + 30 - L = ['' % (width, height, width, height), - '' % BG, - '%s' % (FG, esc(title))] - h = SIZE / 2 - for a, b in pairs: - (xa, ya), (xb, yb) = pos[a], pos[b] - if xa > xb: - (xa, ya), (xb, yb) = (xb, yb), (xa, ya) - L.append('' % (esc(a), esc(b), xa + h, ya, xb - h, yb, LINE)) - # Children grouped by their pair of parents, in file order. - sibships = {} - for s in order: - f, mo = members[s]["father"], members[s]["mother"] - f, mo = (f if f in pos else None), (mo if mo in pos else None) - if f or mo: - sibships.setdefault((f, mo), []).append(s) - for (f, mo), kids in sibships.items(): - if f and mo: - ox, oy = (pos[f][0] + pos[mo][0]) / 2, (pos[f][1] + pos[mo][1]) / 2 - else: - ox, oy = pos[f or mo][0], pos[f or mo][1] + h - top = min(pos[k][1] for k in kids) - h - bar = top - 22 - xs = [pos[k][0] for k in kids] + [ox] - parts = ['' % (ox, oy, ox, bar), - '' % (min(xs), bar, max(xs), bar)] - parts += ['' % (pos[k][0], bar, pos[k][0], pos[k][1] - h) - for k in kids] - L.append('%s' % (esc(f or "."), esc(mo or "."), esc(" ".join(kids)), LINE, "".join(parts))) - for s in order: - L.append(symbol(s, members[s], pos[s][0], pos[s][1], gen[s])) - L.append('Square: male. Circle: female. Diamond: sex not known. Filled: ' - 'sequenced. A line joins partners; children hang from it.' % (legend_y, FG)) - L.append("") - return "\n".join(L) + "\n" - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("path", help="a family folder, members.tsv or pedigree.tsv") - ap.add_argument("--title", help="default: the folder's name") - a = ap.parse_args(argv) - try: - order, members, src = load(a.path) - title = a.title or os.path.basename(os.path.dirname(os.path.abspath(src))) or "Pedigree" - sys.stdout.write(draw(order, members, title)) - except (OSError, ValueError) as e: - print("ERROR: %s" % e, file=sys.stderr) - return 2 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/rust/crates/og-pedigree/src/layout.rs b/rust/crates/og-pedigree/src/layout.rs index d2bd31b80..d3a4b6cee 100644 --- a/rust/crates/og-pedigree/src/layout.rs +++ b/rust/crates/og-pedigree/src/layout.rs @@ -1,2 +1,635 @@ -//! The drawing layout of a pedigree. Empty until its unit of the 1.0.0 plan -//! lands. +//! The drawing of a pedigree: where each person goes, the lines that join +//! partners and drop to their children, and the SVG of it all. The family +//! page draws the same geometry itself, so each person is a link +//! ([`page_layout`]); `og live family svg` writes the picture ([`draw`]). +//! +//! It only draws what the file says. It reads no genotype and checks no +//! relationship: step 38's relationship check does that. +//! +//! The file is `members.tsv` (step 38's table of every person the pedigree +//! names, with `in_pedigree` and `sequenced`) or, when there is none, +//! `pedigree.tsv` (the rows a person wrote; parents and partners named +//! without a row are added, as step 38 adds them, by [`crate::build_members`]). +//! +//! The usual pedigree symbols: a square for a male, a circle for a female, +//! a diamond when the sex is not known. A filled symbol is a sequenced +//! person (`members.tsv`'s `sequenced` column; from `pedigree.tsv`, a person +//! with a row of their own). A horizontal line joins two partners, or the +//! two parents of a child; a line drops from it to a bar over their +//! children. Each generation is one row, parents above their children. +//! +//! Every symbol is a ``, every partner line a ``, and every line of descent a ``, so a test can read the drawing back. + +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use indexmap::IndexMap; +use pycompat::text::{lower, strip}; + +/// A symbol's width and height. +pub const SIZE: i64 = 36; +/// From one person to the next in a row. +pub const STEP_X: i64 = 120; +/// From one generation to the next. +pub const STEP_Y: i64 = 130; +pub const MARGIN: i64 = 40; +/// Room for the title. +pub const TOP: i64 = 50; +const BG: &str = "#16181c"; +const FG: &str = "#e6e6e6"; +const LINE: &str = "#c0c4cc"; +const FILL: &str = "#7aa2f7"; +/// The files of a family folder that declare it, in the order they are read. +pub const MARKERS: [&str; 2] = ["members.tsv", "pedigree.tsv"]; + +/// One person as the drawing needs them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Person { + /// `male`, `female` or `unknown`. + pub sex: String, + pub father: Option, + pub mother: Option, + pub partner: Option, + pub sequenced: bool, +} + +impl Person { + fn rel(&self, k: &str) -> Option<&str> { + match k { + "father" => self.father.as_deref(), + "mother" => self.mother.as_deref(), + _ => self.partner.as_deref(), + } + } +} + +/// Everyone the file names, in its order. +pub type People = IndexMap; + +/// A value the file leaves empty. +fn blank(v: Option<&str>) -> bool { + v.is_none_or(|v| matches!(strip(v, None), "" | "." | "-9" | "0" | "NA" | "na")) +} + +fn sex_of(v: &str) -> String { + let s = lower(strip(v, None)); + if s == "male" || s == "female" { + s + } else { + "unknown".into() + } +} + +/// The header of a tab-separated file: its first line, a leading `#` +/// dropped. +fn header(line: &str) -> Vec { + line.trim_start_matches('#') + .split('\t') + .map(str::to_owned) + .collect() +} + +/// A line's values by column, short lines padded with empty values. +fn record<'a>(head: &'a [String], line: &'a str) -> HashMap<&'a str, &'a str> { + let vals: Vec<&str> = line.split('\t').collect(); + let mut out = HashMap::new(); + for (i, h) in head.iter().enumerate() { + out.insert(h.as_str(), vals.get(i).copied().unwrap_or("")); + } + out +} + +fn lines(path: &Path) -> Result, String> { + let p = path.to_string_lossy(); + crate::text_lines(&p).map_err(|d| d.lines.join("; ")) +} + +/// The people of `members.tsv`: a row with an empty name or a name already +/// seen is skipped; a sex other than male or female is `unknown`. +fn read_members_tsv(path: &Path, text: &[String]) -> Result { + let head = header(text.first().map_or("", String::as_str)); + if !head.iter().any(|h| h == "sample") { + return Err(format!( + "{}: the first line must be a header with a 'sample' column", + path.display() + )); + } + let mut out = People::new(); + for line in text.iter().skip(1) { + if strip(line, None).is_empty() || line.starts_with('#') { + continue; + } + let r = record(&head, line); + let s = strip(r["sample"], None); + if blank(Some(s)) || out.contains_key(s) { + continue; + } + let name = |k: &str| { + let v = r.get(k).copied(); + if blank(v) { + None + } else { + v.map(|v| strip(v, None).to_owned()) + } + }; + out.insert( + s.to_owned(), + Person { + sex: sex_of(r.get("sex").copied().unwrap_or("")), + father: name("father"), + mother: name("mother"), + partner: name("partner"), + sequenced: strip(r.get("sequenced").copied().unwrap_or(""), None) == "yes", + }, + ); + } + Ok(out) +} + +/// The people of a pedigree file, read by step 38's own rules; any problem +/// with the file is an error naming them all. +fn read_pedigree_tsv(path: &Path) -> Result { + let p = path.to_string_lossy(); + let (rows, errors) = crate::read_pedigree(&p).map_err(|d| d.lines.join("; "))?; + if !errors.is_empty() { + return Err(errors.join("; ")); + } + let (members, more) = crate::build_members(&rows, &HashMap::new(), &p, "the samplesheet"); + if !more.is_empty() { + return Err(more.join("; ")); + } + Ok(members + .into_iter() + .map(|(s, m)| { + let sex = match m.sex.as_deref() { + Some(x @ ("male" | "female")) => x.to_owned(), + _ => "unknown".into(), + }; + ( + s, + Person { + sex, + father: m.father, + mother: m.mother, + partner: m.partner, + sequenced: m.row, + }, + ) + }) + .collect()) +} + +/// The people of a family folder (its `members.tsv`, else its +/// `pedigree.tsv`) or of one such file, and the file read. A file whose +/// header has a `sequenced` column is read as `members.tsv`. +pub fn load(path: &Path) -> Result<(People, PathBuf), String> { + if path.is_dir() { + for name in MARKERS { + let p = path.join(name); + if p.is_file() { + return load(&p); + } + } + return Err(format!( + "{} holds no members.tsv or pedigree.tsv", + path.display() + )); + } + let text = lines(path)?; + let head = header(text.first().map_or("", String::as_str)); + let people = if head.iter().any(|h| h == "sequenced") { + read_members_tsv(path, &text)? + } else { + read_pedigree_tsv(path)? + }; + Ok((people, path.to_path_buf())) +} + +/// The pairs drawn joined: partners, and the two parents of a child, in +/// file order, each pair once. +pub fn couples(people: &People) -> Vec<(String, String)> { + let mut seen: HashSet<(String, String)> = HashSet::new(); + let mut out = Vec::new(); + for (s, m) in people { + for (a, b) in [ + (Some(s.as_str()), m.rel("partner")), + (m.rel("father"), m.rel("mother")), + ] { + let (Some(a), Some(b)) = (a, b) else { continue }; + if !people.contains_key(a) || !people.contains_key(b) { + continue; + } + let key = if a <= b { + (a.to_owned(), b.to_owned()) + } else { + (b.to_owned(), a.to_owned()) + }; + if seen.insert(key) { + out.push((a.to_owned(), b.to_owned())); + } + } + } + out +} + +/// The generation of each person: 0 with no parent named, else one below +/// the lower parent. A partner with no parent named moves down to the +/// other partner's row; two partners who both have parents keep their own +/// rows (a line then joins them across rows). +pub fn generations(people: &People, pairs: &[(String, String)]) -> HashMap { + let mut generation: HashMap = people.keys().map(|s| (s.clone(), 0)).collect(); + let parents = |s: &str| -> Vec { + let m = &people[s]; + [&m.father, &m.mother] + .into_iter() + .flatten() + .filter(|p| people.contains_key(p.as_str())) + .cloned() + .collect() + }; + for _ in 0..people.len() + 2 { + let mut changed = false; + for s in people.keys() { + let g = parents(s) + .iter() + .map(|p| generation[p] + 1) + .chain([generation[s]]) + .max() + .unwrap_or(0); + if g != generation[s] { + generation.insert(s.clone(), g); + changed = true; + } + } + for (a, b) in pairs { + for (x, y) in [(a, b), (b, a)] { + if parents(x).is_empty() && generation[x] < generation[y] { + let g = generation[y]; + generation.insert(x.clone(), g); + changed = true; + } + } + } + if !changed { + break; + } + } + generation +} + +/// Where everyone goes. +#[derive(Debug, Clone, PartialEq)] +pub struct Layout { + /// The centre of each person's symbol. + pub pos: HashMap, + pub generation: HashMap, + /// The pairs drawn joined ([`couples`]). + pub pairs: Vec<(String, String)>, + /// People in the longest row. + pub widest: i64, + /// Rows: the last generation plus one. + pub ngen: i64, +} + +/// The centre of each symbol. People with a parent drawn go under their +/// parents (by the mean place of the parents in their row); one without (a +/// partner who married in, or a whole first generation) goes beside the +/// partner already placed, else in file order. Each row is centred on the +/// widest. +pub fn layout(people: &People) -> Layout { + let pairs = couples(people); + let generation = generations(people, &pairs); + let mut partners: HashMap<&str, Vec<&str>> = HashMap::new(); + for (a, b) in &pairs { + partners.entry(a).or_default().push(b); + partners.entry(b).or_default().push(a); + } + let rank: HashMap<&str, usize> = people + .keys() + .enumerate() + .map(|(i, s)| (s.as_str(), i)) + .collect(); + let mut gens: Vec = generation + .values() + .copied() + .collect::>() + .into_iter() + .collect(); + gens.sort_unstable(); + let mut rows: Vec<(i64, Vec)> = Vec::new(); + let mut x_of: HashMap = HashMap::new(); + for &g in &gens { + let row_people: Vec<&str> = people + .keys() + .filter(|s| generation[*s] == g) + .map(String::as_str) + .collect(); + let key = |s: &str, x_of: &HashMap| -> Option { + let m = &people[s]; + let ps: Vec = [&m.father, &m.mother] + .into_iter() + .flatten() + .filter_map(|p| x_of.get(p).map(|&x| x as f64)) + .collect(); + (!ps.is_empty()).then(|| ps.iter().sum::() / ps.len() as f64) + }; + let mut anchored: Vec<(f64, usize, &str)> = row_people + .iter() + .filter_map(|&s| key(s, &x_of).map(|k| (k, rank[s], s))) + .collect(); + anchored.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); + let mut row: Vec = Vec::new(); + let mut placed: HashSet<&str> = HashSet::new(); + fn put<'a>( + s: &'a str, + g: i64, + row: &mut Vec, + placed: &mut HashSet<&'a str>, + partners: &HashMap<&'a str, Vec<&'a str>>, + unanchored: &dyn Fn(&str) -> bool, + generation: &HashMap, + ) { + if !placed.insert(s) { + return; + } + row.push(s.to_owned()); + for &p in partners.get(s).map_or(&[][..], Vec::as_slice) { + if generation.get(p) == Some(&g) && unanchored(p) { + put(p, g, row, placed, partners, unanchored, generation); + } + } + } + let unanchored = |p: &str| key(p, &x_of).is_none(); + let order: Vec<&str> = anchored + .iter() + .map(|a| a.2) + .chain( + row_people + .iter() + .copied() + .filter(|&s| key(s, &x_of).is_none()), + ) + .collect(); + for s in order { + put( + s, + g, + &mut row, + &mut placed, + &partners, + &unanchored, + &generation, + ); + } + for (i, s) in row.iter().enumerate() { + x_of.insert(s.clone(), i); + } + rows.push((g, row)); + } + let widest = rows.iter().map(|(_, r)| r.len()).max().unwrap_or(0) as i64; + let mut pos = HashMap::new(); + for (g, row) in &rows { + let shift = (widest - row.len() as i64) as f64 * STEP_X as f64 / 2.0; + for (i, s) in row.iter().enumerate() { + pos.insert( + s.clone(), + ( + MARGIN as f64 + SIZE as f64 / 2.0 + shift + (i as i64 * STEP_X) as f64, + TOP as f64 + SIZE as f64 / 2.0 + (g * STEP_Y) as f64, + ), + ); + } + } + let ngen = rows.last().map_or(0, |(g, _)| g + 1); + Layout { + pos, + generation, + pairs, + widest, + ngen, + } +} + +/// The children of each pair of parents drawn, in file order: (father, +/// mother, children), a parent who is not drawn left out. +pub fn sibships( + people: &People, + pos: &HashMap, +) -> Vec<(Option, Option, Vec)> { + let mut out: Vec<(Option, Option, Vec)> = Vec::new(); + for (s, m) in people { + let drawn = |p: &Option| p.clone().filter(|p| pos.contains_key(p)); + let (f, mo) = (drawn(&m.father), drawn(&m.mother)); + if f.is_none() && mo.is_none() { + continue; + } + match out.iter_mut().find(|(a, b, _)| *a == f && *b == mo) { + Some(e) => e.2.push(s.clone()), + None => out.push((f, mo, vec![s.clone()])), + } + } + out +} + +/// One sibship's lines: from the parents (the middle of their partner +/// line, or under a lone parent) down to a bar over the children, the bar, +/// and one line down to each child. Each line is `[x1, y1, x2, y2]`. +fn descent_lines( + f: Option<&str>, + mo: Option<&str>, + kids: &[String], + pos: &HashMap, +) -> Vec<[f64; 4]> { + let h = SIZE as f64 / 2.0; + let (ox, oy) = match (f, mo) { + (Some(f), Some(mo)) => ((pos[f].0 + pos[mo].0) / 2.0, (pos[f].1 + pos[mo].1) / 2.0), + _ => { + let p = pos[f.or(mo).unwrap_or_default()]; + (p.0, p.1 + h) + } + }; + let bar = kids.iter().map(|k| pos[k].1).fold(f64::INFINITY, f64::min) - h - 22.0; + let xs: Vec = kids.iter().map(|k| pos[k].0).chain([ox]).collect(); + let (lo, hi) = ( + xs.iter().copied().fold(f64::INFINITY, f64::min), + xs.iter().copied().fold(f64::NEG_INFINITY, f64::max), + ); + let mut out = vec![[ox, oy, ox, bar], [lo, bar, hi, bar]]; + out.extend(kids.iter().map(|k| [pos[k].0, bar, pos[k].0, pos[k].1 - h])); + out +} + +/// One person of [`PageLayout`]. +#[derive(Debug, Clone, PartialEq)] +pub struct PagePerson { + pub sample: String, + pub x: f64, + pub y: f64, + pub generation: i64, + pub sex: String, + pub sequenced: bool, +} + +/// The drawing's geometry as data, for the family page, which draws each +/// person as a link: the people with their centre, then every line +/// (partner lines first, then the descent lines of each sibship), each +/// number rounded to one decimal. +#[derive(Debug, Clone, PartialEq)] +pub struct PageLayout { + pub width: i64, + pub height: i64, + pub size: i64, + pub people: Vec, + pub lines: Vec<[f64; 4]>, +} + +fn round1(v: f64) -> f64 { + pycompat::fmt::round(v, 1).unwrap_or(v) +} + +/// The geometry [`draw`] draws, as the family page draws it. +pub fn page_layout(people: &People) -> PageLayout { + let l = layout(people); + let h = SIZE as f64 / 2.0; + let persons = people + .iter() + .map(|(s, m)| PagePerson { + sample: s.clone(), + x: round1(l.pos[s].0), + y: round1(l.pos[s].1), + generation: l.generation[s], + sex: m.sex.clone(), + sequenced: m.sequenced, + }) + .collect(); + let mut lines = Vec::new(); + for (a, b) in &l.pairs { + let (pa, pb) = (l.pos[a], l.pos[b]); + // the two ends in order of x, then y + let ((xa, ya), (xb, yb)) = if (pa.0, pa.1) <= (pb.0, pb.1) { + (pa, pb) + } else { + (pb, pa) + }; + lines.push([xa + h, ya, xb - h, yb]); + } + for (f, mo, kids) in sibships(people, &l.pos) { + lines.extend(descent_lines(f.as_deref(), mo.as_deref(), &kids, &l.pos)); + } + PageLayout { + width: 2 * MARGIN + SIZE + (l.widest - 1) * STEP_X, + height: TOP + SIZE + (l.ngen - 1) * STEP_Y + 30, + size: SIZE, + people: persons, + lines: lines.into_iter().map(|ln| ln.map(round1)).collect(), + } +} + +fn esc(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +fn symbol(s: &str, m: &Person, x: f64, y: f64, g: i64) -> String { + let h = SIZE as f64 / 2.0; + let fill = if m.sequenced { FILL } else { BG }; + let shape = match m.sex.as_str() { + "male" => format!( + r#" format!(r#" format!( + r#"{shape} fill="{fill}" stroke="{LINE}" stroke-width="2"/>{e}{e}: {sex}, {said}"#, + sex = m.sex, + seq = if m.sequenced { "yes" } else { "no" }, + ty = y + h + 16.0, + said = if m.sequenced { + "sequenced" + } else { + "not sequenced" + }, + ) +} + +/// The pedigree as an SVG, titled `title`; an error when the file names +/// no one. +pub fn draw(people: &People, title: &str) -> Result { + if people.is_empty() { + return Err("no person to draw".into()); + } + let l = layout(people); + let width = 2 * MARGIN + SIZE + (l.widest - 1) * STEP_X; + let legend_y = TOP + SIZE + (l.ngen - 1) * STEP_Y + 50; + let height = legend_y + 30; + let mut out: Vec = vec![ + format!( + r#""# + ), + format!(r#""#), + format!( + r#"{}"#, + esc(title) + ), + ]; + let h = SIZE as f64 / 2.0; + for (a, b) in &l.pairs { + let (mut pa, mut pb) = (l.pos[a], l.pos[b]); + if pa.0 > pb.0 { + std::mem::swap(&mut pa, &mut pb); + } + out.push(format!( + r#""#, + esc(a), + esc(b), + pa.0 + h, + pa.1, + pb.0 - h, + pb.1 + )); + } + for (f, mo, kids) in sibships(people, &l.pos) { + let mut parts = String::new(); + for [x1, y1, x2, y2] in descent_lines(f.as_deref(), mo.as_deref(), &kids, &l.pos) { + let _ = write!( + parts, + r#""# + ); + } + out.push(format!( + r#"{parts}"#, + esc(f.as_deref().unwrap_or(".")), + esc(mo.as_deref().unwrap_or(".")), + esc(&kids.join(" ")) + )); + } + for (s, m) in people { + let (x, y) = l.pos[s]; + out.push(symbol(s, m, x, y, l.generation[s])); + } + out.push(format!( + r#"Square: male. Circle: female. Diamond: sex not known. Filled: sequenced. A line joins partners; children hang from it."# + )); + out.push("".into()); + Ok(out.join("\n") + "\n") +} diff --git a/tests/test_live_family_depth.py b/tests/test_live_family_depth.py deleted file mode 100644 index 61e3d4c00..000000000 --- a/tests/test_live_family_depth.py +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/env python3 -"""The family page's depth (bin/owngenome_live_family.py family.json), on the demo's invented trio -(tests/fixtures/live/demo/DEMO-FAMILY) and on small made-up summaries. - -Checks: - 1. each child's stretches are its _segments.tsv row for row, and the chromosome lengths - the painting scales by are GRCh38's chr1 to chr22 and chrX; - 2. the compound-het pairs come in trans first, and every de novo row is there; - 3. a family without the trio files says they are not there (trio_files) and lists nothing; - 4. the members side by side: one row per gene or system any member's summary.json has (the CYP2D6 - callers and CPIC's CYP2D6 phenotype each a row of their own), one - column per member, '.' where a member's summary lacks the row, "no summary yet" for a member - without one; CPIC's phenotype quoted, never bare; - 5. a member's summary.json that links out of its run folder is never read; - 6. family.json passes the wording gate (live_results.FORBIDDEN); - 7. a value from a stale section is flagged stale in its own cell, and only that one; - 8. every blood group system step 41 typed gets a row (ABO and RhD first, then the rest in order); - 9. each child's trio status: assessed with both parents sequenced and the file here; not assessed - with the reason when a parent is not sequenced or not in the joint call; a missing file and a - file linking out of the folder told apart, the link's target never written; - 10. compound-het pairs come only from the child's step 23 candidates: assessed, not, or not known; - 11. a member's summary.json of a shape the report never writes (sections a list, CYP2D6 calls a - string, a CPIC gene named by a list) costs that member's column only: it says "summary not - readable" there, and every other member's column is as before. -Each check runs again on a planted mistake and must fail there: the segments read from the wrong -child (1), the pairs left in file order (2), a trio file planted (3), a member's summary dropped (4), -the inside check bypassed (5), "ok" planted in a row (6), the stale section made complete (7), the -systems list emptied (8), the link replaced by a plain file (9), a bad summary read -silently as empty (11). A check that cannot fail is no check. - -Run: python3 tests/test_live_family_depth.py -""" -import copy -import json -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import live_results # noqa: E402 -import owngenome_live # noqa: E402 -import owngenome_live_family as fam # noqa: E402 - -DEMO = os.path.join(REPO, "tests", "fixtures", "live", "demo", "DEMO-FAMILY") -FAILS = [] - - -def check(desc, ok, detail=""): - print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:800]}")) - if not ok: - FAILS.append(desc) - - -def control(desc, still_passes): - check(f"control: {desc}", not still_passes, "the check passed on the planted mistake") - - -def segment_problems(page, d): - rows = fam.read_tsv(os.path.join(d, "DEMO-003_segments.tsv")) - got = page["segments"].get("DEMO-003") or [] - out = [] - if [(r["side"], r["chrom"], r["start"], r["end"]) for r in got] != \ - [(r["side"], r["chrom"], r["start"], r["end"]) for r in rows] or not rows: - out.append(f"{len(got)} stretches against {len(rows)} rows") - if [c for c, _ in page["chrom_lengths"]] != [f"chr{i}" for i in range(1, 23)] + ["chrX"]: - out.append(f"lengths {page['chrom_lengths'][:3]}") - return out - - -def pair_problems(page): - phases = [r["phase"] for r in page["compound_het"]] - out = [] - if not phases or phases[0] != "in trans" or phases != sorted(phases, key=lambda p: fam.PHASE_ORDER.get(p, 1)): - out.append(f"order {phases}") - if len(page["de_novo"]) != 2: - out.append(f"{len(page['de_novo'])} de novo rows") - return out - - -def summary(cyp, cpic, hap, y=None): - S = {"cpic": {"state": "ok", "data": {"genes": [{"gene": g, "diplotype": dip, "phenotype": ph} for g, dip, ph in cpic]}}, - "haplogroup": {"state": "ok", "data": {"haplogroup": hap}}, - "bloodtype": {"state": "ok", "data": {"abo": "A1", "rhd": "D+"}}} - if y: - S["y_tree"] = {"state": "ok", "data": {"haplogroup": y}} - return {"cyp2d6": {"calls": {"pypgx": cyp, "Cyrius": cyp}, "agree": True}, "sections": S} - - -def side_problems(sb): - by = {(r["group"], r["label"]): r["values"] for r in sb["rows"]} - want = {("Medicines", "CYP2D6 (callers)"): ["pypgx *1/*4, Cyrius *1/*4 (callers agree)", "pypgx *1/*1, Cyrius *1/*1 (callers agree)", - "no summary yet"], - ("Medicines", "CYP2C19"): ['*1/*2 "Intermediate Metabolizer"', ".", "no summary yet"], - ("Medicines", "CYP2D6"): ['*1/*4 "Intermediate Metabolizer"', ".", "no summary yet"], - ("Medicines", "DPYD"): ['Reference/Reference "Normal Metabolizer"'] * 2 + ["no summary yet"], - ("Haplogroups", "Y chromosome"): ["I-M253", ".", "no summary yet"], - ("Blood groups", "ABO"): ["A1", "A1", "no summary yet"]} - out = [f"{k}: {by.get(k)} != {v}" for k, v in want.items() if by.get(k) != v] - if sb["members"] != ["A", "B", "C"]: - out.append(f"members {sb['members']}") - groups = [r["group"] for r in sb["rows"]] - if groups != sorted(groups, key=["Medicines", "HLA", "Haplogroups", "Blood groups"].index): - out.append(f"groups out of order {groups}") - return out - - -def stale_summary(state): - S = summary("*1/*1", [("DPYD", "Reference/Reference", "Normal Metabolizer")], "H1") - S["sections"]["bloodtype"] = {"state": state, "data": { - "abo": "A1", "rhd": "D+", "systems": [{"system": "ABO", "phenotype": "A1"}, {"system": "RhD", "phenotype": "D+"}, - {"system": "Kell", "phenotype": "K-k+"}, - {"system": "Duffy", "phenotype": "Fy(a+b-)"}]}} - return S - - -def source_summary(key, state): - """Every side-by-side source present and complete, with PharmCAT's CYP2D6 call, the Y haplogroup from - y_haplogroup alone (the fallback), and one section, key, in the given state.""" - S = summary("*1/*1", [("DPYD", "Reference/Reference", "Normal Metabolizer")], "H1") - S["cyp2d6"]["calls"]["PharmCAT"] = "*1/*1" - S["sections"]["hla"] = {"state": "ok", "data": {"loci": [{"gene": "HLA-A", "alleles": ["A*01:01", "A*02:01"]}]}} - S["sections"]["y_haplogroup"] = {"state": "ok", "data": {"haplogroup": "R-M269"}} - S["sections"][key]["state"] = state - return S - - -def stale_problems(sb): - """Member A's blood groups come from a stale section; B's from a complete one; C has no summary.""" - out = [] - for r in sb["rows"]: - want = [r["group"] == "Blood groups", False, False] - if r.get("stale") != want: - out.append(f"{r['label']}: stale {r.get('stale')} != {want}") - return out or ([] if sb["rows"] else ["no rows"]) - - -def blood_problems(sb): - labels = [r["label"] for r in sb["rows"] if r["group"] == "Blood groups"] - want = ["ABO", "RhD", "Kell", "Duffy"] - vals = {r["label"]: r["values"][0] for r in sb["rows"] if r["group"] == "Blood groups"} - out = [] if labels == want else [f"blood rows {labels} != {want}"] - if vals.get("Kell") != "K-k+" or vals.get("Duffy") != "Fy(a+b-)": - out.append(f"values {vals}") - return out - - -def trio_problems(st, want): - out = [] - for child, key, assessed, words in want: - got = (st.get(child) or {}).get(key) or {} - if got.get("assessed") is not assessed or words not in (got.get("why") or ""): - out.append(f"{child} {key}: {got} (want assessed={assessed}, '{words}')") - return out - - -def main(): - work = tempfile.mkdtemp(prefix="test_live_family_depth.") - try: - page = fam.family_page("DEMO-FAMILY", DEMO) - bad = segment_problems(page, DEMO) - check("1. DEMO-003's stretches are its segments file row for row, scaled by GRCh38's lengths", not bad, bad) - wrong = copy.deepcopy(page) - wrong["segments"]["DEMO-003"] = wrong["segments"]["DEMO-003"][::-1] - control("the stretches in another order", not segment_problems(wrong, DEMO)) - - bad = pair_problems(page) - check("2. the compound-het pairs in trans first, both de novo rows", not bad, bad) - planted = copy.deepcopy(page) - planted["compound_het"] = fam.read_tsv(os.path.join(DEMO, "compound_het.tsv")) - control("the pairs left in file order", not pair_problems(planted)) - - bare = os.path.join(work, "bare") - os.makedirs(bare) - shutil.copy(os.path.join(DEMO, "members.tsv"), bare) - b = fam.family_page("bare", bare) - ok = lambda p: (p["trio_files"] == {"de_novo.tsv": False, "compound_het.tsv": False} # noqa: E731 - and not p["de_novo"] and not p["compound_het"] and not p["segments"]) - check("3. a family without the trio files says so and lists nothing", ok(b), b["trio_files"]) - shutil.copy(os.path.join(DEMO, "de_novo.tsv"), bare) - control("a planted de_novo.tsv", ok(fam.family_page("bare", bare))) - - sums = {"A": summary("*1/*4", [("CYP2D6", "*1/*4", "Intermediate Metabolizer"), - ("CYP2C19", "*1/*2", "Intermediate Metabolizer"), - ("DPYD", "Reference/Reference", "Normal Metabolizer")], "H1", "I-M253"), - "B": summary("*1/*1", [("DPYD", "Reference/Reference", "Normal Metabolizer")], "U5"), - "C": None} - sb = fam.side_by_side(["A", "B", "C"], sums) - bad = side_problems(sb) - check("4. side by side: a row per gene or system, '.' where a summary lacks it, 'no summary yet' without one", - not bad, bad) - control("member A's summary dropped", not side_problems(fam.side_by_side(["A", "B", "C"], dict(sums, A=None)))) - - run = os.path.join(work, "root", "A") - os.makedirs(run) - outside = os.path.join(work, "outside.json") - with open(outside, "w") as f: - json.dump(sums["A"], f) - os.symlink(outside, os.path.join(run, "summary.json")) - - class Live: - root = os.path.realpath(os.path.join(work, "root")) - inside = staticmethod(owngenome_live.Live.inside) - check("5. a summary.json linking out of its run is not read", fam.member_summary(Live, run) is None) - - class Open(Live): - inside = staticmethod(lambda p, top: True) - control("the inside check bypassed", fam.member_summary(Open, run) is None) - - page["side_by_side"] = sb - bad = live_results.wording_problems(page) - check("6. family.json has no word that reads as an all-clear", not bad, bad) - planted = copy.deepcopy(page) - planted["side_by_side"]["rows"][0]["values"][0] = "ok" - control("an ok in a side-by-side row", not live_results.wording_problems(planted)) - - sums7 = {"A": stale_summary("stale"), "B": summary("*1/*1", [], "U5"), "C": None} - sb7 = fam.side_by_side(["A", "B", "C"], sums7) - bad = stale_problems(sb7) - check("7. a stale section's values are flagged stale in their own cells, and no other value is", not bad, bad) - control("the stale section made complete", - not stale_problems(fam.side_by_side(["A", "B", "C"], dict(sums7, A=stale_summary("ok"))))) - - bad = blood_problems(sb7) - check("8. every blood group system typed gets a row: ABO, RhD, then Kell and Duffy in order", not bad, bad) - emptied = stale_summary("stale") - emptied["sections"]["bloodtype"]["data"]["systems"] = [] - control("the systems list emptied", not blood_problems(fam.side_by_side(["A", "B", "C"], dict(sums7, A=emptied)))) - - flags = {"cpic": {"CYP2D6 (callers)", "DPYD"}, "hla": {"HLA-A"}, "haplogroup": {"Mitochondrial"}, - "y_haplogroup": {"Y chromosome"}} - bad = [] - for key in flags: - sb = fam.side_by_side(["A"], {"A": source_summary(key, "stale")}) - got = {r["label"] for r in sb["rows"] if r["stale"][0]} - if got != flags[key]: - bad.append(f"{key} stale flags {sorted(got)} != {sorted(flags[key])}") - check("7. each stale source flags its own cells only: CPIC (and PharmCAT's CYP2D6 call), HLA, the " - "haplogroups (the Y fallback too)", not bad, bad) - sb = fam.side_by_side(["A"], {"A": source_summary("cpic", "ok")}) - control("the CPIC section made complete", {r["label"] for r in sb["rows"] if r["stale"][0]} == flags["cpic"]) - - short = stale_summary("ok") - short["sections"]["bloodtype"]["data"]["systems"] = [x for x in short["sections"]["bloodtype"]["data"]["systems"] - if x["system"] != "Kell"] - merged = lambda a, b_: [r["label"] for r in fam.side_by_side(["A", "B"], {"A": a, "B": b_})["rows"] # noqa: E731 - if r["group"] == "Blood groups"] - got = merged(short, stale_summary("ok")) - kidd = lambda s, drop: [x for x in s + [{"system": "Kidd", "phenotype": "Jk(a+b+)"}] # noqa: E731 - if x["system"] != drop] - a_, b2 = stale_summary("ok"), stale_summary("ok") - a_["sections"]["bloodtype"]["data"]["systems"] = kidd(a_["sections"]["bloodtype"]["data"]["systems"], "Duffy") - b2["sections"]["bloodtype"]["data"]["systems"] = kidd(b2["sections"]["bloodtype"]["data"]["systems"], "Kell") - got2 = merged(a_, b2) - check("8. a system only a later member has keeps step 41's order, also when each lacks one", - got == ["ABO", "RhD", "Kell", "Duffy"] and got2 == ["ABO", "RhD", "Kell", "Duffy", "Kidd"], (got, got2)) - saved, fam.STEP41_SYSTEMS = fam.STEP41_SYSTEMS, () - try: - control("step 41's order taken away", merged(a_, b2) == ["ABO", "RhD", "Kell", "Duffy", "Kidd"]) - finally: - fam.STEP41_SYSTEMS = saved - sys.path.insert(0, os.path.join(REPO, "tests")) - from og_binary import og_binary - step41 = subprocess.run([og_binary(), "bloodtype", "systems"], capture_output=True, text=True, check=True) - check("8. the page's order of the systems is step 41's (og bloodtype systems)", - list(fam.STEP41_SYSTEMS) == step41.stdout.split(), (fam.STEP41_SYSTEMS, step41.stdout)) - control("a system moved in the page's order", - list(fam.STEP41_SYSTEMS[1:2] + fam.STEP41_SYSTEMS[:1] + fam.STEP41_SYSTEMS[2:]) == step41.stdout.split()) - - tri = os.path.join(work, "trio") - shutil.copytree(DEMO, tri) - with open(os.path.join(tri, "members.tsv"), "a") as f: - f.write("DEMO-004\tfemale\tDEMO-001\tDEMO-005\t\tyes\tyes\nDEMO-005\tfemale\t\t\t\tyes\tno\n") - plan_p = os.path.join(tri, "family_plan.tsv") - with open(plan_p) as f: - plan_text = f.read() - with open(plan_p, "w") as f: - f.write(plan_text.replace("DEMO-001,DEMO-002,DEMO-003\t", "DEMO-001,DEMO-002,DEMO-003,DEMO-004\t")) - secret = os.path.join(work, "elsewhere-private-name.tsv") - shutil.copy(os.path.join(DEMO, "compound_het.tsv"), secret) - os.remove(os.path.join(tri, "compound_het.tsv")) - os.symlink(secret, os.path.join(tri, "compound_het.tsv")) - want = [("DEMO-003", "de_novo", True, "both parents"), - ("DEMO-004", "de_novo", False, "DEMO-005 (the mother) is not"), - ("DEMO-003", "compound_het", False, "link out of the family folder"), - ("DEMO-004", "compound_het", False, "link out of the family folder")] - tp = fam.family_page("trio", tri) - bad = trio_problems(tp["trio_status"], want) - if tp["trio_file_state"] != {"de_novo.tsv": "here", "compound_het.tsv": "rejected"}: - bad.append(f"file state {tp['trio_file_state']}") - if "elsewhere-private-name" in json.dumps(tp): - bad.append("the link's target is in family.json") - check("9. trio status per child: assessed, a parent not sequenced, a file linking out refused", not bad, bad) - with open(plan_p, "w") as f: - f.write(plan_text) - out_of_call = trio_problems(fam.family_page("trio", tri)["trio_status"], - [("DEMO-004", "de_novo", False, "DEMO-004 is not in the joint call")]) - check("9. a child left out of the joint call is not assessed, and the page says so", not out_of_call, out_of_call) - os.remove(os.path.join(tri, "de_novo.tsv")) - miss = trio_problems(fam.family_page("trio", tri)["trio_status"], - [("DEMO-003", "de_novo", False, "de_novo.tsv is not here")]) - check("9. a missing de_novo.tsv is not assessed, said apart from a refused one", not miss, miss) - os.remove(os.path.join(tri, "compound_het.tsv")) - shutil.copy(secret, os.path.join(tri, "compound_het.tsv")) - control("the link replaced by a plain file", not trio_problems(fam.family_page("trio", tri)["trio_status"], want[2:])) - shutil.copy(os.path.join(DEMO, "de_novo.tsv"), tri) - with open(plan_p, "w") as f: - f.write(plan_text.replace("DEMO-001,DEMO-002,DEMO-003\t", "DEMO-002,DEMO-003,DEMO-004\t")) - st9 = fam.family_page("trio", tri)["trio_status"] - bad = trio_problems(st9, [("DEMO-003", "de_novo", False, "DEMO-001 (the father) is not"), - ("DEMO-003", "compound_het", True, "a parent is sequenced"), - ("DEMO-004", "de_novo", False, "DEMO-001 (the father) is not"), - ("DEMO-004", "compound_het", False, "need a parent sequenced and in the joint call")]) - check("9. a sequenced parent left out of the joint call: no de novo, and no pairs when it was the only one", - not bad, bad) - with open(plan_p, "w") as f: - f.write(plan_text) - control("the parent back in the joint call", not trio_problems( - fam.family_page("trio", tri)["trio_status"], [("DEMO-003", "de_novo", False, "DEMO-001 (the father) is not")])) - - st10 = fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": "absent", "DEMO-004": "here"}) - bad = trio_problems(st10, [("DEMO-003", "compound_het", False, "wrote no candidates for DEMO-003"), - ("DEMO-004", "compound_het", False, "need a parent")]) - st10 = fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": "here"}) - bad += trio_problems(st10, [("DEMO-003", "compound_het", True, "candidates (step 23) are here")]) - for g, words in (("newer", "newer than compound_het.tsv"), (None, "run is not on this site")): - st10 = fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": g}) - bad += trio_problems(st10, [("DEMO-003", "compound_het", None, words)]) - check("10. compound-het pairs by the child's step 23 candidates: here assessed, absent not, newer or no run " - "not known", not bad, bad) - control("the candidates not known read as absent", - not trio_problems(fam.with_candidates(copy.deepcopy(st9), {"DEMO-003": None}), - [("DEMO-003", "compound_het", False, "wrote no candidates")])) - run3 = os.path.join(work, "root", "DEMO-003") - os.makedirs(os.path.join(run3, "clinical")) - vcf = os.path.join(run3, "clinical", "DEMO-003_clinical.vcf.gz") - none_yet = fam.candidates_given(Live, run3, "DEMO-003") - open(vcf, "w").close() - os.utime(vcf, (1000, 1000)) - here, newer = fam.candidates_given(Live, run3, "DEMO-003", 2000), fam.candidates_given(Live, run3, "DEMO-003", 500) - os.remove(vcf) - os.symlink(outside, vcf) - got = (none_yet, here, newer, fam.candidates_given(Live, run3, "DEMO-003"), fam.candidates_given(Live, None, "DEMO-003")) - check("10. candidates: none in the run, here, newer than compound_het.tsv, a link out of the run, no run", - got == ("absent", "here", "newer", "absent", None), got) - control("the inside check bypassed for the candidates", fam.candidates_given(Open, run3, "DEMO-003") == "absent") - - def malformed_problems(): - out = [] - for bad in ({"sections": ["not", "a", "mapping"]}, {"cyp2d6": "CYP2D6 *1/*4"}, - {"sections": {"cpic": {"state": "ok", "data": {"genes": [{"gene": ["CYP2D6"]}]}}}}): - try: - got = fam.side_by_side(["A", "B", "C"], dict(sums, B=bad)) - except Exception as e: # noqa: BLE001 -- the very thing under test - out.append(f"{bad}: raised {type(e).__name__}") - continue - cols = {r["label"]: r["values"] for r in got["rows"]} - want = {r["label"]: r["values"] for r in fam.side_by_side(["A", "B", "C"], dict(sums, B=None))["rows"]} - for label, v in want.items(): - if cols.get(label, [None] * 3)[0] != v[0] or cols.get(label, [None] * 3)[1] != "summary not readable": - out.append(f"{bad}: {label} {cols.get(label)}") - return out - bad = malformed_problems() - check("11. a member's summary of a shape the report never writes costs that member's column only", not bad, bad) - real = fam.side_values - - def quiet(summary): # the planted mistake: a bad summary read as one with no rows, never said - try: - return real(summary) - except (AttributeError, TypeError): - return [] - fam.side_values = quiet - try: - control("a bad summary shown as '.' in every row, as if it had none of them", not malformed_problems()) - finally: - fam.side_values = real - finally: - shutil.rmtree(work, ignore_errors=True) - if FAILS: - print(f"\n{len(FAILS)} check(s) failed") - return 1 - print("\nAll checks passed") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_pedigree_svg.py b/tests/test_pedigree_svg.py deleted file mode 100755 index ea2c4ac30..000000000 --- a/tests/test_pedigree_svg.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python3 -"""bin/pedigree_svg.py on tests/fixtures/families_live. - -Checks, reading the drawing back from its data- attributes and coordinates: - 1. family_a (members.tsv, three generations): every person is drawn once, - with the sex and sequenced state of the file; each generation is one - row, parents above their children; - 2. each line of descent starts at its two parents (the middle of their - partner line) and ends on top of each child it names, and the parents - it names are the ones this test expects (written here, not read from - the fixture); - 3. family_b (pedigree.tsv only): the couple is drawn joined, both filled; - 4. a person whose sex is not known is a diamond; the command line writes - the same SVG and exits 2 on a folder without a pedigree. -Checks 1 and 2 are run again on a copy of the fixture with two parents -swapped and must fail there: a check that cannot fail is no check. - -Run: python3 tests/test_pedigree_svg.py -""" -import os -import shutil -import subprocess -import sys -import tempfile -import xml.etree.ElementTree as ET - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import pedigree_svg # noqa: E402 - -FIX = os.path.join(REPO, "tests", "fixtures", "families_live") -NS = "{http://www.w3.org/2000/svg}" -FAILS = [] - -# What family_a declares, written out by hand: child -> (father, mother). -EXPECTED = {"child_a": ("father_a", "mother_a"), "child_b": ("father_a", "mother_a"), - "father_a": ("gf_a", "gm_a"), "mother_a": (".", "gm_m")} -SEX = {"child_a": "female", "child_b": "male", "father_a": "male", "mother_a": "female", "gf_a": "male", - "gm_a": "female", "gm_m": "female"} -SEQUENCED = {"gm_m": "no"} - - -def check(name, ok, detail=""): - print(("[ OK ] " if ok else "[FAIL] ") + name + ("" if ok else " %s" % (detail,))) - if not ok: - FAILS.append(name) - - -def control(name, passed): - """A check run on a planted mistake: it must fail there.""" - print(("[ OK ] control: " if not passed else "[FAIL] control: ") + name + - (" was caught" if not passed else " was NOT caught")) - if passed: - FAILS.append("control: " + name) - - -def parse(svg): - root = ET.fromstring(svg) - people = {} - for g in root.iter(NS + "g"): - if g.get("class") == "person": - shape = [c for c in g if c.tag in (NS + "rect", NS + "circle", NS + "polygon")][0] - if shape.tag == NS + "rect": - cx, cy = float(shape.get("x")) + 18, float(shape.get("y")) + 18 - elif shape.tag == NS + "circle": - cx, cy = float(shape.get("cx")), float(shape.get("cy")) - else: - pts = [tuple(map(float, p.split(","))) for p in shape.get("points").split()] - cx, cy = pts[0][0], (pts[0][1] + pts[2][1]) / 2 - people.setdefault(g.get("data-sample"), []).append( - {"sex": g.get("data-sex"), "gen": int(g.get("data-gen")), "seq": g.get("data-sequenced"), - "shape": shape.tag[len(NS):], "x": cx, "y": cy, "fill": shape.get("fill")}) - descents = [] - for g in root.iter(NS + "g"): - if g.get("class") == "descent": - lines = [{k: float(ln.get(k)) for k in ("x1", "y1", "x2", "y2")} for ln in g.iter(NS + "line")] - descents.append({"father": g.get("data-father"), "mother": g.get("data-mother"), - "children": g.get("data-children").split(), "lines": lines}) - couples = [(ln.get("data-a"), ln.get("data-b")) for ln in root.iter(NS + "line") if ln.get("class") == "couple"] - return people, descents, couples - - -def people_ok(people): - bad = [s for s in SEX if len(people.get(s, [])) != 1 or people[s][0]["sex"] != SEX[s] - or people[s][0]["seq"] != SEQUENCED.get(s, "yes") - or people[s][0]["shape"] != {"male": "rect", "female": "circle"}[SEX[s]]] - return not bad and set(people) == set(SEX), bad or sorted(set(people) ^ set(SEX)) - - -def rows_ok(people): - bad = [] - for child, (f, m) in EXPECTED.items(): - for p in (f, m): - if p != "." and not people[p][0]["y"] < people[child][0]["y"]: - bad.append((p, child)) - ys = {} - for s, v in people.items(): - ys.setdefault(v[0]["gen"], set()).add(v[0]["y"]) - return not bad and all(len(y) == 1 for y in ys.values()), (bad, ys) - - -def descent_ok(people, descents): - """Every child of EXPECTED hangs from a descent that names its expected parents, starts at - their middle and ends on the child's symbol.""" - bad, seen = [], {} - for d in descents: - for c in d["children"]: - seen[c] = (d["father"], d["mother"]) - ps = [people[p][0] for p in (d["father"], d["mother"]) if p != "."] - ox = sum(p["x"] for p in ps) / len(ps) - first = d["lines"][0] - if abs(first["x1"] - ox) > 0.5 or abs(first["x2"] - ox) > 0.5: - bad.append(("start", d["father"], d["mother"])) - for c in d["children"]: - p = people[c][0] - if not any(abs(ln["x1"] - p["x"]) < 0.5 and abs(ln["y2"] - (p["y"] - 18)) < 0.5 for ln in d["lines"]): - bad.append(("end", c)) - if seen != EXPECTED: - bad.append(("parents", seen)) - return not bad, bad - - -def run_checks(folder): - order, members, _ = pedigree_svg.load(folder) - people, descents, couples = parse(pedigree_svg.draw(order, members, "family_a")) - return [people_ok(people), rows_ok(people), descent_ok(people, descents)], couples - - -def main(): - work = tempfile.mkdtemp(prefix="pedigree-svg-", dir=os.environ.get("TMPDIR")) - try: - results, couples = run_checks(os.path.join(FIX, "family_a")) - names = ["every person drawn once with the file's sex and sequenced state", - "one row per generation, parents above their children", - "each line of descent runs from the expected parents to each child"] - for n, (ok, detail) in zip(names, results): - check(n, ok, detail) - check("partners joined: father_a with mother_a, gf_a with gm_a", - {frozenset(c) for c in couples} == {frozenset(("father_a", "mother_a")), frozenset(("gf_a", "gm_a"))}, - couples) - - # Planted mistake: child_a's parents swapped with father_a's parents. - bad = os.path.join(work, "swapped") - os.makedirs(bad) - with open(os.path.join(FIX, "family_a", "members.tsv")) as f: - text = f.read() - lines = text.split("\n") - for i, ln in enumerate(lines): - c = ln.split("\t") - if c[0] == "child_a": - c[2], c[3] = "gf_a", "gm_a" - elif c[0] == "father_a": - c[2], c[3] = "", "" - lines[i] = "\t".join(c) - with open(os.path.join(bad, "members.tsv"), "w") as f: - f.write("\n".join(lines)) - results, _ = run_checks(bad) - control("child_a drawn under its grandparents", results[2][0] and results[1][0]) - - order, members, _ = pedigree_svg.load(os.path.join(FIX, "family_b")) - people, descents, couples = parse(pedigree_svg.draw(order, members, "family_b")) - check("family_b from pedigree.tsv: a joined, filled couple and no descent", - couples == [("partner_x", "partner_y")] and not descents - and [people[s][0]["seq"] for s in ("partner_x", "partner_y")] == ["yes", "yes"], (couples, people)) - - unk = os.path.join(work, "unknown") - os.makedirs(unk) - with open(os.path.join(unk, "pedigree.tsv"), "w") as f: - f.write("sample\tsex\tfather\tmother\tpartner\nkid_u\tunknown\tdad_u\tmum_u\t.\n") - order, members, _ = pedigree_svg.load(unk) - people, descents, _ = parse(pedigree_svg.draw(order, members)) - check("a sex not known is a diamond; named parents without a row are drawn not sequenced", - people["kid_u"][0]["shape"] == "polygon" and people["dad_u"][0]["seq"] == "no" - and descents and descents[0]["children"] == ["kid_u"], people) - - r = subprocess.run([sys.executable, os.path.join(REPO, "bin", "pedigree_svg.py"), - os.path.join(FIX, "family_a")], capture_output=True, text=True, timeout=60) - order, members, _ = pedigree_svg.load(os.path.join(FIX, "family_a")) - check("the command line writes the same SVG", r.returncode == 0 and r.stdout == - pedigree_svg.draw(order, members, "family_a"), r.stderr) - empty = os.path.join(work, "empty") - os.makedirs(empty) - r = subprocess.run([sys.executable, os.path.join(REPO, "bin", "pedigree_svg.py"), empty], - capture_output=True, text=True, timeout=60) - check("a folder without a pedigree exits 2 and says why", r.returncode == 2 and "members.tsv" in r.stderr, - (r.returncode, r.stderr)) - finally: - shutil.rmtree(work, ignore_errors=True) - if FAILS: - print("%d check(s) failed: %s" % (len(FAILS), ", ".join(FAILS))) - return 1 - print("all checks passed") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From ae127c4a0374e2db398c385413388a8ec685b227 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:25:13 +0100 Subject: [PATCH 2/3] refactor(og-live): build the family page in og live family og live family state, page, families, files and svg give the family's state, family.json, the run list's families, the files a family serves and pedigree.svg. The cargo tests carry the checks of the Python tests this replaces, each with a planted mistake it must catch. Bead: D1 --- rust/Cargo.lock | 5 + rust/crates/og-live/Cargo.toml | 11 +- rust/crates/og-live/src/family.rs | 1618 +++++++++++++++++++- rust/crates/og-live/tests/family_layout.rs | 451 ++++++ rust/crates/og-live/tests/family_page.rs | 1425 +++++++++++++++++ 5 files changed, 3500 insertions(+), 10 deletions(-) create mode 100644 rust/crates/og-live/tests/family_layout.rs create mode 100644 rust/crates/og-live/tests/family_page.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6bfd68843..13e7018e8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -845,9 +845,14 @@ version = "0.1.0" dependencies = [ "indexmap", "jiff", + "og-bloodtype", + "og-io", "og-nf", + "og-pedigree", + "og-render", "og-runlog", "pycompat", + "regex", ] [[package]] diff --git a/rust/crates/og-live/Cargo.toml b/rust/crates/og-live/Cargo.toml index e23bd6872..44b85aeb2 100644 --- a/rust/crates/og-live/Cargo.toml +++ b/rust/crates/og-live/Cargo.toml @@ -15,9 +15,16 @@ pycompat.workspace = true og-nf.workspace = true og-runlog.workspace = true jiff.workspace = true - -[dev-dependencies] indexmap.workspace = true +og-io.workspace = true +# The family page: the pedigree and its drawing, the couple screen's caveat +# and step 41's order of the blood group systems, each from its one source. +og-pedigree.workspace = true +og-render.workspace = true +og-bloodtype.workspace = true [lints] workspace = true + +[dev-dependencies] +regex.workspace = true diff --git a/rust/crates/og-live/src/family.rs b/rust/crates/og-live/src/family.rs index ab31e08b1..eb75706ec 100644 --- a/rust/crates/og-live/src/family.rs +++ b/rust/crates/og-live/src/family.rs @@ -1,12 +1,1614 @@ -//! `og live family`: the live site's family page. +//! `og live family`: the live site's family page, from the files the +//! family steps wrote. //! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! A family folder under GENOME_DIR (one holding `members.tsv` or +//! `pedigree.tsv`, as steps 38 and 39 and the Nextflow family workflow +//! leave it) is a run of kind `family`. Its page shows the pedigree (drawn +//! from that file at request time, [`og_pedigree::layout`]), each child's +//! chromosome map (`_segments.svg`), the relationship check, the +//! shared segments and the couple carrier screen, as the steps wrote them. +//! +//! ```text +//! og live family state DIR [--name NAME] +//! state.json's plugins.family of the family in DIR: its members, each +//! child with its parents and map, the relationship and shared-segment +//! rows, each couple's counts, the plan and the files the page may open. +//! og live family page DIR [--name NAME] [--root ROOT] [--input FILE|-] +//! family.json: the state, the pedigree's layout (so each person is a +//! link), the relationship and shared-segment rows with their measured +//! numbers, each couple's gene rows, each child's inherited stretches +//! (_segments.tsv), the de novo and compound-het candidates +//! (de_novo.tsv, compound_het.tsv) with whether each child was assessed +//! and why not, and GRCh38's chromosome lengths. INPUT (JSON) holds what +//! only the site knows: "runs" (each member run's entry, copied into +//! family.json as it is), "candidates" (each child's run folder: the +//! compound-het line then says whether step 23 wrote the candidates step +//! 38 pairs) and "summaries" (each member's run folder: the members side +//! by side from each summary.json). A run folder is read only inside +//! ROOT. Exit 1 when a file of the family cannot be read. +//! og live family families ROOT +//! every family folder under ROOT as /runs.json lists it: name, dir, +//! kind, members ([] while the pedigree cannot be read) and unreadable +//! (while family.json cannot be built from the folder). +//! og live family files DIR +//! the files of the family folder the page may open, as a JSON list. +//! Exit 2 when the folder holds no pedigree that names anyone. +//! og live family svg DIR [--title TEXT] +//! the pedigree as an SVG (default title: the folder's name). Exit 2 when +//! the folder holds no pedigree that names anyone. +//! ``` +//! +//! It computes nothing about genetics: it reads the files the steps wrote, +//! counts their rows and draws the pedigree the file declares. A file of +//! the family folder that links out of it is never read, and the page never +//! shows where it points. + +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::io::{self, BufReader, Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::time::SystemTime; + +use indexmap::IndexMap; +use og_io::{Lines, Utf8}; +use og_pedigree::layout::{self, People}; +use pycompat::json::{Dump, Value, dumps, loads}; +use pycompat::value::{get, py_str, truthy}; + +const USAGE: &str = "usage: og live family state DIR [--name NAME]\n \ + og live family page DIR [--name NAME] [--root ROOT] [--input FILE|-]\n \ + og live family families ROOT\n \ + og live family files DIR\n \ + og live family svg DIR [--title TEXT]\n"; + +/// The family's result files the page may open, besides the drawn pedigree +/// and the per-child and per-couple files (named from the members). +const FIXED: [&str; 7] = [ + "family_report.md", + "ibd_report.md", + "relationships.tsv", + "ibd_pairs.tsv", + "family_plan.tsv", + "members.tsv", + "pedigree.tsv", +]; +pub const TRIO_FILES: [&str; 2] = ["de_novo.tsv", "compound_het.tsv"]; +const SEGMENT_KEYS: [&str; 10] = [ + "side", + "parent", + "chrom", + "start", + "end", + "length_bp", + "sites", + "discordant", + "ancestor", + "basis", +]; +const DE_NOVO_KEYS: [&str; 12] = [ + "child", + "chrom", + "pos", + "ref", + "alt", + "child_gt", + "father_gt", + "mother_gt", + "child_dp", + "child_gq", + "child_ab", + "quality", +]; +const COMP_KEYS: [&str; 7] = [ + "child", + "gene", + "variant_a", + "variant_b", + "a_from", + "b_from", + "phase", +]; +const REL_KEYS: [&str; 10] = [ + "sample_a", + "sample_b", + "declared", + "expected_relatedness", + "relatedness", + "ibs0", + "ibs2", + "sites", + "verdict", + "reason", +]; +const IBD_KEYS: [&str; 9] = [ + "sample_a", + "sample_b", + "declared", + "segments", + "shared_cm", + "ibd2_cm", + "longest_cm", + "relatedness", + "degree", +]; +const COUPLE_KEYS: [&str; 10] = [ + "gene", + "inheritance", + "condition", + "result", + "offspring_risk", + "partner_a", + "findings_a", + "partner_b", + "findings_b", + "explanation", +]; +/// GRCh38's chr1 to chr22 and chrX, which the painting of each child's +/// stretches scales by (UCSC hg38 chrom.sizes). +pub const GRCH38: [(&str, i64); 23] = [ + ("chr1", 248956422), + ("chr2", 242193529), + ("chr3", 198295559), + ("chr4", 190214555), + ("chr5", 181538259), + ("chr6", 170805979), + ("chr7", 159345973), + ("chr8", 145138636), + ("chr9", 138394717), + ("chr10", 133797422), + ("chr11", 135086622), + ("chr12", 133275309), + ("chr13", 114364328), + ("chr14", 107043718), + ("chr15", 101991189), + ("chr16", 90338345), + ("chr17", 83257441), + ("chr18", 80373285), + ("chr19", 58617616), + ("chr20", 64444167), + ("chr21", 46709983), + ("chr22", 50818468), + ("chrX", 156040895), +]; +/// The groups of the side-by-side table, in its order. +const GROUPS: [&str; 4] = ["Medicines", "HLA", "Haplogroups", "Blood groups"]; + +/// A file of the family that exists but cannot be read: family.json is +/// then not built. The text names the kind of error, never a path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Unreadable(pub String); + +impl Unreadable { + fn of(e: &io::Error) -> Unreadable { + Unreadable( + match e.kind() { + io::ErrorKind::PermissionDenied => "PermissionError", + io::ErrorKind::NotFound => "FileNotFoundError", + io::ErrorKind::IsADirectory => "IsADirectoryError", + _ => "OSError", + } + .into(), + ) + } +} + +// --- paths ------------------------------------------------------------------------------ + +/// `os.path.realpath`: links followed as far as they go, `..` taken after +/// them; a part that does not exist is kept as written. +pub fn realpath(p: &Path) -> PathBuf { + fn resolve(p: &Path, depth: usize) -> PathBuf { + let mut out = PathBuf::from("/"); + for c in p.components() { + match c { + Component::Prefix(_) | Component::RootDir | Component::CurDir => {} + Component::ParentDir => { + out.pop(); + } + Component::Normal(name) => { + let cand = out.join(name); + let link = + fs::symlink_metadata(&cand).is_ok_and(|m| m.file_type().is_symlink()); + out = match fs::read_link(&cand) { + Ok(t) if link && depth < 40 => resolve(&out.join(t), depth + 1), + _ => cand, + }; + } + } + } + out + } + resolve(&absolute(p), 0) +} + +/// `os.path.abspath`: against the working folder, `.` and `..` taken by +/// their names alone. +fn absolute(p: &Path) -> PathBuf { + let joined = if p.is_absolute() { + p.to_path_buf() + } else { + std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from("/")) + .join(p) + }; + let mut out = PathBuf::from("/"); + for c in joined.components() { + match c { + Component::ParentDir => { + out.pop(); + } + Component::Normal(n) => out.push(n), + _ => {} + } + } + out +} + +/// Whether `path`, links followed, is `top` or inside it. +pub fn inside(path: &Path, top: &Path) -> bool { + realpath(path).starts_with(realpath(top)) +} + +fn is_file(p: &Path) -> bool { + fs::metadata(p).is_ok_and(|m| m.is_file()) +} + +/// A folder holding `members.tsv` or `pedigree.tsv`. +pub fn is_family(d: &Path) -> bool { + d.is_dir() && layout::MARKERS.iter().any(|m| is_file(&d.join(m))) +} + +/// A run name: a letter or digit, then letters, digits, `.`, `_` and `-`. +pub fn valid_name(n: &str) -> bool { + let mut cs = n.chars(); + cs.next().is_some_and(|c| c.is_ascii_alphanumeric()) + && cs.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) +} + +/// The family folders under `root`, `(name, dir)` sorted by name. +pub fn family_dirs(root: &Path) -> Vec<(String, PathBuf)> { + let Ok(rd) = fs::read_dir(root) else { + return Vec::new(); + }; + let mut names: Vec = rd + .filter_map(|e| e.ok()?.file_name().into_string().ok()) + .collect(); + names.sort(); + names + .into_iter() + .filter_map(|n| { + let d = root.join(&n); + (!n.starts_with('.') && valid_name(&n) && inside(&d, root) && is_family(&d)) + .then_some((n, d)) + }) + .collect() +} + +// --- the family's files ----------------------------------------------------------------- + +/// A row of a tab-separated file by its header's names. +pub type Row = IndexMap; + +/// The rows of a tab-separated file with a header line, comments and blank +/// lines skipped, short rows padded; none when it is not there or is a link +/// out of its own folder (the page serves only what the family folder +/// itself holds). +pub fn read_tsv(path: &Path) -> Result, Unreadable> { + let dir = absolute(path) + .parent() + .map(Path::to_path_buf) + .unwrap_or_default(); + if !is_file(path) || !inside(path, &dir) { + return Ok(Vec::new()); + } + let f = fs::File::open(path).map_err(|e| Unreadable::of(&e))?; + let mut lines = Lines::new(BufReader::new(f), Utf8::Replace); + let mut line = String::new(); + let head: Vec = if lines.read_line(&mut line).map_err(|e| Unreadable::of(&e))? { + line.trim_end_matches(['\r', '\n']) + .trim_start_matches('#') + .split('\t') + .map(str::to_owned) + .collect() + } else { + vec![String::new()] + }; + let mut rows = Vec::new(); + while lines.read_line(&mut line).map_err(|e| Unreadable::of(&e))? { + let l = line.trim_end_matches(['\r', '\n']); + if pycompat::text::strip(l, None).is_empty() || l.starts_with('#') { + continue; + } + let vals: Vec<&str> = l.split('\t').collect(); + let mut r = Row::new(); + for (i, h) in head.iter().enumerate() { + r.insert(h.clone(), vals.get(i).copied().unwrap_or("").to_owned()); + } + rows.push(r); + } + Ok(rows) +} + +fn cell<'a>(r: &'a Row, k: &str) -> Option<&'a str> { + r.get(k).map(String::as_str) +} + +fn s(v: &str) -> Value { + Value::Str(v.to_owned()) +} + +fn opt(v: Option<&str>) -> Value { + v.map_or(Value::Null, s) +} + +fn obj(pairs: [(&str, Value); N]) -> Value { + Value::Object(pairs.into_iter().map(|(k, v)| (k.to_owned(), v)).collect()) +} + +fn keep(r: &Row, keys: &[&str]) -> Value { + Value::Object( + keys.iter() + .map(|k| ((*k).to_owned(), opt(cell(r, k)))) + .collect(), + ) +} + +fn as_obj(v: &mut Value) -> &mut IndexMap { + match v { + Value::Object(o) => o, + _ => unreachable!("built as an object"), + } +} + +/// The people of the family folder, or why the pedigree cannot be read. +pub fn members_of(d: &Path) -> Result { + let (people, _) = layout::load(d)?; + if people.is_empty() { + return Err("the pedigree names no one".into()); + } + Ok(people) +} + +/// `(a, b, couple__)` of each pair of partners, `a` and `b` sorted by +/// name as step 39 names them. +pub fn couple_names(people: &People) -> Vec<(String, String, String)> { + layout::couples(people) + .into_iter() + .filter(|(a, b)| { + people[a].partner.as_deref() == Some(b.as_str()) + || people[b].partner.as_deref() == Some(a.as_str()) + }) + .map(|(a, b)| { + let (x, y) = if a <= b { (a, b) } else { (b, a) }; + let base = format!("couple_{x}_{y}"); + (x, y, base) + }) + .collect() +} + +/// The file names of the family folder the page may open: those that are +/// there, inside the folder. +pub fn served_files(d: &Path, people: &People) -> Vec { + let mut names: Vec = FIXED.iter().map(|f| (*f).to_owned()).collect(); + names.extend(people.keys().map(|s| format!("{s}_segments.svg"))); + for (_, _, base) in couple_names(people) { + names.push(format!("{base}.md")); + names.push(format!("{base}.tsv")); + } + let mut out: Vec = names + .into_iter() + .collect::>() + .into_iter() + .filter(|n| is_file(&d.join(n)) && inside(&d.join(n), d)) + .collect(); + out.sort(); + out +} + +fn people_rows(people: &People) -> Vec { + people + .iter() + .map(|(s_, m)| { + obj([ + ("sample", s(s_)), + ("sex", s(&m.sex)), + ("father", opt(m.father.as_deref())), + ("mother", opt(m.mother.as_deref())), + ("partner", opt(m.partner.as_deref())), + ("sequenced", Value::Bool(m.sequenced)), + ]) + }) + .collect() +} + +/// state.json's `plugins.family` on the family's page. +pub fn family_state(name: &str, d: &Path) -> Result { + let people = match members_of(d) { + Ok(p) => p, + Err(e) => { + return Ok(obj([ + ("kind", s("family")), + ( + "error", + Value::Str(format!("the pedigree could not be read: {e}")), + ), + ])); + } + }; + let files = served_files(d, &people); + let url = format!("/family/{name}/"); + let link = |f: &str| { + if files.iter().any(|x| x == f) { + Value::Str(format!("{url}{f}")) + } else { + Value::Null + } + }; + let children: Vec = people + .iter() + .filter(|(_, m)| m.father.is_some() || m.mother.is_some()) + .map(|(c, m)| { + obj([ + ("child", s(c)), + ("father", opt(m.father.as_deref())), + ("mother", opt(m.mother.as_deref())), + ("segments", link(&format!("{c}_segments.svg"))), + ]) + }) + .collect(); + let rel = read_tsv(&d.join("relationships.tsv"))? + .iter() + .map(|r| { + obj([ + ("a", opt(cell(r, "sample_a"))), + ("b", opt(cell(r, "sample_b"))), + ("declared", opt(cell(r, "declared"))), + ("verdict", opt(cell(r, "verdict"))), + ("reason", opt(cell(r, "reason"))), + ]) + }) + .collect(); + let ibd = read_tsv(&d.join("ibd_pairs.tsv"))? + .iter() + .map(|r| { + obj([ + ("a", opt(cell(r, "sample_a"))), + ("b", opt(cell(r, "sample_b"))), + ("declared", opt(cell(r, "declared"))), + ("shared_cm", opt(cell(r, "shared_cm"))), + ("degree", opt(cell(r, "degree"))), + ]) + }) + .collect(); + let mut couples = Vec::new(); + for (a, b, base) in couple_names(&people) { + let rows = read_tsv(&d.join(format!("{base}.tsv")))?; + let count = |w: &str| rows.iter().filter(|r| cell(r, "result") == Some(w)).count() as i128; + let md = format!("{base}.md"); + couples.push(obj([ + ("a", s(&a)), + ("b", s(&b)), + ("report", link(&md)), + ("table", link(&format!("{base}.tsv"))), + ( + "screened", + Value::Bool(!rows.is_empty() || files.contains(&md)), + ), + ("shared", Value::Int(count("shared"))), + ("x_linked", Value::Int(count("x_linked"))), + ])); + } + let plan = read_tsv(&d.join("family_plan.tsv"))? + .iter() + .map(|r| { + obj([ + ("analysis", opt(cell(r, "analysis"))), + ("members", opt(cell(r, "members"))), + ("decision", opt(cell(r, "decision"))), + ("reason", opt(cell(r, "reason"))), + ]) + }) + .collect(); + let reports = files + .iter() + .filter(|f| f.ends_with(".md")) + .map(|f| obj([("name", s(f)), ("url", Value::Str(format!("{url}{f}")))])) + .collect(); + Ok(obj([ + ("kind", s("family")), + ("name", s(name)), + ("pedigree", Value::Str(format!("{url}pedigree.svg"))), + ("members", Value::Array(people_rows(&people))), + ("children", Value::Array(children)), + ("relationships", Value::Array(rel)), + ("ibd_pairs", Value::Array(ibd)), + ("couples", Value::Array(couples)), + ("plan", Value::Array(plan)), + ("reports", Value::Array(reports)), + ("files", Value::Array(files.iter().map(|f| s(f)).collect())), + ])) +} + +/// The pedigree drawing's geometry as data, so the page draws each person +/// as a link. +pub fn layout_value(people: &People) -> Value { + let l = layout::page_layout(people); + obj([ + ("width", Value::Int(l.width.into())), + ("height", Value::Int(l.height.into())), + ("size", Value::Int(l.size.into())), + ( + "people", + Value::Array( + l.people + .iter() + .map(|p| { + obj([ + ("sample", s(&p.sample)), + ("x", Value::Float(p.x)), + ("y", Value::Float(p.y)), + ("gen", Value::Int(p.generation.into())), + ("sex", s(&p.sex)), + ("sequenced", Value::Bool(p.sequenced)), + ]) + }) + .collect(), + ), + ), + ( + "lines", + Value::Array( + l.lines + .iter() + .map(|ln| Value::Array(ln.iter().map(|v| Value::Float(*v)).collect())) + .collect(), + ), + ), + ]) +} + +/// Step 38 writes the verdict "ok" for a pair that matches its declared +/// relation. The page says that in words: no word on the page reads as an +/// all-clear (state.json keeps the file's own value). +pub fn verdict_words(v: &Value) -> Value { + match v { + Value::Str(x) if pycompat::text::lower(pycompat::text::strip(x, None)) == "ok" => { + s("matches the declared relation") + } + _ => v.clone(), + } +} + +fn with_verdict_words(rows: &mut Value) { + if let Value::Array(rows) = rows { + for r in rows { + let o = as_obj(r); + let w = verdict_words(o.get("verdict").unwrap_or(&Value::Null)); + o.insert("verdict".into(), w); + } + } +} + +/// family.json without what only the site knows (the member runs): the +/// state, the layout, the full pair rows and couple rows, each child's +/// stretches and the trio tables. +pub fn family_page(name: &str, d: &Path) -> Result { + let st = family_state(name, d)?; + let mut out = obj([ + ("state", st.clone()), + ("layout", Value::Null), + ("relationships", Value::Array(Vec::new())), + ("ibd_pairs", Value::Array(Vec::new())), + ("couples", Value::Object(IndexMap::new())), + ("couple_note", s(og_render::notes::COUPLE_NOTE)), + ("segments", Value::Object(IndexMap::new())), + ("de_novo", Value::Array(Vec::new())), + ("compound_het", Value::Array(Vec::new())), + ("trio_files", Value::Object(IndexMap::new())), + ("side_by_side", Value::Null), + ]); + if get(&st, "error").ok().flatten().is_some() { + return Ok(out); + } + // read again, as the state just did: a pedigree that stopped reading in between is unreadable + let people = members_of(d).map_err(Unreadable)?; + let o = as_obj(&mut out); + o.insert("layout".into(), layout_value(&people)); + let rows = |f: &str, keys: &[&str]| -> Result { + Ok(Value::Array( + read_tsv(&d.join(f))? + .iter() + .map(|r| keep(r, keys)) + .collect(), + )) + }; + o.insert( + "relationships".into(), + rows("relationships.tsv", &REL_KEYS)?, + ); + o.insert("ibd_pairs".into(), rows("ibd_pairs.tsv", &IBD_KEYS)?); + let mut couples = IndexMap::new(); + for (a, b, base) in couple_names(&people) { + let mut rs: Vec = read_tsv(&d.join(format!("{base}.tsv")))?; + rs.sort_by_key(|r| match cell(r, "result") { + Some("shared") => 0, + Some("x_linked") => 1, + Some("one_partner") => 2, + _ => 3, + }); + couples.insert( + format!("{a} {b}"), + Value::Array(rs.iter().map(|r| keep(r, &COUPLE_KEYS)).collect()), + ); + } + o.insert("couples".into(), Value::Object(couples)); + for (k, v) in child_depth(d, &people)? { + o.insert(k, v); + } + o.insert( + "chrom_lengths".into(), + Value::Array( + GRCH38 + .iter() + .map(|(c, n)| Value::Array(vec![s(c), Value::Int((*n).into())])) + .collect(), + ), + ); + let mut state = st; + if let Some(r) = as_obj(&mut state).get_mut("relationships") { + with_verdict_words(r); + } + o.insert("state".into(), state); + if let Some(r) = o.get_mut("relationships") { + with_verdict_words(r); + } + Ok(out) +} + +/// `here`, `rejected` (a link out of the family folder: it is not read, and +/// the page never shows where it points) or `missing`. +pub fn file_state(d: &Path, f: &str) -> &'static str { + let p = d.join(f); + if fs::symlink_metadata(&p).is_err() { + return "missing"; + } + if !inside(&p, &absolute(d)) { + return "rejected"; + } + if is_file(&p) { "here" } else { "missing" } +} + +/// Each child's stretches (step 38's `_segments.tsv`) and the trio +/// tables, as the steps wrote them: the compound-het pairs in trans first. +fn child_depth(d: &Path, people: &People) -> Result, Unreadable> { + let mut segs = IndexMap::new(); + for (c, m) in people { + if m.father.is_none() && m.mother.is_none() { + continue; + } + let rows = read_tsv(&d.join(format!("{c}_segments.tsv")))?; + if !rows.is_empty() { + segs.insert( + c.clone(), + Value::Array(rows.iter().map(|r| keep(r, &SEGMENT_KEYS)).collect()), + ); + } + } + let files: IndexMap<&str, &str> = TRIO_FILES.iter().map(|f| (*f, file_state(d, f))).collect(); + let mut comp = read_tsv(&d.join("compound_het.tsv"))?; + comp.sort_by_key(|r| match cell(r, "phase") { + Some("in trans") => 0, + Some("in cis") => 2, + _ => 1, + }); + let de_novo = read_tsv(&d.join("de_novo.tsv"))?; + let status = trio_status(d, people, &files)?; + Ok(vec![ + ("segments".into(), Value::Object(segs)), + ( + "de_novo".into(), + Value::Array(de_novo.iter().map(|r| keep(r, &DE_NOVO_KEYS)).collect()), + ), + ( + "compound_het".into(), + Value::Array(comp.iter().map(|r| keep(r, &COMP_KEYS)).collect()), + ), + ( + "trio_files".into(), + Value::Object( + files + .iter() + .map(|(f, st)| ((*f).to_owned(), Value::Bool(*st == "here"))) + .collect(), + ), + ), + ( + "trio_file_state".into(), + Value::Object( + files + .iter() + .map(|(f, st)| ((*f).to_owned(), s(st))) + .collect(), + ), + ), + ("trio_status".into(), status), + ]) +} + +fn line(assessed: Option, why: String) -> Value { + obj([ + ("assessed", assessed.map_or(Value::Null, Value::Bool)), + ("why", Value::Str(why)), + ]) +} + +/// Per child, whether step 38 assessed it for de novo candidates and +/// compound-het pairs, and why not. The rule is step 38's (`og family +/// report`): de novo candidates need both parents sequenced, compound-het +/// pairs a sequenced parent; a file that is not here, or is a link out of +/// the folder, means no child was assessed. +fn trio_status( + d: &Path, + people: &People, + files: &IndexMap<&str, &str>, +) -> Result { + let mut plan: HashMap, Row> = HashMap::new(); + for r in read_tsv(&d.join("family_plan.tsv"))? { + plan.insert(cell(&r, "analysis").map(str::to_owned), r); + } + let pl = |k: &str| plan.get(&Some(k.to_owned())); + let filled = |r: Option<&Row>, k: &str| { + r.and_then(|r| cell(r, k)) + .filter(|v| !v.is_empty()) + .map(str::to_owned) + }; + let trio = pl("trio_phasing"); + let absent = |f: &str| { + if files[f] == "rejected" { + return format!( + "{f} is a link out of the family folder, so this page does not read it" + ); + } + let mut why = format!("{f} is not here"); + if let Some(dec) = filled(trio, "decision") { + let reason = filled(trio, "reason") + .map(|r| format!(" ({r})")) + .unwrap_or_default(); + why.push_str(&format!(": trio phasing {dec}{reason}")); + } + why + }; + let joint = pl("joint_call"); + let called: Option> = (joint.and_then(|r| cell(r, "decision")) == Some("runs")) + .then(|| { + joint + .and_then(|r| cell(r, "members")) + .unwrap_or("") + .split(',') + .map(str::to_owned) + .collect() + }); + let seq = |p: Option<&str>| { + p.filter(|p| !p.is_empty()) + .and_then(|p| people.get(p).map(|m| (p, m))) + .is_some_and(|(p, m)| m.sequenced && called.as_ref().is_none_or(|c| c.contains(p))) + }; + let need = if called.is_none() { + "sequenced" + } else { + "sequenced and in the joint call" + }; + let not_called = |p: &str| { + if people[p].sequenced { + format!("{p} is not in the joint call (family_plan.tsv)") + } else { + format!("{p} is not sequenced") + } + }; + let gaps = |m: &layout::Person| { + [ + ("father", m.father.as_deref()), + ("mother", m.mother.as_deref()), + ] + .into_iter() + .filter(|(_, p)| !seq(*p)) + .map(|(w, p)| match p { + None => format!("no {w} is named"), + Some(p) => format!("{p} (the {w}) is not"), + }) + .collect::>() + .join("; ") + }; + let mut out = IndexMap::new(); + for (c, m) in people { + if m.father.is_none() && m.mother.is_none() { + continue; + } + let de_novo = if files["de_novo.tsv"] != "here" { + line(Some(false), absent("de_novo.tsv")) + } else if !seq(Some(c)) { + line(Some(false), not_called(c)) + } else if seq(m.father.as_deref()) && seq(m.mother.as_deref()) { + line(Some(true), "both parents are sequenced".into()) + } else { + line( + Some(false), + format!("de novo candidates need both parents {need}: {}", gaps(m)), + ) + }; + let comp = if files["compound_het.tsv"] != "here" { + line(Some(false), absent("compound_het.tsv")) + } else if !seq(Some(c)) { + line(Some(false), not_called(c)) + } else if seq(m.father.as_deref()) || seq(m.mother.as_deref()) { + line( + Some(true), + "a parent is sequenced; pairs come only from the child's clinical-filter candidates (step 23), so \ + with none given no pair is listed" + .into(), + ) + } else { + line( + Some(false), + format!("compound-het pairs need a parent {need}"), + ) + }; + out.insert( + c.clone(), + obj([("de_novo", de_novo), ("compound_het", comp)]), + ); + } + Ok(Value::Object(out)) +} + +/// Whether the clinical filter (step 23) wrote the child's candidates, the +/// only variants step 38 pairs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Given { + /// There, and not newer than `compound_het.tsv`. + Here, + /// The run is here without them (a file linking out of the run counts + /// as absent). + Absent, + /// Written after `compound_het.tsv`, so step 38 may not have had them. + Newer, +} + +/// [`Given`] of `sample`'s run folder `d`, read only inside `root`; `None` +/// when the run is not on this site, so the page cannot tell. +pub fn candidates_given( + root: &Path, + d: Option<&Path>, + sample: &str, + since: Option, +) -> Option { + let d = d?; + let p = d.join("clinical").join(format!("{sample}_clinical.vcf.gz")); + if !(is_file(&p) && inside(&p, d) && inside(&p, root)) { + return Some(Given::Absent); + } + let newer = since.is_some_and(|t| { + fs::metadata(&p) + .and_then(|m| m.modified()) + .is_ok_and(|m| m > t) + }); + Some(if newer { Given::Newer } else { Given::Here }) +} + +/// The trio status with each child's compound-het line told by its step 23 +/// candidates. `assessed` is null where the page cannot tell. +pub fn with_candidates(status: &mut Value, given: &HashMap>) { + let Value::Object(status) = status else { + return; + }; + for (child, st) in status.iter_mut() { + let st = as_obj(st); + let assessed = st + .get("compound_het") + .and_then(|c| get(c, "assessed").ok().flatten()) + .is_some_and(truthy); + if !assessed { + continue; + } + let new = match given.get(child).copied().flatten() { + Some(Given::Here) => line( + Some(true), + "a parent is sequenced and the child's clinical-filter candidates (step 23) are here; pairs come \ + only from them" + .into(), + ), + Some(Given::Absent) => line( + Some(false), + format!( + "the clinical filter (step 23) wrote no candidates for {child} (clinical/{child}_clinical.vcf.gz), \ + so step 38 had no variant to pair" + ), + ), + Some(Given::Newer) => line( + None, + format!( + "{child}'s clinical-filter candidates (step 23) are newer than compound_het.tsv, so step 38 may \ + not have paired them" + ), + ), + None => line( + None, + format!( + "pairs come only from the child's clinical-filter candidates (step 23), and {child}'s run is not \ + on this site to tell whether step 23 wrote them" + ), + ), + }; + st.insert("compound_het".into(), new); + } +} + +/// A member run's summary.json, or `None`: none yet, unreadable, not an +/// object, or a link out of the run folder or out of `root`. +pub fn member_summary(root: &Path, d: Option<&Path>) -> Option { + let d = d?; + let p = d.join("summary.json"); + if !is_file(&p) || !inside(&p, d) || !inside(&p, root) { + return None; + } + let mut text = String::new(); + fs::File::open(&p).ok()?.read_to_string(&mut text).ok()?; + match loads(&text) { + Ok(v @ Value::Object(_)) => Some(v), + _ => None, + } +} + +// --- the members side by side ----------------------------------------------------------- + +/// A summary of a shape the report never writes: that member's column says +/// so, and only that column. +#[derive(Debug)] +struct Malformed; + +type Got = Result; + +fn empty_obj() -> &'static Value { + static E: std::sync::OnceLock = std::sync::OnceLock::new(); + E.get_or_init(|| Value::Object(IndexMap::new())) +} + +/// `d.get(k)`, `None` for a missing key; a `d` that is not an object is +/// malformed. +fn at<'a>(d: &'a Value, k: &str) -> Got<&'a Value> { + static NULL: Value = Value::Null; + Ok(get(d, k).map_err(|_| Malformed)?.unwrap_or(&NULL)) +} + +/// `v or {}`. +fn or_obj(v: &Value) -> &Value { + if truthy(v) { v } else { empty_obj() } +} + +/// What `for x in v or []` walks: a list's items, a string's characters, an +/// object's keys; anything else that is true cannot be walked. +fn walk(v: &Value) -> Got> { + if !truthy(v) { + return Ok(Vec::new()); + } + match v { + Value::Array(a) => Ok(a.clone()), + Value::Str(x) => Ok(x.chars().map(|c| Value::Str(c.to_string())).collect()), + Value::Object(o) => Ok(o.keys().map(|k| s(k)).collect()), + _ => Err(Malformed), + } +} + +fn text(v: &Value) -> Got { + py_str(v).map_err(|_| Malformed) +} + +/// A section's data while it is complete or stale (`{}` when it has none), +/// else `None`. +fn section(summary: &Value, key: &str) -> Got> { + let sec = or_obj(at(or_obj(at(summary, "sections")?), key)?); + let state = at(sec, "state")?; + let data = at(sec, "data")?; + Ok(matches!(state, Value::Str(x) if x == "ok" || x == "stale").then(|| or_obj(data).clone())) +} + +fn stale(summary: &Value, key: &str) -> Got { + let sec = or_obj(at(or_obj(at(summary, "sections")?), key)?); + Ok(matches!(at(sec, "state")?, Value::Str(x) if x == "stale")) +} + +/// One cell of the side-by-side table: group, label, value, stale. +type Cell = (String, Value, Value, bool); + +/// The cells of one member's summary.json, as its report prints them; +/// nothing computed. A value is stale when its section is older than its +/// input (state `stale`). +fn side_values(summary: Option<&Value>) -> Got> { + let summary = summary.map_or(empty_obj(), or_obj); + let mut out: Vec = Vec::new(); + let c = or_obj(at(summary, "cyp2d6")?); + let calls_v = or_obj(at(c, "calls")?); + let Value::Object(calls_o) = calls_v else { + return Err(Malformed); + }; + let mut calls = Vec::new(); + for (k, v) in calls_o { + if truthy(v) { + calls.push(format!("{k} {}", text(v)?)); + } + } + let calls = calls.join(", "); + if !calls.is_empty() { + let agree = at(c, "agree")?; + let words = if truthy(agree) { + " (callers agree)" + } else if *agree == Value::Bool(false) { + " (callers differ)" + } else { + "" + }; + // summary.json's cyp2d6 block reads pypgx and Cyrius only when complete; its PharmCAT call is cpic's + let old = truthy(at(calls_v, "PharmCAT")?) && stale(summary, "cpic")?; + out.push(( + "Medicines".into(), + s("CYP2D6 (callers)"), + Value::Str(calls + words), + old, + )); + } + let cpic = section(summary, "cpic")?; + for g in walk(at(cpic.as_ref().map_or(empty_obj(), or_obj), "genes")?)? { + let ph = at(&g, "phenotype")?; + let dip = at(&g, "diplotype")?; + let dip = if truthy(dip) { text(dip)? } else { ".".into() }; + let ph = if truthy(ph) { + format!(" \"{}\"", text(ph)?) + } else { + String::new() + }; + let label = at(&g, "gene")?.clone(); + out.push(( + "Medicines".into(), + label, + Value::Str(dip + &ph), + stale(summary, "cpic")?, + )); + } + let hla = section(summary, "hla")?; + for l in walk(at(hla.as_ref().map_or(empty_obj(), or_obj), "loci")?)? { + let alleles = at(&l, "alleles")?; + let mut names = Vec::new(); + for a in walk(alleles)? { + match a { + Value::Str(x) => names.push(x), + _ => return Err(Malformed), + } + } + let typed = names.join(" "); + let value = if typed.is_empty() { + s("not typed") + } else { + Value::Str(typed) + }; + out.push(( + "HLA".into(), + at(&l, "gene")?.clone(), + value, + stale(summary, "hla")?, + )); + } + let dot = s("."); + let or_dot = |v: &Value| if truthy(v) { v.clone() } else { dot.clone() }; + if let Some(mt) = section(summary, "haplogroup")? { + out.push(( + "Haplogroups".into(), + s("Mitochondrial"), + or_dot(at(&mt, "haplogroup")?), + stale(summary, "haplogroup")?, + )); + } + let tree = section(summary, "y_tree")?; + let ykey = if tree.as_ref().is_some_and(truthy) { + "y_tree" + } else { + "y_haplogroup" + }; + let y = match tree { + Some(t) if truthy(&t) => Some(t), + _ => section(summary, "y_haplogroup")?, + }; + if let Some(y) = y { + out.push(( + "Haplogroups".into(), + s("Y chromosome"), + or_dot(at(&y, "haplogroup")?), + stale(summary, ykey)?, + )); + } + if let Some(bt) = section(summary, "bloodtype")? { + let old = stale(summary, "bloodtype")?; + let mut systems = Vec::new(); + for x in walk(at(&bt, "systems")?)? { + if truthy(at(&x, "system")?) { + systems.push(x); + } + } + // {system: phenotype}, a later row of one system winning + let mut by: Vec<(Value, Value)> = Vec::new(); + for x in &systems { + let k = at(x, "system")?.clone(); + if matches!(k, Value::Array(_) | Value::Object(_)) { + return Err(Malformed); + } + let v = or_dot(at(x, "phenotype")?); + match by.iter_mut().find(|(o, _)| same_key(o, &k)) { + Some(e) => e.1 = v, + None => by.push((k, v)), + } + } + let look = |name: &str| { + by.iter() + .find(|(k, _)| same_key(k, &s(name))) + .map_or(dot.clone(), |e| e.1.clone()) + }; + let named = |k: &str, name: &str| -> Got { + let v = at(&bt, k)?; + Ok(if truthy(v) { v.clone() } else { look(name) }) + }; + out.push(("Blood groups".into(), s("ABO"), named("abo", "ABO")?, old)); + out.push(("Blood groups".into(), s("RhD"), named("rhd", "RhD")?, old)); + // every other system step 41 typed, in its order (Kell, Duffy, Kidd, MNS...) + for x in &systems { + let k = at(x, "system")?; + if !matches!(k, Value::Str(v) if v == "ABO" || v == "RhD") { + let v = by + .iter() + .find(|(o, _)| same_key(o, k)) + .map_or(dot.clone(), |e| e.1.clone()); + out.push(("Blood groups".into(), k.clone(), v, old)); + } + } + } + Ok(out) +} + +/// Whether two labels are the same key, as Python compares them: numbers +/// by value (`True` is 1), strings by text. +fn same_key(a: &Value, b: &Value) -> bool { + let num = |v: &Value| match v { + Value::Bool(x) => Some(f64::from(u8::from(*x))), + Value::Int(i) => Some(*i as f64), + Value::Float(x) => Some(*x), + _ => None, + }; + match (a, b) { + (Value::Null, Value::Null) => true, + (Value::Str(x), Value::Str(y)) => x == y, + _ => matches!((num(a), num(b)), (Some(x), Some(y)) if x == y), + } +} + +/// Step 41's own order of the blood group systems (`og bloodtype systems`). +pub fn step41_order() -> Vec { + og_bloodtype::rules::SYSTEMS + .iter() + .map(|(name, _)| name.clone()) + .collect() +} + +/// The members' results in one table: `{members, rows: [{group, label, +/// values, stale}]}`. A member with no summary.json yet says so in its +/// column; a row a member's summary does not have is `.`. `stale` holds, +/// per member, whether that value comes from a section older than its +/// input. A summary whose fields are not of the shape the report writes +/// costs its own column only: it says so there, and the page still draws. +pub fn side_by_side(names: &[String], summaries: &HashMap>) -> Value { + side_by_side_in(names, summaries, &step41_order()) +} + +/// [`side_by_side`] with the blood group systems in `step41`'s order. +pub fn side_by_side_in( + names: &[String], + summaries: &HashMap>, + step41: &[String], +) -> Value { + let mut per: HashMap<&str, Vec> = HashMap::new(); + let mut bad: HashSet<&str> = HashSet::new(); + for n in names { + let sum = summaries.get(n).and_then(Option::as_ref); + match side_values(sum) { + // a label that is a list or an object cannot key the table's rows + Ok(got) + if !got + .iter() + .any(|c| matches!(c.1, Value::Array(_) | Value::Object(_))) => + { + per.insert(n, got); + } + _ => { + per.insert(n, Vec::new()); + bad.insert(n); + } + } + } + let same = |a: &(String, Value), g: &str, l: &Value| a.0 == g && same_key(&a.1, l); + // Each member's rows keep their own order (step 41's for the blood groups): a row first seen in a later + // member goes right after the row before it in that member's list, not at the end. + let mut keys: Vec<(String, Value)> = Vec::new(); + for n in names { + let mut prev: Option<(String, Value)> = None; + for (g, label, _, _) in &per[n.as_str()] { + if !keys.iter().any(|k| same(k, g, label)) { + let at = match &prev { + Some(p) => keys + .iter() + .position(|k| same(k, &p.0, &p.1)) + .map_or(keys.len(), |i| i + 1), + None => keys.len(), + }; + keys.insert(at, (g.clone(), label.clone())); + } + prev = Some((g.clone(), label.clone())); + } + } + let rank41 = |l: &Value| match l { + Value::Str(x) => step41.iter().position(|s_| s_ == x).unwrap_or(step41.len()), + _ => step41.len(), + }; + let mut ranked: Vec<(usize, usize, usize, (String, Value))> = keys + .into_iter() + .enumerate() + .map(|(i, k)| { + let g = GROUPS.iter().position(|x| *x == k.0).unwrap_or(9); + let b = if k.0 == "Blood groups" { + rank41(&k.1) + } else { + 0 + }; + (g, b, i, k) + }) + .collect(); + ranked.sort_by_key(|r| (r.0, r.1, r.2)); + let rows: Vec = ranked + .into_iter() + .map(|(_, _, _, (g, label))| { + let mut values = Vec::new(); + let mut stale = Vec::new(); + for n in names { + let (v, st) = if bad.contains(n.as_str()) { + (s("summary not readable"), false) + } else if summaries.get(n).and_then(Option::as_ref).is_none() { + (s("no summary yet"), false) + } else { + match per[n.as_str()] + .iter() + .find(|c| c.0 == g && same_key(&c.1, &label)) + { + Some(c) => (c.2.clone(), c.3), + None => (s("."), false), + } + }; + values.push(v); + stale.push(Value::Bool(st)); + } + obj([ + ("group", Value::Str(g)), + ("label", label), + ("values", Value::Array(values)), + ("stale", Value::Array(stale)), + ]) + }) + .collect(); + obj([ + ( + "members", + Value::Array(names.iter().map(|n| s(n)).collect()), + ), + ("rows", Value::Array(rows)), + ]) +} + +// --- with what the site knows ----------------------------------------------------------- + +/// A run folder by member name, from the input's `candidates` or +/// `summaries`; a member left out (or null) has no run on this site. +fn dirs(input: &Value, key: &str) -> Option>> { + let Value::Object(m) = get(input, key).ok().flatten()? else { + return None; + }; + Some( + m.iter() + .map(|(k, v)| { + ( + k.clone(), + if let Value::Str(d) = v { + Some(PathBuf::from(d)) + } else { + None + }, + ) + }) + .collect(), + ) +} + +/// family.json as the site serves it: [`family_page`], then the member runs +/// of `input` (`runs`, copied as they are), each child's compound-het line +/// told by its run's step 23 candidates (`candidates`), and the members +/// side by side from each sequenced member's summary.json (`summaries`). +/// A run folder is read only inside `root`. +pub fn site_page(name: &str, d: &Path, root: &Path, input: &Value) -> Result { + let mut out = family_page(name, d)?; + let root = realpath(root); + let o = as_obj(&mut out); + if let Some(runs) = get(input, "runs").ok().flatten() { + o.insert("runs".into(), runs.clone()); + } + if let Some(given_dirs) = dirs(input, "candidates") { + let comp = d.join("compound_het.tsv"); + let since = if is_file(&comp) { + fs::metadata(&comp).and_then(|m| m.modified()).ok() + } else { + None + }; + let mut given = HashMap::new(); + if let Some(Value::Object(st)) = o.get("trio_status") { + for c in st.keys() { + let dir = given_dirs.get(c).cloned().flatten(); + given.insert( + c.clone(), + dir.and_then(|dir| candidates_given(&root, Some(&dir), c, since)), + ); + } + } + if let Some(st) = o.get_mut("trio_status") { + with_candidates(st, &given); + } + } + if let Some(sum_dirs) = dirs(input, "summaries") { + let members = o + .get("state") + .and_then(|st| get(st, "members").ok().flatten()); + let seq: Vec = match members { + Some(Value::Array(ms)) => ms + .iter() + .filter(|m| get(m, "sequenced").ok().flatten().is_some_and(truthy)) + .filter_map(|m| match get(m, "sample").ok().flatten() { + Some(Value::Str(x)) => Some(x.clone()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + let sums: HashMap> = seq + .iter() + .map(|n| { + let dir = sum_dirs.get(n).cloned().flatten(); + ( + n.clone(), + dir.and_then(|dir| member_summary(&root, Some(&dir))), + ) + }) + .collect(); + o.insert( + "side_by_side".into(), + if seq.is_empty() { + Value::Null + } else { + side_by_side(&seq, &sums) + }, + ); + } + Ok(out) +} + +/// Each family folder under `root` as /runs.json lists it. `members` is [] +/// while the pedigree cannot be read; `unreadable` while family.json cannot +/// be built from the folder. +pub fn families(root: &Path) -> Vec { + family_dirs(root) + .into_iter() + .map(|(n, d)| { + let people = members_of(&d).map(|p| people_rows(&p)).unwrap_or_default(); + let readable = !people.is_empty() + && fs::read_dir(&d).is_ok() + && family_page(&n, &d).is_ok_and(|p| { + let st = get(&p, "state").ok().flatten().unwrap_or(&Value::Null); + get(st, "error").ok().flatten().is_none() + }); + obj([ + ("name", s(&n)), + ("dir", Value::Str(d.to_string_lossy().into_owned())), + ("kind", s("family")), + ("members", Value::Array(people)), + ("unreadable", Value::Bool(!readable)), + ]) + }) + .collect() +} + +// --- the command line ------------------------------------------------------------------- + +struct Args { + pos: Vec, + opts: HashMap, +} + +fn parse(args: &[String], known: &[&str]) -> Result { + let mut a = Args { + pos: Vec::new(), + opts: HashMap::new(), + }; + let mut it = args.iter(); + while let Some(x) = it.next() { + if let Some(name) = x.strip_prefix("--") { + if !known.contains(&name) { + return Err(format!("unknown option --{name}")); + } + let v = it.next().ok_or_else(|| format!("--{name} needs a value"))?; + a.opts.insert(name.to_owned(), v.clone()); + } else { + a.pos.push(x.clone()); + } + } + Ok(a) +} + +fn json_line(v: &Value) -> String { + dumps(v, &Dump::default()).unwrap_or_else(|_| "null".into()) + "\n" +} + +fn folder_name(d: &Path) -> String { + absolute(d) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// `og live family ARGS`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + let _ = write!(err, "{USAGE}"); + return 2; + }; + let known: &[&str] = match verb.as_str() { + "state" => &["name"], + "page" => &["name", "root", "input"], + "families" => &[], + "svg" => &["title"], + "files" => &[], + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => { + let _ = writeln!(err, "{USAGE}og live family: unknown verb '{other}'"); + return 2; + } + }; + let a = match parse(&args[1..], known) { + Ok(a) if a.pos.len() == 1 => a, + Ok(_) => { + let _ = writeln!(err, "{USAGE}og live family {verb}: one folder, please"); + return 2; + } + Err(e) => { + let _ = writeln!(err, "{USAGE}og live family {verb}: {e}"); + return 2; + } + }; + let d = PathBuf::from(&a.pos[0]); + let name = a + .opts + .get("name") + .cloned() + .unwrap_or_else(|| folder_name(&d)); + let answer = match verb.as_str() { + "state" => family_state(&name, &d).map(|v| json_line(&v)), + "families" => Ok(json_line(&Value::Array(families(&d)))), + "files" => match members_of(&d) { + Ok(p) => Ok(json_line(&Value::Array( + served_files(&d, &p).iter().map(|f| s(f)).collect(), + ))), + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + return 2; + } + }, + "svg" => { + let title = a + .opts + .get("title") + .cloned() + .unwrap_or_else(|| folder_name(&d)); + match members_of(&d).and_then(|p| layout::draw(&p, &title)) { + Ok(svg) => Ok(svg), + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + return 2; + } + } + } + _ => { + let input = match a.opts.get("input").map(String::as_str) { + None => Value::Object(IndexMap::new()), + Some(f) => { + let mut text = String::new(); + let read = if f == "-" { + io::stdin().read_to_string(&mut text) + } else { + fs::File::open(f).and_then(|mut h| h.read_to_string(&mut text)) + }; + match read + .map_err(|e| e.to_string()) + .and_then(|_| loads(&text).map_err(|e| e.to_string())) + { + Ok(v @ Value::Object(_)) => v, + Ok(_) | Err(_) => { + let _ = + writeln!(err, "og live family page: --input is not a JSON object"); + return 2; + } + } + } + }; + let root = a.opts.get("root").map_or_else( + || d.parent().map(Path::to_path_buf).unwrap_or_default(), + PathBuf::from, + ); + site_page(&name, &d, &root, &input).map(|v| json_line(&v)) + } + }; + match answer { + Ok(text) => match out.write_all(text.as_bytes()) { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og live family: cannot write: {e}"); + 1 + } + }, + Err(Unreadable(kind)) => { + let _ = writeln!( + err, + "og live family: a file of the family cannot be read ({kind})" + ); + 1 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names() { + assert!(valid_name("family_a") && valid_name("A.b-1")); + assert!(!valid_name("") && !valid_name(".x") && !valid_name("-x") && !valid_name("a b")); + } + + #[test] + fn keys_compare_as_python_does() { + assert!( + same_key(&Value::Int(1), &Value::Bool(true)) + && same_key(&Value::Float(1.0), &Value::Int(1)) + ); + assert!(!same_key(&s("1"), &Value::Int(1)) && same_key(&Value::Null, &Value::Null)); + } -use std::io::Write; + #[test] + fn walking_a_value() { + assert_eq!(walk(&s("ab")).unwrap(), vec![s("a"), s("b")]); + assert!(walk(&Value::Int(3)).is_err() && walk(&Value::Int(0)).unwrap().is_empty()); + } -/// `og live family ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og live family: not ported yet"); - 2 + #[test] + fn verdict() { + assert_eq!( + verdict_words(&s(" OK ")), + s("matches the declared relation") + ); + assert_eq!(verdict_words(&s("mismatch")), s("mismatch")); + assert_eq!(verdict_words(&Value::Null), Value::Null); + } } diff --git a/rust/crates/og-live/tests/family_layout.rs b/rust/crates/og-live/tests/family_layout.rs new file mode 100644 index 000000000..882a39bcc --- /dev/null +++ b/rust/crates/og-live/tests/family_layout.rs @@ -0,0 +1,451 @@ +//! The pedigree drawing (`og_pedigree::layout`, `og live family svg`) on +//! tests/fixtures/families_live, read back from its data- attributes and +//! coordinates; moved here from tests/test_pedigree_svg.py, check for check: +//! +//! 1. family_a (members.tsv, three generations): every person is drawn +//! once, with the sex and sequenced state of the file; each generation is +//! one row, parents above their children; +//! 2. each line of descent starts at its two parents (the middle of their +//! partner line) and ends on top of each child it names, and the parents +//! it names are the ones this test expects (written here, not read from +//! the fixture); +//! 3. family_b (pedigree.tsv only): the couple is drawn joined, both filled; +//! 4. a person whose sex is not known is a diamond; the command line writes +//! the same SVG and exits 2 on a folder without a pedigree. +//! +//! Then the family page's geometry (`page_layout`, family.json's `layout`) +//! is the drawing's, number for number. +//! +//! Checks 1 and 2 run again on a copy of the fixture with two parents +//! swapped and must fail there: a check that cannot fail is no check. + +use og_pedigree::layout::{self, SIZE, load, page_layout}; +use regex::Regex; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn fix(name: &str) -> PathBuf { + repo().join("tests/fixtures/families_live").join(name) +} + +fn tmp(name: &str) -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!( + "family-layout-{name}-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + d +} + +/// What family_a declares, written out by hand: child, father, mother. +const EXPECTED: [(&str, &str, &str); 4] = [ + ("child_a", "father_a", "mother_a"), + ("child_b", "father_a", "mother_a"), + ("father_a", "gf_a", "gm_a"), + ("mother_a", ".", "gm_m"), +]; +const SEX: [(&str, &str); 7] = [ + ("child_a", "female"), + ("child_b", "male"), + ("father_a", "male"), + ("mother_a", "female"), + ("gf_a", "male"), + ("gm_a", "female"), + ("gm_m", "female"), +]; + +#[derive(Debug, Clone)] +struct Drawn { + sex: String, + generation: i64, + seq: String, + shape: String, + x: f64, + y: f64, +} + +#[derive(Debug, Clone)] +struct Descent { + father: String, + mother: String, + children: Vec, + lines: Vec<[f64; 4]>, +} + +struct Svg { + people: BTreeMap>, + descents: Vec, + couples: Vec<(String, String, [f64; 4])>, +} + +fn num(s: &str) -> f64 { + s.parse().unwrap() +} + +fn parse(svg: &str) -> Svg { + let person = Regex::new( + r#"(?:> = BTreeMap::new(); + for c in person.captures_iter(svg) { + let h = SIZE as f64 / 2.0; + let (shape, x, y) = if let Some(x) = c.get(5) { + ("rect", num(x.as_str()) + h, num(&c[6]) + h) + } else if let Some(x) = c.get(7) { + ("circle", num(x.as_str()), num(&c[8])) + } else { + let pts: Vec<(f64, f64)> = c[9] + .split(' ') + .map(|p| { + let (a, b) = p.split_once(',').unwrap(); + (num(a), num(b)) + }) + .collect(); + ("polygon", pts[0].0, (pts[0].1 + pts[2].1) / 2.0) + }; + people.entry(c[1].to_owned()).or_default().push(Drawn { + sex: c[2].to_owned(), + generation: c[3].parse().unwrap(), + seq: c[4].to_owned(), + shape: shape.to_owned(), + x, + y, + }); + } + let line = Regex::new(r#""#) + .unwrap(); + let descent = + Regex::new(r#"]*>(.*?)"#) + .unwrap(); + let descents = descent + .captures_iter(svg) + .map(|c| Descent { + father: c[1].to_owned(), + mother: c[2].to_owned(), + children: c[3].split(' ').map(str::to_owned).collect(), + lines: line + .captures_iter(&c[4]) + .map(|l| [num(&l[1]), num(&l[2]), num(&l[3]), num(&l[4])]) + .collect(), + }) + .collect(); + let couple = Regex::new( + r#" Vec { + let mut out = Vec::new(); + for (s, sex) in SEX { + let want_seq = if s == "gm_m" { "no" } else { "yes" }; + let want_shape = if sex == "male" { "rect" } else { "circle" }; + match svg.people.get(s).map(Vec::as_slice) { + Some([d]) if d.sex == sex && d.seq == want_seq && d.shape == want_shape => {} + got => out.push(format!("{s}: {got:?}")), + } + } + let names: BTreeSet<&str> = svg.people.keys().map(String::as_str).collect(); + if names != SEX.iter().map(|(s, _)| *s).collect() { + out.push(format!("people {names:?}")); + } + out +} + +fn row_problems(svg: &Svg) -> Vec { + let mut out = Vec::new(); + let y = |s: &str| svg.people[s][0].y; + for (child, f, m) in EXPECTED { + for p in [f, m] { + if p != "." && y(p) >= y(child) { + out.push(format!("{p} not above {child}")); + } + } + } + let mut ys: BTreeMap> = BTreeMap::new(); + for v in svg.people.values() { + ys.entry(v[0].generation) + .or_default() + .insert(format!("{}", v[0].y)); + } + if ys.values().any(|y| y.len() != 1) { + out.push(format!("rows {ys:?}")); + } + out +} + +fn descent_problems(svg: &Svg) -> Vec { + let mut out = Vec::new(); + let mut seen: BTreeMap = BTreeMap::new(); + for d in &svg.descents { + for c in &d.children { + seen.insert(c.clone(), (d.father.clone(), d.mother.clone())); + } + let ps: Vec<&Drawn> = [&d.father, &d.mother] + .into_iter() + .filter(|p| *p != ".") + .map(|p| &svg.people[p][0]) + .collect(); + let ox = ps.iter().map(|p| p.x).sum::() / ps.len() as f64; + let first = d.lines[0]; + if (first[0] - ox).abs() > 0.5 || (first[2] - ox).abs() > 0.5 { + out.push(format!("start of {} {}", d.father, d.mother)); + } + for c in &d.children { + let p = &svg.people[c][0]; + let h = SIZE as f64 / 2.0; + if !d + .lines + .iter() + .any(|l| (l[0] - p.x).abs() < 0.5 && (l[3] - (p.y - h)).abs() < 0.5) + { + out.push(format!("end at {c}")); + } + } + } + let want: BTreeMap = EXPECTED + .iter() + .map(|(c, f, m)| ((*c).into(), ((*f).into(), (*m).into()))) + .collect(); + if seen != want { + out.push(format!("parents {seen:?}")); + } + out +} + +fn drawing(d: &Path, title: &str) -> String { + let (people, _) = load(d).unwrap(); + layout::draw(&people, title).unwrap() +} + +fn cli(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = og_live::family::cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +#[test] +fn family_a_is_drawn_as_declared() { + let svg = parse(&drawing(&fix("family_a"), "family_a")); + assert_eq!( + people_problems(&svg), + Vec::::new(), + "every person drawn once with the file's sex and sequenced state" + ); + assert_eq!( + row_problems(&svg), + Vec::::new(), + "one row per generation, parents above their children" + ); + assert_eq!( + descent_problems(&svg), + Vec::::new(), + "each line of descent from the expected parents to each child" + ); + let couples: BTreeSet> = svg + .couples + .iter() + .map(|(a, b, _)| [a.clone(), b.clone()].into_iter().collect()) + .collect(); + let want: BTreeSet> = [["father_a", "mother_a"], ["gf_a", "gm_a"]] + .iter() + .map(|p| p.iter().map(|s| (*s).to_owned()).collect()) + .collect(); + assert_eq!( + couples, want, + "partners joined: father_a with mother_a, gf_a with gm_a" + ); +} + +#[test] +fn control_child_drawn_under_its_grandparents() { + // child_a's parents swapped for father_a's, and father_a left without parents + let bad = tmp("swapped"); + let text = fs::read_to_string(fix("family_a").join("members.tsv")).unwrap(); + let lines: Vec = text + .split('\n') + .map(|ln| { + let mut c: Vec<&str> = ln.split('\t').collect(); + match c.first().copied() { + Some("child_a") => { + c[2] = "gf_a"; + c[3] = "gm_a"; + } + Some("father_a") => { + c[2] = ""; + c[3] = ""; + } + _ => {} + } + c.join("\t") + }) + .collect(); + fs::write(bad.join("members.tsv"), lines.join("\n")).unwrap(); + let svg = parse(&drawing(&bad, "family_a")); + assert!( + !descent_problems(&svg).is_empty() || !row_problems(&svg).is_empty(), + "control: child_a drawn under its grandparents was NOT caught" + ); +} + +#[test] +fn family_b_from_its_pedigree_is_a_joined_filled_couple() { + let svg = parse(&drawing(&fix("family_b"), "family_b")); + let couples: Vec<(&str, &str)> = svg + .couples + .iter() + .map(|(a, b, _)| (a.as_str(), b.as_str())) + .collect(); + assert_eq!(couples, [("partner_x", "partner_y")]); + assert!(svg.descents.is_empty()); + assert!( + ["partner_x", "partner_y"] + .iter() + .all(|s| svg.people[*s][0].seq == "yes"), + "{:?}", + svg.people + ); +} + +#[test] +fn unknown_sex_is_a_diamond_and_named_parents_are_not_sequenced() { + let d = tmp("unknown"); + fs::write( + d.join("pedigree.tsv"), + "sample\tsex\tfather\tmother\tpartner\nkid_u\tunknown\tdad_u\tmum_u\t.\n", + ) + .unwrap(); + let svg = parse(&drawing(&d, "Pedigree")); + assert_eq!(svg.people["kid_u"][0].shape, "polygon"); + assert_eq!(svg.people["dad_u"][0].seq, "no"); + assert_eq!(svg.descents.len(), 1); + assert_eq!(svg.descents[0].children, ["kid_u"]); +} + +#[test] +fn the_command_line_draws_the_same_and_refuses_an_empty_folder() { + let a = fix("family_a"); + let (code, out, err) = cli(&["svg", a.to_str().unwrap()]); + assert_eq!( + (code, out.as_str()), + (0, drawing(&a, "family_a").as_str()), + "{err}" + ); + let (code, _, err) = cli(&["svg", a.to_str().unwrap(), "--title", "xa<b & "c""), "{svg}"); +} + +/// family.json's layout and the SVG agree number for number: each +/// person's centre, every partner line and every line of descent. +fn geometry_problems(d: &Path) -> Vec { + let (people, _) = load(d).unwrap(); + let svg = parse(&layout::draw(&people, "t").unwrap()); + let page = page_layout(&people); + let mut out = Vec::new(); + for p in &page.people { + let drawn = &svg.people[&p.sample][0]; + if (drawn.x, drawn.y, drawn.generation) != (p.x, p.y, p.generation) || drawn.sex != p.sex { + out.push(format!( + "{}: page ({}, {}) svg ({}, {})", + p.sample, p.x, p.y, drawn.x, drawn.y + )); + } + } + let mut lines: Vec<[f64; 4]> = svg.couples.iter().map(|(_, _, l)| *l).collect(); + lines.extend(svg.descents.iter().flat_map(|d| d.lines.clone())); + if lines != page.lines { + out.push(format!("lines: page {:?} svg {:?}", page.lines, lines)); + } + if page.size != SIZE || page.people.len() != svg.people.len() { + out.push(format!("size {} people {}", page.size, page.people.len())); + } + out +} + +#[test] +fn the_page_geometry_is_the_drawings() { + for d in [ + fix("family_a"), + fix("family_b"), + repo().join("tests/fixtures/live/demo/DEMO-FAMILY"), + ] { + assert_eq!( + geometry_problems(&d), + Vec::::new(), + "{}", + d.display() + ); + } + let (people, _) = load(&fix("family_a")).unwrap(); + let page = page_layout(&people); + // three rows of 2, 2 and 2 or 3 people at most: 2 * 40 + 36 + 2 * 120 wide, 50 + 36 + 2 * 130 + 30 high + assert_eq!((page.width, page.height), (356, 376)); +} + +#[test] +fn control_a_moved_symbol_breaks_the_geometry_check() { + // the same folder, one partner line drawn from the other partner: the page and the drawing then differ + let d = tmp("moved"); + fs::copy(fix("family_a").join("members.tsv"), d.join("members.tsv")).unwrap(); + let (people, _) = load(&d).unwrap(); + let svg = layout::draw(&people, "t") + .unwrap() + .replacen(r#"x1="136.0""#, r#"x1="137.0""#, 1); + let parsed = parse(&svg); + let page = page_layout(&people); + let first = parsed.couples[0].2; + assert_ne!( + first, page.lines[0], + "control: a moved partner line was NOT caught" + ); +} diff --git a/rust/crates/og-live/tests/family_page.rs b/rust/crates/og-live/tests/family_page.rs new file mode 100644 index 000000000..601eb30b0 --- /dev/null +++ b/rust/crates/og-live/tests/family_page.rs @@ -0,0 +1,1425 @@ +//! `og live family` (og-live's family module) on tests/fixtures/families_live +//! and on the demo's invented trio (tests/fixtures/live/demo/DEMO-FAMILY). +//! Moved here from tests/test_live_family_depth.py and from the checks of +//! tests/test_owngenome_live_family.py that read the family plugin's data +//! (that file keeps the checks of the Python site that serves it): +//! +//! - the family folders are listed as runs of kind family, a sample's folder +//! is not; family_a's state: its members, each child with its parents and +//! map, the relationship check, the shared segments, the couple screen +//! counted from step 39's table, the plan and exactly the result files +//! that exist (a file no step writes and a link out of the folder are not +//! among them); a members file with a header and no rows is an error; +//! - each child's stretches are its `_segments.tsv` row for row, and +//! the chromosome lengths the painting scales by are GRCh38's chr1 to +//! chr22 and chrX; the compound-het pairs come in trans first, and every +//! de novo row is there; a family without the trio files says so and +//! lists nothing; +//! - the members side by side: one row per gene or system any member's +//! summary.json has, '.' where a member's summary lacks the row, "no +//! summary yet" for a member without one, a stale value flagged in its +//! own cell, every blood group system step 41 typed in step 41's order, +//! and a summary of a shape the report never writes costing its member's +//! column only; a summary.json linking out of its run is never read; +//! - each child's trio status, and the compound-het line told by the child's +//! step 23 candidates; +//! - a file of the family that cannot be read: no family.json, the family +//! listed unreadable. +//! +//! Each check runs again on a planted mistake and must fail there. The +//! wording gate on family.json stays in tests/test_live_results.py, with +//! the rest of that gate. + +use og_live::family::{ + Given, candidates_given, cli, families, family_page, family_state, member_summary, read_tsv, + side_by_side, side_by_side_in, site_page, step41_order, with_candidates, +}; +use pycompat::json::{Dump, Value, dumps, loads}; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, SystemTime}; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn demo() -> PathBuf { + repo().join("tests/fixtures/live/demo/DEMO-FAMILY") +} + +fn tmp(name: &str) -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!( + "family-page-{name}-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + fs::canonicalize(&d).unwrap() +} + +fn copy_tree(from: &Path, to: &Path) { + fs::create_dir_all(to).unwrap(); + for e in fs::read_dir(from).unwrap() { + let e = e.unwrap(); + let (src, dst) = (e.path(), to.join(e.file_name())); + let ft = e.file_type().unwrap(); + if ft.is_symlink() { + std::os::unix::fs::symlink(fs::read_link(&src).unwrap(), &dst).unwrap(); + } else if ft.is_dir() { + copy_tree(&src, &dst); + } else { + fs::copy(&src, &dst).unwrap(); + } + } +} + +/// tests/fixtures/families_live as a GENOME_DIR, with a sample run in no family. +fn genome() -> PathBuf { + let root = tmp("genome"); + copy_tree(&repo().join("tests/fixtures/families_live"), &root); + fs::create_dir_all(root.join("loner/logs")).unwrap(); + root +} + +fn j(text: &str) -> Value { + loads(text).unwrap_or_else(|e| panic!("{e}: {text}")) +} + +fn show(v: &Value) -> String { + dumps(v, &Dump::default()).unwrap() +} + +fn at<'a>(v: &'a Value, path: &[&str]) -> &'a Value { + static NULL: Value = Value::Null; + let mut v = v; + for k in path { + v = match v { + Value::Object(o) => o.get(*k).unwrap_or(&NULL), + Value::Array(a) => k + .parse::() + .ok() + .and_then(|i| a.get(i)) + .unwrap_or(&NULL), + _ => &NULL, + }; + } + v +} + +fn list(v: &Value) -> &[Value] { + match v { + Value::Array(a) => a, + _ => &[], + } +} + +fn st(v: &Value) -> &str { + match v { + Value::Str(s) => s, + _ => "", + } +} + +fn state(root: &Path, name: &str) -> Value { + family_state(name, &root.join(name)).unwrap() +} + +// --- the family's state --------------------------------------------------------------------- + +const CHILDREN: &str = r#"[{"child": "child_a", "father": "father_a", "mother": "mother_a", + "segments": "/family/family_a/child_a_segments.svg"}, + {"child": "child_b", "father": "father_a", "mother": "mother_a", "segments": null}, + {"child": "father_a", "father": "gf_a", "mother": "gm_a", "segments": null}, + {"child": "mother_a", "father": null, "mother": "gm_m", "segments": null}]"#; +const FILES: [&str; 9] = [ + "child_a_segments.svg", + "couple_father_a_mother_a.md", + "couple_father_a_mother_a.tsv", + "family_plan.tsv", + "family_report.md", + "ibd_pairs.tsv", + "ibd_report.md", + "members.tsv", + "relationships.tsv", +]; + +fn children_problems(s: &Value) -> Vec { + let mut out = Vec::new(); + if *at(s, &["children"]) != j(CHILDREN) { + out.push(format!("children {}", show(at(s, &["children"])))); + } + let seq: Vec<(&str, bool)> = list(at(s, &["members"])) + .iter() + .map(|m| { + ( + st(at(m, &["sample"])), + *at(m, &["sequenced"]) == Value::Bool(true), + ) + }) + .collect(); + let want = [ + ("child_a", true), + ("child_b", true), + ("father_a", true), + ("mother_a", true), + ("gf_a", true), + ("gm_a", true), + ("gm_m", false), + ]; + if seq != want { + out.push(format!("members {seq:?}")); + } + out +} + +#[test] +fn the_family_folders_are_runs_of_kind_family() { + let root = genome(); + let fams = families(&root); + let names: Vec<&str> = fams.iter().map(|f| st(at(f, &["name"]))).collect(); + assert_eq!( + names, + ["family_a", "family_b"], + "child_a and loner are not families" + ); + assert!( + fams.iter() + .all(|f| st(at(f, &["kind"])) == "family" + && *at(f, &["unreadable"]) == Value::Bool(false)) + ); + assert_eq!( + st(at(&fams[0], &["dir"])), + root.join("family_a").to_str().unwrap() + ); + assert_eq!(list(at(&fams[0], &["members"])).len(), 7); + // the command line lists the same + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli( + &["families".into(), root.to_string_lossy().into_owned()], + &mut out, + &mut err, + ); + assert_eq!(code, 0); + assert_eq!(j(&String::from_utf8(out).unwrap()), Value::Array(fams)); +} + +#[test] +fn family_a_state_is_what_the_steps_wrote() { + let root = genome(); + let s = state(&root, "family_a"); + assert_eq!(children_problems(&s), Vec::::new()); + let rel: Vec<(&str, &str, &str)> = list(at(&s, &["relationships"])) + .iter() + .map(|r| { + ( + st(at(r, &["a"])), + st(at(r, &["b"])), + st(at(r, &["verdict"])), + ) + }) + .collect(); + assert_eq!( + rel, + [("child_a", "father_a", "ok"), ("child_a", "child_b", "ok")], + "the relationship check as step 38 wrote it" + ); + assert_eq!( + *at(&s, &["ibd_pairs"]), + j( + r#"[{"a": "child_a", "b": "child_b", "declared": "full siblings", "shared_cm": "2550.0", "degree": "first (full siblings)"}]"# + ), + "the shared segments as step 38 wrote them" + ); + assert_eq!( + *at(&s, &["couples"]), + j( + r#"[{"a": "father_a", "b": "mother_a", "report": "/family/family_a/couple_father_a_mother_a.md", + "table": "/family/family_a/couple_father_a_mother_a.tsv", "screened": true, "shared": 1, "x_linked": 1}, + {"a": "gf_a", "b": "gm_a", "report": null, "table": null, "screened": false, "shared": 0, "x_linked": 0}]"# + ), + "the couple screen counted from step 39's table, the grandparents not screened" + ); + let plan: Vec<&str> = list(at(&s, &["plan"])) + .iter() + .map(|r| st(at(r, &["analysis"]))) + .collect(); + assert_eq!(plan, ["joint_call", "couple_carrier"]); + let files: Vec<&str> = list(at(&s, &["files"])).iter().map(st).collect(); + assert_eq!( + files, FILES, + "exactly the result files that exist (notes.txt is written by no step)" + ); + assert_eq!(st(at(&s, &["pedigree"])), "/family/family_a/pedigree.svg"); + let b = state(&root, "family_b"); + assert_eq!( + *at(&b, &["couples"]), + j( + r#"[{"a": "partner_x", "b": "partner_y", "report": null, "table": null, "screened": false, "shared": 0, "x_linked": 0}]"# + ), + "family_b (pedigree.tsv only): a couple not screened yet" + ); +} + +#[test] +fn control_a_swapped_parent_in_the_state() { + let root = genome(); + let p = root.join("family_a/members.tsv"); + let text = fs::read_to_string(&p).unwrap(); + fs::write( + &p, + text.replace( + "child_a\tfemale\tfather_a\tmother_a", + "child_a\tfemale\tgf_a\tmother_a", + ), + ) + .unwrap(); + assert!( + !children_problems(&state(&root, "family_a")).is_empty(), + "control: a swapped parent was NOT caught" + ); +} + +#[test] +fn a_link_out_of_the_folder_is_not_listed_or_read() { + let root = genome(); + let rep = root.join("family_a/ibd_report.md"); + fs::remove_file(&rep).unwrap(); + std::os::unix::fs::symlink(repo().join("README.md"), &rep).unwrap(); + let outside = root.join("outside-folder-name.tsv"); + fs::write(&outside, "sample_a\tsample_b\tverdict\nx\ty\tok\n").unwrap(); + let rel = root.join("family_a/relationships.tsv"); + fs::remove_file(&rel).unwrap(); + std::os::unix::fs::symlink(&outside, &rel).unwrap(); + let s = state(&root, "family_a"); + let files: Vec<&str> = list(at(&s, &["files"])).iter().map(st).collect(); + assert!( + !files.contains(&"ibd_report.md") && !files.contains(&"relationships.tsv"), + "{files:?}" + ); + assert!( + list(at(&s, &["relationships"])).is_empty(), + "a table linking out of the folder is not read" + ); + assert!(!show(&s).contains("outside-folder-name")); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli( + &[ + "files".into(), + root.join("family_a").to_string_lossy().into_owned(), + ], + &mut out, + &mut err, + ); + assert_eq!(code, 0); + assert_eq!( + j(&String::from_utf8(out).unwrap()), + *at(&s, &["files"]), + "og live family files is the state's list" + ); +} + +#[test] +fn a_members_file_with_no_rows_is_an_error() { + let root = genome(); + let empty = root.join("family_empty"); + fs::create_dir_all(&empty).unwrap(); + let head = fs::read_to_string(root.join("family_a/members.tsv")).unwrap(); + fs::write( + empty.join("members.tsv"), + head.lines().next().unwrap().to_owned() + "\n", + ) + .unwrap(); + let s = state(&root, "family_empty"); + assert!( + st(at(&s, &["error"])).starts_with("the pedigree could not be read: "), + "{}", + show(&s) + ); + let fam = families(&root) + .into_iter() + .find(|f| st(at(f, &["name"])) == "family_empty") + .unwrap(); + assert!( + list(at(&fam, &["members"])).is_empty() && *at(&fam, &["unreadable"]) == Value::Bool(true) + ); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli( + &["svg".into(), empty.to_string_lossy().into_owned()], + &mut out, + &mut err, + ); + assert_eq!(code, 2, "no pedigree to draw"); + let page = family_page("family_empty", &empty).unwrap(); + assert_eq!(*at(&page, &["layout"]), Value::Null); +} + +/// chmod 0 blocks the read unless the test runs as root. +fn blocked(p: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(p, fs::Permissions::from_mode(0o000)).unwrap(); + fs::read(p).is_err() +} + +fn unblock(p: &Path) { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(p, fs::Permissions::from_mode(0o644)).unwrap(); +} + +#[test] +fn a_file_that_cannot_be_read_gives_no_page_and_an_unreadable_family() { + let root = genome(); + let d = root.join("family_a"); + let rel = d.join("relationships.tsv"); + if !blocked(&rel) { + unblock(&rel); + eprintln!("[SKIP] chmod 0 does not stop this user reading"); + return; + } + let page = family_page("family_a", &d); + let fams = families(&root); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli( + &["page".into(), d.to_string_lossy().into_owned()], + &mut out, + &mut err, + ); + unblock(&rel); + assert!(page.is_err(), "family.json is not built"); + let err = String::from_utf8(err).unwrap(); + assert!( + code == 1 && out.is_empty() && err.contains("PermissionError") && !err.contains("/"), + "{code} {err}" + ); + let a = &fams[0]; + assert!( + *at(a, &["unreadable"]) == Value::Bool(true) && list(at(a, &["members"])).len() == 7, + "{}", + show(a) + ); + let reopened = families(&root); + assert_eq!( + *at(&reopened[0], &["unreadable"]), + Value::Bool(false), + "readable again at once" + ); + let mem = d.join("members.tsv"); + if blocked(&mem) { + let fams = families(&root); + unblock(&mem); + assert!( + list(at(&fams[0], &["members"])).is_empty() + && *at(&fams[0], &["unreadable"]) == Value::Bool(true) + ); + } else { + unblock(&mem); + } +} + +// --- the depth of the page: stretches, trio tables ---------------------------------------- + +fn segment_problems(page: &Value, d: &Path) -> Vec { + let rows = read_tsv(&d.join("DEMO-003_segments.tsv")).unwrap(); + let got = list(at(page, &["segments", "DEMO-003"])); + let mut out = Vec::new(); + let key = |r: &Value| ["side", "chrom", "start", "end"].map(|k| st(at(r, &[k])).to_owned()); + let want: Vec<[String; 4]> = rows + .iter() + .map(|r| ["side", "chrom", "start", "end"].map(|k| r.get(k).cloned().unwrap_or_default())) + .collect(); + if rows.is_empty() || got.iter().map(key).collect::>() != want { + out.push(format!( + "{} stretches against {} rows", + got.len(), + rows.len() + )); + } + let chroms: Vec<&str> = list(at(page, &["chrom_lengths"])) + .iter() + .map(|c| st(at(c, &["0"]))) + .collect(); + let want: Vec = (1..=22) + .map(|i| format!("chr{i}")) + .chain(["chrX".into()]) + .collect(); + if chroms != want { + out.push(format!("lengths {chroms:?}")); + } + if *at(page, &["chrom_lengths", "0", "1"]) != Value::Int(248956422) { + out.push("chr1's length".into()); + } + out +} + +fn pair_problems(page: &Value) -> Vec { + let rank = |p: &str| match p { + "in trans" => 0, + "in cis" => 2, + _ => 1, + }; + let phases: Vec<&str> = list(at(page, &["compound_het"])) + .iter() + .map(|r| st(at(r, &["phase"]))) + .collect(); + let mut sorted = phases.clone(); + sorted.sort_by_key(|p| rank(p)); + let mut out = Vec::new(); + if phases.first() != Some(&"in trans") || phases != sorted { + out.push(format!("order {phases:?}")); + } + if list(at(page, &["de_novo"])).len() != 2 { + out.push("de novo rows".into()); + } + out +} + +#[test] +fn stretches_and_trio_tables_as_the_steps_wrote_them() { + let page = family_page("DEMO-FAMILY", &demo()).unwrap(); + assert_eq!(segment_problems(&page, &demo()), Vec::::new()); + let mut wrong = page.clone(); + if let Value::Object(o) = &mut wrong + && let Some(Value::Object(s)) = o.get_mut("segments") + && let Some(Value::Array(a)) = s.get_mut("DEMO-003") + { + a.reverse(); + } + assert!( + !segment_problems(&wrong, &demo()).is_empty(), + "control: the stretches in another order" + ); + assert_eq!( + pair_problems(&page), + Vec::::new(), + "the compound-het pairs in trans first, both de novo rows" + ); + let mut planted = page.clone(); + let file_order: Vec = read_tsv(&demo().join("compound_het.tsv")) + .unwrap() + .iter() + .map(|r| { + Value::Object( + r.iter() + .map(|(k, v)| (k.clone(), Value::Str(v.clone()))) + .collect(), + ) + }) + .collect(); + if let Value::Object(o) = &mut planted { + o.insert("compound_het".into(), Value::Array(file_order)); + } + assert!( + !pair_problems(&planted).is_empty(), + "control: the pairs left in file order" + ); + // the de novo row's own columns only: the parents' depth and quality are not on the page + let row = at(&page, &["de_novo", "0"]); + assert!( + *at(row, &["father_dp"]) == Value::Null && st(at(row, &["child_ab"])) == "0.47", + "{}", + show(row) + ); +} + +fn bare_ok(p: &Value) -> bool { + *at(p, &["trio_files"]) == j(r#"{"de_novo.tsv": false, "compound_het.tsv": false}"#) + && list(at(p, &["de_novo"])).is_empty() + && list(at(p, &["compound_het"])).is_empty() + && *at(p, &["segments"]) == j("{}") +} + +#[test] +fn a_family_without_the_trio_files_says_so() { + let bare = tmp("bare"); + fs::copy(demo().join("members.tsv"), bare.join("members.tsv")).unwrap(); + assert!(bare_ok(&family_page("bare", &bare).unwrap())); + fs::copy(demo().join("de_novo.tsv"), bare.join("de_novo.tsv")).unwrap(); + assert!( + !bare_ok(&family_page("bare", &bare).unwrap()), + "control: a planted de_novo.tsv" + ); +} + +// --- the members side by side ------------------------------------------------------------- + +fn summary(cyp: &str, cpic: &[(&str, &str, &str)], hap: &str, y: Option<&str>) -> Value { + let genes: Vec = cpic + .iter() + .map(|(g, d, p)| format!(r#"{{"gene": "{g}", "diplotype": "{d}", "phenotype": "{p}"}}"#)) + .collect(); + let y = y + .map(|y| format!(r#", "y_tree": {{"state": "ok", "data": {{"haplogroup": "{y}"}}}}"#)) + .unwrap_or_default(); + j(&format!( + r#"{{"cyp2d6": {{"calls": {{"pypgx": "{cyp}", "Cyrius": "{cyp}"}}, "agree": true}}, "sections": {{ + "cpic": {{"state": "ok", "data": {{"genes": [{}]}}}}, + "haplogroup": {{"state": "ok", "data": {{"haplogroup": "{hap}"}}}}, + "bloodtype": {{"state": "ok", "data": {{"abo": "A1", "rhd": "D+"}}}}{y}}}}}"#, + genes.join(", ") + )) +} + +fn set(v: &mut Value, path: &[&str], x: Value) { + let mut v = v; + for k in &path[..path.len() - 1] { + v = match v { + Value::Object(o) => o + .entry((*k).to_owned()) + .or_insert_with(|| Value::Object(Default::default())), + _ => panic!("not an object at {k}"), + }; + } + if let Value::Object(o) = v { + o.insert(path[path.len() - 1].to_owned(), x); + } +} + +fn stale_summary(state: &str) -> Value { + let mut s = summary( + "*1/*1", + &[("DPYD", "Reference/Reference", "Normal Metabolizer")], + "H1", + None, + ); + set( + &mut s, + &["sections", "bloodtype"], + j(&format!( + r#"{{"state": "{state}", "data": {{"abo": "A1", "rhd": "D+", "systems": [{{"system": "ABO", "phenotype": "A1"}}, + {{"system": "RhD", "phenotype": "D+"}}, {{"system": "Kell", "phenotype": "K-k+"}}, + {{"system": "Duffy", "phenotype": "Fy(a+b-)"}}]}}}}"# + )), + ); + s +} + +/// Every side-by-side source present and complete, with PharmCAT's CYP2D6 +/// call, the Y haplogroup from y_haplogroup alone (the fallback), and one +/// section, `key`, in the given state. +fn source_summary(key: &str, state: &str) -> Value { + let mut s = summary( + "*1/*1", + &[("DPYD", "Reference/Reference", "Normal Metabolizer")], + "H1", + None, + ); + set( + &mut s, + &["cyp2d6", "calls", "PharmCAT"], + Value::Str("*1/*1".into()), + ); + set( + &mut s, + &["sections", "hla"], + j( + r#"{"state": "ok", "data": {"loci": [{"gene": "HLA-A", "alleles": ["A*01:01", "A*02:01"]}]}}"#, + ), + ); + set( + &mut s, + &["sections", "y_haplogroup"], + j(r#"{"state": "ok", "data": {"haplogroup": "R-M269"}}"#), + ); + set( + &mut s, + &["sections", key, "state"], + Value::Str(state.into()), + ); + s +} + +fn names(v: &[&str]) -> Vec { + v.iter().map(|s| (*s).to_owned()).collect() +} + +fn sums(pairs: &[(&str, Option)]) -> HashMap> { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect() +} + +fn abc() -> HashMap> { + sums(&[ + ( + "A", + Some(summary( + "*1/*4", + &[ + ("CYP2D6", "*1/*4", "Intermediate Metabolizer"), + ("CYP2C19", "*1/*2", "Intermediate Metabolizer"), + ("DPYD", "Reference/Reference", "Normal Metabolizer"), + ], + "H1", + Some("I-M253"), + )), + ), + ( + "B", + Some(summary( + "*1/*1", + &[("DPYD", "Reference/Reference", "Normal Metabolizer")], + "U5", + None, + )), + ), + ("C", None), + ]) +} + +fn row<'a>(sb: &'a Value, group: &str, label: &str) -> Option<&'a Value> { + list(at(sb, &["rows"])) + .iter() + .find(|r| st(at(r, &["group"])) == group && st(at(r, &["label"])) == label) +} + +fn values(sb: &Value, group: &str, label: &str) -> Vec { + row(sb, group, label) + .map(|r| { + list(at(r, &["values"])) + .iter() + .map(|v| st(v).to_owned()) + .collect() + }) + .unwrap_or_default() +} + +fn side_problems(sb: &Value) -> Vec { + let want: [(&str, &str, [&str; 3]); 6] = [ + ( + "Medicines", + "CYP2D6 (callers)", + [ + "pypgx *1/*4, Cyrius *1/*4 (callers agree)", + "pypgx *1/*1, Cyrius *1/*1 (callers agree)", + "no summary yet", + ], + ), + ( + "Medicines", + "CYP2C19", + ["*1/*2 \"Intermediate Metabolizer\"", ".", "no summary yet"], + ), + ( + "Medicines", + "CYP2D6", + ["*1/*4 \"Intermediate Metabolizer\"", ".", "no summary yet"], + ), + ( + "Medicines", + "DPYD", + [ + "Reference/Reference \"Normal Metabolizer\"", + "Reference/Reference \"Normal Metabolizer\"", + "no summary yet", + ], + ), + ( + "Haplogroups", + "Y chromosome", + ["I-M253", ".", "no summary yet"], + ), + ("Blood groups", "ABO", ["A1", "A1", "no summary yet"]), + ]; + let mut out: Vec = want + .iter() + .filter(|(g, l, v)| values(sb, g, l) != *v) + .map(|(g, l, v)| format!("{g} {l}: {:?} != {v:?}", values(sb, g, l))) + .collect(); + if *at(sb, &["members"]) != j(r#"["A", "B", "C"]"#) { + out.push("members".into()); + } + let order = ["Medicines", "HLA", "Haplogroups", "Blood groups"]; + let groups: Vec = list(at(sb, &["rows"])) + .iter() + .map(|r| { + order + .iter() + .position(|g| *g == st(at(r, &["group"]))) + .unwrap() + }) + .collect(); + if !groups.is_sorted() { + out.push(format!("groups out of order {groups:?}")); + } + out +} + +#[test] +fn the_members_side_by_side() { + let sb = side_by_side(&names(&["A", "B", "C"]), &abc()); + assert_eq!(side_problems(&sb), Vec::::new()); + let mut dropped = abc(); + dropped.insert("A".into(), None); + assert!( + !side_problems(&side_by_side(&names(&["A", "B", "C"]), &dropped)).is_empty(), + "control: member A's summary dropped" + ); +} + +#[test] +fn a_summary_linking_out_of_its_run_is_never_read() { + let root = tmp("root"); + let run = root.join("A"); + fs::create_dir_all(&run).unwrap(); + let outside = root + .parent() + .unwrap() + .join(format!("outside-{}.json", std::process::id())); + fs::write(&outside, show(abc()["A"].as_ref().unwrap())).unwrap(); + std::os::unix::fs::symlink(&outside, run.join("summary.json")).unwrap(); + assert_eq!( + member_summary(&root, Some(&run)), + None, + "a summary.json linking out of its run is not read" + ); + // control: the same file inside the run is read + fs::remove_file(run.join("summary.json")).unwrap(); + fs::copy(&outside, run.join("summary.json")).unwrap(); + fs::remove_file(&outside).unwrap(); + assert!( + member_summary(&root, Some(&run)).is_some(), + "control: the inside check reads a plain file" + ); + // a run folder that links out of the root: never read either + let elsewhere = tmp("elsewhere"); + fs::copy(run.join("summary.json"), elsewhere.join("summary.json")).unwrap(); + std::os::unix::fs::symlink(&elsewhere, root.join("B")).unwrap(); + assert_eq!(member_summary(&root, Some(&root.join("B"))), None); + fs::write(run.join("summary.json"), "[1, 2]").unwrap(); + assert_eq!( + member_summary(&root, Some(&run)), + None, + "a summary that is not an object" + ); + fs::write(run.join("summary.json"), "{not json").unwrap(); + assert_eq!(member_summary(&root, Some(&run)), None); +} + +fn stale_problems(sb: &Value) -> Vec { + let mut out: Vec = list(at(sb, &["rows"])) + .iter() + .filter_map(|r| { + let want = j(&format!( + "[{}, false, false]", + st(at(r, &["group"])) == "Blood groups" + )); + (*at(r, &["stale"]) != want) + .then(|| format!("{}: {}", st(at(r, &["label"])), show(at(r, &["stale"])))) + }) + .collect(); + if list(at(sb, &["rows"])).is_empty() { + out.push("no rows".into()); + } + out +} + +fn blood_problems(sb: &Value) -> Vec { + let rows: Vec<&Value> = list(at(sb, &["rows"])) + .iter() + .filter(|r| st(at(r, &["group"])) == "Blood groups") + .collect(); + let labels: Vec<&str> = rows.iter().map(|r| st(at(r, &["label"]))).collect(); + let mut out = Vec::new(); + if labels != ["ABO", "RhD", "Kell", "Duffy"] { + out.push(format!("blood rows {labels:?}")); + } + if values(sb, "Blood groups", "Kell") + .first() + .map(String::as_str) + != Some("K-k+") + || values(sb, "Blood groups", "Duffy") + .first() + .map(String::as_str) + != Some("Fy(a+b-)") + { + out.push("values".into()); + } + out +} + +fn abc7(a: Value) -> HashMap> { + sums(&[ + ("A", Some(a)), + ("B", Some(summary("*1/*1", &[], "U5", None))), + ("C", None), + ]) +} + +#[test] +fn a_stale_value_is_flagged_in_its_own_cell() { + let sb = side_by_side(&names(&["A", "B", "C"]), &abc7(stale_summary("stale"))); + assert_eq!(stale_problems(&sb), Vec::::new()); + let complete = side_by_side(&names(&["A", "B", "C"]), &abc7(stale_summary("ok"))); + assert!( + !stale_problems(&complete).is_empty(), + "control: the stale section made complete" + ); + let label = |r: &Value| st(at(r, &["label"])).to_owned(); + for (key, want) in [ + ("cpic", vec!["CYP2D6 (callers)", "DPYD"]), + ("hla", vec!["HLA-A"]), + ("haplogroup", vec!["Mitochondrial"]), + ("y_haplogroup", vec!["Y chromosome"]), + ] { + let sb = side_by_side( + &names(&["A"]), + &sums(&[("A", Some(source_summary(key, "stale")))]), + ); + let mut got: Vec = list(at(&sb, &["rows"])) + .iter() + .filter(|r| *at(r, &["stale", "0"]) == Value::Bool(true)) + .map(label) + .collect(); + got.sort(); + assert_eq!( + got, want, + "{key}: each stale source flags its own cells only" + ); + } + let sb = side_by_side( + &names(&["A"]), + &sums(&[("A", Some(source_summary("cpic", "ok")))]), + ); + assert!( + list(at(&sb, &["rows"])) + .iter() + .all(|r| *at(r, &["stale", "0"]) == Value::Bool(false)), + "control: the CPIC section made complete flags nothing" + ); +} + +#[test] +fn every_blood_group_system_in_step_41s_order() { + let sb = side_by_side(&names(&["A", "B", "C"]), &abc7(stale_summary("stale"))); + assert_eq!(blood_problems(&sb), Vec::::new()); + let mut emptied = stale_summary("stale"); + set( + &mut emptied, + &["sections", "bloodtype", "data", "systems"], + j("[]"), + ); + assert!( + !blood_problems(&side_by_side(&names(&["A", "B", "C"]), &abc7(emptied))).is_empty(), + "control: the systems list emptied" + ); + let with_systems = |drop: &str, kidd: bool| { + let mut s = stale_summary("ok"); + let mut systems: Vec = + list(at(&s, &["sections", "bloodtype", "data", "systems"])).to_vec(); + if kidd { + systems.push(j(r#"{"system": "Kidd", "phenotype": "Jk(a+b+)"}"#)); + } + systems.retain(|x| st(at(x, &["system"])) != drop); + set( + &mut s, + &["sections", "bloodtype", "data", "systems"], + Value::Array(systems), + ); + s + }; + let merged = |a: Value, b: Value, order: &[String]| -> Vec { + let sb = side_by_side_in( + &names(&["A", "B"]), + &sums(&[("A", Some(a)), ("B", Some(b))]), + order, + ); + list(at(&sb, &["rows"])) + .iter() + .filter(|r| st(at(r, &["group"])) == "Blood groups") + .map(|r| st(at(r, &["label"])).to_owned()) + .collect() + }; + let order = step41_order(); + assert_eq!( + merged(with_systems("Kell", false), stale_summary("ok"), &order), + ["ABO", "RhD", "Kell", "Duffy"] + ); + let got = merged( + with_systems("Duffy", true), + with_systems("Kell", true), + &order, + ); + assert_eq!( + got, + ["ABO", "RhD", "Kell", "Duffy", "Kidd"], + "a system only a later member has keeps step 41's order" + ); + assert_ne!( + merged(with_systems("Duffy", true), with_systems("Kell", true), &[]), + ["ABO", "RhD", "Kell", "Duffy", "Kidd"], + "control: step 41's order taken away" + ); + assert_eq!( + &order[..5], + ["ABO", "RhD", "RhCE", "Kell", "Duffy"], + "the page's order is og bloodtype systems'" + ); +} + +type Summaries = HashMap>; + +fn malformed_problems(read: &dyn Fn(&[String], &Summaries) -> Value) -> Vec { + let mut out = Vec::new(); + let three = names(&["A", "B", "C"]); + for bad in [ + r#"{"sections": ["not", "a", "mapping"]}"#, + r#"{"cyp2d6": "CYP2D6 *1/*4"}"#, + r#"{"sections": {"cpic": {"state": "ok", "data": {"genes": [{"gene": ["CYP2D6"]}]}}}}"#, + ] { + let mut s = abc(); + s.insert("B".into(), Some(j(bad))); + let got = read(&three, &s); + let mut none = abc(); + none.insert("B".into(), None); + let want = side_by_side(&three, &none); + for r in list(at(&want, &["rows"])) { + let (g, l) = (st(at(r, &["group"])), st(at(r, &["label"]))); + let cols = values(&got, g, l); + if cols.first().map(String::as_str) != Some(st(at(r, &["values", "0"]))) + || cols.get(1).map(String::as_str) != Some("summary not readable") + { + out.push(format!("{bad}: {l} {cols:?}")); + } + } + } + out +} + +#[test] +fn a_summary_of_a_shape_never_written_costs_its_column_only() { + assert_eq!( + malformed_problems(&|n, s| side_by_side(n, s)), + Vec::::new() + ); + // the planted mistake: a bad summary read as one with no rows, never said + let quiet = |n: &[String], s: &HashMap>| { + let mut s = s.clone(); + s.insert("B".into(), Some(j("{}"))); + side_by_side(n, &s) + }; + assert!( + !malformed_problems(&quiet).is_empty(), + "control: a bad summary shown as '.' in every row" + ); +} + +// --- each child's trio status --------------------------------------------------------------- + +fn trio_problems(status: &Value, want: &[(&str, &str, Option, &str)]) -> Vec { + want.iter() + .filter_map(|(child, key, assessed, words)| { + let got = at(status, &[child, key]); + let a = match at(got, &["assessed"]) { + Value::Bool(b) => Some(*b), + _ => None, + }; + (a != *assessed || !st(at(got, &["why"])).contains(words)) + .then(|| format!("{child} {key}: {}", show(got))) + }) + .collect() +} + +struct Trio { + dir: PathBuf, + plan: PathBuf, + plan_text: String, + secret: PathBuf, +} + +/// The demo's trio with a second child, DEMO-004, whose mother DEMO-005 is +/// not sequenced, and compound_het.tsv a link out of the folder. +fn trio() -> Trio { + let work = tmp("trio"); + let dir = work.join("trio"); + copy_tree(&demo(), &dir); + let mut m = fs::read_to_string(dir.join("members.tsv")).unwrap(); + m.push_str( + "DEMO-004\tfemale\tDEMO-001\tDEMO-005\t\tyes\tyes\nDEMO-005\tfemale\t\t\t\tyes\tno\n", + ); + fs::write(dir.join("members.tsv"), m).unwrap(); + let plan = dir.join("family_plan.tsv"); + let plan_text = fs::read_to_string(&plan).unwrap(); + fs::write( + &plan, + plan_text.replace( + "DEMO-001,DEMO-002,DEMO-003\t", + "DEMO-001,DEMO-002,DEMO-003,DEMO-004\t", + ), + ) + .unwrap(); + let secret = work.join("elsewhere-private-name.tsv"); + fs::copy(dir.join("compound_het.tsv"), &secret).unwrap(); + fs::remove_file(dir.join("compound_het.tsv")).unwrap(); + std::os::unix::fs::symlink(&secret, dir.join("compound_het.tsv")).unwrap(); + Trio { + dir, + plan, + plan_text, + secret, + } +} + +fn status(t: &Trio) -> (Value, Value) { + let p = family_page("trio", &t.dir).unwrap(); + (at(&p, &["trio_status"]).clone(), p) +} + +const ASSESSED: [(&str, &str, Option, &str); 4] = [ + ("DEMO-003", "de_novo", Some(true), "both parents"), + ( + "DEMO-004", + "de_novo", + Some(false), + "DEMO-005 (the mother) is not", + ), + ( + "DEMO-003", + "compound_het", + Some(false), + "link out of the family folder", + ), + ( + "DEMO-004", + "compound_het", + Some(false), + "link out of the family folder", + ), +]; + +#[test] +fn each_childs_trio_status() { + let t = trio(); + let (s, page) = status(&t); + assert_eq!(trio_problems(&s, &ASSESSED), Vec::::new()); + assert_eq!( + *at(&page, &["trio_file_state"]), + j(r#"{"de_novo.tsv": "here", "compound_het.tsv": "rejected"}"#) + ); + assert!( + !show(&page).contains("elsewhere-private-name"), + "the link's target is not in family.json" + ); + fs::write(&t.plan, &t.plan_text).unwrap(); + assert_eq!( + trio_problems( + &status(&t).0, + &[( + "DEMO-004", + "de_novo", + Some(false), + "DEMO-004 is not in the joint call" + )] + ), + Vec::::new() + ); + fs::remove_file(t.dir.join("de_novo.tsv")).unwrap(); + assert_eq!( + trio_problems( + &status(&t).0, + &[( + "DEMO-003", + "de_novo", + Some(false), + "de_novo.tsv is not here" + )] + ), + Vec::::new(), + "a missing de_novo.tsv, said apart from a refused one" + ); + fs::remove_file(t.dir.join("compound_het.tsv")).unwrap(); + fs::copy(&t.secret, t.dir.join("compound_het.tsv")).unwrap(); + assert!( + !trio_problems(&status(&t).0, &ASSESSED[2..]).is_empty(), + "control: the link replaced by a plain file" + ); + fs::copy(demo().join("de_novo.tsv"), t.dir.join("de_novo.tsv")).unwrap(); + fs::write( + &t.plan, + t.plan_text.replace( + "DEMO-001,DEMO-002,DEMO-003\t", + "DEMO-002,DEMO-003,DEMO-004\t", + ), + ) + .unwrap(); + let want = [ + ( + "DEMO-003", + "de_novo", + Some(false), + "DEMO-001 (the father) is not", + ), + ( + "DEMO-003", + "compound_het", + Some(true), + "a parent is sequenced", + ), + ( + "DEMO-004", + "de_novo", + Some(false), + "DEMO-001 (the father) is not", + ), + ( + "DEMO-004", + "compound_het", + Some(false), + "need a parent sequenced and in the joint call", + ), + ]; + assert_eq!( + trio_problems(&status(&t).0, &want), + Vec::::new(), + "a sequenced parent left out of the joint call" + ); + fs::write(&t.plan, &t.plan_text).unwrap(); + assert!( + !trio_problems(&status(&t).0, &want[..1]).is_empty(), + "control: the parent back in the joint call" + ); +} + +fn left_out_of_the_joint_call() -> Value { + let t = trio(); + fs::remove_file(t.dir.join("compound_het.tsv")).unwrap(); + fs::copy(&t.secret, t.dir.join("compound_het.tsv")).unwrap(); + fs::write( + &t.plan, + t.plan_text.replace( + "DEMO-001,DEMO-002,DEMO-003\t", + "DEMO-002,DEMO-003,DEMO-004\t", + ), + ) + .unwrap(); + status(&t).0 +} + +fn given(pairs: &[(&str, Option)]) -> HashMap> { + pairs.iter().map(|(k, v)| ((*k).to_owned(), *v)).collect() +} + +#[test] +fn compound_het_pairs_by_the_childs_step_23_candidates() { + let st9 = left_out_of_the_joint_call(); + let told = |g: &[(&str, Option)]| { + let mut s = st9.clone(); + with_candidates(&mut s, &given(g)); + s + }; + let mut bad = trio_problems( + &told(&[ + ("DEMO-003", Some(Given::Absent)), + ("DEMO-004", Some(Given::Here)), + ]), + &[ + ( + "DEMO-003", + "compound_het", + Some(false), + "wrote no candidates for DEMO-003", + ), + ("DEMO-004", "compound_het", Some(false), "need a parent"), + ], + ); + bad.extend(trio_problems( + &told(&[("DEMO-003", Some(Given::Here))]), + &[( + "DEMO-003", + "compound_het", + Some(true), + "candidates (step 23) are here", + )], + )); + for (g, words) in [ + (Some(Given::Newer), "newer than compound_het.tsv"), + (None, "run is not on this site"), + ] { + bad.extend(trio_problems( + &told(&[("DEMO-003", g)]), + &[("DEMO-003", "compound_het", None, words)], + )); + } + assert_eq!(bad, Vec::::new()); + assert!( + !trio_problems( + &told(&[("DEMO-003", None)]), + &[( + "DEMO-003", + "compound_het", + Some(false), + "wrote no candidates" + )] + ) + .is_empty(), + "control: the candidates not known read as absent" + ); + + let root = tmp("candidates"); + let run3 = root.join("DEMO-003"); + fs::create_dir_all(run3.join("clinical")).unwrap(); + let vcf = run3.join("clinical/DEMO-003_clinical.vcf.gz"); + let none_yet = candidates_given(&root, Some(&run3), "DEMO-003", None); + fs::write(&vcf, "").unwrap(); + let t = |s: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(s); + fs::File::options() + .write(true) + .open(&vcf) + .unwrap() + .set_modified(t(1000)) + .unwrap(); + let here = candidates_given(&root, Some(&run3), "DEMO-003", Some(t(2000))); + let newer = candidates_given(&root, Some(&run3), "DEMO-003", Some(t(500))); + let outside = root + .parent() + .unwrap() + .join(format!("outside-{}.vcf.gz", std::process::id())); + fs::write(&outside, "").unwrap(); + fs::remove_file(&vcf).unwrap(); + std::os::unix::fs::symlink(&outside, &vcf).unwrap(); + let linked = candidates_given(&root, Some(&run3), "DEMO-003", None); + let got = [ + none_yet, + here, + newer, + linked, + candidates_given(&root, None, "DEMO-003", None), + ]; + fs::remove_file(&outside).unwrap(); + assert_eq!( + got, + [ + Some(Given::Absent), + Some(Given::Here), + Some(Given::Newer), + Some(Given::Absent), + None + ] + ); +} + +#[test] +fn the_site_page_adds_runs_candidates_and_summaries() { + let t = trio(); + let root = t.dir.parent().unwrap().to_path_buf(); + for n in ["DEMO-001", "DEMO-002", "DEMO-003"] { + fs::create_dir_all(root.join(n).join("clinical")).unwrap(); + } + fs::write( + root.join("DEMO-001/summary.json"), + show(abc()["A"].as_ref().unwrap()), + ) + .unwrap(); + fs::write( + root.join("DEMO-002/summary.json"), + r#"{"sections": ["not", "a", "mapping"]}"#, + ) + .unwrap(); + fs::remove_file(t.dir.join("compound_het.tsv")).unwrap(); + fs::copy(&t.secret, t.dir.join("compound_het.tsv")).unwrap(); + let dir = |n: &str| Value::Str(root.join(n).to_string_lossy().into_owned()); + let mut input = j(r#"{"runs": {"DEMO-001": {"url": "/runs/DEMO-001/", "child": false}}}"#); + set(&mut input, &["candidates", "DEMO-003"], dir("DEMO-003")); + for n in ["DEMO-001", "DEMO-002", "DEMO-003"] { + set(&mut input, &["summaries", n], dir(n)); + } + let page = site_page("trio", &t.dir, &root, &input).unwrap(); + assert_eq!( + *at(&page, &["runs"]), + *at(&input, &["runs"]), + "the member runs as the site gave them" + ); + let keys: Vec<&str> = match &page { + Value::Object(o) => o.keys().map(String::as_str).collect(), + _ => Vec::new(), + }; + assert_eq!(keys.last(), Some(&"runs")); + assert_eq!( + trio_problems( + at(&page, &["trio_status"]), + &[ + ( + "DEMO-003", + "compound_het", + Some(false), + "wrote no candidates for DEMO-003" + ), + ( + "DEMO-004", + "compound_het", + None, + "DEMO-004's run is not on this site" + ), + ] + ), + Vec::::new(), + "candidates told only for the children the site named" + ); + let sb = at(&page, &["side_by_side"]); + assert_eq!( + *at(sb, &["members"]), + j(r#"["DEMO-001", "DEMO-002", "DEMO-003", "DEMO-004"]"#) + ); + assert_eq!( + values(sb, "Blood groups", "ABO"), + [ + "A1", + "summary not readable", + "no summary yet", + "no summary yet" + ] + ); + // the command line reads the same input on stdin's stand-in, a file + let f = root.join("input.json"); + fs::write(&f, show(&input)).unwrap(); + let args: Vec = [ + "page", + t.dir.to_str().unwrap(), + "--name", + "trio", + "--root", + root.to_str().unwrap(), + "--input", + f.to_str().unwrap(), + ] + .iter() + .map(|a| (*a).to_owned()) + .collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!( + cli(&args, &mut out, &mut err), + 0, + "{}", + String::from_utf8_lossy(&err) + ); + assert_eq!(j(&String::from_utf8(out).unwrap()), page); + // without the input: the page as family_page builds it, no runs, nobody side by side + let plain = family_page("trio", &t.dir).unwrap(); + assert!(*at(&plain, &["runs"]) == Value::Null && *at(&plain, &["side_by_side"]) == Value::Null); +} + +#[test] +fn ok_reads_in_words_on_the_page_and_as_written_in_the_state() { + let page = family_page("DEMO-FAMILY", &demo()).unwrap(); + for rows in [ + at(&page, &["relationships"]), + at(&page, &["state", "relationships"]), + ] { + assert!( + list(rows) + .iter() + .all(|r| st(at(r, &["verdict"])) == "matches the declared relation"), + "{}", + show(rows) + ); + } + let s = family_state("DEMO-FAMILY", &demo()).unwrap(); + assert!( + list(at(&s, &["relationships"])) + .iter() + .all(|r| st(at(r, &["verdict"])) == "ok") + ); +} From 0fa6183ad4b0791806499919b2f970a0ade90e23 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:25:13 +0100 Subject: [PATCH 3/3] refactor(live): relay the family page and pedigree to og The family plugin asks og live family for every page it serves; the Python pedigree drawing and family report code and their tests are removed. Bead: D1 --- CHANGELOG.md | 1 + bin/owngenome_live_family.py | 557 +++++----------------------------- scripts/ci/e2e-case-paths.tsv | 36 +-- tests/demo/build_live_demo.py | 26 +- tests/parity/remnants.tsv | 1 - tests/test_family_report.py | 48 +-- 6 files changed, 119 insertions(+), 550 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dda3da6ec..3e52d9465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The live site's family page and its pedigree drawing are now built by `og live family`, in Rust. `bin/owngenome_live_family.py` asks `og` for a family's state, its `family.json`, the list of files it serves and `pedigree.svg`, and no longer reads the family's tables itself; `bin/pedigree_svg.py` and `bin/family_report.py` are gone. The page shows the same thing: `family.json` and the drawing were compared with the old code on the demo family and the test families, field by field and number by number, and matched. The live site therefore needs `og` (`scripts/setup.sh --og` installs it); without it a family's `family.json` answers 500, its `pedigree.svg` answers 503, and the run list marks each family unreadable. A family file that cannot be read still gives no page and an unreadable family. - Step 25 and the Nextflow `PRS_SUMMARY` withhold a score's percentile when the genotypes pgsc_calc scored list variants only. Every site such a VCF leaves out adds nothing to the sum, so the sum leans away from the panel's sums, which come from complete genotypes, and the percentile was not comparable. Each score keeps its raw sum. Its `Percentile` is `NA`, its `Effect` is empty, and a new column before `Input`, `Percentile_Note`, says `percentile withheld: the input lists variants only`. The decision reads the genotypes, not the kind of input: at least one genotyped position in five must be a reference call (0/0), the rule step 26b applies to the panel's sites. Genotypes from a gVCF and chip VCFs from `scripts/chip-to-vcf.sh`, which list their reference calls, keep their percentiles. `og prs table` takes the genotypes with a new option, `--target VCF`, and prints the NOTE that step 25 printed before; the NOTE now follows the genotypes too, so a gVCF without reference blocks at the score positions gets it. `og summary collect` passes the note on as `percentile_note`. When every percentile was withheld, both reports say "Percentile withheld: the input lists variants only" and why, and the summary's `not_assessed` list says the percentiles were withheld, in place of the lines for a run without the panel, which told the user to install a panel the run had used. The output gains one column; no column, option or file name changes. - A third-party outage no longer fails the Freshness check of a pull request. Since 2026-11-18 rest.ensembl.org has not answered, and the pull-request job's dry run of `og dev freshness` failed on every pull request that touched `versions.env` or `og-fresh`. `og dev freshness --unreachable-warns` now lists a lookup whose upstream did not answer (no connection, a timeout, HTTP 5xx or 429 after the retries, or HTTP 403) under "Upstreams that did not answer" as a warning, and its database row reads `NO ANSWER`; it does not change the exit. The 403 counts because rest.ensembl.org came back on 2026-11-19 answering 403 to GitHub's runners and 200 elsewhere, which still failed the pull-request job. A 404, an empty answer or an answer in an odd shape still exits 1. `freshness.yml` passes the flag only in the pull-request dry run. Without the flag the report and the exit are as before, and the monthly report on main still counts an outage as an error in the issue. - Three statements in the docs were wrong. Troubleshooting said exit code 137 confirms a kill for memory, and checked it with `docker ps -a --latest`. 137 is any SIGKILL, and the step scripts run their containers with `--rm`, so that command shows an older, unrelated container. The page now says how to confirm a kill for memory: `docker events --filter event=oom`, the kernel log, or `.State.OOMKilled` for a container kept without `--rm`. The run-state and troubleshooting pages said `bin/og` does not run on a Mac and told Mac users to build `og` with cargo or run it in a container. `bin/og` runs the native macOS build that `scripts/setup.sh --og` installs, and the pages now say so; on a Mac, `og run headroom` reports the machine's memory as unknown. Troubleshooting, the FAQ and the lessons-learned page said the step scripts run their containers as root, and lessons-learned gave `:latest` images and `chmod 777` as current fixes. The step scripts run their containers as you; those entries are now marked as past fixes and name what the pipeline does today. Documentation only: no option, output or behaviour changes. diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index 47725f49c..6fb723535 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -3,7 +3,7 @@ A family folder under GENOME_DIR (one holding members.tsv or pedigree.tsv, as steps 38 and 39 and the Nextflow family workflow leave it) is listed as a run of kind "family". Its page shows the pedigree, drawn from that file at request -time by bin/pedigree_svg.py, each child's chromosome map +time by `og live family svg`, each child's chromosome map (_segments.svg), the relationship check, the shared segments and the couple carrier screen, as the steps wrote them. A sample's page links to the family it is in. @@ -35,24 +35,30 @@ {"kind": "member", "families": [], "unknown": true} when no readable family lists the sample but a family's pedigree could not be read, null otherwise. -The plugin computes nothing about genetics: it reads the files the steps -wrote, counts their rows and draws the pedigree the file declares. +og builds every answer (`og live family`, rust/crates/og-live/src/family.rs): +state.json's part, family.json, the family list and the drawing. og reads the +files the steps wrote, counts their rows and draws the pedigree the file +declares; it computes nothing about genetics. The plugin adds what only the +site knows (each member run's progress, attention count and child flag) and +sends og's answer. OG names og when set, else bin/og. While og cannot list the +families, every family folder counts as unreadable, so every run is treated +as a child (fail closed). This is Python only while the site is: the Rust +port of the live server replaces this file. """ +import json import os import re +import subprocess import sys HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, HERE) -import pedigree_svg # noqa: E402 +# OG names og when set (tests/og_binary.py reads it the same way), else bin/og. +OG = os.environ.get("OG") or os.path.join(HERE, "og") +OG_SECONDS = 120 NAME = r"[A-Za-z0-9][A-Za-z0-9._-]*" NAME_RE = re.compile("^%s$" % NAME) MARKERS = ("members.tsv", "pedigree.tsv") -# The family's result files the page may open, besides the drawn pedigree and the -# per-child and per-couple files (named from the members). -FIXED = ("family_report.md", "ibd_report.md", "relationships.tsv", "ibd_pairs.tsv", "family_plan.tsv", - "members.tsv", "pedigree.tsv") TYPES = {".svg": "image/svg+xml", ".md": "text/plain; charset=utf-8", ".tsv": "text/plain; charset=utf-8"} @@ -79,461 +85,60 @@ def family_dirs(root): return out -def read_tsv(path): - """Rows of a tab-separated file with a header line, as dicts; [] when it is not there or is a - link out of its own folder (family.json serves only what the family folder itself holds).""" - if not os.path.isfile(path) or not inside(path, os.path.dirname(os.path.abspath(path))): - return [] - rows = [] - with open(path, errors="replace") as f: - head = f.readline().rstrip("\r\n").lstrip("#").split("\t") - for line in f: - line = line.rstrip("\r\n") - if line.strip() and not line.startswith("#"): - vals = line.split("\t") - rows.append(dict(zip(head, vals + [""] * (len(head) - len(vals))))) - return rows - - -def members_of(d): - order, members, _ = pedigree_svg.load(d) - if not order: - raise ValueError("the pedigree names no one") - return order, members - - -def couple_names(order, members): - """couple__ of each pair of partners, a and b sorted by name as step 39 names them.""" - out = [] - for a, b in pedigree_svg.couples(order, members): - if members[a]["partner"] == b or members[b]["partner"] == a: - x, y = sorted((a, b)) - out.append((x, y, "couple_%s_%s" % (x, y))) - return out +def og(args, stdin=None, og_bin=None): + """og live family ARGS: (exit code, stdout, its first stderr line); 127 when og is not on this host.""" + try: + p = subprocess.run([og_bin or OG, "live", "family"] + args, input=stdin, capture_output=True, + timeout=OG_SECONDS, check=False) + except FileNotFoundError: + return 127, b"", "og is not on this host: run scripts/setup.sh --og" + except (OSError, subprocess.TimeoutExpired) as e: + return 1, b"", type(e).__name__ + return p.returncode, p.stdout, (p.stderr.decode(errors="replace").strip().splitlines() or [""])[0] -def served_files(d, order, members): - """The file names of the family folder the page may open: those that exist.""" - names = list(FIXED) + ["%s_segments.svg" % s for s in order] - for _, _, base in couple_names(order, members): - names += [base + ".md", base + ".tsv"] - return sorted(n for n in set(names) if os.path.isfile(os.path.join(d, n)) and inside(os.path.join(d, n), d)) +def og_json(args, stdin=None, og_bin=None): + code, out, err = og(args, stdin, og_bin) + if code != 0: + raise OSError(err or "og live family %s exited %s" % (args[0], code)) + return json.loads(out) def family_state(name, d): - try: - order, members = members_of(d) - except (OSError, ValueError) as e: - return {"kind": "family", "error": "the pedigree could not be read: %s" % e} - files = served_files(d, order, members) - url = "/family/%s/" % name - - def link(f): - return url + f if f in files else None - - people = [{"sample": s, "sex": members[s]["sex"], "father": members[s]["father"], - "mother": members[s]["mother"], "partner": members[s]["partner"], - "sequenced": members[s]["sequenced"]} for s in order] - children = [{"child": p["sample"], "father": p["father"], "mother": p["mother"], - "segments": link("%s_segments.svg" % p["sample"])} - for p in people if p["father"] or p["mother"]] - rel = [{"a": r.get("sample_a"), "b": r.get("sample_b"), "declared": r.get("declared"), - "verdict": r.get("verdict"), "reason": r.get("reason")} - for r in read_tsv(os.path.join(d, "relationships.tsv"))] - ibd = [{"a": r.get("sample_a"), "b": r.get("sample_b"), "declared": r.get("declared"), - "shared_cm": r.get("shared_cm"), "degree": r.get("degree")} - for r in read_tsv(os.path.join(d, "ibd_pairs.tsv"))] - couples = [] - for a, b, base in couple_names(order, members): - rows = read_tsv(os.path.join(d, base + ".tsv")) - results = [r.get("result") for r in rows] - couples.append({"a": a, "b": b, "report": link(base + ".md"), "table": link(base + ".tsv"), - "screened": bool(rows) or base + ".md" in files, - "shared": results.count("shared"), "x_linked": results.count("x_linked")}) - plan = [{"analysis": r.get("analysis"), "members": r.get("members"), "decision": r.get("decision"), - "reason": r.get("reason")} for r in read_tsv(os.path.join(d, "family_plan.tsv"))] - return {"kind": "family", "name": name, "pedigree": url + "pedigree.svg", "members": people, - "children": children, "relationships": rel, "ibd_pairs": ibd, "couples": couples, "plan": plan, - "reports": [{"name": f, "url": url + f} for f in files if f.endswith(".md")], - "files": files} - - -def layout_of(order, members): - """The pedigree drawing's geometry (bin/pedigree_svg.py layout) as data, so the page draws each - person as a link: people with their centre, partner lines and the descent lines of each sibship.""" - pos, gen, pairs, widest, ngen = pedigree_svg.layout(order, members) - h = pedigree_svg.SIZE / 2 - people = [{"sample": s, "x": round(pos[s][0], 1), "y": round(pos[s][1], 1), "gen": gen[s], "sex": members[s]["sex"], - "sequenced": bool(members[s]["sequenced"])} for s in order] - lines = [] - for a, b in pairs: - (xa, ya), (xb, yb) = sorted((pos[a], pos[b])) - lines.append([xa + h, ya, xb - h, yb]) - sibships = {} - for s in order: - f, mo = members[s]["father"], members[s]["mother"] - f, mo = (f if f in pos else None), (mo if mo in pos else None) - if f or mo: - sibships.setdefault((f, mo), []).append(s) - for (f, mo), kids in sibships.items(): - if f and mo: - ox, oy = (pos[f][0] + pos[mo][0]) / 2, (pos[f][1] + pos[mo][1]) / 2 - else: - ox, oy = pos[f or mo][0], pos[f or mo][1] + h - bar = min(pos[k][1] for k in kids) - h - 22 - xs = [pos[k][0] for k in kids] + [ox] - lines += [[ox, oy, ox, bar], [min(xs), bar, max(xs), bar]] - lines += [[pos[k][0], bar, pos[k][0], pos[k][1] - h] for k in kids] - width = 2 * pedigree_svg.MARGIN + pedigree_svg.SIZE + (widest - 1) * pedigree_svg.STEP_X - height = pedigree_svg.TOP + pedigree_svg.SIZE + (ngen - 1) * pedigree_svg.STEP_Y + 30 - return {"width": width, "height": height, "size": pedigree_svg.SIZE, "people": people, - "lines": [[round(v, 1) for v in ln] for ln in lines]} - - -def family_page(name, d): - """family.json without the member runs: the state, the layout, the full pair rows and couple rows.""" - st = family_state(name, d) - import render_report # the couple screen's caveat, word for word - out = {"state": st, "layout": None, "relationships": [], "ibd_pairs": [], "couples": {}, - "couple_note": render_report.COUPLE_NOTE, "segments": {}, "de_novo": [], "compound_het": [], - "trio_files": {}, "side_by_side": None} - if st.get("error"): - return out - order, members = members_of(d) - out["layout"] = layout_of(order, members) - keep = ("sample_a", "sample_b", "declared", "expected_relatedness", "relatedness", "ibs0", "ibs2", "sites", - "verdict", "reason") - out["relationships"] = [{k: r.get(k) for k in keep} for r in read_tsv(os.path.join(d, "relationships.tsv"))] - keep = ("sample_a", "sample_b", "declared", "segments", "shared_cm", "ibd2_cm", "longest_cm", "relatedness", - "degree") - out["ibd_pairs"] = [{k: r.get(k) for k in keep} for r in read_tsv(os.path.join(d, "ibd_pairs.tsv"))] - order_of = {"shared": 0, "x_linked": 1, "one_partner": 2} - for a, b, base in couple_names(order, members): - rows = read_tsv(os.path.join(d, base + ".tsv")) - keep = ("gene", "inheritance", "condition", "result", "offspring_risk", "partner_a", "findings_a", - "partner_b", "findings_b", "explanation") - rows = sorted(({k: r.get(k) for k in keep} for r in rows), key=lambda r: order_of.get(r.get("result"), 3)) - out["couples"]["%s %s" % (a, b)] = rows - out.update(child_depth(d, order, members)) - import report_charts # GRCh38's lengths, for the painting of each child's stretches - out["chrom_lengths"] = [[c, n] for c, n in report_charts.GRCH38 if c != "chrY"] - # Step 38 writes the verdict "ok" for a pair that matches its declared relation. The page says that - # in words: no word on the page reads as an all-clear (state.json keeps the file's own value). - out["state"] = dict(st, relationships=[dict(r, verdict=verdict_words(r.get("verdict"))) - for r in st.get("relationships") or []]) - for r in out["relationships"]: - r["verdict"] = verdict_words(r.get("verdict")) - return out + """state.json's plugins.family on the family's page (og live family state).""" + return og_json(["state", d, "--name", name]) -SEGMENT_KEYS = ("side", "parent", "chrom", "start", "end", "length_bp", "sites", "discordant", "ancestor", "basis") -DE_NOVO_KEYS = ("child", "chrom", "pos", "ref", "alt", "child_gt", "father_gt", "mother_gt", "child_dp", "child_gq", - "child_ab", "quality") -COMP_KEYS = ("child", "gene", "variant_a", "variant_b", "a_from", "b_from", "phase") -PHASE_ORDER = {"in trans": 0, "unknown": 1, "in cis": 2} - - -def child_depth(d, order, members): - """Each child's stretches (step 38's _segments.tsv) and the trio tables, as the steps wrote them: - {segments: {child: [rows]}, de_novo: [rows], compound_het: [rows, in trans first]}.""" - segs = {} - for s in order: - if not (members[s]["father"] or members[s]["mother"]): - continue - rows = read_tsv(os.path.join(d, "%s_segments.tsv" % s)) - if rows: - segs[s] = [{k: r.get(k) for k in SEGMENT_KEYS} for r in rows] - files = {f: file_state(d, f) for f in TRIO_FILES} - comp = [{k: r.get(k) for k in COMP_KEYS} for r in read_tsv(os.path.join(d, "compound_het.tsv"))] - comp.sort(key=lambda r: PHASE_ORDER.get(r.get("phase"), 1)) - return {"segments": segs, - "de_novo": [{k: r.get(k) for k in DE_NOVO_KEYS} for r in read_tsv(os.path.join(d, "de_novo.tsv"))], - "compound_het": comp, - # read_tsv's own test: a link out of the family folder counts as not there - "trio_files": {f: files[f] == "here" for f in TRIO_FILES}, - "trio_file_state": files, - "trio_status": trio_status(d, order, members, files)} - - -TRIO_FILES = ("de_novo.tsv", "compound_het.tsv") -# Step 41's own order of the blood group systems, as `og bloodtype systems` -# prints it (tests/test_live_family_depth.py checks the two agree). -STEP41_SYSTEMS = ("ABO", "RhD", "RhCE", "Kell", "Duffy", "Kidd", "MNS", "Lutheran", "Diego", "Dombrock", "Colton", - "Vel", "Langereis", "Junior", "H", "Secretor", *(f"HPA-{n}" for n in range(1, 36)), - "HNA-1", "HNA-3", "HNA-4", "HNA-5") - - -def file_state(d, f): - """'here', 'rejected' (a link out of the family folder: read_tsv does not read it, and the page never - shows where it points) or 'missing'.""" - p = os.path.join(d, f) - if not os.path.lexists(p): - return "missing" - if not inside(p, os.path.abspath(d)): - return "rejected" - return "here" if os.path.isfile(p) else "missing" - - -def trio_status(d, order, members, files): - """Per child, whether step 38 assessed it for de novo candidates and compound-het pairs, and why not: - {child: {"de_novo": {"assessed": bool, "why": text}, "compound_het": {...}}}. The rule is step 38's - (og family report): de novo candidates need both parents sequenced, compound-het pairs a sequenced - parent; a file that is not here, or is a link out of the folder, means no child was assessed.""" - plan = {r.get("analysis"): r for r in read_tsv(os.path.join(d, "family_plan.tsv"))} - trio = plan.get("trio_phasing") or {} - - def absent(f): - if files[f] == "rejected": - return "%s is a link out of the family folder, so this page does not read it" % f - why = "%s is not here" % f - if trio.get("decision"): - why += ": trio phasing %s%s" % (trio["decision"], " (%s)" % trio["reason"] if trio.get("reason") else "") - return why - - joint = plan.get("joint_call") or {} - called = set((joint.get("members") or "").split(",")) if joint.get("decision") == "runs" else None - - def seq(p): - return bool(p) and p in members and bool(members[p]["sequenced"]) and (called is None or p in called) - - need = "sequenced" if called is None else "sequenced and in the joint call" - - def not_called(p): - if not members[p]["sequenced"]: - return "%s is not sequenced" % p - return "%s is not in the joint call (family_plan.tsv)" % p - - def gaps(m): - return "; ".join(("no %s is named" % w) if not m[w] else "%s (the %s) is not" % (m[w], w) - for w in ("father", "mother") if not seq(m[w])) - - out = {} - for s in order: - m = members[s] - if not (m["father"] or m["mother"]): - continue - st = {} - if files["de_novo.tsv"] != "here": - st["de_novo"] = {"assessed": False, "why": absent("de_novo.tsv")} - elif not seq(s): - st["de_novo"] = {"assessed": False, "why": not_called(s)} - elif seq(m["father"]) and seq(m["mother"]): - st["de_novo"] = {"assessed": True, "why": "both parents are sequenced"} - else: - st["de_novo"] = {"assessed": False, "why": "de novo candidates need both parents %s: %s" % (need, gaps(m))} - if files["compound_het.tsv"] != "here": - st["compound_het"] = {"assessed": False, "why": absent("compound_het.tsv")} - elif not seq(s): - st["compound_het"] = {"assessed": False, "why": not_called(s)} - elif seq(m["father"]) or seq(m["mother"]): - st["compound_het"] = {"assessed": True, "why": "a parent is sequenced; pairs come only from the child's " - "clinical-filter candidates (step 23), so with none given no pair is listed"} - else: - st["compound_het"] = {"assessed": False, "why": "compound-het pairs need a parent %s" % need} - out[s] = st - return out +def family_page(name, d, site=None, root=None, og_bin=None): + """family.json (og live family page). SITE holds what only the site knows: "runs" (each member run's entry), + "candidates" and "summaries" (each child's and member's run folder, read only inside ROOT).""" + args = ["page", d, "--name", name] + if site is None: + return og_json(args, None, og_bin) + args += ["--root", root or os.path.dirname(os.path.abspath(d)), "--input", "-"] + return og_json(args, json.dumps(site).encode(), og_bin) -def candidates_given(live, d, sample, since=None): - """Whether the clinical filter (step 23) wrote 's candidates, the only variants step 38 pairs: - "here", "absent" (the run is here without them; a file linking out of the run counts as absent), - "newer" (written after since, compound_het.tsv's mtime, so step 38 may not have had them) or None - (the run is not on this site: the page cannot tell).""" - if not d: +def draw(d, title, og_bin=None): + """The pedigree's SVG (og live family svg), or None when the folder holds no pedigree that names anyone.""" + code, out, err = og(["svg", d, "--title", title], None, og_bin) + if code == 2: return None - p = os.path.join(d, "clinical", "%s_clinical.vcf.gz" % sample) - if not (os.path.isfile(p) and live.inside(p, os.path.realpath(d)) and live.inside(p, live.root)): - return "absent" - return "newer" if since is not None and os.path.getmtime(p) > since else "here" - - -def with_candidates(status, given): - """trio_status with the compound-het line told by the child's step 23 candidates ({child: one of - candidates_given's answers}). assessed is None where the page cannot tell.""" - for child, st in status.items(): - c = st.get("compound_het") or {} - if not c.get("assessed"): - continue - g = given.get(child) - if g == "here": - c["why"] = ("a parent is sequenced and the child's clinical-filter candidates (step 23) are here; " - "pairs come only from them") - elif g == "absent": - st["compound_het"] = {"assessed": False, "why": "the clinical filter (step 23) wrote no candidates for " - "%s (clinical/%s_clinical.vcf.gz), so step 38 had no variant to pair" % (child, child)} - elif g == "newer": - st["compound_het"] = {"assessed": None, "why": "%s's clinical-filter candidates (step 23) are newer than " - "compound_het.tsv, so step 38 may not have paired them" % child} - else: - st["compound_het"] = {"assessed": None, "why": "pairs come only from the child's clinical-filter " - "candidates (step 23), and %s's run is not on this site to tell whether step 23 " - "wrote them" % child} - return status - - -def _sec(summary, key): - sec = ((summary or {}).get("sections") or {}).get(key) or {} - return sec.get("data") or {} if sec.get("state") in ("ok", "stale") else None - - -def _stale(summary, *keys): - secs = (summary or {}).get("sections") or {} - return any(((secs.get(k) or {}).get("state")) == "stale" for k in keys) - - -def side_values(summary): - """[(group, label, value, stale)] of one member's summary.json, as its report prints them; nothing - computed. stale is true when the value comes from a section older than its input (state "stale").""" - out = [] - c = (summary or {}).get("cyp2d6") or {} - calls = ", ".join("%s %s" % (k, v) for k, v in (c.get("calls") or {}).items() if v) - if calls: - agree = c.get("agree") - # summary.json's cyp2d6 block reads pypgx and Cyrius only when complete; its PharmCAT call is cpic's - out.append(("Medicines", "CYP2D6 (callers)", calls + (" (callers agree)" if agree else " (callers differ)" - if agree is False else ""), - bool((c.get("calls") or {}).get("PharmCAT")) and _stale(summary, "cpic"))) - cpic = _sec(summary, "cpic") - for g in (cpic or {}).get("genes") or []: - ph = g.get("phenotype") - out.append(("Medicines", g.get("gene"), "%s%s" % (g.get("diplotype") or ".", ' "%s"' % ph if ph else ""), - _stale(summary, "cpic"))) - for l in (_sec(summary, "hla") or {}).get("loci") or []: - out.append(("HLA", l.get("gene"), " ".join(l.get("alleles") or []) or "not typed", _stale(summary, "hla"))) - mt = _sec(summary, "haplogroup") - if mt is not None: - out.append(("Haplogroups", "Mitochondrial", mt.get("haplogroup") or ".", _stale(summary, "haplogroup"))) - ykey = "y_tree" if _sec(summary, "y_tree") else "y_haplogroup" - y = _sec(summary, "y_tree") or _sec(summary, "y_haplogroup") - if y is not None: - out.append(("Haplogroups", "Y chromosome", y.get("haplogroup") or ".", _stale(summary, ykey))) - bt = _sec(summary, "bloodtype") - if bt is not None: - old = _stale(summary, "bloodtype") - systems = [x for x in bt.get("systems") or [] if x.get("system")] - by = {x["system"]: x.get("phenotype") or "." for x in systems} - out.append(("Blood groups", "ABO", bt.get("abo") or by.get("ABO", "."), old)) - out.append(("Blood groups", "RhD", bt.get("rhd") or by.get("RhD", "."), old)) - # every other system step 41 typed, in its order (Kell, Duffy, Kidd, MNS...) - for x in systems: - if x["system"] not in ("ABO", "RhD"): - out.append(("Blood groups", x["system"], by[x["system"]], old)) + if code != 0: + raise OSError(err or "og live family svg exited %s" % code) return out -def side_by_side(names, summaries): - """The members' results in one table: {members, rows: [{group, label, values, stale}]}. A member with - no summary.json yet says so in its column; a row a member's summary does not have is '.'. stale holds, - per member, whether that value comes from a section older than its input. A summary whose fields are - not of the shape the report writes costs its own column only: it says so there, and the family page - still draws.""" - per, bad = {}, set() - for n in names: - try: - got = side_values(summaries.get(n)) - for g, label, _, _ in got: - hash((g, label)) # a label that is a list or a dict cannot key the table's rows - per[n] = got - except (AttributeError, KeyError, TypeError, ValueError, IndexError): - per[n], bad = [], bad | {n} - # Each member's rows keep their own order (step 41's for the blood groups): a row first seen in a later - # member goes right after the row before it in that member's list, not at the end. - keys = [] - for n in names: - prev = None - for g, label, _, _ in per[n]: - if (g, label) not in keys: - keys.insert(keys.index(prev) + 1 if prev else len(keys), (g, label)) - prev = (g, label) - groups = ["Medicines", "HLA", "Haplogroups", "Blood groups"] - step41 = {s: i for i, s in enumerate(STEP41_SYSTEMS)} # for systems two members each lack one of - pos = {k: i for i, k in enumerate(keys)} - keys.sort(key=lambda k: (groups.index(k[0]) if k[0] in groups else 9, - step41.get(k[1], len(step41)) if k[0] == "Blood groups" else 0, pos[k])) - rows = [{"group": g, "label": label, "values": [], "stale": []} for g, label in keys] - for r in rows: - for n in names: - if n in bad: - r["values"].append("summary not readable") - r["stale"].append(False) - elif summaries.get(n) is None: - r["values"].append("no summary yet") - r["stale"].append(False) - else: - hit = next((x for x in per[n] if (x[0], x[1]) == (r["group"], r["label"])), None) - r["values"].append(hit[2] if hit else ".") - r["stale"].append(bool(hit and hit[3])) - return {"members": list(names), "rows": rows} - - -def verdict_words(v): - return "matches the declared relation" if (v or "").strip().lower() == "ok" else v - - -def member_summary(live, d): - """A member run's summary.json as a dict, or None (none yet, unreadable, or a link out of the run).""" - import json - p = os.path.join(d, "summary.json") if d else None - if not p or not os.path.isfile(p) or not live.inside(p, os.path.realpath(d)) or not live.inside(p, live.root): - return None - try: - with open(p) as f: - got = json.load(f) - except (OSError, ValueError): - return None - return got if isinstance(got, dict) else None - - -_BUILT = {} - - -def folder_stamp(d): - """What a read of the family folder depends on: each entry's name, mode, size, mtime and ctime (a chmod changes - the ctime alone). None when the folder cannot be listed.""" - try: - names = sorted(os.listdir(d)) - except OSError: - return None - out = [] - for n in names: - try: - st = os.stat(os.path.join(d, n)) - out.append((n, st.st_mode, st.st_size, st.st_mtime_ns, st.st_ctime_ns)) - except OSError: - out.append((n, None)) - return tuple(out) - - -def family_readable(name, d): - """Whether family.json's family part (family_page) builds from the folder's files as they are now, with its - pedigree read. Cached by folder_stamp, so a file that reads again counts at once, with no request needed.""" - stamp = folder_stamp(d) - hit = _BUILT.get(d) - if stamp is not None and hit and hit[0] == stamp: - return hit[1] - try: - ok = stamp is not None and not family_page(name, d)["state"].get("error") - except Exception: # any file of the folder that cannot be read - ok = False - _BUILT[d] = (stamp, ok) - return ok - - def families(root): """Each family folder under root as /runs.json lists it: {"name", "dir", "kind", "members", "unreadable"}. members is [] while the pedigree cannot be read; unreadable while family.json cannot be built from the folder.""" - out = [] - for n, d in family_dirs(root): - try: - order, members = members_of(d) - people = [{"sample": s, "sex": members[s]["sex"], "father": members[s]["father"], - "mother": members[s]["mother"], "partner": members[s]["partner"], - "sequenced": members[s]["sequenced"]} for s in order] - except (OSError, ValueError): - people = [] - out.append({"name": n, "dir": d, "kind": "family", "members": people, - "unreadable": not people or not family_readable(n, d)}) - return out + if not family_dirs(root): + return [] + try: + return og_json(["families", root]) + except (OSError, ValueError) as e: + print(f"owngenome_live: the families could not be read: {e}", file=sys.stderr) + return [{"name": n, "dir": d, "kind": "family", "members": [], "unreadable": True} for n, d in family_dirs(root)] def member_state(root, sample): @@ -574,17 +179,20 @@ def page_data(req, m): return req.send_json(out) def page_json(req, name, d): - out = family_page(name, d) live = getattr(req, "live", None) - runs = {} + runs, candidates, summaries = {}, {}, {} listed = live.runs() if live else {} - for p in out["state"].get("members") or []: + for p in family_state(name, d).get("members") or []: r = listed.get(p["sample"]) if live else None - if not r or r.get("kind") == "family": + if not r: + continue + rd = r["dir"] or os.path.join(live.root, r["name"]) + summaries[p["sample"]] = rd + if r.get("kind") == "family": continue + candidates[p["sample"]] = rd st = live.base_state(r) planned = [s for s in st["steps"] if s["state"] != "not planned"] - rd = r["dir"] or os.path.join(live.root, r["name"]) n, mtime = live.attention(rd) # A child whose own page keeps the report closed has no count of flagged cards here: the count changes # with which cards the unopened results flag, and a parent's page would show it. @@ -595,24 +203,11 @@ def page_json(req, name, d): runs[p["sample"]] = {"url": "/runs/%s/" % p["sample"], "state": st["run"]["state"], "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"), "attention": n, "results": mtime is not None, "child": live.child(p["sample"], listed)} - out["runs"] = runs - given = {} - comp_p = os.path.join(d, "compound_het.tsv") - since = os.path.getmtime(comp_p) if os.path.isfile(comp_p) else None - for c in out.get("trio_status") or {}: - r = live.run(c) if live else None - given[c] = (candidates_given(live, r["dir"] or os.path.join(live.root, r["name"]), c, since) - if r and r.get("kind") != "family" else None) - with_candidates(out.get("trio_status") or {}, given) - # The members side by side, from each sequenced member's summary.json alone (read as the home - # page's attention count reads it: never a file linking out of its run). - seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")] - sums = {} - for s_ in seq: - r = live.run(s_) if live else None - sums[s_] = member_summary(live, r["dir"] or os.path.join(live.root, r["name"])) if r else None - out["side_by_side"] = side_by_side(seq, sums) if seq else None - return out + # og tells each child's compound-het line by its run's step 23 candidates, and puts the members side by + # side from each sequenced member's summary.json alone (as the home page's attention count reads it: + # never a file linking out of its run). + return family_page(name, d, {"runs": runs, "candidates": candidates, "summaries": summaries}, + live.root if live else None) def hook(run_dir, state, events): if not run_dir: @@ -628,16 +223,22 @@ def serve(req, m): if not d or not is_family(d) or not inside(d, root()): app.log("family", name, f, 404) return req.send_body(404, "text/plain; charset=utf-8", "not found\n") - try: - order, members = members_of(d) - except (OSError, ValueError): - app.log("family", name, f, 404) - return req.send_body(404, "text/plain; charset=utf-8", "not found\n") if f == "pedigree.svg": + try: + svg = draw(d, name) + except OSError as e: + print(f"owngenome_live: pedigree.svg of {name} not drawn: {e}", file=sys.stderr) + app.log("family", name, f, 503) + return req.send_body(503, "text/plain; charset=utf-8", "The family page needs og on this host\n") + if svg is None: + app.log("family", name, f, 404) + return req.send_body(404, "text/plain; charset=utf-8", "not found\n") app.log("family", name, f, 200) - return req.send_body(200, TYPES[".svg"], pedigree_svg.draw(order, members, name)) + return req.send_body(200, TYPES[".svg"], svg) + code, out, _ = og(["files", d]) + files = json.loads(out) if code == 0 else [] p = os.path.join(d, f) - if f not in served_files(d, order, members) or not inside(p, d): + if f not in files or not inside(p, d): app.log("family", name, f, 404) return req.send_body(404, "text/plain; charset=utf-8", "not found\n") with open(p, "rb") as fh: diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index d7344bc7b..77bf2f307 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -51,9 +51,9 @@ alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py alignment-markdup-gvcf-sex-4-bwamem2 scripts/02a-alignment-bwamem2.sh scripts/ci/fake-docker/lib.sh ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh -annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh -annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh +annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh +annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh bash-step-08-hla - bash-step-09-str assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh @@ -81,23 +81,23 @@ cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/4 cn-guard-1-mapq0 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh cn-intervals-1-catalog assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/og modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh bin/og -couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh rust/crates/og-dev/src/fixture.rs -families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv -family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv +family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bin/og followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv bin/og scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh bin/og -followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh +followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og genome-lookup-1 bin/og scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bin/og rust/crates/og-dev/src/fixture.rs @@ -110,21 +110,21 @@ hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes. hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh i411-callability-overlap-1-mates modules/local/mosdepth/* scripts/16b-mosdepth.sh scripts/lib/common.sh -ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/og rust/crates/og-dev/src/fixture.rs scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/og rust/crates/og-dev/src/fixture.rs scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/og bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh +mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/og modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh scripts/generate-report.sh rust/crates/og-dev/src/fixture.rs mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bin/og -nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz -nf-wiring-2-sv-str-tools bin/family_report.py bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* +nf-wiring-2-sv-str-tools bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs @@ -149,10 +149,10 @@ prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summar prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-1-sample-qc bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-3-cram assets/callability_genes.tsv bin/collect_summary.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bin/og reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bin/og -report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh bin/og reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py index 0fb0ee4c0..340ef8cae 100644 --- a/tests/demo/build_live_demo.py +++ b/tests/demo/build_live_demo.py @@ -90,7 +90,6 @@ import live_results # noqa: E402 import owngenome_live # noqa: E402 import owngenome_live_family # noqa: E402 -import pedigree_svg # noqa: E402 import run_headroom # noqa: E402 import run_state # noqa: E402 @@ -244,6 +243,7 @@ def frame_key(st): def replay(root, held, drop_hook=None, og=OG): """(frames of DEMO-001, the family page's state, the app) with the clock at each message.""" + os.environ["OG"] = og # the family plugin the server loads runs og (OG, else bin/og): this one app = owngenome_live.App() cfg = owngenome_live.configure(["--root", root, "--state-dir", os.path.join(root, ".owngenome-live"), "--eta-cores", "16", "--eta-box", "demo", "--quiet"], app) @@ -352,9 +352,11 @@ def unroot(v): return v -def family_json(root, index): - """family.json as bin/owngenome_live_family.py serves it, its links relative to runs/DEMO-FAMILY/.""" - out = owngenome_live_family.family_page(FAMILY, os.path.join(root, FAMILY)) +def family_json(root, index, og=OG): + """family.json as bin/owngenome_live_family.py serves it, its links relative to runs/DEMO-FAMILY/: og puts the + members side by side from each one's summary.json.""" + out = owngenome_live_family.family_page(FAMILY, os.path.join(root, FAMILY), + {"summaries": {n: os.path.join(root, n) for n in os.listdir(root)}}, root, og) by = {r["name"]: r for r in index["runs"]} out["runs"] = {p["sample"]: {k: by[p["sample"]][k] for k in ("url", "state", "planned", "done", "attention", "results")} for p in out["state"].get("members") or [] if p["sample"] in by} @@ -363,15 +365,6 @@ def family_json(root, index): r["child"] = owngenome_live.child_of(index.get("families"), name) if r["child"] and (by[name].get("levels") or {}).get("act"): r["attention"] = None - seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")] - sums = {} - for name in seq: - try: - with open(os.path.join(root, name, "summary.json")) as f: - sums[name] = json.load(f) - except OSError: - sums[name] = None - out["side_by_side"] = owngenome_live_family.side_by_side(seq, sums) return relink(out) @@ -487,12 +480,11 @@ def write_site(out, root, frames, fam, og=OG): with open(os.path.join(fd, "index.html"), "w") as f: f.write(run_page(FAMILY, plugin_js, "family.html")) write_json(os.path.join(fd, "frames.json"), {"interval_ms": INTERVAL_MS, "frames": [{"t": fam["t"], "state": fam["state"]}]}) - write_json(os.path.join(fd, "family.json"), family_json(root, fam["index"])) + write_json(os.path.join(fd, "family.json"), family_json(root, fam["index"], og)) famdir, ffiles = os.path.join(out, "family", FAMILY), os.path.join(root, FAMILY) os.makedirs(famdir) - order, members, _ = pedigree_svg.load(ffiles) - with open(os.path.join(famdir, "pedigree.svg"), "w") as f: - f.write(pedigree_svg.draw(order, members, FAMILY)) + with open(os.path.join(famdir, "pedigree.svg"), "wb") as f: + f.write(owngenome_live_family.draw(ffiles, FAMILY, og)) for name in (fam["state"]["plugins"].get("family") or {}).get("files") or []: shutil.copy(os.path.join(ffiles, name), os.path.join(famdir, name)) return drawn diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv index 7e5c03fb3..3d6d3a00a 100644 --- a/tests/parity/remnants.tsv +++ b/tests/parity/remnants.tsv @@ -3,6 +3,5 @@ # names listed here and nothing else, that every name has a user, and that # the importers are exactly these (docs/rust.md). # filenamesimporterslane -bin/family_report.py COLUMNS,MISSING,SEX,NAME,BINS,missing,read_pedigree,build_members,bad_name,kinship_table,declared_relationship,degree bin/pedigree_svg.py,tests/test_family_report.py R4 bin/genome_lookup.py GENE_KEYS,RSID_RE,norm_chrom,parse_query,names_of,walk_mentions,flat bin/results_index.py R5c bin/collect_summary.py HERE,ASSETS_DIR,SECTIONS,og,collect,STARS,COMMON_AF,RUN_ALL_STEPS,open_text,info_map,read_tsv,hla_quality,HLA_MAX_CANDIDATES,EXPLORE_DOCS,csq_rank bin/owngenome_live.py,bin/render_report.py,bin/results_index.py,tests/demo/make_demo_sample.py,tests/test_collect_summary.py,tests/test_live_journal.py,tests/test_live_results.py,tests/test_render_report.py,tests/test_run_state.py,tests/test_viewer.py R6b diff --git a/tests/test_family_report.py b/tests/test_family_report.py index f056e76b7..a61774129 100644 --- a/tests/test_family_report.py +++ b/tests/test_family_report.py @@ -1,11 +1,8 @@ #!/usr/bin/env python3 # og-leg: OG_REQUIRED """og family (step 38 and the Nextflow FAMILY workflow) on synthetic -pedigrees, somalier tables and joint VCFs, and the pedigree rules other -Python tools still import from bin/family_report.py. - -Without og (OG unset) only check 2 runs, on bin/family_report.py. -With OG naming the og binary, every check runs. +pedigrees, somalier tables and joint VCFs. With OG naming the og binary, +every check runs; without it none does. Checks: 1. prepare: a trio with a grandparent named but not sequenced writes @@ -14,8 +11,8 @@ (no header, an unknown column, a female father, a person who is their own parent, a loop, a second row, a sex the samplesheet contradicts, a gVCF whose sample is another, fewer than two sequenced samples); - 2. the expected relatedness of parent and child, full and half siblings, - grandparent, first cousins and partners; + 2. (the expected relatedness of each relation is og-pedigree's cargo test + expected_relatedness); 3. check: somalier numbers that fit the trio pass; the same numbers with two names swapped in the pedigree exit 1 and name the pair; too few sites is not_checked, not a pass and not a stop; a first-degree pair @@ -32,6 +29,7 @@ Run: OG=rust/target/release/og python3 tests/test_family_report.py """ import gzip +import json import os import subprocess import sys @@ -40,8 +38,6 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OG = os.environ.get("OG", "") PAR = os.path.join(REPO, "assets", "par_grch38.bed") -sys.path.insert(0, os.path.join(REPO, "bin")) -import family_report as fr # noqa: E402 FAILS = [] @@ -142,25 +138,6 @@ def test_prepare(): rc == 2 and "holds the sample HG003, not 'dad'" in out and "bcftools reheader" in out, out) -def test_kinship(): - ped = { - "gp": (None, None), "gm": (None, None), - "f": ("gp", "gm"), "u": ("gp", "gm"), "m": (None, None), "w": (None, None), - "a": ("f", "m"), "b": ("f", "m"), "h": ("f", "w"), "c": ("u", "w2"), "w2": (None, None), - } - members = {k: {"father": v[0], "mother": v[1], "sex": None, "partner": None} for k, v in ped.items()} - members["m"]["partner"] = "f" - phi = fr.kinship_table(members) - cases = [("a", "f", 0.5, "parent-child"), ("a", "b", 0.5, "full siblings"), ("a", "h", 0.25, "half siblings"), - ("a", "gp", 0.25, "second degree"), ("a", "c", 0.125, "third degree"), ("f", "m", 0.0, "partners"), - ("m", "w", 0.0, "unrelated")] - for x, y, want, label in cases: - got = 2 * phi(x, y) - lab = fr.declared_relationship(x, y, members, got) - check("expected relatedness of %s and %s is %.3f, %s (got %.3f, %s)" % (x, y, want, label, got, lab), - abs(got - want) < 1e-9 and lab == label) - - PAIRS_HEAD = ("#sample_a\tsample_b\trelatedness\tibs0\tibs2\tconcordance\thets_a\thets_b\thets_ab\tshared_hets" "\thom_alts_a\thom_alts_b\tshared_hom_alts\tn\tx_ibs0\tx_ibs2\texpected_relatedness\n") @@ -413,11 +390,11 @@ def test_families_file(): rc == 0 and [r["sample"] for r in rows] == want, out + str(rows)) check("%s's pedigree has the pedigree columns, no family column" % fam, rows and list(rows[0]) == ["sample", "sex", "father", "mother", "partner"], str(rows[:1])) - rows, errors = fr.read_pedigree(os.path.join(d, "family_a", "pedigree.tsv")) - members, more = fr.build_members(rows, {}, "x") - check("the written pedigree reads back: child_a's parents are father_a and mother_a", - not errors + more and (members["child_a"]["father"], members["child_a"]["mother"]) == ("father_a", "mother_a"), - str(errors + more)) + p = subprocess.run([OG, "live", "family", "state", os.path.join(d, "family_a")], capture_output=True, text=True) + st = json.loads(p.stdout) if p.returncode == 0 else {} + kid = next((m for m in st.get("members") or [] if m["sample"] == "child_a"), {}) + check("the written pedigree reads back (og live family state): child_a's parents are father_a and mother_a", + "error" not in st and (kid.get("father"), kid.get("mother")) == ("father_a", "mother_a"), p.stdout + p.stderr) # a file without the column gives the same members.tsv as before the column existed args = ["prepare", "--pedigree", os.path.join(REPO, "assets", "pedigree.example.tsv"), "--out-dir", @@ -560,10 +537,9 @@ def test_plan(): if __name__ == "__main__": - test_kinship() if not OG: - print("skip: OG is not set (OG=rust/target/release/og): only the kinship check ran") - sys.exit(1 if FAILS or os.environ.get("OG_REQUIRED") else 0) + print("skip: OG is not set (OG=rust/target/release/og): no check ran") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) test_prepare() test_check() test_report()