From 48505ed74fa39429aff8ad681bd23a0dc0d8a3ac Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:06:00 +0100 Subject: [PATCH 1/3] fix(pgx): require 10x in the CYP2D6 flanks before the depth check passes The CYP2D6 depth check read only the share of MAPQ 0 reads, so flanks at 6x with every read mapped well passed as fit for a copy-number call, and two callers could agree on too few reads. Flanks under 10x at MAPQ >= 1 now make the verdict unreliable, with the message "agreement not counted (depth)". Multi-mapped reads are still named first. Bead: pgx-09 Bead: D10 --- rust/crates/og-bam/src/cyp2d6.rs | 27 ++++- rust/crates/og-bam/tests/i411_cyp2d6_gate.rs | 121 +++++++++++++++++++ 2 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 rust/crates/og-bam/tests/i411_cyp2d6_gate.rs diff --git a/rust/crates/og-bam/src/cyp2d6.rs b/rust/crates/og-bam/src/cyp2d6.rs index cc33dd51d..63e5a6148 100644 --- a/rust/crates/og-bam/src/cyp2d6.rs +++ b/rust/crates/og-bam/src/cyp2d6.rs @@ -15,10 +15,13 @@ //! verdict, `ok` or `unreliable`; the shares never change it. Exit 0 //! whatever the verdict, 2 on unreadable input. //! -//! One deliberate change from the Python: a region listed twice in a +//! Two deliberate changes from the Python. A region listed twice in a //! mosdepth file counts once, and listed twice with two different depths it //! is refused (exit 2). The Python added every copy to the totals, so two -//! depths for one region were silently averaged. +//! depths for one region were silently averaged. And flanks under 10x at +//! MAPQ >= 1 make the verdict `unreliable` (`agreement not counted +//! (depth)`): the Python looked only at the MAPQ 0 share, so a shallow +//! region passed and two callers could agree on too few reads. use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; @@ -68,6 +71,9 @@ const UNRELIABLE: &str = "CYP2D6 copy number unreliable: reads are multi-mapped (was this BAM aligned to a reference with ALT contigs?)"; const NO_DEPTH: &str = "CYP2D6 copy number unreliable: no reads in the flanks of CYP2D6 \ (chr22:42.05-42.25 Mb), so its depth cannot be compared"; +const SHALLOW: &str = "CYP2D6 copy number unreliable, agreement not counted (depth): \ + the flanks of CYP2D6 (chr22:42.05-42.25 Mb) have under 10x at MAPQ >= 1, \ + too few reads for a copy-number call"; /// A depth fit for a copy-number call has at least this share of its reads /// at MAPQ >= 1, at CYP2D6 and in the flanks. @@ -75,6 +81,10 @@ const MIN_MAPQ1_SHARE: f64 = 0.6; /// Below this ratio of CYP2D6 to flank depth (all reads) the gene has too /// few reads for its MAPQ share to mean anything: both copies deleted. const MIN_GENE_RATIO: f64 = 0.15; +/// A depth fit for a copy-number call has at least this mean depth in the +/// flanks at MAPQ >= 1, the reads the callers use. Below it two callers can +/// agree on noise, so their agreement does not count. +pub const MIN_FLANK_DEPTH: f64 = 10.0; /// The verdict on the four mean depths. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -84,6 +94,8 @@ pub enum Verdict { MultiMapped, /// No reads in the flanks, so nothing to compare CYP2D6 with. NoFlankDepth, + /// Flanks under 10x at MAPQ >= 1: too few reads for a copy-number call. + ShallowFlanks, } impl Verdict { @@ -99,6 +111,7 @@ impl Verdict { Verdict::Ok => OK, Verdict::MultiMapped => UNRELIABLE, Verdict::NoFlankDepth => NO_DEPTH, + Verdict::ShallowFlanks => SHALLOW, } } } @@ -116,7 +129,8 @@ pub struct Depths { /// MAPQ 0: the aligner placed them on more than one sequence. A deletion /// leaves few reads at CYP2D6, but those map uniquely and the flanks are /// untouched, so it passes; below a gene/flank ratio of 0.15 the gene share -/// is not read at all. +/// is not read at all. Reads that map well are not enough: flanks under 10x +/// at MAPQ >= 1 are unreliable too (multi-mapped reads are named first). pub fn assess(all: Depths, q1: Depths) -> Verdict { if all.flank <= 0.0 { return Verdict::NoFlankDepth; @@ -132,6 +146,8 @@ pub fn assess(all: Depths, q1: Depths) -> Verdict { || (ratio_all >= MIN_GENE_RATIO && share_gene < MIN_MAPQ1_SHARE) { Verdict::MultiMapped + } else if q1.flank < MIN_FLANK_DEPTH { + Verdict::ShallowFlanks } else { Verdict::Ok } @@ -460,6 +476,11 @@ mod tests { assert_eq!(assess(d(6.0, 30.0), d(0.0, 29.0)), Verdict::MultiMapped); // No gene depth at all: the gene share is taken as 1. assert_eq!(assess(d(0.0, 30.0), d(0.0, 29.0)), Verdict::Ok); + // Flanks under 10x at MAPQ >= 1: shallow, whatever the share. + assert_eq!(assess(d(6.0, 6.0), d(6.0, 6.0)), Verdict::ShallowFlanks); + assert_eq!(assess(d(12.0, 12.0), d(12.0, 10.0)), Verdict::Ok); + // Multi-mapped reads are named before the depth. + assert_eq!(assess(d(6.0, 6.0), d(6.0, 2.0)), Verdict::MultiMapped); } #[test] diff --git a/rust/crates/og-bam/tests/i411_cyp2d6_gate.rs b/rust/crates/og-bam/tests/i411_cyp2d6_gate.rs new file mode 100644 index 000000000..b2b253d66 --- /dev/null +++ b/rust/crates/og-bam/tests/i411_cyp2d6_gate.rs @@ -0,0 +1,121 @@ +//! The CYP2D6 depth check needs real depth in the flanks (#411, pgx-09). +//! +//! `og pgx cyp2d6-check` looked only at the share of reads with MAPQ 0, so +//! flanks at 6x with every read mapped uniquely passed as "fit for a +//! copy-number call", and step 36 then counted pypgx and Cyrius agreeing on +//! those few reads as a confirmed CYP2D6 call. The flanks must now have +//! 10x or more at MAPQ >= 1, the reads the callers use, before the check +//! passes; under that the status is `unreliable` and the message, which step +//! 36 writes as its reason, says `agreement not counted (depth)`. +//! +//! These cases pin: +//! +//! 1. flanks at 6x, cleanly mapped: `unreliable`, `agreement not counted +//! (depth)`; +//! 2. flanks at 12x: `ok`, the agreement counts; +//! 3. the edge: 9.99x at MAPQ >= 1 fails, 10x passes, and the floor reads +//! the MAPQ >= 1 depth, not the depth of all reads; +//! 4. multi-mapped reads on shallow flanks keep the multi-mapped message, +//! which names the cause the user can fix. + +use std::fs; +use std::path::{Path, PathBuf}; + +use og_bam::cyp2d6::{MIN_FLANK_DEPTH, cli}; + +const OK: &str = "CYP2D6 depth is fit for a copy-number call"; +const NOT_COUNTED: &str = "agreement not counted (depth)"; +const MULTI: &str = "reads are multi-mapped"; + +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Self { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("i411-cyp2d6-gate-{name}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + Scratch(dir) + } + + fn path(&self, name: &str) -> String { + self.0.join(name).display().to_string() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// A mosdepth regions file over the regions `cyp2d6-bed` prints. +fn regions(gene: f64, flank: f64) -> String { + format!( + "chr22\t42050000\t42100000\tflank\t{flank:.2}\n\ + chr22\t42123192\t42132032\tCYP2D6\t{gene:.2}\n\ + chr22\t42200000\t42250000\tflank\t{flank:.2}\n" + ) +} + +/// `(status, message)` of `og pgx cyp2d6-check` on the two depth tables, +/// each `(gene, flank)` mean depth, all reads first, then MAPQ >= 1. +fn check(name: &str, all: (f64, f64), q1: (f64, f64)) -> (String, String) { + let s = Scratch::new(name); + let (q0, q1p, out) = (s.path("q0.bed"), s.path("q1.bed"), s.path("check.tsv")); + fs::write(&q0, regions(all.0, all.1)).unwrap(); + fs::write(&q1p, regions(q1.0, q1.1)).unwrap(); + let argv: Vec = ["--all", &q0, "--mapq1", &q1p, "--out", &out] + .iter() + .map(|a| (*a).to_owned()) + .collect(); + let (mut o, mut e) = (Vec::new(), Vec::new()); + let code = cli("check", &argv, &mut o, &mut e); + assert_eq!(code, 0, "{name}: {}", String::from_utf8_lossy(&e)); + let text = fs::read_to_string(&out).unwrap(); + let get = |k: &str| { + text.lines() + .find_map(|l| l.strip_prefix(&format!("{k}\t"))) + .unwrap_or_else(|| panic!("{name}: no {k} row in {text}")) + .to_owned() + }; + (get("status"), get("message")) +} + +#[test] +fn flanks_at_6x_do_not_count_the_agreement() { + let (status, message) = check("6x", (4.0, 6.0), (3.8, 6.0)); + assert_eq!(status, "unreliable"); + assert!(message.contains(NOT_COUNTED), "{message}"); + assert!(message.contains("under 10x at MAPQ >= 1"), "{message}"); +} + +#[test] +fn flanks_at_12x_count() { + let (status, message) = check("12x", (7.5, 12.0), (7.0, 12.0)); + assert_eq!(status, "ok"); + assert_eq!(message, OK); +} + +#[test] +fn the_floor_is_10x_at_mapq1() { + assert!((MIN_FLANK_DEPTH - 10.0).abs() < f64::EPSILON); + let (status, message) = check("9.99x", (6.0, 9.99), (6.0, 9.99)); + assert_eq!(status, "unreliable", "9.99x: {message}"); + assert!(message.contains(NOT_COUNTED), "{message}"); + let (status, _) = check("10x", (6.0, 10.0), (6.0, 10.0)); + assert_eq!(status, "ok", "10x at MAPQ >= 1 counts"); + // 11x of all reads, 9x of them at MAPQ >= 1 (a share of 0.82, not + // multi-mapped): the callers see 9x, so the agreement does not count. + let (status, message) = check("mapq1", (7.0, 11.0), (6.5, 9.0)); + assert_eq!(status, "unreliable", "{message}"); + assert!(message.contains(NOT_COUNTED), "{message}"); +} + +#[test] +fn multi_mapped_reads_are_named_before_the_depth() { + // Half the flank reads at MAPQ 0, and 4x left at MAPQ >= 1. + let (status, message) = check("multi", (5.0, 8.0), (2.0, 4.0)); + assert_eq!(status, "unreliable"); + assert!(message.contains(MULTI), "{message}"); + assert!(!message.contains(NOT_COUNTED), "{message}"); +} From d9ad330f2aaf376d1942a9c68eb3235fde034f60 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:06:00 +0100 Subject: [PATCH 2/3] fix(pgx): flag weak pypgx evidence in the step 36 consensus table A pypgx CYP2D6 call whose defining variants are not all PASS, or not of good quality, now reads "weak evidence (...)" in the Evidence column of pgx_consensus.tsv. A call Cyrius agrees with is still passed: the flag shows the weakness, it does not withhold the call. Bead: pgx-01 --- rust/crates/og-steps/src/pgx_rules.rs | 132 ++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 10 deletions(-) diff --git a/rust/crates/og-steps/src/pgx_rules.rs b/rust/crates/og-steps/src/pgx_rules.rs index b999f9af5..a6ad63be2 100644 --- a/rust/crates/og-steps/src/pgx_rules.rs +++ b/rust/crates/og-steps/src/pgx_rules.rs @@ -21,8 +21,12 @@ //! `--optitype`, only when OptiType gives the same pair; with //! `--require-optitype` and no OptiType result, neither gene. //! - CYP2D6: only when pypgx (step 32) and Cyrius (step 21) give the same -//! diplotype and the depth check passed. One depth-based call alone can be -//! a false whole-gene deletion that changes a prescription. +//! diplotype and the depth check passed (which needs 10x in the flanks). +//! One depth-based call alone can be a false whole-gene deletion that +//! changes a prescription. A pypgx call whose defining variants are not +//! all `PASS` or not of good quality (step 32's `Defining_filter` and +//! `Quality`) reads `weak evidence (...)` in the Evidence column; Cyrius's +//! agreement still passes it. //! //! The consensus table (`Gene Result Outside_call Reason Evidence`) says //! for each of the three genes what each caller said and what was passed. @@ -716,10 +720,26 @@ fn cell(row: &pycompat::csv::DictRow, key: &str) -> String { strip(row.get(key).flatten().unwrap_or("")).to_owned() } -/// pypgx's CYP2D6 diplotype (`""` when its summary has no CYP2D6 row). -fn read_pypgx(path: &str) -> Result { +/// pypgx's CYP2D6 diplotype (`""` when its summary has no CYP2D6 row) and +/// what is weak in the evidence for it: step 32's `Defining_filter` when it +/// is not `PASS` and its `Quality` when it is not `ok`, `"; "`-joined +/// (`""` when neither is, or the summary is older than those columns). +fn read_pypgx(path: &str) -> Result<(String, String), Error> { let got = dict_rows(path, |row| { - (cell(row, "Gene") == "CYP2D6").then(|| cell(row, "Diplotype")) + (cell(row, "Gene") == "CYP2D6").then(|| { + let filter = cell(row, "Defining_filter"); + let quality = cell(row, "Quality"); + let fine: [(String, &[&str]); 2] = [ + (filter, &["", "PASS", "none", "N/A"]), + (quality, &["", "ok", "N/A"]), + ]; + let weak: Vec = fine + .into_iter() + .filter(|(v, ok)| !ok.contains(&v.as_str())) + .map(|(v, _)| v) + .collect(); + (cell(row, "Diplotype"), weak.join("; ")) + }) })?; Ok(got.unwrap_or_default()) } @@ -758,13 +778,18 @@ pub fn cyp2d6_row( let cy = cyrius.map(read_cyrius).transpose()?; let dc = depth.map(read_depth_check).transpose()?; - let pg_ok = pg.as_deref().is_some_and(is_call); + let pg_ok = pg.as_ref().is_some_and(|(d, _)| is_call(d)); let cy_ok = cy.as_ref().is_some_and(|(g, f)| is_call(g) && f == "PASS"); + // A call on weak evidence is flagged, not withheld: Cyrius reads the + // BAM on its own, so its agreement is the second line of evidence. let pg_txt = match &pg { None => "not run".to_owned(), - Some(p) if pg_ok => p.clone(), - Some(p) if p.is_empty() => "no call (no CYP2D6 row)".to_owned(), - Some(p) => format!("no call ({p})"), + Some((p, weak)) if pg_ok && !weak.is_empty() => { + format!("{p}, weak evidence ({weak})") + } + Some((p, _)) if pg_ok => p.clone(), + Some((p, _)) if p.is_empty() => "no call (no CYP2D6 row)".to_owned(), + Some((p, _)) => format!("no call ({p})"), }; let cy_txt = match &cy { None => "not run".to_owned(), @@ -804,7 +829,7 @@ pub fn cyp2d6_row( if !(pg_ok && cy_ok) { return withheld(ONE_CALLER); } - let (pg, cy) = (pg.unwrap_or_default(), cy.unwrap_or_default()); + let (pg, cy) = (pg.unwrap_or_default().0, cy.unwrap_or_default()); if !same_diplotype(&pg, &cy.0) { return withheld("pypgx and Cyrius disagree"); } @@ -1233,6 +1258,93 @@ mod tests { let _ = std::fs::remove_dir_all(d); } + #[test] + fn shallow_flanks_do_not_count_an_agreement() { + // What `og pgx cyp2d6-check` writes for flanks under 10x (og-bam's + // tests/i411_cyp2d6_gate.rs pins that side). + let d = tmp("shallow"); + let pg = put(&d, "p.tsv", "Gene\tDiplotype\nCYP2D6\t*1/*2\n"); + let head = "Sample\tGenotype\tFilter\n"; + let cy = put(&d, "c.tsv", &format!("{head}S\t*2/*1\tPASS\n")); + let shallow = put( + &d, + "shallow.tsv", + "metric\tvalue\nstatus\tunreliable\nmessage\tCYP2D6 copy number unreliable, \ + agreement not counted (depth): the flanks of CYP2D6 (chr22:42.05-42.25 Mb) \ + have under 10x at MAPQ >= 1, too few reads for a copy-number call\n\ + flank_depth_mapq1\t6.00\n", + ); + let row = cyp2d6_row(Some(&pg), Some(&cy), Some(&shallow)).unwrap(); + assert_eq!(row.decision, Decision::Withheld("indeterminate")); + assert!( + row.reason.contains("agreement not counted (depth)"), + "{}", + row.reason + ); + let _ = std::fs::remove_dir_all(d); + } + + #[test] + fn weak_pypgx_evidence_is_flagged_not_withheld() { + let d = tmp("weak"); + let header = "Gene\tDiplotype\tPhenotype\tCNV_call\tSource\tStatus\tDefining_variants\t\ + Min_GQ\tWeakest_variant\tDefining_filter\tQuality\n"; + let weak = put( + &d, + "weak.tsv", + &format!( + "{header}CYP2D6\t*1/*4\tIntermediate Metabolizer\tNormal\tBAM\tcalled\t\ + 1/2 in the VCF\t8\tchr22:42128945 C>T GQ 8 AD 3,2\t\ + not PASS: chr22:42128945 C>T RefCall\tlow-confidence: check the reads\n" + ), + ); + let no_gq = put( + &d, + "nogq.tsv", + &format!( + "{header}CYP2D6\t*1/*4\tIntermediate Metabolizer\tNormal\tBAM\tcalled\t\ + 2/2 in the VCF\tN/A\tN/A\tPASS\tGQ not in the VCF: check the reads\n" + ), + ); + let clean = put( + &d, + "clean.tsv", + &format!( + "{header}CYP2D6\t*1/*4\tIntermediate Metabolizer\tNormal\tBAM\tcalled\t\ + 2/2 in the VCF\t40\tchr22:42128945 C>T GQ 40 AD 15,14\tPASS\tok\n" + ), + ); + let head = "Sample\tGenotype\tFilter\n"; + let cy = put(&d, "c.tsv", &format!("{head}S\t*4/*1\tPASS\n")); + let ok = put(&d, "ok.tsv", "status\tok\nmessage\t\n"); + + let row = cyp2d6_row(Some(&weak), Some(&cy), Some(&ok)).unwrap(); + assert_eq!(row.decision, Decision::Passed("*1/*4".to_owned())); + assert_eq!(row.reason, "pypgx and Cyrius agree; the depth check passed"); + assert_eq!( + row.evidence, + "pypgx: *1/*4, weak evidence (not PASS: chr22:42128945 C>T RefCall; \ + low-confidence: check the reads); Cyrius: *4/*1; depth check: ok" + ); + + let row = cyp2d6_row(Some(&no_gq), Some(&cy), Some(&ok)).unwrap(); + assert_eq!(row.decision, Decision::Passed("*1/*4".to_owned())); + assert_eq!( + row.evidence, + "pypgx: *1/*4, weak evidence (GQ not in the VCF: check the reads); \ + Cyrius: *4/*1; depth check: ok" + ); + + // One caller only: still withheld, still flagged. + let row = cyp2d6_row(Some(&weak), None, Some(&ok)).unwrap(); + assert_eq!(row.decision, Decision::Withheld("indeterminate")); + assert!(row.evidence.contains("weak evidence"), "{}", row.evidence); + + let row = cyp2d6_row(Some(&clean), Some(&cy), Some(&ok)).unwrap(); + assert_eq!(row.evidence, "pypgx: *1/*4; Cyrius: *4/*1; depth check: ok"); + let _ = std::fs::remove_dir_all(d); + } + #[test] fn both_files_or_neither() { let d = tmp("both"); From 7e6654efff98b59e5155d6cb6874677e7900241f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:06:00 +0100 Subject: [PATCH 3/3] docs(pgx): say the CYP2D6 depth check needs 10x flanks and how weak pypgx evidence shows Bead: pgx-09 Bead: pgx-01 --- CHANGELOG.md | 1 + docs/21-cyrius.md | 2 +- docs/32-pypgx.md | 5 +++-- docs/36-pgx-consensus.md | 6 +++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb7a130e7..831389120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Two CYP2D6 callers no longer count as agreeing on too few reads. The CYP2D6 depth check (`og pgx cyp2d6-check`, steps 21 and 32) looked only at the share of reads with MAPQ 0, so flanks at 6x with every read mapped well passed as fit for a copy-number call, and step 36 passed pypgx and Cyrius's shared diplotype to PharmCAT. The check now also needs 10x or more in the flanks at MAPQ >= 1. Under that its status is `unreliable` and its message says `agreement not counted (depth)`: pypgx's CYP2D6 row reads `Indeterminate`, the Cyrius call gets Filter `CYP2D6_depth_unreliable`, and step 36 withholds CYP2D6 with that message as its reason. Multi-mapped reads are still named first. Step 36's consensus table also shows weak pypgx evidence: when step 32's `Defining_filter` is not `PASS` or its `Quality` is not `ok`, the `Evidence` column reads `pypgx: , weak evidence (...)`. A call Cyrius agrees with is still passed. The file names and columns are unchanged; the `Evidence` text and the check's verdict on shallow flanks are new. Part of #411 (pgx-09, pgx-01). - A third-party outage no longer fails the Freshness check of a pull request. Since 2026-11-18 rest.ensembl.org has not answered, and the pull-request job's dry run of `og dev freshness` failed on every pull request that touched `versions.env` or `og-fresh`. `og dev freshness --unreachable-warns` now lists a lookup whose upstream did not answer (no connection, a timeout, HTTP 5xx or 429 after the retries, or HTTP 403) under "Upstreams that did not answer" as a warning, and its database row reads `NO ANSWER`; it does not change the exit. The 403 counts because rest.ensembl.org came back on 2026-11-19 answering 403 to GitHub's runners and 200 elsewhere, which still failed the pull-request job. A 404, an empty answer or an answer in an odd shape still exits 1. `freshness.yml` passes the flag only in the pull-request dry run. Without the flag the report and the exit are as before, and the monthly report on main still counts an outage as an error in the issue. - Three statements in the docs were wrong. Troubleshooting said exit code 137 confirms a kill for memory, and checked it with `docker ps -a --latest`. 137 is any SIGKILL, and the step scripts run their containers with `--rm`, so that command shows an older, unrelated container. The page now says how to confirm a kill for memory: `docker events --filter event=oom`, the kernel log, or `.State.OOMKilled` for a container kept without `--rm`. The run-state and troubleshooting pages said `bin/og` does not run on a Mac and told Mac users to build `og` with cargo or run it in a container. `bin/og` runs the native macOS build that `scripts/setup.sh --og` installs, and the pages now say so; on a Mac, `og run headroom` reports the machine's memory as unknown. Troubleshooting, the FAQ and the lessons-learned page said the step scripts run their containers as root, and lessons-learned gave `:latest` images and `chmod 777` as current fixes. The step scripts run their containers as you; those entries are now marked as past fixes and name what the pipeline does today. Documentation only: no option, output or behaviour changes. - Lint's ShellCheck job now checks every tracked shell script, not only `scripts/`: the e2e cases and their library, the fake-docker and parity tests, `tests/*.sh`, `bin/og` and the fake tools, 482 files in all, at every ShellCheck level. The 265 findings these files hold today are listed in `tests/shellcheck-baseline.txt` and still pass; any other finding fails lint. A listed finding is matched by its file, its code and the text of its line, so it keeps matching when lines above it move, and a new line with the same mistake still fails. Each run plants an unquoted `$1` in a new test script and on a new line of a listed script, and fails if either passes or if the file search misses the new script or `bin/og`. The `scripts/` check at warning level stays as it was. Part of #411 (plat-10). diff --git a/docs/21-cyrius.md b/docs/21-cyrius.md index 7e8fbb5cd..318613ea3 100644 --- a/docs/21-cyrius.md +++ b/docs/21-cyrius.md @@ -49,7 +49,7 @@ With `run-all.sh`, a default run plans it when the install matches this version 1. Validates that the sorted BAM, its index and the Cyrius install exist 2. Checks the depth at CYP2D6 against its flanks ([depth check](32-pypgx.md#cyp2d6-depth-check)), with `${MOSDEPTH_IMAGE}` 3. Creates a manifest file listing the BAM path (Cyrius requires this) and runs `python3 -m cyrius --genome 38` (GRCh38) from the install, with no network -4. When the depth check found multi-mapped reads, sets the call's Filter to `CYP2D6_depth_unreliable`: [step 36](36-pgx-consensus.md) then does not pass it on. Cyrius writes into `cyrius/partial/`, and its TSV moves to `${SAMPLE}_cyp2d6.tsv` only after this, so a run that stops earlier leaves no call for step 36 +4. When the depth check failed (multi-mapped reads, or flanks under 10x at MAPQ >= 1), sets the call's Filter to `CYP2D6_depth_unreliable`: [step 36](36-pgx-consensus.md) then does not pass it on. Cyrius writes into `cyrius/partial/`, and its TSV moves to `${SAMPLE}_cyp2d6.tsv` only after this, so a run that stops earlier leaves no call for step 36 5. Parses the output TSV to display the called diplotype ## Output diff --git a/docs/32-pypgx.md b/docs/32-pypgx.md index 9fcf28455..6d187fdb3 100644 --- a/docs/32-pypgx.md +++ b/docs/32-pypgx.md @@ -139,6 +139,7 @@ The BAM-based calling uses both `--variants` and `--depth-of-coverage` per the u A copy-number call from depth is only as good as the depth. Before pypgx runs, the step measures, with mosdepth (`${MOSDEPTH_IMAGE}`), the mean depth over CYP2D6 (chr22:42,123,193-42,132,032, the region Cyrius uses) and over two 50 kb flanks outside the CYP2D6-CYP2D8 cluster (chr22:42.05-42.10 Mb and 42.20-42.25 Mb), once for all reads and once for reads with MAPQ >= 1 (callers ignore MAPQ 0 reads). `og pgx cyp2d6-check` then judges it (the regions come from `og pgx cyp2d6-bed`): - **unreliable** when more than 40% of the reads at CYP2D6 (if it has at least 15% of the flank depth) or in the flanks have MAPQ 0: the aligner placed them on more than one sequence. The step writes `CYP2D6 copy number unreliable: reads are multi-mapped (was this BAM aligned to a reference with ALT contigs?)` to its log and to `${SAMPLE}_cyp2d6_depth_check.tsv`, and the summary's CYP2D6 row says `Indeterminate` (pypgx's own call stays in `CYP2D6/results.zip`). +- **unreliable** too when the flanks have under 10x at MAPQ >= 1, the reads the callers use. The message is `CYP2D6 copy number unreliable, agreement not counted (depth): the flanks of CYP2D6 (chr22:42.05-42.25 Mb) have under 10x at MAPQ >= 1, too few reads for a copy-number call`, and the summary's CYP2D6 row says `Indeterminate` as above. At that depth two callers can agree on noise, so their agreement does not count. Multi-mapped reads are named first when both apply, because that cause can be fixed by realigning. - **ok** otherwise. A real deletion leaves few reads at CYP2D6, but those map uniquely and the flanks keep their depth, so it passes. Beside the verdict, the check gives three more figures, which do not change it. Cyrius's region includes REP6 and the 3' flank of the gene, where a sizeable share of the depth is MAPQ 0 even on a clean BAM, so its MAPQ >= 1 share reads worse than the gene is. `og pgx cyp2d6-measure` counts, from the BAM itself, the depth over that region and over the gene body alone, CYP2D6's MANE Select transcript (chr22:42,126,499-42,130,810, without REP6), for all reads, MAPQ >= 1 and MAPQ >= 20, leaving out the reads mosdepth leaves out (unmapped, secondary, QC-failed, duplicate). The check file then has `gene_mapq1_share` (the share the verdict reads), `gene_mapq20_share`, `body_depth_all`, `body_mapq1_share` and `body_mapq20_share`. A share reads `not measured` when the measurement did not run, and `no reads` when there was no depth to divide by. The two shares on real 30x genomes are still to be measured; the verdict rule stays the one above until they are. @@ -164,7 +165,7 @@ All output is written to `${GENOME_DIR}/${SAMPLE}/pypgx/`. | Column | Description | |---|---| | Gene | Gene symbol | -| Diplotype | Star allele call (e.g., \*1/\*4); `Indeterminate` for CYP2D6 when the depth check found multi-mapped reads; `FAILED` for a gene pypgx wrote no result for; `N/A` for a gene not callable on this reference (see Status) | +| Diplotype | Star allele call (e.g., \*1/\*4); `Indeterminate` for CYP2D6 when the depth check failed (multi-mapped reads, or flanks under 10x); `FAILED` for a gene pypgx wrote no result for; `N/A` for a gene not callable on this reference (see Status) | | Phenotype | Metabolizer status (e.g., Intermediate Metabolizer). `no pypgx phenotype rule` for a gene pypgx has no phenotype rule for (no `PhenotypeMethod` in its own gene table; in 0.26.0 the 17 genes listed under [Weakly supported genes](#weakly-supported-genes)), where pypgx itself writes `Indeterminate` | | CNV_call | The copy-number call pypgx itself made from read depth (its `CNV` field, for example `Normal` or `WholeDel1`). BAM-based genes only; `N/A` for VCF-based genes, and `N/A (CYP2D6 depth check)` for CYP2D6 when the depth check failed. Earlier versions guessed a Yes/No flag from the allele names, which flagged any name containing `*5` and missed the GSTM1/GSTT1/CYP2A6 whole-gene deletions | | Source | BAM (SV genes) or VCF (variant-based genes) | @@ -235,7 +236,7 @@ The two tools are complementary. PharmCAT provides drug recommendations for the - pypgx does not produce drug recommendations directly. Its CYP2D6 call reaches PharmCAT, and so the CPIC recommendations of step 27, only through [step 36](36-pgx-consensus.md), when Cyrius (step 21, after `setup.sh --cyrius`) gives the same diplotype and the depth check passed. Alone, it is shown in the reports as a probable call and goes no further. Step 27 also lists the genes PharmCAT could not call but pypgx did; for the other genes consult [CPIC guidelines](https://cpicpgx.org/guidelines/). - **The image and the bundle must be the same release.** The pinned pair is image `pypgx:0.26.0--pyh7e72e81_0` with the `0.26.0` branch of pypgx-bundle; with that pair all 23 genes of the earlier list, including its four BAM-based ones, were called on a real 30x genome (see [lessons learned](lessons-learned.md)). The 0.27.0 image against the 0.26.0 bundle failed every gene. Bump both together and rerun a known sample before trusting the new calls. - **GSTT1 needs an ALT contig, which the default reference does not have.** In GRCh38, GSTT1 lies on `chr22_KI270879v1_alt`. The pipeline's reference, the no-ALT analysis set, has no such contig (it leaves ALT contigs out so CYP2D6 and the other BAM genes keep their depth, see [realignment](realignment.md)). The step then prints a notice, leaves GSTT1 out of depth preparation (otherwise it fails for every BAM-based gene) and writes GSTT1 with Status `not callable on this reference`, not `FAILED`: nothing went wrong, the reference cannot hold the gene. Both reports list it as not assessable on this reference. The other 28 genes are called. The rule comes from the table: a row with `callable_without_alt` `no` and its `alt_contig`. -- **A copy-number call is only as good as the depth it reads.** On a reference with ALT contigs and an aligner that is not run ALT-aware, reads at CYP2D6 split between the primary and ALT copies, depth on the primary drops, and pypgx can report a deletion that is not there. The [CYP2D6 depth check](#cyp2d6-depth-check) marks such a call `Indeterminate`, and step 36 reports CYP2D6 to PharmCAT only when two callers agree. +- **A copy-number call is only as good as the depth it reads.** On a reference with ALT contigs and an aligner that is not run ALT-aware, reads at CYP2D6 split between the primary and ALT copies, depth on the primary drops, and pypgx can report a deletion that is not there. The [CYP2D6 depth check](#cyp2d6-depth-check) marks such a call `Indeterminate`, as it does a call from flanks under 10x, and step 36 reports CYP2D6 to PharmCAT only when two callers agree. - Individual gene failures do not stop the pipeline. However, if **all** genes fail, the script exits with status 1 before generating the summary TSV — this signals a systemic problem (e.g., wrong BAM path, corrupted index, missing pypgx-bundle). Rerun with verbose output to identify the root cause. For partial failures, check the summary TSV for "FAILED" entries. A gene that failed in this run reads `FAILED` even when an earlier run called it: the earlier run's result is removed before pypgx starts. ## Maintenance diff --git a/docs/36-pgx-consensus.md b/docs/36-pgx-consensus.md index 983343a36..893df0d10 100644 --- a/docs/36-pgx-consensus.md +++ b/docs/36-pgx-consensus.md @@ -5,7 +5,7 @@ Decides what the BAM-based callers tell PharmCAT. PharmCAT (step 7) reads a VCF, and from a VCF it types neither the HLA genes nor CYP2D6, so its drug guidance for them (abacavir, carbamazepine, allopurinol, codeine, tramadol, tamoxifen and many more) could never fire. This step writes PharmCAT's [outside-call file](https://pharmcat.org/using/Outside-Call-Format/): - **HLA-A and HLA-B** from T1K (step 8), cut to two fields (`*57:01`), when every allele has a T1K quality above 0. T1K can print several equally likely candidates for one allele, comma-separated. When those candidates, or an allele with an expression suffix (`*57:01:01N`, a null allele), leave open whether one of the alleles PharmCAT acts on is there (HLA-A\*31:01, HLA-B\*15:02, \*57:01 or \*58:01), the gene is `indeterminate` and is not passed, with a reason such as `ambiguous: B*57:01 possible, B*58:01 possible (T1K ties *57:01 and *58:01 on one allele)`. The answer does not depend on the order T1K printed the candidates in, and an OptiType call that matches one candidate does not decide it. A tie or a null that names none of those alleles changes no PharmCAT answer: the gene is passed, the first candidate in sorted order stands in for the tie and a null goes without its suffix, and the reason says so. A null that names none of those alleles never holds back a sure B\*57:01 on the other copy. When step 8 also ran OptiType, the opt-in second caller (`HLA_QC=true`, `--hla_qc true` in Nextflow), a gene passes only when both callers give the same pair at two fields; OptiType never calls a null, so it is compared with the names PharmCAT reads (T1K's `*02:53N` against OptiType's `*02:53` agrees here); see [step 8](08-hla-typing.md#second-caller-optitype-opt-in). One allele reads as homozygous, the way T1K reports a homozygous gene. -- **CYP2D6** only when pypgx (step 32) and Cyrius (step 21, once `setup.sh --cyrius` installed it) give the same diplotype **and** the [CYP2D6 depth check](32-pypgx.md#cyp2d6-depth-check) passed. One caller alone, a disagreement, or multi-mapped reads leave CYP2D6 `indeterminate`, and nothing reaches PharmCAT. +- **CYP2D6** only when pypgx (step 32) and Cyrius (step 21, once `setup.sh --cyrius` installed it) give the same diplotype **and** the [CYP2D6 depth check](32-pypgx.md#cyp2d6-depth-check) passed. The depth check passes only when the flanks of CYP2D6 have 10x or more at MAPQ >= 1 and the reads are not multi-mapped. One caller alone, a disagreement, multi-mapped reads or flanks under 10x leave CYP2D6 `indeterminate`, and nothing reaches PharmCAT. It also writes a consensus table that says, for each of the three genes, what every caller said and what was passed on. Step 27 copies it into the CPIC recommendations. @@ -54,7 +54,7 @@ All output is written to `${GENOME_DIR}/${SAMPLE}/pgx_consensus/`. | `${SAMPLE}_outside_calls.tsv` | PharmCAT's outside-call file: `genediplotype`, one line per gene passed on; empty when none was. Step 7 passes it with `-po` when it is not empty. | | `${SAMPLE}_pgx_consensus.tsv` | `Gene`, `Result` (the diplotype, `indeterminate` or `not typed`), `Outside_call` (`yes` or `no`), `Reason`, `Evidence` (what each caller said) | -For CYP2D6 the `Reason` is one of: the depth check's message (multi-mapped reads), `no caller made a call`, `one caller only: a second caller must agree (Cyrius: setup.sh --cyrius)`, `pypgx and Cyrius disagree`, `the CYP2D6 depth check did not run`, or `pypgx and Cyrius agree; the depth check passed`. +For CYP2D6 the `Reason` is one of: the depth check's message (multi-mapped reads, or `agreement not counted (depth)` for flanks under 10x), `no caller made a call`, `one caller only: a second caller must agree (Cyrius: setup.sh --cyrius)`, `pypgx and Cyrius disagree`, `the CYP2D6 depth check did not run`, or `pypgx and Cyrius agree; the depth check passed`. ## Runtime @@ -65,7 +65,7 @@ For CYP2D6 the `Reason` is one of: the depth check's message (multi-mapped reads - Without Cyrius, CYP2D6 is always `indeterminate` here: that is the point, not a fault. Install Cyrius (`setup.sh --cyrius`, non-commercial licence) to get an agreed call; `run-all.sh` then runs step 21 in a default run. - **A probable call, never an outside call.** When pypgx alone called CYP2D6 (Cyrius did not run, or made no usable call), the depth check passed, and every variant that defines its star alleles is in the VCF as a `PASS` call at or above the GQ floor (step 32's `Defining_filter` and `Quality` columns), both reports show pypgx's diplotype and phenotype labelled `probable (one caller: pypgx)`. This table still says `indeterminate` for it and nothing reaches PharmCAT, so no drug guidance rests on it. The rule is the probable call of `og summary collect` (`rust/crates/og-summary/src/collect/pgx.rs`); a failed depth check, a usable Cyrius call (then the two callers decide), a defining variant that is missing, not `PASS`, below the GQ floor or without a GQ, a star allele pypgx's table lacks, or a summary from before these columns gives no probable call. - A Cyrius call counts only with Filter `PASS`; `None`, an ambiguous call (two diplotypes) or `CYP2D6_depth_unreliable` is no call. -- When pypgx and Cyrius agree and the depth check passed, CYP2D6 is passed whatever step 32's `Defining_filter` and `Quality` columns say: Cyrius calls from the BAM on its own, so its agreement is the second line of evidence. Both reports show those columns as a flag beside the call. The cargo test `rust/crates/og-steps/tests/i411_hla_ties.rs` pins this rule. +- When pypgx and Cyrius agree and the depth check passed, CYP2D6 is passed whatever step 32's `Defining_filter` and `Quality` columns say: Cyrius calls from the BAM on its own, so its agreement is the second line of evidence. Weak evidence is shown, not hidden: when `Defining_filter` is not `PASS` (or `none`) or `Quality` is not `ok`, the `Evidence` column reads `pypgx: *1/*4, weak evidence (; )`, naming only the column that is weak, whether the call was passed or not. A summary from before those columns gets no flag. Both reports show those columns as a flag beside the call. The cargo test `rust/crates/og-steps/tests/i411_hla_ties.rs` pins this rule, and the unit test `weak_pypgx_evidence_is_flagged_not_withheld` in `pgx_rules.rs` pins the flag. - HLA agreement between T1K and OptiType is extra evidence, not an independent check: both read the same alignment, so reads that never reach the HLA windows (an allele that drops out or maps elsewhere) can make them agree on the wrong pair, most often a false homozygote. The consensus table names the callers that decided each gene. - The two diplotypes must be the same up to the order of the alleles. The one passed to PharmCAT is pypgx's. - PharmCAT turns the HLA alleles into its own phenotypes (`*57:01 positive`, `*58:01 negative` and so on) and gives the drug guidance for them; step 27 marks those genes `[outside call]`.