From 1069e604a6c7af5b90976442b79ac8ec89d65b37 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:06:13 +0100 Subject: [PATCH 1/5] feat(sv): og sv survivor-prep keeps only the calls a caller makes Step 22 handed SURVIVOR every PASS record, so a Delly record genotyped 0/0, a Delly call whose sample filter says LowQual, or a CNVpytor segment no quality rule had read counted as a second caller. The new verb drops a record whose FILTER or first-sample FT is not PASS, or whose genotype calls an allele but no ALT (0/0, 0|0, 0, 0/.). A no-call and a sites-only record keep their vote. For CNVpytor it applies the e-value and MAPQ 0 rule of og sv add-callers, now shared constants. Bead: sv-05 Bead: sv-06 Bead: D14 --- rust/crates/og-steps/src/sv/add_callers.rs | 12 +- rust/crates/og-steps/src/sv/survivor_prep.rs | 279 ++++++++++++++++++- rust/crates/og-steps/src/sv/tests.rs | 180 ++++++++++++ 3 files changed, 461 insertions(+), 10 deletions(-) diff --git a/rust/crates/og-steps/src/sv/add_callers.rs b/rust/crates/og-steps/src/sv/add_callers.rs index fa3cb4107..00667b31a 100644 --- a/rust/crates/og-steps/src/sv/add_callers.rs +++ b/rust/crates/og-steps/src/sv/add_callers.rs @@ -98,6 +98,14 @@ const STANDARD_INFO: [(&str, &str); 7] = [ ), ]; +/// CNVpytor's quality rule, the defaults of `--max-eval` and `--max-q0`: a +/// call counts when its e-value (`pytorP1`) is below [`MAX_EVAL`] and its +/// share of MAPQ 0 reads (`pytorQ0`) below [`MAX_Q0`]. Step 22's +/// `og sv survivor-prep --caller cnvpytor` applies the same two numbers. +pub const MAX_EVAL: f64 = 0.01; +/// See [`MAX_EVAL`]. +pub const MAX_Q0: f64 = 0.5; + /// The INFO keys an added record keeps, in this order. const KEPT: [&str; 4] = ["END", "SVTYPE", "CIPOS", "CIEND"]; @@ -530,8 +538,8 @@ fn opts(argv: &[String]) -> Result { sex, par_bed: given("--par-bed"), cnvpytor: given("--cnvpytor"), - max_eval: float("--max-eval", 0.01)?, - max_q0: float("--max-q0", 0.5)?, + max_eval: float("--max-eval", MAX_EVAL)?, + max_q0: float("--max-q0", MAX_Q0)?, min_overlap: float("--min-overlap", 0.5)?, }) } diff --git a/rust/crates/og-steps/src/sv/survivor_prep.rs b/rust/crates/og-steps/src/sv/survivor_prep.rs index 4f17375a3..314b78193 100644 --- a/rust/crates/og-steps/src/sv/survivor_prep.rs +++ b/rust/crates/og-steps/src/sv/survivor_prep.rs @@ -1,13 +1,276 @@ //! `og sv survivor-prep`: one caller's structural variants made ready for -//! SURVIVOR's merge in step 22. +//! SURVIVOR's merge in step 22, so that only a call the sample carries +//! counts as that caller's vote. //! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! SURVIVOR counts a caller as support for every record of its input, +//! whatever the genotype says. Before this verb, step 22 kept every record +//! whose FILTER was `PASS` or `.`, so a PASS Delly record genotyped `0/0` +//! (Delly tested the site and found the reference), a Delly call whose own +//! sample filter (FORMAT `FT`) says `LowQual`, or a CNVpytor segment no +//! quality rule had looked at, each made a Manta-only call read "2 +//! callers". A record gives no vote when: +//! +//! - its FILTER is not `PASS` or `.`; +//! - its first sample's `FT` is there and is not `PASS` or `.`; +//! - its first sample's genotype calls an allele but no ALT: `0/0`, `0|0`, +//! haploid `0`, `0/.`. A no-call (`./.`), a record without `GT` and a +//! sites-only record keep their vote: nothing in them says the call is +//! absent, and SURVIVOR writes `./.` itself; +//! - for `--caller cnvpytor`, its e-value (`pytorP1`) is not below +//! [`MAX_EVAL`] or its share of MAPQ 0 reads (`pytorQ0`) is not below +//! [`MAX_Q0`]: the rule step 05's `og sv add-callers` applies to the same +//! calls. A missing or unreadable value is no evidence, so the call does +//! not vote. +//! +//! The kept records are written as plain text with one sample column named +//! after the caller: SURVIVOR names its output columns after the input +//! samples, and three inputs that all name the sample would give one name +//! three times. A sites-only input gets `GT` `./.`. The header passes +//! through; a header without `GT` or without the CNVpytor tags is read the +//! same as one with them. A summary line goes to stderr. + +use std::fmt; +use std::fs::File; +use std::io::{self, BufRead, BufWriter, Write}; + +use og_filters::args::{self, ArgError, Spec}; + +use super::add_callers::{MAX_EVAL, MAX_Q0}; +use super::{first_gt, info_value}; + +const USAGE: &str = "usage: og sv survivor-prep --caller NAME IN.vcf[.gz]|- OUT.vcf|- +"; + +const SPEC: Spec = Spec { + prog: "og sv survivor-prep", + usage: USAGE, + options: &["--caller"], + positionals: &["IN", "OUT"], + required: &["--caller"], +}; + +const GT_HEADER: &str = "##FORMAT="; -use std::io::Write; +/// Why a record gives no vote. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NoVote { + /// FILTER is not `PASS` or `.`. + Filter, + /// The first sample's `FT` is not `PASS` or `.`. + SampleFilter, + /// The first sample's genotype calls an allele, none of them ALT. + Reference, + /// A CNVpytor call that fails the quality rule. + Quality, +} + +/// Whether the genotype calls an allele (`0`, `1`, ...) but no ALT: `0/0`, +/// `0|0`, `0`, `0/.`. `./.`, `.` and an empty value call none. +pub fn reference_only(gt: &str) -> bool { + let mut called_ref = false; + for a in gt.split(['/', '|']) { + if a.is_empty() || !a.bytes().all(|b| b.is_ascii_digit()) { + continue; + } + if a.bytes().all(|b| b == b'0') { + called_ref = true; + } else { + return false; + } + } + called_ref +} + +/// The value of the first sample's FORMAT key `key`, when FORMAT has it. +fn sample_value<'a>(fields: &[&'a str], key: &str) -> Option<&'a str> { + if fields.len() < 10 { + return None; + } + let i = fields[8].split(':').position(|k| k == key)?; + Some(fields[9].split(':').nth(i).unwrap_or(".")) +} + +/// A number of INFO, or NaN (missing, a flag or not a number), which no +/// threshold passes. +fn info_number(info: &str, key: &str) -> f64 { + pycompat::num::parse_float(info_value(info, key)).unwrap_or(f64::NAN) +} + +/// Why the record `fields` (at least 8) gives no vote, or `None` when it +/// does. `cnvpytor` applies CNVpytor's quality rule. +pub fn verdict(fields: &[&str], cnvpytor: bool) -> Option { + if !matches!(fields[6], "PASS" | ".") { + return Some(NoVote::Filter); + } + if let Some(ft) = sample_value(fields, "FT") + && !matches!(ft, "PASS" | "." | "") + { + return Some(NoVote::SampleFilter); + } + if fields.len() >= 10 && reference_only(first_gt(fields)) { + return Some(NoVote::Reference); + } + if cnvpytor { + let info = fields[7]; + let ok = info_number(info, "pytorP1") < MAX_EVAL && info_number(info, "pytorQ0") < MAX_Q0; + if !ok { + return Some(NoVote::Quality); + } + } + None +} + +/// What one run read and kept. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Counts { + pub records: usize, + pub kept: usize, + pub filter: usize, + pub sample_filter: usize, + pub reference: usize, + pub quality: usize, +} + +impl fmt::Display for Counts { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} records, {} vote; no vote: {} FILTER not PASS, {} sample FT not PASS, {} genotyped reference", + self.records, self.kept, self.filter, self.sample_filter, self.reference + )?; + if self.quality > 0 { + write!(f, ", {} below CNVpytor's quality rule", self.quality)?; + } + Ok(()) + } +} + +/// A failure that ends the run with exit 1. +#[derive(Debug)] +pub enum Error { + Read(String, io::Error), + Write(String, io::Error), + Short(String, usize), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Read(p, e) => write!(f, "cannot read {p}: {e}"), + Error::Write(p, e) => write!(f, "cannot write {p}: {e}"), + Error::Short(p, n) => write!(f, "{p} line {n}: a record needs at least 8 columns"), + } + } +} + +/// Reads `input` (named `path`, for messages) and writes the records that +/// vote to `out`, with one sample column named `caller`. +pub fn prep( + input: impl BufRead, + path: &str, + caller: &str, + out: &mut impl Write, +) -> Result { + let w = |e| Error::Write("the output".to_owned(), e); + let cnvpytor = caller == "cnvpytor"; + let mut c = Counts::default(); + let mut sites = false; + let mut has_gt = false; + for (i, line) in input.lines().enumerate() { + let line = line.map_err(|e| Error::Read(path.to_owned(), e))?; + let line = line.trim_end_matches('\r'); + if line.is_empty() { + continue; + } + if line.starts_with("##") { + has_gt |= line.starts_with("##FORMAT= = line.split('\t').collect(); + if line.starts_with("#CHROM") { + sites = f.len() < 10; + if sites && !has_gt { + writeln!(out, "{GT_HEADER}").map_err(w)?; + } + let head = f.get(..8).unwrap_or(&f).join("\t"); + writeln!(out, "{head}\tFORMAT\t{caller}").map_err(w)?; + continue; + } + if f.len() < 8 { + return Err(Error::Short(path.to_owned(), i + 1)); + } + c.records += 1; + match verdict(&f, cnvpytor) { + Some(NoVote::Filter) => c.filter += 1, + Some(NoVote::SampleFilter) => c.sample_filter += 1, + Some(NoVote::Reference) => c.reference += 1, + Some(NoVote::Quality) => c.quality += 1, + None => { + c.kept += 1; + let site = f[..8].join("\t"); + if sites || f.len() < 10 { + writeln!(out, "{site}\tGT\t./.").map_err(w)?; + } else { + writeln!(out, "{site}\t{}\t{}", f[8], f[9]).map_err(w)?; + } + } + } + } + out.flush().map_err(w)?; + Ok(c) +} + +/// Runs the verb on files: `out_path` `-` is stdout. +fn run(input: &str, out_path: &str, caller: &str, stdout: &mut dyn Write) -> Result { + let reader = og_io::open(input).map_err(|e| Error::Read(input.to_owned(), e))?; + if out_path == "-" { + let mut w = BufWriter::new(stdout); + return prep(reader, input, caller, &mut w); + } + // Written beside the target and renamed, so a failed run leaves no + // half file for SURVIVOR to read. + let part = format!("{out_path}.part"); + let file = File::create(&part).map_err(|e| Error::Write(part.clone(), e))?; + let mut w = BufWriter::new(file); + let c = prep(reader, input, caller, &mut w).inspect_err(|_| { + let _ = std::fs::remove_file(&part); + })?; + drop(w); + std::fs::rename(&part, out_path).map_err(|e| Error::Write(out_path.to_owned(), e))?; + Ok(c) +} -/// `og sv survivor-prep ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og sv survivor-prep: not ported yet"); - 2 +/// `og sv survivor-prep --caller NAME IN OUT`: exit 0, 1 for an input or +/// output problem, 2 for a usage error. +pub fn cli(argv: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let p = match args::parse(&SPEC, argv) { + Ok(p) => p, + Err(ArgError::Help) => { + let _ = write!(out, "{USAGE}"); + return 0; + } + Err(ArgError::Usage(m)) => { + let _ = writeln!(err, "{USAGE}og sv survivor-prep: error: {m}"); + return 2; + } + }; + let caller = p.get("--caller").unwrap_or(""); + if caller.is_empty() || caller.bytes().any(|b| b.is_ascii_whitespace() || b == b':') { + let _ = writeln!( + err, + "{USAGE}og sv survivor-prep: error: argument --caller: a caller name is one word, got {caller:?}" + ); + return 2; + } + match run(&p.pos[0], &p.pos[1], caller, out) { + Ok(c) => { + let _ = writeln!(err, "og sv survivor-prep {caller}: {c}"); + 0 + } + Err(e) => { + let _ = writeln!(err, "ERROR: {e}"); + 1 + } + } } diff --git a/rust/crates/og-steps/src/sv/tests.rs b/rust/crates/og-steps/src/sv/tests.rs index 08099ff6a..76132152a 100644 --- a/rust/crates/og-steps/src/sv/tests.rs +++ b/rust/crates/og-steps/src/sv/tests.rs @@ -340,3 +340,183 @@ fn a_missing_table_is_a_message() { assert!(!d.join("o.tsv").exists()); std::fs::remove_dir_all(d).unwrap(); } + +// og sv survivor-prep: which records vote in step 22's SURVIVOR merge. + +use super::survivor_prep::{self, NoVote}; + +fn prep_text(input: &str, caller: &str) -> (String, survivor_prep::Counts) { + let mut out = Vec::new(); + let c = survivor_prep::prep(input.as_bytes(), "in.vcf", caller, &mut out).unwrap(); + (String::from_utf8(out).unwrap(), c) +} + +const SV_HEAD: &str = "##fileformat=VCFv4.2\n##FORMAT=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHG002\n"; + +fn sv_rec(id: &str, filter: &str, info: &str, format: &str, sample: &str) -> String { + format!( + "chr20\t1000\t{id}\tN\t\t60\t{filter}\tSVTYPE=DEL;END=5000;{info}\t{format}\t{sample}\n" + ) +} + +#[test] +fn a_reference_genotype_gives_no_vote() { + for gt in ["0/0", "0|0", "0", "0/.", ".|0", "00"] { + assert!(survivor_prep::reference_only(gt), "{gt} calls no ALT"); + } + for gt in ["0/1", "1/1", "1", "./1", "0|2", "./.", ".", ""] { + assert!( + !survivor_prep::reference_only(gt), + "{gt} is not reference only" + ); + } + let input = [ + SV_HEAD.to_owned(), + sv_rec("REF00", "PASS", "CT=3to5", "GT:FT", "0/0:PASS"), + sv_rec("REF0", "PASS", "CT=3to5", "GT", "0"), + sv_rec("HALF", "PASS", "CT=3to5", "GT", "0/."), + sv_rec("HET", "PASS", "CT=3to5", "GT:FT", "0/1:PASS"), + sv_rec("NOCALL", ".", "CT=3to5", "GT", "./."), + ] + .concat(); + let (out, c) = prep_text(&input, "delly"); + // Red-first control: before this verb, step 22 kept every PASS record, + // so REF00 made a Manta-only call read "2 callers". + assert_eq!(ids(&out), ["HET", "NOCALL"]); + assert_eq!((c.records, c.kept, c.reference), (5, 2, 3)); +} + +#[test] +fn a_sample_filter_or_filter_gives_no_vote() { + let input = [ + SV_HEAD.to_owned(), + sv_rec("LOWQ", "PASS", "CT=3to5", "GT:GQ:FT", "0/1:30:LowQual"), + sv_rec("FTDOT", "PASS", "CT=3to5", "GT:GQ:FT", "0/1:30:."), + sv_rec("SHORTCOL", "PASS", "CT=3to5", "GT:GQ:FT", "0/1"), + sv_rec("SITELOW", "LowQual", "CT=3to5", "GT:FT", "0/1:PASS"), + ] + .concat(); + let (out, c) = prep_text(&input, "delly"); + assert_eq!(ids(&out), ["FTDOT", "SHORTCOL"]); + assert_eq!((c.sample_filter, c.filter), (1, 1)); + let line = sv_rec("X", "PASS", "", "GT:FT", "1/1:LowQual"); + let v: Vec<&str> = line.trim_end().split('\t').collect(); + assert_eq!( + survivor_prep::verdict(&v, false), + Some(NoVote::SampleFilter) + ); +} + +#[test] +fn cnvpytor_votes_by_its_quality_rule() { + let cnv = |id: &str, p1: &str, q0: &str, gt: &str| { + sv_rec( + id, + "PASS", + &format!("pytorP1={p1};pytorQ0={q0};pytorPN=0"), + "GT", + gt, + ) + }; + let input = [ + SV_HEAD.to_owned(), + cnv("GOOD", "1e-08", "0.01", "./."), + cnv("NOISY", "0.5", "0", "./."), + cnv("MAPQ0", "1e-08", "0.9", "./."), + cnv("EDGE", "0.01", "0", "./."), + cnv("NAN", "nan", "0", "./."), + sv_rec("NOTAGS", "PASS", "SVLEN=-4000", "GT", "./."), + ] + .concat(); + let (out, c) = prep_text(&input, "cnvpytor"); + // ./. is how the table-made VCF writes every call: it keeps the vote. + assert_eq!(ids(&out), ["GOOD"]); + assert_eq!((c.kept, c.quality), (1, 5)); + assert!( + c.to_string().ends_with("5 below CNVpytor's quality rule"), + "{c}" + ); + // The rule is CNVpytor's own: another caller's records keep their vote. + let (out, c) = prep_text(&input, "delly"); + assert_eq!(c.kept, 6, "{out}"); + assert_eq!( + survivor_prep::verdict(&["c", "1", ".", "N", "", ".", "PASS", "."], true), + Some(NoVote::Quality) + ); +} + +#[test] +fn a_header_without_gt_or_samples_does_not_stop_it() { + // Sites only and no GT declared: SURVIVOR still gets a GT column. + let sites = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\nchr20\t1000\tS1\tN\t\t.\tPASS\tSVTYPE=DEL;END=5000\n"; + let (out, c) = prep_text(sites, "tiddit"); + assert_eq!(c.kept, 1); + assert_eq!( + out, + "##fileformat=VCFv4.2\n##FORMAT=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\ttiddit\nchr20\t1000\tS1\tN\t\t.\tPASS\tSVTYPE=DEL;END=5000\tGT\t./.\n" + ); + // A sample column whose FORMAT has no GT keeps its vote, and the + // header gets no second GT line. + let no_gt = "##fileformat=VCFv4.2\n##FORMAT=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\nchr20\t1000\tC1\tN\t\t.\tPASS\tSVTYPE=DUP;END=5000\tCN\t3\t2\n"; + let (out, c) = prep_text(no_gt, "manta"); + assert_eq!(c.kept, 1); + assert_eq!(out.matches("##FORMAT= = a.iter().map(|s| (*s).to_owned()).collect(); + let (mut o, mut e) = (Vec::new(), Vec::new()); + let code = survivor_prep::cli(&argv, &mut o, &mut e); + ( + code, + String::from_utf8(o).unwrap(), + String::from_utf8(e).unwrap(), + ) + }; + let (code, _, err) = run(&["--caller", "delly", &input, &out]); + assert_eq!(code, 0, "{err}"); + assert_eq!(ids(&std::fs::read_to_string(&out).unwrap()), ["HET"]); + assert!( + err.contains("og sv survivor-prep delly: 2 records, 1 vote;") + && err.contains("1 genotyped reference"), + "{err}" + ); + let (code, stdout, _) = run(&["--caller", "delly", &input, "-"]); + assert_eq!((code, ids(&stdout)), (0, vec!["HET".to_owned()])); + let (code, _, err) = run(&[ + "--caller", + "delly", + &d.join("none.vcf").to_string_lossy(), + &out, + ]); + assert!(code == 1 && err.starts_with("ERROR: cannot read "), "{err}"); + let short = put(&d, "short.vcf", &format!("{SV_HEAD}chr20\t1000\tX\n")); + let bad_out = d.join("bad.vcf").to_string_lossy().into_owned(); + let (code, _, err) = run(&["--caller", "delly", &short, &bad_out]); + assert!( + code == 1 && err.contains("line 4: a record needs at least 8 columns"), + "{err}" + ); + assert!(!d.join("bad.vcf").exists() && !d.join("bad.vcf.part").exists()); + assert_eq!(run(&["--caller", "a b", &input, &out]).0, 2); + assert_eq!(run(&["--caller", "delly", &input]).0, 2); + assert_eq!(run(&[&input, &out]).0, 2); + std::fs::remove_dir_all(d).unwrap(); +} From fb037e08dd72661564e5d43134c644b2cd55155e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:16:13 +0100 Subject: [PATCH 2/5] fix(sv): step 22 prepares each caller's votes with og sv survivor-prep The script and the Nextflow SURVIVOR_PREP each carried the same bcftools view -f PASS,. and awk. Both now call og sv survivor-prep, so a 0/0 or FT=LowQual Delly record and a CNVpytor call below its quality rule no longer reach SURVIVOR. The VCF step 22 makes from CNVpytor's table carries the table's e-value, q0 and pN under CNVpytor's INFO names, so the same rule reads both inputs. Bead: sv-05 Bead: sv-06 Bead: sv-40 Bead: D14 --- docs/22-survivor-merge.md | 32 ++++++++++++++++++----- modules/local/survivor_merge/main.nf | 18 ++++++------- scripts/22-survivor-merge.sh | 38 +++++++++++++++------------- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/docs/22-survivor-merge.md b/docs/22-survivor-merge.md index f441d7a9d..2c59dbf95 100644 --- a/docs/22-survivor-merge.md +++ b/docs/22-survivor-merge.md @@ -4,6 +4,8 @@ Keeps the structural variants (SVs) that two or more callers agree on. SURVIVOR merge pairs two calls when both of their breakpoints lie within 1,000 bp of each other, their SV type and strands agree, and the event is at least 50 bp long. Each consensus record says how many callers support it and which. That is caller agreement, not validation (see Why). +A caller votes only with the calls it makes (see [Which records vote](#which-records-vote)): a record the caller genotyped as reference, or filtered, is not a second caller. + ## Why Individual SV callers each have distinct biases and false-positive profiles: @@ -14,18 +16,19 @@ Individual SV callers each have distinct biases and false-positive profiles: An SV that two callers report is less likely to be one caller's artefact. Agreement is not independent proof, though. Manta and Delly both read split and paired reads from the same alignment, so a misaligned read cluster both of them see can reach the consensus. Only CNVpytor reads a different signal, depth, and its breakpoints are coarse (see Limitations). -The step used to bin calls by chromosome, `int(POS/1000)` and SV type. That split one deletion called at positions 999 and 1001 over two bins, and counted two deletions of very different size that start in one bin as agreement. SURVIVOR compares both breakpoints, so neither happens: the e2e case `tests/e2e/sv-mito-telomere-steps-1-sv-merge.sh` runs both situations on the synthetic three-caller set in `tests/fixtures/sv/`. +The step used to bin calls by chromosome, `int(POS/1000)` and SV type. That split one deletion called at positions 999 and 1001 over two bins, and counted two deletions of very different size that start in one bin as agreement. SURVIVOR compares both breakpoints, so neither happens: the e2e case `tests/e2e/sv-mito-telomere-steps-1-sv-merge.sh` runs both situations on the synthetic three-caller set in `tests/fixtures/sv/`. The e2e case `tests/e2e/sv-merge-votes.sh` runs the records that must not vote on the same set. ## Tool - **SURVIVOR** 1.0.7 (Jeffares et al., Nat Commun 2017), `SURVIVOR merge` -- **bcftools** for the PASS filter before the merge and the sorted, indexed output +- **og** (`og sv survivor-prep`) for the records each caller votes with +- **bcftools** for the sorted, indexed output Not chosen: `truvari collapse`, which also compares sequence similarity. It would need a comparison run against SURVIVOR on real calls first. ## Docker Image -- `SURVIVOR_IMAGE` (the merge), `BCFTOOLS_IMAGE` (filter, sort, index) +- `SURVIVOR_IMAGE` (the merge), `BCFTOOLS_IMAGE` (runs `og` for the records that vote; sort, index) Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. @@ -53,19 +56,34 @@ The Nextflow pipeline (`survivor_merge` in `--tools`, with at least two of `mant ## What the Script Does Internally -1. Turns CNVpytor's table into a VCF when step 18 left no VCF -2. Writes each caller's PASS (or unfiltered) records as plain VCF under `sv_merged/inputs/`, with one sample column named after the caller. SURVIVOR names its output columns after the input samples, so three inputs that all name the sample would give one name three times +1. Turns CNVpytor's table into a VCF when step 18 left no VCF. Each record carries the table's e-value, `q0` and `pN` under the INFO names CNVpytor's own VCF uses (`pytorP1`, `pytorQ0`, `pytorPN`) +2. Writes the records each caller votes with (below) as plain VCF under `sv_merged/inputs/`, with one sample column named after the caller (`og sv survivor-prep --caller NAME IN OUT`, which prints how many records it left out and why). SURVIVOR names its output columns after the input samples, so three inputs that all name the sample would give one name three times 3. Runs `SURVIVOR merge inputs/sv_files.txt 1000 2 1 1 0 50`: maximum breakpoint distance 1,000 bp, support from 2 callers, same type, same strands, no size-scaled distance, minimum size 50 bp 4. Checks that SURVIVOR wrote a VCF (it exits 0 when it cannot open an input), then sorts, compresses and indexes it 5. Prints the count, and the count per type and caller combination +### Which records vote + +SURVIVOR counts a caller as support for every record of its input. A record goes in only when it is a call the caller makes: + +| Record | Votes | Why | +|---|---|---| +| FILTER `PASS` or `.`, genotype with an ALT allele (`0/1`, `1/1`, `1`) | yes | a call | +| genotype `./.`, or no `GT`, or a sites-only VCF | yes | nothing says the call is absent; the VCF made from CNVpytor's table writes `./.` for every call | +| FILTER other than `PASS` or `.` (Delly's `LowQual`) | no | the caller filtered it | +| sample filter `FT` other than `PASS` or `.` (Delly's per-sample `LowQual`) | no | the caller filtered it for this sample | +| genotype with a called allele and no ALT (`0/0` phased or not, haploid `0`, `0/.`) | no | the caller tested the site and found the reference | +| a CNVpytor call with e-value (`pytorP1`) 0.01 or more, `q0` (`pytorQ0`) 0.5 or more, or either value missing | no | CNVpytor's quality rule, the one step 5 applies when it adds CNVpytor's calls ([step 18](18-cnvpytor.md#filtering)) | + +Delly genotypes every site it reports, so a PASS Delly record with `0/0` is common. Before this rule, such a record next to a Manta call read as two callers. + ## Output | File | Contents | |---|---| | `${SAMPLE}_sv_consensus.vcf.gz` | Consensus SVs supported by 2+ callers, with `SUPP` and `SUPP_VEC` in INFO | | `${SAMPLE}_sv_consensus.vcf.gz.tbi` | Tabix index | -| `inputs/sv_files.txt`, `inputs/.vcf` | The PASS records SURVIVOR read, in `SUPP_VEC` order | +| `inputs/sv_files.txt`, `inputs/.vcf` | The records each caller voted with, which SURVIVOR read, in `SUPP_VEC` order | All output is written to `${GENOME_DIR}/${SAMPLE}/sv_merged/`. @@ -101,7 +119,7 @@ docker run --rm -v "${GENOME_DIR}:/genome" "${BCFTOOLS_IMAGE}" \ - Single-caller SVs are discarded even if they are real. If you suspect a specific SV, check the individual caller outputs directly. - CNVpytor's calls are depth-only and their breakpoints are coarse (its bin size), so a CNVpytor call often lies more than 1 kb from the Manta or Delly breakpoint of the same event and does not pair with it. -- The CNVpytor VCF made from its table carries no genotype and no quality. +- The CNVpytor VCF made from its table carries no genotype (`./.`); its e-value and `q0` come from the table. - A breakend (BND) record pairs with a breakend, an inversion (INV) or a translocation (TRA) of another caller: SURVIVOR 1.0.7 lets a BND match those two types. That includes a lone Manta inversion junction of 1 Mb or more, which step 4 keeps as its two breakends instead of a whole-span inversion. ## Notes diff --git a/modules/local/survivor_merge/main.nf b/modules/local/survivor_merge/main.nf index 3a808f3be..7503e8c78 100644 --- a/modules/local/survivor_merge/main.nf +++ b/modules/local/survivor_merge/main.nf @@ -2,8 +2,11 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SURVIVOR_MERGE — SV consensus: the calls two or more callers agree on ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - SURVIVOR_PREP each caller's PASS (or unfiltered) records as plain VCF, - one sample column named after the caller (bcftools) + SURVIVOR_PREP the calls each caller votes with, as plain VCF, one sample + column named after the caller (og sv survivor-prep): no + record whose FILTER or sample FT is not PASS, none whose + genotype is reference only (0/0, 0|0, 0, 0/.), and for + CNVpytor none that fails step 05's e-value and q0 rule SURVIVOR_MERGE `SURVIVOR merge LIST 1000 2 1 1 0 50`: both breakpoints within 1,000 bp, same type and strands, >= 50 bp, kept when 2+ callers support it; SUPP and SUPP_VEC say which @@ -30,15 +33,10 @@ process SURVIVOR_PREP { script: // SURVIVOR names its output columns after the input samples: three // columns all named after the sample would be one name three times. + // og (bin/) reads the gzip input itself; scripts/22-survivor-merge.sh + // runs the same verb. """ - bcftools view -f PASS,. ${vcf} | awk -v s=${caller} 'BEGIN { FS = OFS = "\\t" } - /^##/ { print; next } - /^#CHROM/ { - if (NF < 10) { print "##FORMAT="; sites = 1; print \$1, \$2, \$3, \$4, \$5, \$6, \$7, \$8, "FORMAT", s } - else print \$1, \$2, \$3, \$4, \$5, \$6, \$7, \$8, \$9, s - next - } - { if (sites) print \$1, \$2, \$3, \$4, \$5, \$6, \$7, \$8, "GT", "./."; else print \$1, \$2, \$3, \$4, \$5, \$6, \$7, \$8, \$9, \$10 }' > ${caller}.vcf + og sv survivor-prep --caller ${caller} ${vcf} ${caller}.vcf cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/scripts/22-survivor-merge.sh b/scripts/22-survivor-merge.sh index 83a6d2ddb..119daf33e 100755 --- a/scripts/22-survivor-merge.sh +++ b/scripts/22-survivor-merge.sh @@ -17,6 +17,11 @@ # callers' inversions and translocations and count as votes for an event GRIDSS # never typed. # +# A caller votes only with the calls it makes: og sv survivor-prep drops a +# record whose FILTER or sample FT is not PASS, or whose genotype is reference +# only (0/0, 0|0, 0, 0/.), and for CNVpytor a call that fails step 05's quality +# rule (e-val < 0.01, q0 < 0.5). A no-call (./.) and a sites-only record vote. +# # SUPP counts callers that agree, not independent proof: Manta and Delly both # read split and paired reads, so they share alignment errors. AnnotSV (step 05) # annotates the callers' own calls; the reports join SUPP onto them. @@ -43,7 +48,9 @@ echo "============================================" echo "" # CNVpytor's own table, when step 18 left no VCF: one DEL or DUP record per -# CNV, with a sample column, so SURVIVOR reads it as it reads the others. +# CNV, with a sample column, so SURVIVOR reads it as it reads the others. The +# e-value (column 5), q0 (9) and pN (10) go into INFO under the names CNVpytor's +# VCF uses, so its quality rule reads both inputs the same. CNV_TXT="${S}/cnvpytor/${SAMPLE}_cnvs.txt" CNV_VCF="${S}/cnvpytor/${SAMPLE}_cnvs.vcf.gz" if [ ! -f "$CNV_VCF" ] && [ -f "$CNV_TXT" ]; then @@ -53,14 +60,17 @@ if [ ! -f "$CNV_VCF" ] && [ -f "$CNV_TXT" ]; then echo "##INFO=" echo "##INFO=" echo "##INFO=" + echo "##INFO=" + echo "##INFO=" + echo "##INFO=" echo "##FORMAT=" awk '{printf "##contig=\n", $1, $2}' "${REF_FASTA}.fai" printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "$SAMPLE" - awk '$1 == "deletion" || $1 == "duplication" { + awk -F'\t' '$1 == "deletion" || $1 == "duplication" { split($2, a, ":"); split(a[2], b, "-") t = ($1 == "duplication") ? "DUP" : "DEL" len = (t == "DEL") ? -(b[2] - b[1]) : b[2] - b[1] - printf "%s\t%s\t.\tN\t<%s>\t.\tPASS\tSVTYPE=%s;END=%s;SVLEN=%d\tGT\t./.\n", a[1], b[1], t, t, b[2], len + printf "%s\t%s\t.\tN\t<%s>\t.\tPASS\tSVTYPE=%s;END=%s;SVLEN=%d;pytorP1=%s;pytorQ0=%s;pytorPN=%s\tGT\t./.\n", a[1], b[1], t, t, b[2], len, $5, $9, $10 }' "$CNV_TXT" } > "${IN_DIR}/cnvpytor_txt.vcf" run_in --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" bash -euo pipefail -c \ @@ -85,22 +95,14 @@ for c in "${CALLERS[@]}"; do echo " [--] ${name}: no ${vcf#"$S"/}" continue fi - # PASS (or unfiltered) records as plain text, which SURVIVOR reads, with - # one sample column named after the caller: SURVIVOR names its output - # columns after the input samples, and three columns all named after the - # sample would be one name three times. - run_in --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" bash -euo pipefail -c ' - bcftools view -f PASS,. "$1" | awk -v s="$3" '\''BEGIN { FS = OFS = "\t" } - /^##/ { print; next } - /^#CHROM/ { - if (NF < 10) { print "##FORMAT="; sites = 1; print $1, $2, $3, $4, $5, $6, $7, $8, "FORMAT", s } - else print $1, $2, $3, $4, $5, $6, $7, $8, $9, s - next - } - { if (sites) print $1, $2, $3, $4, $5, $6, $7, $8, "GT", "./."; else print $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 }'\'' > "$2"' \ - _ "$(cpath "$vcf")" "$(cpath "${IN_DIR}/${name}.vcf")" "$name" + # The calls that vote, as plain text, which SURVIVOR reads, with one + # sample column named after the caller: SURVIVOR names its output columns + # after the input samples, and three columns all named after the sample + # would be one name three times. og prints what it left out. + run_in --cpus 1 --memory 1g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$BCFTOOLS_IMAGE" \ + /pgp-bin/og sv survivor-prep --caller "$name" "$(cpath "$vcf")" "$(cpath "${IN_DIR}/${name}.vcf")" n=$(grep -vc '^#' "${IN_DIR}/${name}.vcf" || true) - echo " [OK] ${name}: ${n} PASS records (${vcf#"$S"/})" + echo " [OK] ${name}: ${n} calls that vote (${vcf#"$S"/})" echo "$(cpath "${IN_DIR}/${name}.vcf")" >> "${IN_DIR}/sv_files.txt" USED+=("$name") done From 220aab74a7d7611d659771e507075b5f2caa7b96 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:16:13 +0100 Subject: [PATCH 3/5] fix(cnvpytor): one quality rule for CNVpytor's calls in step 18's hint Step 18 and its doc told the reader to keep calls with e-value below 0.05 and size over 1 kb, while steps 05 and 22 use e-value below 0.01 and q0 below 0.5. The hint and the doc now give that one rule. Bead: sv-09 Bead: sv-10 Bead: sv-11 --- docs/18-cnvpytor.md | 6 ++++-- scripts/18-cnvpytor.sh | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/18-cnvpytor.md b/docs/18-cnvpytor.md index affed3ee7..69d5044d3 100644 --- a/docs/18-cnvpytor.md +++ b/docs/18-cnvpytor.md @@ -80,9 +80,11 @@ The `1000` parameter is the bin size in base pairs (must be divisible by 100). U - `${SAMPLE}_cnvs.vcf.gz` (+ `.tbi`) — normalized VCF with `SVTYPE=DEL/DUP`, `END`, `SVLEN`, `GT`, and `CN` ## Filtering +One quality rule reads CNVpytor's calls everywhere in the pipeline: e-value (column 5, `pytorP1` in the VCF) below 0.01 and `q0` (column 9, `pytorQ0`) below 0.5. Step 5 adds only those calls to AnnotSV's input (`og sv add-callers`), and in step 22 only those vote in the SV consensus (`og sv survivor-prep`). + ```bash -# Keep confident CNVs (e-value in col 5 < 0.05, size in col 3 > 1 kb) -awk -F'\t' '$5 < 0.05 && $3 > 1000' ${SAMPLE}_cnvs.txt +# The calls steps 5 and 22 use (e-value in col 5 < 0.01, q0 in col 9 < 0.5) +awk -F'\t' '$5 < 0.01 && $9 < 0.5' ${SAMPLE}_cnvs.txt ``` The first call is often a low-`level` artifact spanning the chromosome-start N gap (`pN` near 1.0); the `pN`/`q0` columns let you drop such regions. diff --git a/scripts/18-cnvpytor.sh b/scripts/18-cnvpytor.sh index 61fa83000..04973652e 100755 --- a/scripts/18-cnvpytor.sh +++ b/scripts/18-cnvpytor.sh @@ -134,5 +134,5 @@ echo "Total CNVs called: ${CNV_COUNT}" echo "Calls (TSV): ${OUTPUT_DIR}/${SAMPLE}_cnvs.txt" echo "Calls (VCF): ${OUTPUT_DIR}/${SAMPLE}_cnvs.vcf.gz" echo "" -echo "Filter significant CNVs (e-val1 < 0.05 in col 5, size > 1kb in col 3):" -echo " awk -F'\t' '\$5 < 0.05 && \$3 > 1000' ${OUTPUT_DIR}/${SAMPLE}_cnvs.txt" +echo "The calls steps 05 and 22 use (e-val1 < 0.01 in col 5, q0 < 0.5 in col 9):" +printf '%s\n' " awk -F'\t' '\$5 < 0.01 && \$9 < 0.5' ${OUTPUT_DIR}/${SAMPLE}_cnvs.txt" From e42bedaecceb1627e9e8bb456a2e7f9d769b94b9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:16:13 +0100 Subject: [PATCH 4/5] test(sv): the records that must not vote stay out of the consensus Three Manta-only deletions in the step 22 fixtures sit beside a Delly 0/0 record, a Delly FT=LowQual call and a CNVpytor segment with e-value 0.5. Main's prep made each read as two callers (4 consensus records instead of 1). The new e2e case sv-merge-votes checks them, a ./. CNVpytor call that still votes, and the same calls from CNVpytor's table. Step 22 now runs og, so its existing case lists bin/og and the scope test counts it among the cases a Rust change runs. Bead: sv-05 Bead: sv-06 Bead: sv-40 --- CHANGELOG.md | 1 + scripts/ci/e2e-case-paths.tsv | 3 +- scripts/ci/e2e-case-times.tsv | 1 + tests/e2e/sv-merge-votes.sh | 77 +++++++++++++++++++++++++++++++ tests/fixtures/sv/README.md | 12 ++++- tests/fixtures/sv/cnvpytor.vcf.in | 6 ++- tests/fixtures/sv/delly.vcf.in | 3 ++ tests/fixtures/sv/manta.vcf.in | 3 ++ tests/test_e2e_scope.sh | 9 ++-- 9 files changed, 106 insertions(+), 9 deletions(-) create mode 100755 tests/e2e/sv-merge-votes.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index dda3da6ec..ac4562be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 22's SV consensus no longer counts a record a caller did not call as a second caller. Before, every PASS record went to SURVIVOR, so a PASS Delly record genotyped `0/0`, a Delly call whose sample filter (`FT`) says `LowQual`, or a CNVpytor segment no quality rule had read sat beside a Manta-only call and made it read "2 callers". A new verb, `og sv survivor-prep --caller NAME IN OUT`, now writes each caller's input for both the script and the Nextflow `SURVIVOR_PREP`, in place of the bcftools and awk they each carried. A record votes when its FILTER and its sample `FT` are `PASS` or `.` and its genotype is not reference only (`0/0`, `0|0`, haploid `0`, `0/.`); a no-call (`./.`) and a sites-only record still vote. A CNVpytor call votes when its e-value is below 0.01 and its share of MAPQ 0 reads below 0.5, the rule step 05 already applies when it adds CNVpytor's calls; the VCF step 22 makes from CNVpytor's table now carries the table's e-value, `q0` and `pN` as `pytorP1`, `pytorQ0` and `pytorPN`, so the rule reads both inputs. A header without `GT` or without these tags no longer matters. Step 18's closing hint and `docs/18-cnvpytor.md` gave e-value below 0.05 and size over 1 kb; they now give the same rule. What changes in the output: `sv_merged/inputs/.vcf` holds only the records that vote, and the consensus can hold fewer records. No file, column or option is renamed or removed. Part of #411 (sv-05, sv-06, sv-09, sv-10, sv-11, sv-40). - Step 25 and the Nextflow `PRS_SUMMARY` withhold a score's percentile when the genotypes pgsc_calc scored list variants only. Every site such a VCF leaves out adds nothing to the sum, so the sum leans away from the panel's sums, which come from complete genotypes, and the percentile was not comparable. Each score keeps its raw sum. Its `Percentile` is `NA`, its `Effect` is empty, and a new column before `Input`, `Percentile_Note`, says `percentile withheld: the input lists variants only`. The decision reads the genotypes, not the kind of input: at least one genotyped position in five must be a reference call (0/0), the rule step 26b applies to the panel's sites. Genotypes from a gVCF and chip VCFs from `scripts/chip-to-vcf.sh`, which list their reference calls, keep their percentiles. `og prs table` takes the genotypes with a new option, `--target VCF`, and prints the NOTE that step 25 printed before; the NOTE now follows the genotypes too, so a gVCF without reference blocks at the score positions gets it. `og summary collect` passes the note on as `percentile_note`. When every percentile was withheld, both reports say "Percentile withheld: the input lists variants only" and why, and the summary's `not_assessed` list says the percentiles were withheld, in place of the lines for a run without the panel, which told the user to install a panel the run had used. The output gains one column; no column, option or file name changes. - A third-party outage no longer fails the Freshness check of a pull request. Since 2026-11-18 rest.ensembl.org has not answered, and the pull-request job's dry run of `og dev freshness` failed on every pull request that touched `versions.env` or `og-fresh`. `og dev freshness --unreachable-warns` now lists a lookup whose upstream did not answer (no connection, a timeout, HTTP 5xx or 429 after the retries, or HTTP 403) under "Upstreams that did not answer" as a warning, and its database row reads `NO ANSWER`; it does not change the exit. The 403 counts because rest.ensembl.org came back on 2026-11-19 answering 403 to GitHub's runners and 200 elsewhere, which still failed the pull-request job. A 404, an empty answer or an answer in an odd shape still exits 1. `freshness.yml` passes the flag only in the pull-request dry run. Without the flag the report and the exit are as before, and the monthly report on main still counts an outage as an error in the issue. - Three statements in the docs were wrong. Troubleshooting said exit code 137 confirms a kill for memory, and checked it with `docker ps -a --latest`. 137 is any SIGKILL, and the step scripts run their containers with `--rm`, so that command shows an older, unrelated container. The page now says how to confirm a kill for memory: `docker events --filter event=oom`, the kernel log, or `.State.OOMKilled` for a container kept without `--rm`. The run-state and troubleshooting pages said `bin/og` does not run on a Mac and told Mac users to build `og` with cargo or run it in a container. `bin/og` runs the native macOS build that `scripts/setup.sh --og` installs, and the pages now say so; on a Mac, `og run headroom` reports the machine's memory as unknown. Troubleshooting, the FAQ and the lessons-learned page said the step scripts run their containers as root, and lessons-learned gave `:latest` images and `chmod 777` as current fixes. The step scripts run their containers as you; those entries are now marked as past fixes and name what the pipeline does today. Documentation only: no option, output or behaviour changes. diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index d7344bc7b..6c1c396b3 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -166,7 +166,8 @@ short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carr single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh bin/og str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs -sv-mito-telomere-steps-1-sv-merge scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in +sv-merge-votes bin/og scripts/22-survivor-merge.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in +sv-mito-telomere-steps-1-sv-merge bin/og scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh bin/og sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh sv-mito-telomere-steps-4-clinvar-tier bin/og scripts/23-clinical-filter.sh scripts/ci/fake-docker/lib.sh diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index b773cf399..ab5d048e2 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -162,6 +162,7 @@ short-sample-id-stub shard 48 - single-source-lib-ownership shard 1 - str-catalog-1-strchive shard 12 - str-extra-repeats-1-readings shard 16 - +sv-merge-votes shard 10 - sv-mito-telomere-steps-1-sv-merge shard 5 - sv-mito-telomere-steps-2-mito shard 6 bash-step-20-numt sv-mito-telomere-steps-3-y-haplogroup shard 13 - diff --git a/tests/e2e/sv-merge-votes.sh b/tests/e2e/sv-merge-votes.sh new file mode 100755 index 000000000..d799fa825 --- /dev/null +++ b/tests/e2e/sv-merge-votes.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Step 22 (SURVIVOR merge): a caller votes only with the calls it makes. On the +# synthetic set of tests/fixtures/sv/ (its README has the known answer), three +# Manta-only deletions each have a second record beside them that is no call: +# a PASS Delly record genotyped 0/0, a Delly call whose sample filter (FT) is +# LowQual, and a CNVpytor segment with e-value 0.5. None of them may reach the +# consensus, while a CNVpytor call that passes the quality rule and has +# genotype ./. still pairs with Manta's call (support 2, SUPP_VEC 101). Then the +# same calls come from CNVpytor's table instead of its VCF: the e-value and q0 +# of the table decide the same way. +# +# Red before the fix: main's step 22 kept every PASS record, so the three +# Manta-only deletions read SUPP=2 and the consensus held 5 records, not 2. +. "$(dirname "$0")/lib.sh" + +T="${SAMPLE}svvote" +FIX="${REPO}/tests/fixtures/sv" +rm -rf "${GENOME_DIR:?}/${T}" +mkdir -p "${GENOME_DIR}/${T}/manta/results/variants" "${GENOME_DIR}/${T}/delly" "${GENOME_DIR}/${T}/cnvpytor" "${CASE_TMP}/in" +cp "${FIX}/manta.vcf.in" "${FIX}/delly.vcf.in" "${FIX}/cnvpytor.vcf.in" "${CASE_TMP}/in/" +# The positive control, a 10 kb deletion both Manta and CNVpytor call, CNVpytor +# with genotype ./. as the VCF made from its table writes it. +printf 'chr20\t10700001\tmanta_6\tN\t\t60\tPASS\tSVTYPE=DEL;END=10710001;SVLEN=-10000\tGT\t0/1\n' >> "${CASE_TMP}/in/manta.vcf.in" +printf 'chr20\t10700201\tcnv_3\tN\t\t.\tPASS\tSVTYPE=DEL;END=10710201;SVLEN=-10000;pytorP1=1e-08;pytorQ0=0.01;pytorPN=0\tGT\t./.\n' \ + >> "${CASE_TMP}/in/cnvpytor.vcf.in" +bgz() { # bgz NAME DEST: the VCF NAME, bgzipped and indexed at GENOME_DIR/DEST + docker run --rm -i --user "$(id -u):$(id -g)" -v "${CASE_TMP}/in:/in:ro" -v "${GENOME_DIR}:/genome" "$BCFTOOLS_IMAGE" \ + sh -c "bcftools view -Oz -o /genome/$2 /in/$1 && bcftools index -t /genome/$2" +} +bgz manta.vcf.in "${T}/manta/results/variants/diploidSV.vcf.gz" +bgz delly.vcf.in "${T}/delly/${T}_sv.vcf.gz" +bgz cnvpytor.vcf.in "${T}/cnvpytor/${T}_cnvs.vcf.gz" + +VCF="${T}/sv_merged/${T}_sv_consensus.vcf.gz" +IN="${GENOME_DIR}/${T}/sv_merged/inputs" +consensus() { # POS SUPP SUPP_VEC of every consensus record + bcf query -f '%POS\t%INFO/SUPP\t%INFO/SUPP_VEC\n' "$VCF" 2>/dev/null | tee "${CASE_TMP}/consensus.tsv" +} +at() { # at FROM TO: the consensus records that start in FROM..TO + awk -F'\t' -v a="$1" -v b="$2" '$1 >= a && $1 <= b' "${CASE_TMP}/consensus.tsv" +} +ids() { grep -v '^#' "$1" 2>/dev/null | cut -f3 | tr '\n' ' '; } +votes() { # the checks of one run; $1 names the CNVpytor input + echo "Consensus records (POS SUPP SUPP_VEC):" + consensus + check_eq "$1: one record for the deletion Manta and Delly both call" "$(at 10100999 10101001)" $'10101001\t2\t110' + check_eq "$1: no record for the Manta call beside Delly's 0/0 record" "$(at 10399000 10405000 | grep -c . || true)" 0 + check_eq "$1: no record for the Manta call beside Delly's FT=LowQual call" "$(at 10499000 10504000 | grep -c . || true)" 0 + check_eq "$1: no record for the Manta call beside a CNVpytor e-value of 0.5" "$(at 10599000 10611000 | grep -c . || true)" 0 + check_eq "$1: the ./. CNVpytor call that passes its quality rule still votes" "$(cut -f2,3 <<<"$(at 10699000 10711000)")" $'2\t101' + check_eq "$1: consensus records in all" "$(vcf_count "$VCF")" 2 + check_eq "$1: the Delly records that vote" "$(ids "${IN}/delly.vcf")" "delly_1 delly_2 " + check "$1: the log counts Delly's left-out records" \ + has 'og sv survivor-prep delly: 4 records, 2 vote; no vote: 0 FILTER not PASS, 1 sample FT not PASS, 1 genotyped reference' "$(cat "$STEP_LOG")" + check "$1: the log counts the CNVpytor call below the rule" has 'og sv survivor-prep cnvpytor: .*, 1 below CNVpytor.s quality rule' "$(cat "$STEP_LOG")" +} + +run_step 22-survivor-merge.sh "$T" +check_step_exit 22-survivor-merge.sh +votes "CNVpytor VCF" +check_eq "CNVpytor VCF: the CNVpytor records that vote" "$(ids "${IN}/cnvpytor.vcf")" "cnv_1 cnv_3 " + +# The same three CNVpytor calls from its table (step 18's _cnvs.txt: type, +# region, size, level, e-val1..4, q0, pN, dG) when step 18 left no VCF. +rm -f "${GENOME_DIR}/${T}/cnvpytor/${T}_cnvs.vcf.gz" "${GENOME_DIR}/${T}/cnvpytor/${T}_cnvs.vcf.gz.tbi" +printf 'deletion\tchr20:%s\t10001\t0.5\t%s\t0\t0\t0\t%s\t0\t1000\n' \ + 10300000-10310000 1e-08 0 10600001-10610001 0.5 0 10700201-10710201 1e-08 0.01 \ + > "${GENOME_DIR}/${T}/cnvpytor/${T}_cnvs.txt" +run_step 22-survivor-merge.sh "$T" +check_step_exit 22-survivor-merge.sh +check "the log says the table was turned into a VCF" has 'CNVpytor: converting' "$(cat "$STEP_LOG")" +check "the table's e-value reaches INFO" has 'pytorP1=0.5;pytorQ0=0;pytorPN=0' \ + "$(bcf view -H "${T}/cnvpytor/${T}_cnvs.vcf.gz" 2>/dev/null)" +votes "CNVpytor table" +check_eq "CNVpytor table: the CNVpytor calls that vote" "$(grep -v '^#' "${IN}/cnvpytor.vcf" 2>/dev/null | cut -f2 | tr '\n' ' ')" "10300000 10700201 " + +finish diff --git a/tests/fixtures/sv/README.md b/tests/fixtures/sv/README.md index daec4e0d6..4fa4e8fe7 100644 --- a/tests/fixtures/sv/README.md +++ b/tests/fixtures/sv/README.md @@ -1,8 +1,9 @@ # Synthetic SV calls for step 22 Three caller VCFs on chr20 with a known answer, read by -`tests/e2e/sv-mito-telomere-steps-1-sv-merge.sh` and by the `SURVIVOR_IMAGE` -row of `tests/smoke/commands.tsv`. The records are made up; they describe no +`tests/e2e/sv-mito-telomere-steps-1-sv-merge.sh` and +`tests/e2e/sv-merge-votes.sh`; the `SURVIVOR_IMAGE` row of +`tests/smoke/commands.tsv` uses the first two events. The records are made up; they describe no person. | Event | manta.vcf.in | delly.vcf.in | cnvpytor.vcf.in | Consensus | @@ -10,7 +11,14 @@ person. | one 2 kb deletion, called 2 bp apart across a 1 kb boundary | 10,100,999-10,102,999 | 10,101,001-10,103,001 | | one record, SUPP=2 | | two deletions starting in the same 1 kb window, 600 bp and 5 kb long | 10,200,100-10,200,700 | 10,200,300-10,205,300 | | none: the ends are 4.6 kb apart | | a deletion one caller saw | | | 10,300,000-10,310,000 | none | +| a deletion beside a Delly record genotyped 0/0 | 10,400,001-10,404,001 | 10,400,011-10,404,011, `0/0` | | none: Delly found the reference | +| a deletion beside a Delly call whose sample filter (FT) is LowQual | 10,500,001-10,503,001 | 10,500,011-10,503,011, FT `LowQual` | | none: Delly filtered it | +| a deletion beside a CNVpytor segment with e-value 0.5 | 10,600,001-10,610,001 | | 10,600,001-10,610,001, `./.` | none: CNVpytor's quality rule | The position binning step 22 used before (chromosome, `int(POS/1000)` and SVTYPE) gets the first two wrong: it splits the first event over two windows and counts the second pair as agreement. + +In each of the last three events only the Manta call votes. Step 22 kept every +PASS record before `og sv survivor-prep` (`tests/e2e/sv-merge-votes.sh`), so +each read as two callers and the consensus held 4 records instead of 1. diff --git a/tests/fixtures/sv/cnvpytor.vcf.in b/tests/fixtures/sv/cnvpytor.vcf.in index 366a9a1f4..04d91c5f7 100644 --- a/tests/fixtures/sv/cnvpytor.vcf.in +++ b/tests/fixtures/sv/cnvpytor.vcf.in @@ -5,6 +5,10 @@ ##INFO= ##INFO= ##INFO= +##INFO= +##INFO= +##INFO= ##FORMAT= #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT HG002 -chr20 10300000 cnv_1 N . PASS SVTYPE=DEL;END=10310000;SVLEN=-10000 GT 0/1 +chr20 10300000 cnv_1 N . PASS SVTYPE=DEL;END=10310000;SVLEN=-10000;pytorP1=1e-08;pytorQ0=0;pytorPN=0 GT 0/1 +chr20 10600001 cnv_2 N . PASS SVTYPE=DEL;END=10610001;SVLEN=-10000;pytorP1=0.5;pytorQ0=0;pytorPN=0 GT ./. diff --git a/tests/fixtures/sv/delly.vcf.in b/tests/fixtures/sv/delly.vcf.in index 9838865ca..da11fffef 100644 --- a/tests/fixtures/sv/delly.vcf.in +++ b/tests/fixtures/sv/delly.vcf.in @@ -6,7 +6,10 @@ ##INFO= ##INFO= ##FORMAT= +##FORMAT= ##INFO= #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT HG002 chr20 10101001 delly_1 N 60 PASS SVTYPE=DEL;END=10103001;SVLEN=-2000;CT=3to5 GT 0/1 chr20 10200300 delly_2 N 60 PASS SVTYPE=DEL;END=10205300;SVLEN=-5000;CT=3to5 GT 0/1 +chr20 10400011 delly_3 N 60 PASS SVTYPE=DEL;END=10404011;SVLEN=-4000;CT=3to5 GT:FT 0/0:PASS +chr20 10500011 delly_4 N 60 PASS SVTYPE=DEL;END=10503011;SVLEN=-3000;CT=3to5 GT:FT 0/1:LowQual diff --git a/tests/fixtures/sv/manta.vcf.in b/tests/fixtures/sv/manta.vcf.in index e82edd267..ac02c67db 100644 --- a/tests/fixtures/sv/manta.vcf.in +++ b/tests/fixtures/sv/manta.vcf.in @@ -9,3 +9,6 @@ #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT HG002 chr20 10100999 manta_1 N 60 PASS SVTYPE=DEL;END=10102999;SVLEN=-2000 GT 0/1 chr20 10200100 manta_2 N 60 PASS SVTYPE=DEL;END=10200700;SVLEN=-600 GT 0/1 +chr20 10400001 manta_3 N 60 PASS SVTYPE=DEL;END=10404001;SVLEN=-4000 GT 0/1 +chr20 10500001 manta_4 N 60 PASS SVTYPE=DEL;END=10503001;SVLEN=-3000 GT 0/1 +chr20 10600001 manta_5 N 60 PASS SVTYPE=DEL;END=10610001;SVLEN=-10000 GT 0/1 diff --git a/tests/test_e2e_scope.sh b/tests/test_e2e_scope.sh index 672a77de1..39be6033c 100755 --- a/tests/test_e2e_scope.sh +++ b/tests/test_e2e_scope.sh @@ -153,14 +153,13 @@ printf '%s\n' rust/og/src/main.rs > "${WORK}/changed.txt" if [ -s "${WORK}/rust.txt" ] && cmp -s "${WORK}/og.txt" "${WORK}/rust.txt"; then ok "the real table: a Rust change runs the $(wc -l < "${WORK}/rust.txt" | tr -d ' ') cases a bin/og change runs" else bad "the real table: a Rust change picks $(wc -l < "${WORK}/rust.txt" | tr -d ' ') cases, a bin/og change $(wc -l < "${WORK}/og.txt" | tr -d ' '): $(diff "${WORK}/og.txt" "${WORK}/rust.txt" | grep '^[<>]' | paste -sd' ' -)"; fi -for c in output-contract-1-hg002 traits-panel-1 prs-2-ancestry bash-step-11-roh vendor-vcf-intake-5-docs; do +for c in output-contract-1-hg002 traits-panel-1 prs-2-ancestry bash-step-11-roh vendor-vcf-intake-5-docs sv-mito-telomere-steps-1-sv-merge; do if grep -qx "$c" "${WORK}/rust.txt"; then ok "the real table runs ${c}, which runs og, on a Rust change" else bad "the real table does not run ${c} on a Rust change, though it runs og"; fi done -for c in alignment-markdup-gvcf-sex-4-bwamem2 sv-mito-telomere-steps-1-sv-merge; do - if grep -qx "$c" "${WORK}/rust.txt"; then bad "the real table runs ${c} on a Rust change, though it runs no og" - else ok "the real table leaves out ${c}, which runs no og, on a Rust change"; fi -done +c=alignment-markdup-gvcf-sex-4-bwamem2 +if grep -qx "$c" "${WORK}/rust.txt"; then bad "the real table runs ${c} on a Rust change, though it runs no og" +else ok "the real table leaves out ${c}, which runs no og, on a Rust change"; fi printf '%s\n' tests/fixtures/consumer_export/sites.tsv.gz > "${WORK}/changed.txt" if "$S" select "${WORK}/changed.txt" 2> /dev/null | grep -qx consumer-export-1; then ok "the real table runs consumer-export-1 when its fixture changes" From 85988be3b8e9b0e851d95a0087e0dcdd4ae19b71 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:29:59 +0100 Subject: [PATCH 5/5] fix(sv): step 22 makes the table's CNVpytor VCF again when it lacks the e-value An earlier step 22 wrote cnvpytor/_cnvs.vcf.gz from CNVpytor's table without pytorP1 and pytorQ0, and a rerun reused it, so none of its calls would vote. When the table is there and that VCF's header does not declare pytorP1, the step makes the VCF again. Step 18's own VCF declares it and is kept. The e2e case strips the tags and checks the rerun. Bead: sv-40 --- docs/22-survivor-merge.md | 2 +- scripts/22-survivor-merge.sh | 19 ++++++++++++++++++- tests/e2e/sv-merge-votes.sh | 16 +++++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/22-survivor-merge.md b/docs/22-survivor-merge.md index 2c59dbf95..b1a241d87 100644 --- a/docs/22-survivor-merge.md +++ b/docs/22-survivor-merge.md @@ -56,7 +56,7 @@ The Nextflow pipeline (`survivor_merge` in `--tools`, with at least two of `mant ## What the Script Does Internally -1. Turns CNVpytor's table into a VCF when step 18 left no VCF. Each record carries the table's e-value, `q0` and `pN` under the INFO names CNVpytor's own VCF uses (`pytorP1`, `pytorQ0`, `pytorPN`) +1. Turns CNVpytor's table into a VCF when step 18 left no VCF, or when the VCF there has no `pytorP1` (an earlier version of this step made it from the table without the e-value). Each record carries the table's e-value, `q0` and `pN` under the INFO names CNVpytor's own VCF uses (`pytorP1`, `pytorQ0`, `pytorPN`) 2. Writes the records each caller votes with (below) as plain VCF under `sv_merged/inputs/`, with one sample column named after the caller (`og sv survivor-prep --caller NAME IN OUT`, which prints how many records it left out and why). SURVIVOR names its output columns after the input samples, so three inputs that all name the sample would give one name three times 3. Runs `SURVIVOR merge inputs/sv_files.txt 1000 2 1 1 0 50`: maximum breakpoint distance 1,000 bp, support from 2 callers, same type, same strands, no size-scaled distance, minimum size 50 bp 4. Checks that SURVIVOR wrote a VCF (it exits 0 when it cannot open an input), then sorts, compresses and indexes it diff --git a/scripts/22-survivor-merge.sh b/scripts/22-survivor-merge.sh index 119daf33e..7ff2b584f 100755 --- a/scripts/22-survivor-merge.sh +++ b/scripts/22-survivor-merge.sh @@ -53,7 +53,24 @@ echo "" # VCF uses, so its quality rule reads both inputs the same. CNV_TXT="${S}/cnvpytor/${SAMPLE}_cnvs.txt" CNV_VCF="${S}/cnvpytor/${SAMPLE}_cnvs.vcf.gz" -if [ ! -f "$CNV_VCF" ] && [ -f "$CNV_TXT" ]; then +CNV_MAKE=false +if [ -f "$CNV_TXT" ]; then + if [ ! -f "$CNV_VCF" ]; then + CNV_MAKE=true + else + # A VCF an earlier version of this step made from the table has no + # e-value: none of its calls would vote, so it is made again. Step 18's + # own VCF declares pytorP1 and is kept. + CNV_HEAD=$(run_in "$BCFTOOLS_IMAGE" bcftools view -h "$(cpath "$CNV_VCF")") || CNV_HEAD= + case "$CNV_HEAD" in + *'##INFO=/dev/null | cut -f2 | tr '\n' ' ')" "10300000 10700201 " +# An earlier version of step 22 left the table's VCF without the e-value tags. +# Kept as it is, none of its calls would vote; the step makes it again. +# shellcheck disable=SC2016 # $1 expands inside the container +docker run --rm --network none --user "$(id -u):$(id -g)" -v "${GENOME_DIR}:/genome" "$BCFTOOLS_IMAGE" bash -euo pipefail -c ' + f="/genome/$1/cnvpytor/$1_cnvs.vcf.gz" + bcftools view "$f" | sed -e "/^##INFO=/dev/null)" +run_step 22-survivor-merge.sh "$T" +check_step_exit 22-survivor-merge.sh +check "the log says the old table VCF is made again" has 'has no e-value \(pytorP1\); making it again' "$(cat "$STEP_LOG")" +votes "old table VCF" + finish