From dd9290f8fe342fcd010457258adb70082cecbd67 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:22:06 +0100 Subject: [PATCH 1/5] fix(report): say what a filtered-out MEI call means in the MEI note Step 50 now writes "call filtered out (FLAGS)" for a catalog site whose only calls MEGAnE flagged. The note under the MEI section explained "not callable" and "not seen" only. Both renderers now add one sentence: MEGAnE called an insertion there but flagged it, so it is not a finding, and not proof of absence either. The Python and Rust notes stay the same, byte for byte. Bead: sv-34 --- bin/render_report.py | 4 +++- rust/crates/og-render/src/notes.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/render_report.py b/bin/render_report.py index c53f2324d..a339af688 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -2265,7 +2265,9 @@ def viral_body(d): MEI_NOTE = ("Alu, L1 and SVA copies the genome carries and the reference does not, found and genotyped by " "MEGAnE (Kojima et al. 2023) from short read pairs. Most are common and harmless; the catalog rows are " "insertions known to cause disease. Not callable: too few reads in that window to tell. Not seen is " - "not proof of absence: short reads find an SVA's edges, not its body. Every call: mei/_mei.tsv.") + "not proof of absence: short reads find an SVA's edges, not its body. Call filtered out: MEGAnE called " + "an insertion there but flagged it (LC is low confidence), so it is not a finding, and not proof of " + "absence either. Every call: mei/_mei.tsv.") def mei_lines(d): diff --git a/rust/crates/og-render/src/notes.rs b/rust/crates/og-render/src/notes.rs index e04bc59f7..0440e116f 100644 --- a/rust/crates/og-render/src/notes.rs +++ b/rust/crates/og-render/src/notes.rs @@ -175,7 +175,9 @@ pub const MEI_NOTE: &str = "Alu, L1 and SVA copies the genome carries and the re genotyped by MEGAnE (Kojima et al. 2023) from short read pairs. Most are common and \ harmless; the catalog rows are insertions known to cause disease. Not callable: too few \ reads in that window to tell. Not seen is not proof of absence: short reads find an \ - SVA's edges, not its body. Every call: mei/_mei.tsv."; + SVA's edges, not its body. Call filtered out: MEGAnE called an insertion there but \ + flagged it (LC is low confidence), so it is not a finding, and not proof of absence \ + either. Every call: mei/_mei.tsv."; pub const MCA_NOTE: &str = "Large gains, losses and copy-neutral losses of heterozygosity (CN-LOH) carried by a \ share of the blood cells, from MoChA on the phased heterozygous sites of chr1 to chr22. \ In a UK Biobank study they became more common with age and went with a higher risk of a \ From a012f9950e7966704725da4d19980a6d52a3a3b0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:24:49 +0100 Subject: [PATCH 2/5] test(parity): record that og mei report reads FILTER where its oracle did not Two mei-report cases put a call MEGAnE flagged LC inside the FKTN window: alone (og: call filtered out (LC); bin/mei_report.py: seen), and before a PASS call in the same window (og names the PASS call; the oracle named the first). Each has a golden s.tsv, so the harness fails if og goes back to the oracle's reading, and fails as stale if the golden is changed back to what the oracle writes. The calls table is the same on both sides. Bead: sv-34 --- tests/parity/cases.tsv | 2 + tests/parity/gen/mei-filtered.sh | 43 +++++++++++++++++++ .../golden/mei-report/filtered-pass/s.tsv | 3 ++ tests/parity/golden/mei-report/filtered/s.tsv | 3 ++ 4 files changed, 51 insertions(+) create mode 100755 tests/parity/gen/mei-filtered.sh create mode 100644 tests/parity/golden/mei-report/filtered-pass/s.tsv create mode 100644 tests/parity/golden/mei-report/filtered/s.tsv diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index bd64723bf..614d84329 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -516,6 +516,8 @@ mei-report no-chrom-line mei/nochrom.vcf,mei/sites.tsv,mei/depth.tsv --sample S mei-report not-megane mei/not-megane.vcf,mei/sites.tsv,mei/depth.tsv --sample S --vcf not-megane.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv - bytes PYTHON_IMAGE mei-report bad-depth mei/fktn.vcf,mei/sites.tsv,mei/depth-bad.tsv --sample S --vcf fktn.vcf --sites sites.tsv --depth depth-bad.tsv --out-calls c.tsv --out-sites s.tsv - bytes PYTHON_IMAGE mei-report bad-catalog mei/fktn.vcf,mei/sites-bad.tsv,mei/depth.tsv --sample S --vcf fktn.vcf --sites sites-bad.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv - bytes PYTHON_IMAGE +mei-report filtered gen:mei-filtered.sh,mei/sites.tsv,mei/depth.tsv --sample S --vcf filtered.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE +mei-report filtered-pass gen:mei-filtered.sh,mei/sites.tsv,mei/depth.tsv --sample S --vcf filtered-pass.vcf --sites sites.tsv --depth depth.tsv --out-calls c.tsv --out-sites s.tsv c.tsv,s.tsv bytes PYTHON_IMAGE mei-annotsv-input calls mei/classes.vcf --sample S --vcf classes.vcf --out in.vcf in.vcf bytes PYTHON_IMAGE mei-annotsv-input annotsv-ids mei/fktn.vcf --sample S --vcf fktn.vcf --out in.vcf in.vcf bytes PYTHON_IMAGE mei-annotsv-input empty mei/empty.vcf --sample S --vcf empty.vcf --out in.vcf in.vcf bytes PYTHON_IMAGE diff --git a/tests/parity/gen/mei-filtered.sh b/tests/parity/gen/mei-filtered.sh new file mode 100755 index 000000000..6b023dd25 --- /dev/null +++ b/tests/parity/gen/mei-filtered.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# mei-filtered.sh: writes the MEGAnE VCFs of the mei-report cases that put a +# call MEGAnE flagged inside the FKTN window of tests/parity/inputs/mei/sites.tsv, +# in the current directory. scripts/ci/oracle-diff.sh runs it. +# filtered.vcf an Alu outside the catalog, and the FKTN SVA flagged LC +# filtered-pass.vcf the FKTN SVA flagged LC at 105639400, then a PASS one at +# 105639660 in the same window +# og reads FILTER: a flagged call alone makes the site "call filtered out (LC)", +# and a PASS call in the window wins over a flagged one. The oracle, +# bin/mei_report.py, read any call in the window as seen and named the first, +# so both cases have a golden s.tsv (tests/parity/golden/mei-report/). +set -euo pipefail +export LC_ALL=C + +vcf_head() { + printf '##fileformat=VCFv4.1\n' + printf '##source=MEI search version "v1.0.1 2022/04/12"\n' + printf '##contig=\n' + printf '##ALT=\n' + printf '##FILTER=\n' + printf '##INFO=\n' + printf '##INFO=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' +} + +# sva POS FILTER: an SVA_F call of one copy that MEGAnE identified. +sva() { + printf 'chr9\t%s\t.\tN\t\t.\t%s\tSVTYPE=Retroposon/SVA;MEPRED=PASS;HOMLEN=9;MEI_rpos=%s;MEI=SVA_F,1/1500,+/+;CN_conf=H\tCN\t1\n' \ + "$1" "$2" "$(($1 + 8))" +} + +{ + vcf_head + printf 'chr20\t5040478\t.\tN\t\t.\tPASS\tSVTYPE=SINE/Alu;MEPRED=PASS;HOMLEN=9;MEI_rpos=5040486;MEI=AluYa5,0/288,+/+;CN_conf=H\tCN\t1\n' + sva 105639660 LC +} > filtered.vcf + +{ + vcf_head + sva 105639400 LC + sva 105639660 PASS +} > filtered-pass.vcf diff --git a/tests/parity/golden/mei-report/filtered-pass/s.tsv b/tests/parity/golden/mei-report/filtered-pass/s.tsv new file mode 100644 index 000000000..35801804f --- /dev/null +++ b/tests/parity/golden/mei-report/filtered-pass/s.tsv @@ -0,0 +1,3 @@ +gene chrom start end site element inheritance condition clinvar pmid status mean_depth call +FKTN chr9 105639156 105640156 105639656 SVA autosomal recessive Fukuyama congenital muscular dystrophy 3199 9690476 seen 30.0 chr9:105639660 SVA_F 1 copy PASS +TAF1 chrX 71439993 71440993 71440493 SVA X-linked X-linked dystonia-parkinsonism 9862 29474918 not seen 30.0 . diff --git a/tests/parity/golden/mei-report/filtered/s.tsv b/tests/parity/golden/mei-report/filtered/s.tsv new file mode 100644 index 000000000..dd80509d1 --- /dev/null +++ b/tests/parity/golden/mei-report/filtered/s.tsv @@ -0,0 +1,3 @@ +gene chrom start end site element inheritance condition clinvar pmid status mean_depth call +FKTN chr9 105639156 105640156 105639656 SVA autosomal recessive Fukuyama congenital muscular dystrophy 3199 9690476 call filtered out (LC) 30.0 chr9:105639660 SVA_F 1 copy LC +TAF1 chrX 71439993 71440993 71440493 SVA X-linked X-linked dystonia-parkinsonism 9862 29474918 not seen 30.0 . From 8ed78bd7f71f5b081ef5ac5f66cb6256c36e983c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:24:49 +0100 Subject: [PATCH 3/5] test(parity): compare og report render with the Python of the new MEI note The MEI note changed in both renderers in dd9290f8. The report-render oracle was render_report.py at 65095e97, which still had the old note, so 28 cases would differ from og only by that sentence. The oracle and the modules it imports now come from dd9290f8, where the Python and Rust notes are the same: every report-render case still compares byte for byte. Bead: sv-34 --- tests/parity/gen/report-corpus.sh | 2 +- tests/parity/oracle.ref | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parity/gen/report-corpus.sh b/tests/parity/gen/report-corpus.sh index aa950f941..cc4519b69 100644 --- a/tests/parity/gen/report-corpus.sh +++ b/tests/parity/gen/report-corpus.sh @@ -27,7 +27,7 @@ set -euo pipefail export LC_ALL=C TZ=UTC here=$(cd "$(dirname "$0")" && pwd) root=$(cd "${here}/../../.." && pwd) -ORACLE_SHA=65095e975bea01c74ed92a4f1b8c1bd00d05caaa +ORACLE_SHA=dd9290f8fe342fcd010457258adb70082cecbd67 og=${OG:-${root}/rust/target/release/og} # --- the oracle's modules ------------------------------------------------------------ diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index a53cf25c9..7a7aa3670 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -97,6 +97,6 @@ traits 669a8196cf4c377200f9b4af9c235ef85fb774a1 file:bin/traits_report.py bash . roh-summary 7c1470bc50933f48a1a0dde3c0e0b2b87bc48718 file:scripts/11-roh-analysis.sh sh ./oracle.sh roh summary couple-screen d4d3d4b629fc77a058bbbbd302c75bbfae1a01c2 file:bin/couple_report.py sh -c 'exec sh ./oracle.sh "$0" screen "$@"' couple screen couple-genes d4d3d4b629fc77a058bbbbd302c75bbfae1a01c2 file:bin/couple_report.py sh -c 'exec sh ./oracle.sh "$0" genes "$@"' couple genes -report-render 65095e975bea01c74ed92a4f1b8c1bd00d05caaa file:bin/render_report.py sh ./oracle.sh report render +report-render dd9290f8fe342fcd010457258adb70082cecbd67 file:bin/render_report.py sh ./oracle.sh report render archaic-compare 65095e975bea01c74ed92a4f1b8c1bd00d05caaa file:bin/archaic_compare.py sh -c 'exec python3 "$0" compare "$@"' archaic compare archaic-genomes 65095e975bea01c74ed92a4f1b8c1bd00d05caaa file:bin/archaic_compare.py sh -c 'exec python3 "$0" genomes "$@"' archaic genomes From a66e28a1f56761a9376fa4296c3efac5d552acc1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:26:04 +0100 Subject: [PATCH 4/5] test(report): both reports explain a filtered-out MEI call The MEI note must name every status a catalog row can have, and both reports must show a "call filtered out (LC)" row with the sentence that explains it. Red on the note before dd9290f8: both tests failed. Bead: sv-34 --- .../tests/i411_mei_filtered_status_docs.rs | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 rust/crates/og-render/tests/i411_mei_filtered_status_docs.rs diff --git a/rust/crates/og-render/tests/i411_mei_filtered_status_docs.rs b/rust/crates/og-render/tests/i411_mei_filtered_status_docs.rs new file mode 100644 index 000000000..cb60f972e --- /dev/null +++ b/rust/crates/og-render/tests/i411_mei_filtered_status_docs.rs @@ -0,0 +1,93 @@ +//! The MEI section explains every status step 50 writes (#411 sv-34). +//! +//! `og mei report` reads a catalog site as seen, not seen, not callable, or +//! `call filtered out (FLAGS)` when the only calls MEGAnE made there are ones +//! it flagged. The note under the section explained the first three only, so +//! a reader saw "call filtered out (LC)" with nothing to say what it means. +//! The note now says it: MEGAnE called an insertion there but flagged it, so +//! it is not a finding, and not proof of absence either. +//! +//! `bin/render_report.py` writes the same note; the report-render parity +//! cases (tests/parity/cases.tsv) compare the two byte for byte. + +use og_render::model::Summary; +use og_render::notes::MEI_NOTE; +use og_render::sections::mei; +use og_render::{html, text}; + +/// The sentence that explains the filtered-out status. +const FILTERED: &str = "Call filtered out: MEGAnE called an insertion there but flagged it (LC is low \ + confidence), so it is not a finding, and not proof of absence either."; + +/// A summary whose MEI section ran, with one site of each status. +fn summary() -> String { + r#"{ + "schema_version": 2, "sample": "S1", "generated_utc": "2026-11-20 12:00 UTC", + "run": {"started_utc": "2026-11-20 10:00 UTC", "declared_sex": "male"}, + "databases": {"clinvar_release": "2026-10-01", "cpsr_clinvar_release": null, "hla_database": null}, + "manifest": [], + "sections": { + "coverage": {"title": "Coverage (mosdepth)", "state": "ok", "step": "16b", "source": "qc/S1.mosdepth.summary.txt", + "file_date": "2026-11-20 11:00:00", "note": null, "data": {"mean_depth": 31.25}}, + "mei": {"title": "Mobile element insertions (MEGAnE)", "state": "ok", "step": "50", + "source": "mei/S1_mei_sites.tsv", "file_date": "2026-11-20 11:00:00", "note": null, + "data": {"calls": 2, "by_element": {"Alu": 1, "SVA": 1}, "sites": [ + {"gene": "FKTN", "element": "SVA", "condition": "Fukuyama congenital muscular dystrophy", + "inheritance": "autosomal recessive", "status": "call filtered out (LC)", "mean_depth": "30.0", + "call": "chr9:105639660 SVA_F 1 copy LC"}, + {"gene": "TAF1", "element": "SVA", "condition": "X-linked dystonia-parkinsonism", "inheritance": "X-linked", + "status": "not callable", "mean_depth": "0.0", "call": "."}, + {"gene": "BBS1", "element": "SVA", "condition": "Bardet-Biedl syndrome", "inheritance": "autosomal recessive", + "status": "not seen", "mean_depth": "30.0", "call": "."}]}} + }, + "cyp2d6": null, "not_run": [], "not_assessed": [] +}"# + .to_owned() +} + +/// The note names each status a catalog row can have, and says what a +/// filtered-out call means. +#[test] +fn the_note_explains_every_status() { + assert!(MEI_NOTE.contains(FILTERED), "{MEI_NOTE}"); + for status in [ + "Not callable:", + "Not seen is not proof of absence", + "Call filtered out:", + ] { + assert!( + MEI_NOTE.contains(status), + "{status} is not explained: {MEI_NOTE}" + ); + } +} + +/// Both reports show the filtered-out row and the note that explains it. +#[test] +fn both_reports_explain_a_filtered_out_call() { + let s = Summary::parse(&summary()).expect("a summary"); + let lines = mei::lines(&s.section("mei")).unwrap(); + assert_eq!( + lines[1], + "FKTN SVA (Fukuyama congenital muscular dystrophy; autosomal recessive): call filtered out (LC) \ + (depth 30.0, chr9:105639660 SVA_F 1 copy LC)" + ); + assert_eq!(lines.last().unwrap(), MEI_NOTE); + + let txt = text::report(&s).expect("the text report"); + assert!(txt.contains("call filtered out (LC)"), "{txt}"); + assert!( + txt.contains(FILTERED), + "the text report lacks the sentence:\n{txt}" + ); + + let page = html::report(&s).expect("the HTML report"); + let card = page.split_once("id=\"mei\">").map_or("", |(_, rest)| { + rest.split("").next().unwrap_or("") + }); + assert!(card.contains("call filtered out (LC)"), "{card}"); + assert!( + card.contains(FILTERED), + "the HTML card lacks the sentence:\n{card}" + ); +} From 632a54d544ea2f73f2852bd164c8d2e8255d5512 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:26:37 +0100 Subject: [PATCH 5/5] docs(mei): list the call-filtered-out status wherever the statuses are listed The overview table and the design brief listed seen, not seen and not callable only. docs/50-mei.md now says what the report note explains and names the new parity cases and test. The changelog says what changed in the reports and why not callable keeps its name. Bead: sv-34 --- CHANGELOG.md | 1 + docs/50-mei.md | 4 ++-- docs/design/journal/BRIEF.md | 2 +- docs/pipeline-overview.md | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dda3da6ec..b601e1911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Gene callability (step 16b and the Nextflow `MOSDEPTH` process) no longer counts a fragment twice. The gene pass ran mosdepth with `--fast-mode`, which counts every base from a read's start to its end: where the two mates of a pair overlap, a base counted two reads, and the bases of a deletion in a read counted as read. A gene that fewer than 10 fragments covered could reach 10x and read as read. The gene pass now runs without `--fast-mode`; the genome-wide 500 bp pass keeps it. The file name and columns of `_callability.bed.gz` do not change; its `10X` counts can be lower, so a gene near 90% can move to "not read". Step 16b skips a sample it already ran, so a sample from before this change keeps its old counts: delete `/mosdepth/_callability.bed.gz` and run step 16b again, and only the gene pass runs. On public HG002 30x, 4 of 1,553 genes moved (DHX40, SPATA31C2, GOLGA6L10 and UGT2B17), and the pass takes about 37% more CPU time, 3.5-4 min on a 10-core Mac mini. The step doc and `docs/lessons-learned.md` said `--fast-mode` drops the per-base output; `--no-per-base` does that, and they now say so. A new e2e case, `i411-callability-overlap-1-mates`, runs both paths on a small BAM with overlapping pairs and a deletion. - The output contract check now compares the contract with the last release, so a stable output cannot disappear between releases. Before, `og dev contract --static` compared the code with the contract of the same checkout only: a change that renamed a stable output in the code and in its row, removed a stable param with its row, or made a stable row experimental passed. `og dev contract --baseline FILE` checks every stable row of a release's contract against this one: still listed, still stable, the same format, every leg that wrote the file and every required field. New rows and changes to experimental and internal rows pass. Guard's `rust` job runs it against the contract of the newest `v1.x.y` tag; until 1.0.0 is tagged there is none, and the step says so. The HG002 e2e case now also checks 31 stable files the base run writes whose `e2e` cell was `-` (the clinical summary, the slivar, mitochondrial and pypgx tables, the ClinVar cDNA and deep-intronic tables, the gVCF, the read count, the charts and search index, the PharmCAT JSON and HTML, the coverage beds, the sex check, indexcov, the Stranger VCF of ExpansionHunter, the indexes of the clinical, slivar and mitochondrial VCFs, and the Mutect2 stats of chrM). `run_manifest.tsv`, those three indexes and the Mutect2 stats are now listed on both legs, since the step scripts write them too. No output changes. `.github/CODEOWNERS` now says it routes review requests: main requires its status checks, not an approving review. - Step 50 marks a known disease insertion (the FKTN, TAF1 and BBS1 SVAs of `assets/mei_sites.tsv`) seen only when MEGAnE's call there passed its own filters (FILTER `PASS`). Before, a call MEGAnE had flagged, such as `LC` for low confidence, also made the site read seen, so a call MEGAnE did not trust could read as carrier status. Now a site whose only calls are flagged reads `call filtered out (LC)` (with the call's flags) in the `status` column of `mei/_mei_sites.tsv`, and `call` names the flagged call. A `PASS` call in the same window still makes the site seen. That status value is new; the columns and the other values (seen, not seen, not callable) do not change. A window with zero depth and no call reads not callable, never not seen, as before; a new test now holds that. The calls table `mei/_mei.tsv` does not change: it lists flagged calls with their `filter`, as before. Tests: `rust/crates/og-reports/tests/i411_mei_seen_pass.rs`. Part of #411. +- The MEI section of both reports now says what `call filtered out (LC)` means. Step 50 writes that status when MEGAnE's only calls at a catalog site are ones it flagged, but the note under the section explained not callable and not seen only. The note now adds one sentence: MEGAnE called an insertion there but flagged it, so it is not a finding, and not proof of absence either. The text and HTML reports change by that sentence; no column or file changes. `docs/pipeline-overview.md` and `docs/50-mei.md` list the status too. Two parity cases (`mei-report/filtered`, `mei-report/filtered-pass`) record with a golden that `og mei report` reads FILTER where the Python it replaced did not, and the report-render parity oracle moves to the commit with the new note, so the Rust and Python reports still compare byte for byte. The status `not callable` keeps its name: renaming it to `not assessed` would change an output value of step 50 for no gain in meaning. Tests: `rust/crates/og-render/tests/i411_mei_filtered_status_docs.rs`. Part of #411. - Step 53 (Locityper) no longer shows a pass at CYP2D6 and FCGR. The authors set their quality filter (no warning, a weighted distance of 30 or less, few unexplained reads) on HLA and KIR genes only, but `og locityper report` applied it as pass or fail at every locus, so a CYP2D6 or FCGR call could read as a green pass that nobody had measured. The `filter` column now reads `pass` or `fail` only at a locus whose name starts with `HLA-` or `KIR`; at any other locus it reads `uncalibrated`, and `reason` says what the rule alone gives there (`... by its rule alone it passes`, or `... it fails:` with the reasons). Both reports show `uncalibrated` in grey and count only `pass` rows as passing. The step's last line adds the count of uncalibrated loci. `docs/53-locityper.md` also says that the cut of 30 comes from Locityper 0.17.3, whose weighted distance was a minimizer Jaccard distance, while the 1.7.14 the step runs gives an edit distance. No column or file name changes; the `filter` column has one new value. A cargo test, `rust/crates/og-reports/tests/i411_locityper_uncalibrated.rs`, checks all five shipped loci; 12 parity goldens of `locityper-report` record the new rows. The docs demo sample (`og dev demo-sample`) now writes its step 53 table with `og locityper report` too, so the demo report shows CYP2D6 and FCGR as `uncalibrated` like a real run; `rust/og/tests/port_demo_sample.rs` checks it. - Step 45's Y-STR counts are on FamilyTreeDNA's scale at the 24 markers with a published counting rule. The marker set's regions did not always cover the repeat blocks a lab counts, so DYS19 read 16 where FamilyTreeDNA reads 15 on the same DNA, DYS449 read 36 for 29 and Y-GATA-H4 31 for 11. `assets/ystr_panel.tsv` now carries each marker's offset, from the GRCh38 sequence counted by NIST STRBase's repeat structure, and `og y str-report` adds it. A called marker with no published rule says in its note that its count is on the reference's scale. - `high` confidence now also needs at least 70% of the reads that span the repeat to show the called length (HipSTR's `MALLREADS`). Before, it meant read quality only: a call could be high while most reads showed another length. A record without `MALLREADS` is not high. diff --git a/docs/50-mei.md b/docs/50-mei.md index 1553c314d..489361111 100644 --- a/docs/50-mei.md +++ b/docs/50-mei.md @@ -74,7 +74,7 @@ With `run-all.sh`: `TOOLS=...,mei`; a run lists step 50 only when `TOOLS` names | `mei/${SAMPLE}_mei_depth.tsv` | `samtools bedcov -c` over the catalog windows | | `mei/${SAMPLE}_megane.log` | MEGAnE's log | -Both reports show the count per element and the catalog in their "Mobile element insertions" section; the AnnotSV columns are in the calls table only. +Both reports show the count per element and the catalog in their "Mobile element insertions" section; the AnnotSV columns are in the calls table only. The note under the section says what each status means: not callable is too few reads to tell, not seen is not proof of absence, and call filtered out is an insertion MEGAnE called but flagged (`LC` is low confidence), so it is not a finding and not proof of absence either. AnnotSV classifies deletions and duplications only. In AnnotSV 3.5 the ranking of an insertion is empty (`SVrankingINS` in its `AnnotSV-ranking.tcl` scores nothing), so `acmg_class` is `NA` for every insertion. Read an insertion by its gene and location instead: one inside an exon can break the gene; most in introns or between genes do nothing, though some do (the TAF1 SVA above is intronic). A `benign_af` above a few percent means insertions at that place are common. Without step 50, the report lists mobile element insertions under what the run did not assess. @@ -84,7 +84,7 @@ On HG002's chr20 at 30x (13 million mapped reads) on a 4-CPU GitHub runner, MEGA ## Validation -The e2e case `tests/e2e/mei-1.sh` runs the step on HG002's chr20: the reads of chr20 in GIAB's 60x novoalign BAM, half the pairs (30x), realigned with minimap2 to chr20 alone, the unmapped reads dropped (so the case fits a runner's time), and the k-mer set of that chromosome. Its Alu calls are scored against the GIAB v5.0q insertions of HG002 on chr20 (RepeatMasker SINE/Alu, interspersed, 250 to 400 bp, inside the benchmark regions): recall must be at least 0.70 and precision at least 0.80, within 100 bp. On 2026-10-27 it found 12 of the 15 insertions (0.80), and 13 of its 15 Alu calls inside the benchmark regions matched a GIAB insertion (0.87); with the unmapped reads kept, the same 12 of 15 and 13 of 15. With every read pair within 1 kb of a GIAB Alu insertion removed, it found 0 of 15 and the recall check failed. The same case then runs AnnotSV on the calls with made-up AnnotSV data (the bundle of `tests/fixtures/annotsv/class45.sh` plus one made-up gene over all of chr20) and checks that every call comes back inside that gene, with its location there and class `NA`; the same VCF read without AnnotSV's table fails that check. It also runs the Nextflow pipeline with `--tools mei` and `--annotsv_annotations` on the same BAM and checks that its calls table is the bash step's, byte for byte. `rust/crates/og-reports/tests/optin_mei.rs` plants an SVA in the FKTN window of a synthetic MEGAnE VCF and checks it is seen, and not seen without it, and joins a table in AnnotSV's columns back to the calls. `rust/crates/og-reports/tests/i411_mei_seen_pass.rs` checks that an SVA there that MEGAnE flagged `LC` reads `call filtered out (LC)`, not seen, that a `PASS` call in the same window wins over a flagged one, and that a window with zero depth and no call reads not callable, never not seen. +The e2e case `tests/e2e/mei-1.sh` runs the step on HG002's chr20: the reads of chr20 in GIAB's 60x novoalign BAM, half the pairs (30x), realigned with minimap2 to chr20 alone, the unmapped reads dropped (so the case fits a runner's time), and the k-mer set of that chromosome. Its Alu calls are scored against the GIAB v5.0q insertions of HG002 on chr20 (RepeatMasker SINE/Alu, interspersed, 250 to 400 bp, inside the benchmark regions): recall must be at least 0.70 and precision at least 0.80, within 100 bp. On 2026-10-27 it found 12 of the 15 insertions (0.80), and 13 of its 15 Alu calls inside the benchmark regions matched a GIAB insertion (0.87); with the unmapped reads kept, the same 12 of 15 and 13 of 15. With every read pair within 1 kb of a GIAB Alu insertion removed, it found 0 of 15 and the recall check failed. The same case then runs AnnotSV on the calls with made-up AnnotSV data (the bundle of `tests/fixtures/annotsv/class45.sh` plus one made-up gene over all of chr20) and checks that every call comes back inside that gene, with its location there and class `NA`; the same VCF read without AnnotSV's table fails that check. It also runs the Nextflow pipeline with `--tools mei` and `--annotsv_annotations` on the same BAM and checks that its calls table is the bash step's, byte for byte. `rust/crates/og-reports/tests/optin_mei.rs` plants an SVA in the FKTN window of a synthetic MEGAnE VCF and checks it is seen, and not seen without it, and joins a table in AnnotSV's columns back to the calls. `rust/crates/og-reports/tests/i411_mei_seen_pass.rs` checks that an SVA there that MEGAnE flagged `LC` reads `call filtered out (LC)`, not seen, that a `PASS` call in the same window wins over a flagged one, and that a window with zero depth and no call reads not callable, never not seen. The parity cases `mei-report/filtered` and `mei-report/filtered-pass` (`tests/parity/cases.tsv`) record the same two readings against the Python that `og mei report` replaced, which read any call in the window as seen: each has a golden sites table under `tests/parity/golden/mei-report/`. `rust/crates/og-render/tests/i411_mei_filtered_status_docs.rs` checks that both reports show a `call filtered out (LC)` row with the note that explains it. ## Notes diff --git a/docs/design/journal/BRIEF.md b/docs/design/journal/BRIEF.md index 53b543299..d68c2abd4 100644 --- a/docs/design/journal/BRIEF.md +++ b/docs/design/journal/BRIEF.md @@ -54,7 +54,7 @@ The site has to show the result families below. The report already groups them i | Variant prioritisation (31 slivar) | 4 groups; compound-het pairs with phase (trans/cis/unknown) | expert; most pairs are noise | info | phase is mostly unknown | | Symptom ranking (48 Exomiser, opt-in) | top 10: rank, gene, score, mode of inheritance, variants, ACMG class; the HPO terms used | a ranking to take to a geneticist | review | not a diagnosis; an empty list rules nothing out | | Structural variants (04/15/18/19/22/05) | type (DEL, DUP, INV, BND, INS), position, length, genotype, supporting callers, duphold DHFFC/DHBFC, AnnotSV class 1-5, genes, CDS overlap %, gnomAD-SV AF; counts per caller | 5-10k SVs is normal. **Only class 4-5 in a disease gene matters.** Single-caller BND calls are mostly artefacts | act (class 4-5) / info | short reads miss insertions and inversions | -| Mobile elements (50, opt-in) | 3 catalogued sites (FKTN, TAF1, BBS1): seen / not seen / not callable; about 1,200 Alu/L1/SVA calls | "not seen" is not proof of absence | carrier / review | none | +| Mobile elements (50, opt-in) | 3 catalogued sites (FKTN, TAF1, BBS1): seen / call filtered out / not seen / not callable; about 1,200 Alu/L1/SVA calls | "not seen" is not proof of absence; "call filtered out" is a call MEGAnE flagged, not a finding | carrier / review | none | | Repeat expansions (09/09b) | locus, gene, two allele counts with 95% CI, Stranger status (normal, pre_mutation, full_mutation), normal max, pathogenic min, sized or flanking-only | HTT, FMR1, C9orf72, DMPK and ATXN1 first. **Intermediate = no disease, may expand in children. Full mutation = see a clinical geneticist.** An unsized locus is never shown as normal | act (full) / review (pre or intermediate) / info | short-read sizing fails on long expansions; C9orf72 thresholds vary between labs | | MUC1 VNTR (52, opt-in) | positive / possible / negative / not assessed, sensitivity at this depth | ADTKD-MUC1, a kidney disease | act (positive) / review (possible) | needs confirmation (SNaPshot) | | Mito variants (20) | site, allele fraction (heteroplasmy), ClinVar class, gnomAD AF; poly-C length variants listed apart | above 95% = haplogroup marker; 10-95% = look it up; m.3243A>G or m.8344A>G = specialist | act / review / info | blood under-represents heteroplasmy in other tissues | diff --git a/docs/pipeline-overview.md b/docs/pipeline-overview.md index 8f28913c3..9f4b2b2b3 100644 --- a/docs/pipeline-overview.md +++ b/docs/pipeline-overview.md @@ -24,7 +24,7 @@ | **Traits** | Lactase persistence, caffeine, alcohol flush, bitter taste, earwax, pigmentation, APOE and other replicated variants, each row citing its primary paper | 42 | | **Archaic Ancestry** | The Neanderthal and Denisovan shares and the segments carried, with IBDmix against the Altai and Vindija Neanderthals and the Altai Denisovan (opt-in) | 43 | | **Viral DNA** | Inherited integrated HHV-6 and the DNA of 23 human viruses (herpesviruses, HPV, HBV, HIV-1, polyomaviruses and others) in the reads that do not map to the human genome, with phiX as the lab control (opt-in) | 47 | -| **Mobile Elements** | Alu, L1 and SVA insertions the reference lacks, each genotyped as one or two copies, and three SVA insertions known to cause disease (FKTN, TAF1, BBS1) read as seen, not seen or not callable (opt-in; needs the BAM) | 50 | +| **Mobile Elements** | Alu, L1 and SVA insertions the reference lacks, each genotyped as one or two copies, and three SVA insertions known to cause disease (FKTN, TAF1, BBS1) read as seen, call filtered out (a call MEGAnE flagged), not seen or not callable (opt-in; needs the BAM) | 50 | | **Mosaic Alterations** | Large gains, losses and copy-neutral loss of heterozygosity on chr1 to chr22 in a share of the blood cells, with the cell fraction (opt-in; mosaic loss of Y and X are step 33's) | 51 | ## Pipeline Overview