From 46333726dc16f1d512b66b1f57e6160bcde6594f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:21:14 +0100 Subject: [PATCH 1/5] feat(run): EVERYTHING=true runs every optional step, TIDDIT=true runs script 04a EVERYTHING=true reads assets/everything.tsv: every tool row goes into TOOLS (html_report aside: run-all.sh writes the report itself), every switch row is set to its value, and TOOLS or a switch set to another value stops the run with exit 2. A step skipped because its data is not installed, or because a panel is set to none, stops the run with exit 2 before Nextflow and names the setup command. Skips that come from the sample stay skips: no BAM, no gVCF, a female sample for steps 37 and 45, a BAM that is not PacBio HiFi for 35b. Step 37 on a female sample is now listed as skipped, as step 45 was; the pipeline skipped it already. Step 35b runs og paraphase check-bam on the BAM first, so a BAM that is not PL:PACBIO, or a FASTQ row, is a skip instead of a failed step. TIDDIT=true runs 04a-tiddit.sh after the pipeline, so every optional step is reachable from one command. EVERYTHING and TIDDIT join SWITCHES, the replay list of og recheck, the output contract, and step 04a joins assets/steps.tsv and RUN_ALL_STEPS, which the tests keep equal to what run-all.sh records. Bead: D8 --- assets/output_contract.tsv | 2 + assets/steps.tsv | 1 + bin/collect_summary.py | 2 +- rust/crates/og-recheck/src/rerun.rs | 2 + rust/crates/og-summary/src/collect/model.rs | 8 +-- scripts/run-all.sh | 70 ++++++++++++++++++--- tests/test_run_state.py | 2 +- 7 files changed, 72 insertions(+), 15 deletions(-) diff --git a/assets/output_contract.tsv b/assets/output_contract.tsv index 736705877..4d812d50d 100644 --- a/assets/output_contract.tsv +++ b/assets/output_contract.tsv @@ -425,6 +425,7 @@ switch VIRAL_REF - - - - experimental - - switch MEGANE_KMERS - - - - experimental - - switch MOCHA_DATA - - - - experimental - - switch GRIDSS - - - - stable - - +switch TIDDIT - - - - stable - - switch IMPUTATION - - - - stable - - switch SOMATIC - - - - stable - - switch EXTRA_CALLERS - - - - stable - - @@ -433,6 +434,7 @@ switch FAMILIES - - - - experimental - - switch OWNGENOME_LIVE_URL - - - - experimental - - switch LOCITYPER_DATA - - - - experimental - - switch LOCITYPER_BG_REGION - - - - experimental - - +switch EVERYTHING - - - - stable - - switch NXF_VER - - - - stable - - switch NXF_WORK - - - - stable - - switch NXF_HOME - - - - stable - - diff --git a/assets/steps.tsv b/assets/steps.tsv index 5f8bb5df1..aefdc50b1 100644 --- a/assets/steps.tsv +++ b/assets/steps.tsv @@ -22,6 +22,7 @@ step label tool recorded processes sections upstream hours hours_from hours_low 03c Strelka2 yes 02 03d Octopus yes 02 4 estimate 04 Manta manta yes MANTA manta 02 0.68 measured 0.3666 +04a TIDDIT yes 02 04b GRIDSS yes 02 8 estimate 05 AnnotSV annotsv yes ANNOTSV annotsv 15,19,18 0.03 measured 0.0166 06 ClinVar screen clinvar yes CLINVAR_SCREEN,CLINVAR_CDNA_CHECK clinvar,carrier 03 0.02 measured 0.0083 clinvar_file_date diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 570dc9f01..be50233c5 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -141,7 +141,7 @@ def collect(sample, sample_dir, declared_sex=None, out=None): # equal; the set stays written here because this file also runs where only # bin/ is mounted). A section whose step is not here (variant calling, step # 03, which run-all reuses on purpose) is never marked stale. -RUN_ALL_STEPS = {"03a", "03b", "03c", "03d", "04", "04b", "05", "06", "07", "08", "09", "09b", "10", "11", +RUN_ALL_STEPS = {"03a", "03b", "03c", "03d", "04", "04a", "04b", "05", "06", "07", "08", "09", "09b", "10", "11", "12", "13", "14", "15", "16", "16b", "17", "18", "19", "20", "21", "22", "23", "25", "26", "26b", "27", "28", "29", "30", "31", "32", "33", "35", "35b", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53"} diff --git a/rust/crates/og-recheck/src/rerun.rs b/rust/crates/og-recheck/src/rerun.rs index 130c9a47c..3570c642d 100644 --- a/rust/crates/og-recheck/src/rerun.rs +++ b/rust/crates/og-recheck/src/rerun.rs @@ -152,6 +152,7 @@ pub const SWITCHES: &[&str] = &[ "MEGANE_KMERS", "MOCHA_DATA", "GRIDSS", + "TIDDIT", "IMPUTATION", "SOMATIC", "EXTRA_CALLERS", @@ -160,6 +161,7 @@ pub const SWITCHES: &[&str] = &[ "OWNGENOME_LIVE_URL", "LOCITYPER_DATA", "LOCITYPER_BG_REGION", + "EVERYTHING", "NXF_VER", "NXF_WORK", "NXF_HOME", diff --git a/rust/crates/og-summary/src/collect/model.rs b/rust/crates/og-summary/src/collect/model.rs index 2d6e7c994..cfeef97c9 100644 --- a/rust/crates/og-summary/src/collect/model.rs +++ b/rust/crates/og-summary/src/collect/model.rs @@ -18,10 +18,10 @@ pub const SCHEMA_VERSION: i128 = 2; /// section of another step (variant calling, step 03, which a run reuses on /// purpose) is never stale. pub const RUN_ALL_STEPS: &[&str] = &[ - "03a", "03b", "03c", "03d", "04", "04b", "05", "06", "07", "08", "09", "09b", "10", "11", "12", - "13", "14", "15", "16", "16b", "17", "18", "19", "20", "21", "22", "23", "25", "26", "26b", - "27", "28", "29", "30", "31", "32", "33", "35", "35b", "37", "38", "39", "40", "41", "42", - "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", + "03a", "03b", "03c", "03d", "04", "04a", "04b", "05", "06", "07", "08", "09", "09b", "10", + "11", "12", "13", "14", "15", "16", "16b", "17", "18", "19", "20", "21", "22", "23", "25", + "26", "26b", "27", "28", "29", "30", "31", "32", "33", "35", "35b", "37", "38", "39", "40", + "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", ]; /// What a section's answer is worth. diff --git a/scripts/run-all.sh b/scripts/run-all.sh index 35b2555d9..caa04ba62 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -23,13 +23,18 @@ # the ACMG SF genes; the report's own secondary-findings card from step 23 stays); # HPO=HP:0001250,HP:0001263 (the symptoms as HPO ids: step 48 ranks the variants that could explain them with Exomiser, # after setup.sh --exomiser-data); +# EVERYTHING=true (every optional step of assets/everything.tsv: every --tools name, KIR, HLA_QC, GRIDSS, TIDDIT, +# IMPUTATION, SOMATIC, BENCHMARK and every EXTRA_CALLERS; TOOLS or a switch set to another value stops it with exit 2. +# A step that would be skipped because its data is not installed, or because ANCESTRY_PANEL, ANCESTRY_FINE_DIR or +# ARCHAIC_DIR is none, stops it with exit 2 before Nextflow, naming the setup command; a skip that comes from the +# input stays a skip: no BAM, no gVCF, a female sample for the Y steps, a BAM that is not PacBio HiFi for Paraphase); # REF_FASTA, EH_CATALOG (a file, or bundled), MANTA_CALL_REGIONS, PARASCOPY_POPULATION, PARAPHASE_REGIONS, ANCESTRY_PANEL (--ancestry_ref; none: raw # scores), ANCESTRY_FINE_DIR (--ancestry_fine_panel, step 26b, # always 100 bootstrap replicates; none: skip), ARCHAIC_DIR and ARCHAIC_POP (--archaic_data and --archaic_pop, # step 43; ARCHAIC_DIR=none: skip), PGSC_CALC_DIR (--pgsc_calc), LOCITYPER_DATA (--locityper_data, step 53's # database) and LOCITYPER_BG_REGION (--locityper_bg_region) as for the single steps; the panels, the databases and # pgsc_calc are passed when installed. -# Run as scripts after the pipeline: GRIDSS=true (04b), IMPUTATION=true (14), +# Run as scripts after the pipeline: TIDDIT=true (04a), GRIDSS=true (04b), IMPUTATION=true (14), # SOMATIC=true (29), EXTRA_CALLERS=gatk,freebayes,strelka2,octopus (03a-03d), BENCHMARK=true (needs # EXTRA_CALLERS or a second caller VCF); then the HTML report (24) and the text report, also after a failed pipeline. # FAMILIES=families.tsv (who is related to whom, assets/families.example.tsv): when the sample is in a family, the sex @@ -56,8 +61,8 @@ shift 2; SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) USER_THREADS=${THREADS:-} # shape back word by word and starts run-all.sh with those arguments, never through a shell; it refuses any other. SWITCHES=(GENOME_DIR SKIP_VALIDATION THREADS SKIP_TRIM INTERVALS ALIGN_DIR TOOLS KIR HLA_QC HPO CPSR_SECONDARY_FINDINGS REF_FASTA EH_CATALOG MANTA_CALL_REGIONS PARASCOPY_POPULATION PARAPHASE_REGIONS ANCESTRY_PANEL ANCESTRY_FINE_DIR ARCHAIC_DIR ARCHAIC_POP - PGSC_CALC_DIR ANCIENT_PANEL_DIR VIRAL_REF MEGANE_KMERS MOCHA_DATA GRIDSS IMPUTATION SOMATIC EXTRA_CALLERS BENCHMARK - FAMILIES OWNGENOME_LIVE_URL LOCITYPER_DATA LOCITYPER_BG_REGION NXF_VER NXF_WORK NXF_HOME NXF_OFFLINE) + PGSC_CALC_DIR ANCIENT_PANEL_DIR VIRAL_REF MEGANE_KMERS MOCHA_DATA GRIDSS TIDDIT IMPUTATION SOMATIC EXTRA_CALLERS BENCHMARK + FAMILIES OWNGENOME_LIVE_URL LOCITYPER_DATA LOCITYPER_BG_REGION EVERYTHING NXF_VER NXF_WORK NXF_HOME NXF_OFFLINE) RERUN_CMD="cd $(printf '%q' "$PWD") && "; for v in "${SWITCHES[@]}"; do [ -z "${!v+x}" ] || RERUN_CMD+="${v}=$(printf '%q' "${!v}") "; done RERUN_CMD+=$(printf '%q ' "${SCRIPT_DIR}/run-all.sh" "$SAMPLE" "$SEX" "$@"); RERUN_CMD=${RERUN_CMD% } # With -bg nextflow returns at once, and the steps would be recorded and the reports written before the run ends. @@ -66,6 +71,25 @@ export GENOME_DIR=${GENOME_DIR:?Set GENOME_DIR to your data directory} # shellcheck source=lib/common.sh . "${SCRIPT_DIR}/lib/common.sh" validate_sample "$SAMPLE" +# EVERYTHING=true: every tool row of assets/everything.tsv goes into TOOLS (html_report aside: this script writes +# the report itself, step 24) and every switch row is set to its value. EVHOW keeps each row's install command. +EV="" EVMISS=(); declare -A EVHOW=() +if [[ "${EVERYTHING:-}" =~ ^(true|1)$ ]]; then + EV=1 EVT="" EVF="${PGP_ROOT}/assets/everything.tsv" + [ -s "$EVF" ] || { echo "ERROR: EVERYTHING=true reads ${EVF}, which is missing" >&2; exit 2; } + [ -z "${TOOLS:-}" ] || { echo "ERROR: EVERYTHING=true names every step itself: unset TOOLS (TOOLS=${TOOLS})" >&2; exit 2; } + while IFS=$'\t' read -r kind name value _; do + case "$kind" in + tool) [ "$name" = html_report ] || EVT+="${EVT:+,}${name}"; EVHOW[$name]=$value ;; + input) EVHOW[$name]=$value ;; + switch) if [ -n "${!name:-}" ] && [ "${!name}" != "$value" ] && ! { [ "$value" = true ] && [ "${!name}" = 1 ]; }; then + echo "ERROR: EVERYTHING=true sets ${name}=${value}: unset ${name} (it is ${!name})" >&2; exit 2; fi + printf -v "$name" '%s' "$value" ;; + esac + done < <(grep -v '^#' "$EVF" | tail -n +2) + TOOLS=$EVT + echo "[Everything] EVERYTHING=true: every optional step (${EVF}); one that lacks its data stops the run before Nextflow" +fi G=$GENOME_DIR S="${GENOME_DIR}/${SAMPLE}" LOG_DIR="${GENOME_DIR}/${SAMPLE}/logs" T=${TOOLS:-} C=${EXTRA_CALLERS:-} on() { [[ "${!1:-}" =~ ^(true|1)$ ]]; } SF=${CPSR_SECONDARY_FINDINGS:-true} # as scripts/17-cpsr.sh reads it: unset or empty is true, else true or false only @@ -130,7 +154,18 @@ plan() { # plan "STEP Label" TOOL [REASON]: 0 when TOOL runs [[ -n "$NOBAM" || " 16 16b 10 20 21 04 19 15 22 08 09 09b 18 05 28 32 33 35 35b 37 45 49 " != *" ${1%% *} "* ]] || r="no BAM" [ -z "$t" ] || [[ ",${t// /}," == *",$2,"* ]] || r=${r:-not in TOOLS} if [ -z "$r" ]; then SEL+=("$2") RUNS+=("${1%% *}") TOOL_OF[${1%% *}]=$2; printf ' %-28s runs\n' "$1"; return 0; fi - printf ' %-28s skipped (%s)\n' "$1" "$r"; printf 'step\t%s\tskipped (%s)\n' "${1%% *}" "$r" >> "$STATUS"; return 1 + printf ' %-28s skipped (%s)\n' "$1" "$r"; printf 'step\t%s\tskipped (%s)\n' "${1%% *}" "$r" >> "$STATUS" + evmiss "$1" "$2" "$r"; return 1 +} +# evmiss "STEP Label" TOOL REASON: with EVERYTHING=true, a skip for data that is not installed, or for X=none, stops +# the run before Nextflow (the list prints after the plan). A skip that comes from the input is not one of them. +evmiss() { + local h=${EVHOW[$2]:-setup.sh}; [ "$h" != - ] || h=setup.sh + [ -n "$EV" ] || return 0 + case "$3" in + *=none) EVMISS+=("$1: $3: unset ${3%%=*}") ;; + "data not installed"*|"not installed"*|"opt-in"*) EVMISS+=("$1: $3; install: $h") ;; + esac } arg() { [ ! -e "$2" ] || NF+=("$1" "$2"); } # arg --param FILE: pass FILE when it exists CV="${G}/clinvar/clinvar_pathogenic_chr.vcf.gz" A="${G}/annotations" CAT=${EH_CATALOG:-} @@ -186,8 +221,9 @@ for p in "07 PharmCAT:pharmcat" "27 CPIC lookup:cpic" "11 ROH:roh" "12 Mito hapl plan "06 ClinVar screen" clinvar "$(need "$CV" "${CV}.tbi")" && NF+=(--clinvar "$CV" --clinvar_index "${CV}.tbi") H=$(data_file hla_dat || true) GT=$(data_file gencode_genes || true) plan "08 HLA typing (T1K)" hla_typing "$(need "$H" "$GT")" && NF+=(--hla_dat "$H" --hla_genes "$GT") \ - && if on KIR; then K="${G}/kir/IPD-KIR_${KIR_DB_RELEASE}/kir.dat"; [ -s "$K" ] && NF+=(--kir true --kir_dat "$K") \ - || echo " KIR typing skipped: data not installed: ${K#"$G"/} (setup.sh --kir-data)"; fi + && if on KIR; then K="${G}/kir/IPD-KIR_${KIR_DB_RELEASE}/kir.dat"; if [ -s "$K" ]; then NF+=(--kir true --kir_dat "$K") + else echo " KIR typing skipped: data not installed: ${K#"$G"/} (setup.sh --kir-data)" + [ -z "$EV" ] || EVMISS+=("08 KIR typing (KIR=true): data not installed: ${K#"$G"/}; install: setup.sh --kir-data"); fi; fi [[ " ${SEL[*]} " != *" hla_typing "* ]] || ! on HLA_QC || { NF+=(--hla_qc true); echo " (HLA_QC=true: OptiType checks T1K's HLA-A and HLA-B)"; } plan "09 ExpansionHunter" expansion_hunter "$(need "$CAT")" && NF+=(--expansion_catalog "$CAT") plan "09b Stranger" stranger "$(need "$CAT")" || true @@ -216,7 +252,18 @@ plan "21 Cyrius CYP2D6" cyrius "$([ -n "$CYOK" ] || if [ -e "${CY}/INSTALLED" ] && NF+=(--cyrius_install "$CY") && { [ -n "$T" ] || echo " (planned because setup.sh --cyrius installed it; TOOLS without cyrius leaves it out)"; } plan "35 Parascopy SMN1/SMN2" parascopy "$(optin parascopy || need "${PS}/homology_table/GRCh38.bed.gz")" && NF+=(--parascopy_data "$PS") \ && NF+=(--parascopy_population "${PARASCOPY_POPULATION:-EUR}") -plan "35b Paraphase (PacBio HiFi)" paraphase "$(optin paraphase)" && { [ -z "${PARAPHASE_REGIONS:-}" ] || NF+=(--paraphase_regions "$PARAPHASE_REGIONS"); } +# Step 35b reads PacBio HiFi reads only. og paraphase check-bam, which PARAPHASE_CHECK runs too, says whether every +# read group of the BAM is PL:PACBIO (exit 3: not). Another BAM is a skip, not a failed step; so is a FASTQ row, which +# the pipeline aligns as short reads. A check that cannot run leaves the step to PARAPHASE_CHECK. +hifi() { + [ -n "$NOBAM" ] || return 0 + [ -n "$fb" ] || { echo 'not PacBio HiFi: a FASTQ row is aligned as short reads'; return 0; } + local rc=0 + run_in --cpus 1 --memory 1g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" /pgp-bin/og paraphase check-bam "/genome/${fb#"$G"/}" \ + > /dev/null 2>&1 || rc=$? + [ "$rc" -ne 3 ] || echo 'not PacBio HiFi: a read group of the BAM is not PL:PACBIO' +} +plan "35b Paraphase (PacBio HiFi)" paraphase "$(optin paraphase || hifi)" && { [ -z "${PARAPHASE_REGIONS:-}" ] || NF+=(--paraphase_regions "$PARAPHASE_REGIONS"); } plan "25 PRS" prs "$(need "$(compgen -G "${G}/prs_scores/*.txt.gz" | head -n 1 || echo "${G}/prs_scores/.txt.gz")")" && NF+=(--pgs_scoring "${G}/prs_scores") \ && { PC=${PGSC_CALC_DIR:-${G}/tools/pgsc_calc-${PGSC_CALC_VERSION}}; [ ! -f "${PC}/main.nf" ] || NF+=(--pgsc_calc "$PC"); } # Step 26 is pgsc_calc's projection inside the PRS run, with its panel and the site list setup.sh writes beside it. @@ -228,7 +275,8 @@ AF=${ANCESTRY_FINE_DIR:-${G}/reference/ancestry_fine/${ANCESTRY_FINE_PANEL}} plan "26b Fine-grained ancestry" ancestry_fine "$(if [ "$AF" = none ]; then echo 'ANCESTRY_FINE_DIR=none'; else need "${AF}/model/DONE"; fi)" \ && NF+=(--ancestry_fine_panel "$AF") YD="${G}/reference/yleaf-${YLEAF_DATA_VERSION}/data" -plan "37 Y haplogroup (Yleaf)" y_haplogroup "$(optin y_haplogroup || need "${YD}/hg38/new_positions.txt")" && NF+=(--yleaf_data "$YD") +plan "37 Y haplogroup (Yleaf)" y_haplogroup "$(optin y_haplogroup || if [ "$SEX" = male ]; then need "${YD}/hg38/new_positions.txt" + else echo 'female sample: no Y chromosome'; fi)" && NF+=(--yleaf_data "$YD") # Step 40 reads the gVCF: a row that starts from a VCF without one beside it has none. CS="${G}/reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" plan "40 Raw-data export" consumer_export "$(optin consumer_export || if [ -n "$fv" ] && [ -z "$GROW" ]; then echo 'no gVCF beside the VCF'; else need "$CS"; fi)" \ @@ -292,6 +340,10 @@ case "$LD" in /*) ;; *) LD="${G}/${LD}" ;; esac if [[ ",${T// /}," == *",locityper,"* ]]; then plan "53 Hard loci (Locityper)" locityper "$(if [ -z "$NOBAM" ]; then echo 'no BAM'; else need "${LD}/DONE"; fi)" \ && NF+=(--locityper_data "$LD") && { [ -z "${LOCITYPER_BG_REGION:-}" ] || NF+=(--locityper_bg_region "$LOCITYPER_BG_REGION"); } || true; else KNOWN+=" locityper"; fi for t in ${T//,/ }; do [[ "${KNOWN} " == *" ${t} "* ]] || { echo "ERROR: unknown step '${t}' in TOOLS. Known:${KNOWN}" >&2; exit 2; }; done +if [ "${#EVMISS[@]}" -gt 0 ]; then + { echo "ERROR: EVERYTHING=true runs every optional step, and ${#EVMISS[@]} cannot run (nothing started):"; printf ' %s\n' "${EVMISS[@]}" + echo "Install what is missing, or run without EVERYTHING=true to skip them."; } >&2; exit 2 +fi arg --cytoband "$(data_file cytoband || true)"; ! GEX=$(data_file gencode_exons) || NF+=(--gencode_exons "$GEX"); arg --delly_exclude "$(data_file delly_exclude || true)"; arg --manta_call_regions "${MANTA_CALL_REGIONS:-}" # Step 20's annotation of the chrM calls (MITO_VARIANTS_ANNOTATE): ClinVar with its index, and gnomAD's chrM sites. MC="${G}/clinvar/clinvar_chr.vcf.gz"; if [ -s "$MC" ] && [ -s "${MC}.tbi" ]; then NF+=(--mito_clinvar "$MC"); fi; arg --gnomad_mito "${G}/gnomad_mito/${GNOMAD_MITO_URL##*/}" @@ -336,7 +388,7 @@ if [[ " $* " != *" --publish_dir_mode"* && " $* " != *" -w "* && " $* " != *" -w rm -f "$LP" "${S}/.pgp-link-probe.$$" "${G}/.pgp-link-probe.$$" fi if on SKIP_TRIM; then NF+=(--skip_trim true); fi -for v in GRIDSS:04b-gridss IMPUTATION:14-imputation-prep SOMATIC:29-mutect2-somatic; do if on "${v%%:*}"; then OPTIN+=("${v#*:}.sh"); fi; done +for v in TIDDIT:04a-tiddit GRIDSS:04b-gridss IMPUTATION:14-imputation-prep SOMATIC:29-mutect2-somatic; do if on "${v%%:*}"; then OPTIN+=("${v#*:}.sh"); fi; done for c in ${C//,/ }; do OPTIN+=("$(cd "$SCRIPT_DIR" && compgen -G "03[a-d]-${c}*.sh")") || { echo "ERROR: unknown caller '${c}' in EXTRA_CALLERS (gatk, freebayes, strelka2, octopus)" >&2; exit 2; }; done if ! on BENCHMARK; then :; elif [ -n "$C" ] || compgen -G "${S}/vcf_*/${SAMPLE}.vcf.gz" >/dev/null || [ -f "${S}/vcf_strelka2/results/variants/variants.vcf.gz" ]; then OPTIN+=(benchmark-variants.sh) else echo " benchmark-variants skipped (only one caller VCF: set EXTRA_CALLERS, or run a 03a-03d script first)"; fi diff --git a/tests/test_run_state.py b/tests/test_run_state.py index 03feaec3d..c2bacc8f3 100755 --- a/tests/test_run_state.py +++ b/tests/test_run_state.py @@ -195,7 +195,7 @@ def run_all_steps(): planned = set(re.findall(r'"(\d+b?) ([^":]+):(\w+)"', src)) planned |= set(re.findall(r'plan "(\d+b?) ([^"]+)" (\w+)', src)) other = set(re.findall(r"printf 'step\\t(\d+b?)\\tskipped", src)) - {s for s, _, _ in planned} - scripts = set(re.findall(r"\b\w+:(\d+b?)-[\w-]+\b", src)) # GRIDSS:04b-gridss ... + scripts = set(re.findall(r"\b\w+:(\d+[ab]?)-[\w-]+\b", src)) # TIDDIT:04a-tiddit, GRIDSS:04b-gridss ... m = re.search(r'compgen -G "03\[([a-d-]+)\]', src) callers = {f"03{c}" for c in "abcd"} if m and m.group(1) == "a-d" else set() return planned, other | scripts | callers From fa143760073904d4c3bf9fcfe853ef219dabf7bc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:21:14 +0100 Subject: [PATCH 2/5] feat(dev): og dev everything-check keeps assets/everything.tsv complete assets/everything.tsv lists every optional item: each --tools name of main.nf, each run-all.sh switch, each setup.sh flag, each false-by-default or opt-in parameter of nextflow.config and each optional image of versions.env, as tool, switch, setup, input, image or excluded with the reason. og dev everything-check fails when one of them has no row, when a row names an item that is gone, when conf/everything.config sets other tools or parameters than the rows, or when conf/test_all.config misses a tool. Guard runs it; its planted cases are og-dev unit tests. -profile everything (conf/everything.config) gives the Nextflow path the same tools and switches as EVERYTHING=true. Bead: D8 --- .github/workflows/guard.yml | 8 + assets/everything.tsv | 159 +++++++ conf/everything.config | 23 + nextflow.config | 5 + rust/crates/og-dev/src/everything.rs | 678 ++++++++++++++++++++++++++- 5 files changed, 865 insertions(+), 8 deletions(-) create mode 100644 assets/everything.tsv create mode 100644 conf/everything.config diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 069b4b080..1944a525d 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -910,6 +910,14 @@ jobs: working-directory: . run: bin/og dev remnants + # EVERYTHING=true and -profile everything run every optional step from + # assets/everything.tsv: a new tool, switch, setup flag, false-by-default + # or opt-in parameter, or optional image with no row there fails here. + # Its planted cases are og-dev's everything::tests, in cargo test. + - name: og dev everything-check + working-directory: . + run: bin/og dev everything-check + # main.nf runs the same check before any task: a stale og stops # `nextflow run` with the setup command. Without the check, this stub # run completes and the step fails. diff --git a/assets/everything.tsv b/assets/everything.tsv new file mode 100644 index 000000000..a7ad9a018 --- /dev/null +++ b/assets/everything.tsv @@ -0,0 +1,159 @@ +# assets/everything.tsv: every optional part of a run, and what EVERYTHING=true (scripts/run-all.sh) and +# -profile everything (conf/everything.config) do with it. One row per item; `og dev everything-check` fails when +# main.nf, nextflow.config, run-all.sh, setup.sh or versions.env gains an item with no row here. +# kind: tool (a main.nf --tools name: EVERYTHING names it), switch (a run-all.sh switch: EVERYTHING sets it to +# value), setup (a setup.sh flag: value names the rows whose data it installs), input (a choice or a file only +# the user can give: EVERYTHING leaves it to them), image (an optional image of versions.env: value names the row +# whose step runs it), excluded (left out: reason says why). +# value: tool and input rows: how to install the data, '-' when none is needed; switch rows: the value EVERYTHING +# sets; setup and image rows: the rows they serve, comma separated. +# nextflow: switch rows with a Nextflow parameter: that parameter as conf/everything.config sets it, else '-'. +# Rows named -- stand for a Nextflow parameter that has no run-all.sh switch. +kind name value nextflow reason +tool pharmcat - - step 07 +tool cpic - - step 27 +tool clinvar setup.sh - step 06 +tool pypgx the pypgx-bundle, by hand: docs/00-reference-setup.md - step 32 +tool vep setup.sh --vep-cache - step 13 +tool vcfanno setup.sh --vep-cache, then the score files of docs/00-reference-setup.md - step 30 +tool slivar setup.sh --vep-cache - step 31 +tool clinical_filter setup.sh --vep-cache - step 23 +tool cpsr setup.sh --clinical-data - step 17 +tool roh - - step 11 +tool prs setup.sh - step 25 +tool ancestry setup.sh --ancestry-panel - step 26 +tool ancestry_fine setup.sh --ancestry-fine-panel - step 26b +tool mito_haplogroup - - step 12 +tool hla_typing setup.sh - step 08 +tool expansion_hunter - - step 09 +tool stranger - - step 9b +tool telomere_hunter - - step 10 +tool mosdepth - - step 16b +tool mito_variants - - step 20 +tool cyrius setup.sh --cyrius - step 21; its licence is non-commercial, and installing it accepts that licence +tool manta - - step 04 +tool delly - - step 19 +tool cnvpytor the CNVpytor resources, by hand: docs/00-reference-setup.md - step 18 +tool duphold - - step 15 +tool annotsv setup.sh --annotsv-data - step 05 +tool survivor_merge - - step 22 +tool sample_qc setup.sh --sample-qc-data - step 33 +tool parascopy setup.sh --parascopy-data - step 35 +tool paraphase - - step 35b; a BAM that is not PacBio HiFi is skipped +tool y_haplogroup setup.sh --yleaf-data - step 37; a female sample is skipped +tool html_report - - step 24; run-all.sh writes the report itself after the pipeline +tool multiqc - - step 28 +tool consumer_export setup.sh --consumer-sites - step 40 +tool bloodtype - - step 41 +tool lpa - - step 46 +tool vntyper - - step 52 +tool traits - - step 42 +tool archaic setup.sh --archaic - step 43 +tool ancient setup.sh --ancient-panel - step 44 +tool y_str - - step 45; a female sample is skipped +tool viral setup.sh --viral-data - step 47 +tool chip setup.sh --chip-data - step 49 +tool mei setup.sh --megane-kmers - step 50 +tool mocha setup.sh --mocha-data - step 51 +tool locityper setup.sh --locityper-data - step 53 +input exomiser setup.sh --exomiser-data - step 48 runs when HPO (or --hpo) names the symptoms; naming it without them stops the run +excluded cram_archive - - a CRAM copy of each BAM for the archive: it costs disk and adds no result +switch KIR true --kir true step 08 types the KIR genes too; setup.sh --kir-data +switch HLA_QC true --hla_qc true step 08 runs OptiType beside T1K +switch CPSR_SECONDARY_FINDINGS true --cpsr_secondary_findings true step 17 also screens the ACMG SF genes +switch GRIDSS true - script 04b after the pipeline +switch TIDDIT true - script 04a after the pipeline +switch IMPUTATION true - script 14 after the pipeline +switch SOMATIC true - script 29 after the pipeline +switch EXTRA_CALLERS gatk,freebayes,strelka2,octopus - scripts 03a to 03d after the pipeline +switch BENCHMARK true - benchmark-variants.sh after the extra callers +input HPO setup.sh --exomiser-data - the symptoms as HPO ids; only the user knows them +input FAMILIES - - who is related to whom; steps 38 and 39 run when the family has what they need +input ALIGN_DIR - - the BAM of another aligner (script 02a writes aligned_bwamem2/) +input REF_FASTA - - another GRCh38 build +input EH_CATALOG - - another catalog for steps 09 and 9b +input MANTA_CALL_REGIONS - - the regions Manta calls on +input PARASCOPY_POPULATION - - the Parascopy model; EUR when unset +input PARAPHASE_REGIONS - - the Paraphase regions; all when unset +input ANCESTRY_PANEL - - another panel for step 26; none skips the step, which EVERYTHING=true refuses +input ANCESTRY_FINE_DIR - - another panel for step 26b; none skips the step, which EVERYTHING=true refuses +input ARCHAIC_DIR - - another table for step 43; none skips the step, which EVERYTHING=true refuses +input ARCHAIC_POP - - the 1000 Genomes group of step 43; ALL when unset +input PGSC_CALC_DIR - - another pgsc_calc checkout +input ANCIENT_PANEL_DIR - - another panel for step 44 +input VIRAL_REF - - other viral genomes for step 47 +input MEGANE_KMERS - - another k-mer set for step 50 +input MOCHA_DATA - - another panel for step 51 +input LOCITYPER_DATA - - another database for step 53 +input LOCITYPER_BG_REGION - - another background region for step 53 +input long-read - - scripts 02b, 03e and 04c read ONT or PacBio HiFi reads: run them by hand on long-read data +input --pedigree - - the family file of the Nextflow path (FAMILIES on run-all.sh) +input --medications - - the drugs taken, for step 27's phenoconversion +excluded EVERYTHING - - this switch +excluded TOOLS - - EVERYTHING=true names every tool itself, so it refuses TOOLS +excluded GENOME_DIR - - where the data is, not a step +excluded SKIP_VALIDATION - - skips a check, not a step +excluded THREADS - - a resource cap, not a step +excluded SKIP_TRIM - - skips fastp, so it removes a step +excluded INTERVALS - - calls part of the genome, so it removes work +excluded OWNGENOME_LIVE_URL - - the live site's address, not a step +excluded NXF_VER - - the Nextflow release, not a step +excluded NXF_WORK - - Nextflow's work folder, not a step +excluded NXF_HOME - - Nextflow's home folder, not a step +excluded NXF_OFFLINE - - Nextflow's offline mode, not a step +excluded --allow_unfiltered - - accepts a VCF with no FILTER=PASS record: an input choice, not a step +excluded --skip_trim - - skips fastp, so it removes a step +excluded --prs_keep_ambiguous - - changes which rows of a score count, not which steps run +setup --cyrius cyrius - Cyrius, under its non-commercial licence +setup --parascopy-data parascopy - Parascopy's table and models +setup --kir-data KIR - IPD-KIR +setup --ancestry-panel ancestry - pgsc_calc's ancestry panel +setup --ancestry-fine-panel ancestry_fine - the fine-grained ancestry panel +setup --yleaf-data y_haplogroup - Yleaf's tables +setup --consumer-sites consumer_export - the 23andMe v5 site table +setup --ancient-panel ancient - the AADR panel +setup --viral-data viral - the viral genomes +setup --exomiser-data exomiser,HPO - Exomiser's data +setup --chip-data chip - Ensembl's gene annotation +setup --strchive-data expansion_hunter,stranger - the STRchive catalog; part of a full setup +setup --megane-kmers mei - MEGAnE's k-mer set +setup --mocha-data mocha - MoChA's panel +setup --locityper-data locityper - Locityper's database +setup --vep-cache vep,vcfanno,slivar,clinical_filter - the VEP cache; part of a full setup +setup --annotsv-data annotsv - AnnotSV's data; part of a full setup +setup --archaic archaic - the archaic table +setup --clinical-data cpsr,vep - the VEP caches and the CPSR bundle; part of a full setup +setup --sample-qc-data sample_qc - somalier's sites and VerifyBamID2's panel; part of a full setup +excluded --masked-reference - - another reference: every sample is aligned again on it +excluded --og - - builds og, which a full setup does too +excluded --no-clinical-data - - installs less +excluded --pull-only - - pulls the images only +excluded --refresh - - replaces ClinVar with a newer release +image BWAMEM2_IMAGE ALIGN_DIR - script 02a +image BWA_IMAGE GRIDSS - the classic BWA index of script 04b +image FREEBAYES_IMAGE EXTRA_CALLERS - script 03b +image STRELKA_IMAGE EXTRA_CALLERS - script 03c +image OCTOPUS_IMAGE EXTRA_CALLERS - script 03d +image CLAIR3_IMAGE long-read - script 03e +image TIDDIT_IMAGE TIDDIT - script 04a +image SNIFFLES_IMAGE long-read - script 04c +image HAPPY_IMAGE BENCHMARK - benchmark-variants.sh +image YLEAF_IMAGE y_haplogroup - step 37 +image HIPSTR_IMAGE y_str - step 45 +image OPTITYPE_IMAGE HLA_QC - step 08 with HLA_QC=true +image PARASCOPY_IMAGE parascopy - step 35 +image PARAPHASE_IMAGE paraphase - step 35b +image MEGANE_IMAGE mei - step 50 +image VNTYPER_IMAGE vntyper - step 52 +image LOCITYPER_IMAGE locityper - step 53 +image GLNEXUS_IMAGE FAMILIES - step 38 +image HAP_IBD_IMAGE FAMILIES - step 38 +image EXOMISER_IMAGE exomiser - step 48 +image ADMIXTURE_IMAGE ancestry_fine - step 26b +image BEAGLE_IMAGE ancestry_fine,mocha - steps 26b and 51 +image RFMIX_IMAGE ancestry_fine - step 26b +image BCFTOOLS_HTTPS_IMAGE --ancestry-fine-panel - setup.sh --ancestry-fine-panel +image MOCHA_IMAGE mocha - step 51 +image IBDMIX_IMAGE archaic - step 43 +image EIGENSOFT_IMAGE ancient - step 44 +excluded SAMTOOLS_HTTPS_IMAGE - - docs/quick-test.md reads a CRAM over https with it; no step runs it diff --git a/conf/everything.config b/conf/everything.config new file mode 100644 index 000000000..3e9a055ad --- /dev/null +++ b/conf/everything.config @@ -0,0 +1,23 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config: every optional step +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + nextflow run main.nf -profile docker,everything --input ... --reference ... \ + -- ... + + The tool rows and the switch rows of assets/everything.tsv, as + EVERYTHING=true in scripts/run-all.sh sets them: every --tools name but + exomiser (it runs when --hpo names the symptoms) and cram_archive (an + archive copy of each BAM), plus KIR typing and OptiType. Pass every + database the tools need: main.nf stops before any task and names the + option a tool lacks. `og dev everything-check` fails when this file and + the list differ. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +params { + tools = 'pharmcat,cpic,clinvar,pypgx,vep,vcfanno,slivar,clinical_filter,cpsr,roh,prs,ancestry,ancestry_fine,mito_haplogroup,hla_typing,expansion_hunter,stranger,telomere_hunter,mosdepth,mito_variants,cyrius,manta,delly,cnvpytor,duphold,annotsv,survivor_merge,sample_qc,parascopy,paraphase,y_haplogroup,html_report,multiqc,consumer_export,bloodtype,lpa,vntyper,traits,archaic,ancient,y_str,viral,chip,mei,mocha,locityper' + kir = true + hla_qc = true + cpsr_secondary_findings = true +} diff --git a/nextflow.config b/nextflow.config index dca4b1106..4c902e036 100644 --- a/nextflow.config +++ b/nextflow.config @@ -235,6 +235,11 @@ profiles { test_all { includeConfig 'conf/test_all.config' } + // Every optional step: the tools and switches of assets/everything.tsv + // (EVERYTHING=true in scripts/run-all.sh). Pass the databases they need. + everything { + includeConfig 'conf/everything.config' + } } // Execution reports diff --git a/rust/crates/og-dev/src/everything.rs b/rust/crates/og-dev/src/everything.rs index be8e755e5..8b8c61301 100644 --- a/rust/crates/og-dev/src/everything.rs +++ b/rust/crates/og-dev/src/everything.rs @@ -1,13 +1,675 @@ -//! `og dev everything-check`: `assets/everything.tsv` against every tool, -//! switch and installer. +//! `og dev everything-check [--repo DIR]`: `assets/everything.tsv` against +//! every tool, switch, installer and optional image. //! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! `EVERYTHING=true` (scripts/run-all.sh) and `-profile everything` +//! (conf/everything.config) run every optional step from that one list, so +//! an opt-in that has no row there would be missed with no word. The check +//! reads, as text: +//! +//! - the names of `known_tools` in `main.nf`; +//! - the parameters of `nextflow.config`'s `params` block that default to +//! `false`, and those whose comment says `opt-in`; +//! - the words of `SWITCHES=(...)` in `scripts/run-all.sh`; +//! - the flags of the first `case "${1:-}" in` of `scripts/setup.sh`; +//! - the images of `versions.env` whose comment starts with `optional`; +//! +//! and fails when one of them has no row, when a row names an item that is +//! gone, when `conf/everything.config` sets other tools or parameters than +//! the rows say, or when `conf/test_all.config`'s tools miss one of them (the +//! stub run of every process would then miss it too). +//! +//! Exit 0 when everything agrees, 1 when a check fails (one `FAIL` line per +//! problem, naming the item), 2 when a file cannot be read or an argument is +//! wrong. `--repo` defaults to the current folder. +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; use std::io::Write; +use std::path::Path; + +const USAGE: &str = "usage: og dev everything-check [--repo DIR]\n"; +const LIST: &str = "assets/everything.tsv"; +const KINDS: [&str; 6] = ["tool", "switch", "setup", "input", "image", "excluded"]; + +/// One row of `assets/everything.tsv`. +struct Row { + kind: String, + name: String, + value: String, + nextflow: String, + line: usize, +} + +/// What the repository declares, read from its files. +#[derive(Default)] +struct Items { + tools: BTreeSet, + nf_params: BTreeSet, + switches: BTreeSet, + setup_flags: BTreeSet, + images: BTreeSet, +} + +fn read(repo: &Path, rel: &str) -> Result { + fs::read_to_string(repo.join(rel)).map_err(|e| format!("cannot read {rel}: {e}")) +} + +fn read_rows(text: &str) -> Result, String> { + let mut rows = Vec::new(); + let mut header = false; + for (i, line) in text.lines().enumerate() { + if line.starts_with('#') || line.trim().is_empty() { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if !header { + if c != ["kind", "name", "value", "nextflow", "reason"] { + return Err(format!( + "{LIST} line {}: the header must be kind, name, value, nextflow, reason", + i + 1 + )); + } + header = true; + continue; + } + let [kind, name, value, nextflow, reason] = c.as_slice() else { + return Err(format!( + "{LIST} line {}: {} fields, want 5: kind, name, value, nextflow, reason", + i + 1, + c.len() + )); + }; + if [kind, name, value, nextflow, reason] + .iter() + .any(|f| f.trim().is_empty()) + { + return Err(format!("{LIST} line {}: an empty field", i + 1)); + } + rows.push(Row { + kind: (*kind).to_owned(), + name: (*name).to_owned(), + value: (*value).to_owned(), + nextflow: (*nextflow).to_owned(), + line: i + 1, + }); + } + if rows.is_empty() { + return Err(format!("{LIST}: no rows")); + } + Ok(rows) +} + +/// The quoted words of `def known_tools = [ ... ]` in `main.nf`. +fn known_tools(main_nf: &str) -> Result, String> { + let start = main_nf + .find("def known_tools = [") + .ok_or("main.nf: no `def known_tools = [`")?; + let rest = &main_nf[start + "def known_tools = [".len()..]; + let end = rest.find(']').ok_or("main.nf: known_tools has no `]`")?; + let names: BTreeSet = rest[..end] + .split('\'') + .skip(1) + .step_by(2) + .map(str::to_owned) + .collect(); + if names.len() < 10 { + return Err(format!( + "main.nf: known_tools read as {} names: that is the list read wrong", + names.len() + )); + } + Ok(names) +} + +/// A line split into its code and its `//` comment, outside quotes. +fn split_comment(line: &str) -> (&str, &str) { + let mut quote = None; + let bytes = line.as_bytes(); + for (i, &b) in bytes.iter().enumerate() { + match (quote, b) { + (None, b'\'' | b'"') => quote = Some(b), + (Some(q), _) if b == q => quote = None, + (None, b'/') if bytes.get(i + 1) == Some(&b'/') => { + return (&line[..i], &line[i + 2..]); + } + _ => {} + } + } + (line, "") +} + +/// `name = value` of one config line, with its comment. +fn assignment(line: &str) -> Option<(&str, &str, &str)> { + let (code, comment) = split_comment(line); + let (name, value) = code.split_once('=')?; + let name = name.trim(); + let ok = !name.is_empty() + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); + ok.then(|| (name, value.trim(), comment)) +} + +/// The lines of the first top-level `params {` block of a config. +fn params_block(config: &str) -> Option> { + let mut lines = config.lines(); + lines.find(|l| l.trim_end() == "params {")?; + Some(lines.take_while(|l| !l.starts_with('}')).collect()) +} + +/// The parameters of `nextflow.config` that default to `false` or whose +/// comment says `opt-in`. +fn optional_params(config: &str) -> Result, String> { + let block = params_block(config).ok_or("nextflow.config: no `params {` block")?; + let mut out = BTreeSet::new(); + let mut seen = 0; + for line in block { + if let Some((name, value, comment)) = assignment(line) { + seen += 1; + if value == "false" || comment.contains("opt-in") { + out.insert(name.to_owned()); + } + } + } + if seen < 20 { + return Err(format!( + "nextflow.config: the params block read as {seen} parameters: that is the block read wrong" + )); + } + Ok(out) +} + +/// The words of `SWITCHES=( ... )` in `run-all.sh`. +fn switches(run_all: &str) -> Result, String> { + let start = run_all + .find("SWITCHES=(") + .ok_or("scripts/run-all.sh: no `SWITCHES=(`")?; + let rest = &run_all[start + "SWITCHES=(".len()..]; + let end = rest + .find(')') + .ok_or("scripts/run-all.sh: SWITCHES has no `)`")?; + let words: BTreeSet = rest[..end].split_whitespace().map(str::to_owned).collect(); + if words.len() < 10 { + return Err(format!( + "scripts/run-all.sh: SWITCHES read as {} names: that is the list read wrong", + words.len() + )); + } + Ok(words) +} + +/// The flags of the first `case "${1:-}" in` of `setup.sh`: every `--flag` +/// of each pattern line, up to its `esac`. +fn setup_flags(setup: &str) -> Result, String> { + let mut lines = setup.lines(); + lines + .find(|l| l.trim() == r#"case "${1:-}" in"#) + .ok_or(r#"scripts/setup.sh: no `case "${1:-}" in`"#)?; + let mut out = BTreeSet::new(); + for line in lines.take_while(|l| l.trim() != "esac") { + let t = line.trim(); + let Some(pattern) = t.strip_suffix(')') else { + continue; + }; + if pattern.starts_with("--") { + out.extend(pattern.split('|').map(str::to_owned)); + } + } + if out.len() < 5 { + return Err(format!( + "scripts/setup.sh: its flags read as {}: that is the case read wrong", + out.len() + )); + } + Ok(out) +} + +/// The images of `versions.env` whose comment starts with `optional`. +fn optional_images(env: &str) -> BTreeSet { + env.lines() + .filter_map(|l| { + let (name, rest) = l.split_once('=')?; + let ok = name.ends_with("_IMAGE") + && name + .chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_'); + let (_, comment) = rest.split_once(" # ")?; + (ok && comment.trim_start().starts_with("optional")).then(|| name.to_owned()) + }) + .collect() +} + +/// The comma-separated names of a config's `tools = '...'`. +fn config_tools(config: &str, rel: &str) -> Result, String> { + let block = params_block(config).ok_or(format!("{rel}: no `params {{` block"))?; + let value = block + .iter() + .find_map(|l| assignment(l).filter(|(n, _, _)| *n == "tools")) + .map(|(_, v, _)| v) + .ok_or(format!("{rel}: no `tools =` in its params block"))?; + Ok(value + .trim_matches(|c| c == '\'' || c == '"') + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .collect()) +} + +fn read_items(repo: &Path) -> Result { + let env = read(repo, "versions.env")?; + let images = optional_images(&env); + if images.len() < 5 { + return Err(format!( + "versions.env: {} optional images: that is the file read wrong", + images.len() + )); + } + Ok(Items { + tools: known_tools(&read(repo, "main.nf")?)?, + nf_params: optional_params(&read(repo, "nextflow.config")?)?, + switches: switches(&read(repo, "scripts/run-all.sh")?)?, + setup_flags: setup_flags(&read(repo, "scripts/setup.sh")?)?, + images, + }) +} + +/// Does the repository still declare the item a row names? +fn row_item_exists(row: &Row, items: &Items) -> bool { + let n = row.name.as_str(); + let param = n.strip_prefix("--"); + match row.kind.as_str() { + "tool" => items.tools.contains(n), + "switch" => items.switches.contains(n), + "setup" => items.setup_flags.contains(n), + "image" => items.images.contains(n), + _ if n.ends_with("_IMAGE") => items.images.contains(n), + _ if param.is_some() => { + items.setup_flags.contains(n) || param.is_some_and(|p| items.nf_params.contains(p)) + } + _ if n.chars().any(|c| c.is_ascii_uppercase()) => items.switches.contains(n), + "input" => true, + _ => items.tools.contains(n), + } +} + +/// Every problem, one line each; empty when the list agrees with the repo. +fn problems(repo: &Path) -> Result<(Vec, usize), String> { + let rows = read_rows(&read(repo, LIST)?)?; + let items = read_items(repo)?; + let mut bad = Vec::new(); + let mut by_name: BTreeMap<&str, &Row> = BTreeMap::new(); + for r in &rows { + if !KINDS.contains(&r.kind.as_str()) { + bad.push(format!( + "{LIST} line {}: kind '{}' is none of {}", + r.line, + r.kind, + KINDS.join(", ") + )); + } + if by_name.insert(&r.name, r).is_some() { + bad.push(format!("{LIST} line {}: '{}' has two rows", r.line, r.name)); + } + } + for r in &rows { + if KINDS.contains(&r.kind.as_str()) && !row_item_exists(r, &items) { + bad.push(format!( + "{LIST} line {}: {} row '{}' names nothing the repository declares (renamed or removed?)", + r.line, r.kind, r.name + )); + } + if matches!(r.kind.as_str(), "setup" | "image") { + for target in r.value.split(',') { + if !by_name.contains_key(target) { + bad.push(format!( + "{LIST} line {}: {} row '{}' serves '{target}', which has no row", + r.line, r.kind, r.name + )); + } + } + } + let nf_ok = if r.kind == "switch" { + r.nextflow == "-" + || r.nextflow + .strip_prefix("--") + .and_then(|s| s.split_once(' ')) + .is_some_and(|(p, v)| !p.is_empty() && !v.is_empty() && !v.contains(' ')) + } else { + r.nextflow == "-" + }; + if !nf_ok { + bad.push(format!( + "{LIST} line {}: nextflow '{}' of '{}': a switch row takes '--param value' or '-', any other row '-'", + r.line, r.nextflow, r.name + )); + } + } + // Every item the repository declares has a row. + let has = |name: &str| by_name.contains_key(name); + let nf_named: BTreeSet<&str> = rows + .iter() + .filter_map(|r| r.nextflow.strip_prefix("--")?.split(' ').next()) + .collect(); + for t in &items.tools { + if !has(t) { + bad.push(format!( + "main.nf: tool '{t}' of known_tools has no row in {LIST}" + )); + } + } + for p in &items.nf_params { + if !has(&format!("--{p}")) && !nf_named.contains(p.as_str()) { + bad.push(format!( + "nextflow.config: params.{p} (false by default or opt-in) has no row in {LIST}" + )); + } + } + for s in &items.switches { + if !has(s) { + bad.push(format!( + "scripts/run-all.sh: switch {s} of SWITCHES has no row in {LIST}" + )); + } + } + for f in &items.setup_flags { + if !has(f) { + bad.push(format!("scripts/setup.sh: flag {f} has no row in {LIST}")); + } + } + for i in &items.images { + if !has(i) { + bad.push(format!( + "versions.env: optional image {i} has no row in {LIST}" + )); + } + } + // conf/everything.config sets exactly the tool rows and the switch rows' + // Nextflow parameters. + let want_tools: BTreeSet = rows + .iter() + .filter(|r| r.kind == "tool") + .map(|r| r.name.clone()) + .collect(); + let ev_rel = "conf/everything.config"; + let ev = read(repo, ev_rel)?; + let ev_tools = config_tools(&ev, ev_rel)?; + for t in want_tools.difference(&ev_tools) { + bad.push(format!("{ev_rel}: tools lacks '{t}', a tool row of {LIST}")); + } + for t in ev_tools.difference(&want_tools) { + bad.push(format!( + "{ev_rel}: tools names '{t}', which is no tool row of {LIST}" + )); + } + let want_params: BTreeMap<&str, &str> = rows + .iter() + .filter(|r| r.kind == "switch") + .filter_map(|r| r.nextflow.strip_prefix("--")?.split_once(' ')) + .collect(); + let mut set: BTreeMap = BTreeMap::new(); + for line in params_block(&ev).unwrap_or_default() { + if let Some((name, value, _)) = assignment(line) + && name != "tools" + { + set.insert(name.to_owned(), value.trim_matches('\'').to_owned()); + } + } + for (p, v) in &want_params { + match set.get(*p) { + Some(got) if got == v => {} + Some(got) => bad.push(format!( + "{ev_rel}: {p} = {got}, but its switch row says {v}" + )), + None => bad.push(format!( + "{ev_rel}: does not set {p} = {v} of its switch row" + )), + } + } + for p in set.keys() { + if !want_params.contains_key(p.as_str()) { + bad.push(format!( + "{ev_rel}: sets {p}, which no switch row of {LIST} names" + )); + } + } + // The stub run of every process covers every tool of the everything profile. + let ta_rel = "conf/test_all.config"; + let ta_tools = config_tools(&read(repo, ta_rel)?, ta_rel)?; + for t in want_tools.difference(&ta_tools) { + bad.push(format!( + "{ta_rel}: tools lacks '{t}', so the stub run of every tool misses it" + )); + } + Ok((bad, rows.len())) +} + +/// `og dev everything-check ARGS`: 0 when the list covers every optional +/// item, 1 when it does not, 2 on a usage or read error. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let mut repo = std::path::PathBuf::from("."); + let mut it = args.iter(); + while let Some(a) = it.next() { + match (a.as_str(), it.clone().next()) { + ("--repo", Some(v)) => { + repo = v.into(); + it.next(); + } + ("-h" | "--help", _) => { + let _ = write!(out, "{USAGE}"); + return 0; + } + _ => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og dev everything-check: error: unexpected argument '{a}'" + ); + return 2; + } + } + } + match problems(&repo) { + Ok((bad, n)) if bad.is_empty() => { + let _ = writeln!( + out, + "ok {LIST}: {n} rows cover every tool, switch, setup flag and optional image; conf/everything.config and conf/test_all.config agree" + ); + 0 + } + Ok((bad, _)) => { + for b in &bad { + let _ = writeln!(out, "FAIL {b}"); + } + let _ = writeln!( + out, + "{} problem(s): give each new opt-in a row in {LIST} (kind tool, switch, setup, input, image or excluded with the reason)", + bad.len() + ); + 1 + } + Err(e) => { + let _ = writeln!(err, "og dev everything-check: {e}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + const FILES: [&str; 8] = [ + LIST, + "main.nf", + "nextflow.config", + "scripts/run-all.sh", + "scripts/setup.sh", + "versions.env", + "conf/everything.config", + "conf/test_all.config", + ]; + + fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") + } + + /// A copy of the files the check reads, in a folder of its own. + fn copy(name: &str) -> PathBuf { + let d = + std::env::temp_dir().join(format!("og-dev-everything-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + for f in FILES { + let to = d.join(f); + fs::create_dir_all(to.parent().unwrap()).unwrap(); + fs::copy(repo_root().join(f), &to).unwrap(); + } + d + } + + fn edit(d: &Path, rel: &str, from: &str, to: &str) { + let p = d.join(rel); + let text = fs::read_to_string(&p).unwrap(); + assert!(text.contains(from), "{rel} has no {from:?}"); + fs::write(&p, text.replacen(from, to, 1)).unwrap(); + } + + fn run(d: &Path) -> (u8, String) { + let (mut out, mut err) = (Vec::new(), Vec::new()); + let args = ["--repo".to_owned(), d.display().to_string()]; + let code = cli(&args, &mut out, &mut err); + let mut text = String::from_utf8(out).unwrap(); + text.push_str(&String::from_utf8(err).unwrap()); + (code, text) + } + + fn assert_fails(d: &Path, want: &str) { + let (code, text) = run(d); + let _ = fs::remove_dir_all(d); + assert_eq!(code, 1, "{text}"); + assert!(text.contains(want), "no {want:?} in:\n{text}"); + } + + #[test] + fn the_repository_agrees() { + let d = copy("ok"); + let (code, text) = run(&d); + let _ = fs::remove_dir_all(&d); + assert_eq!(code, 0, "{text}"); + assert!(text.starts_with("ok "), "{text}"); + } + + #[test] + fn a_new_tool_in_known_tools_fails() { + let d = copy("tool"); + edit( + &d, + "main.nf", + "'locityper',\n", + "'locityper',\n 'newtool',\n", + ); + assert_fails(&d, "main.nf: tool 'newtool' of known_tools has no row"); + } + + #[test] + fn a_new_false_param_fails() { + let d = copy("param"); + edit( + &d, + "nextflow.config", + " kir = false", + " foo = false // a new switch\n kir = false", + ); + assert_fails( + &d, + "nextflow.config: params.foo (false by default or opt-in) has no row", + ); + } + + #[test] + fn a_new_opt_in_param_fails() { + let d = copy("optin"); + edit( + &d, + "nextflow.config", + " pedigree = null", + " bar = null // opt-in: a new input\n pedigree = null", + ); + assert_fails( + &d, + "nextflow.config: params.bar (false by default or opt-in) has no row", + ); + } + + #[test] + fn a_new_setup_flag_fails() { + let d = copy("setup"); + edit( + &d, + "scripts/setup.sh", + "|--archaic)", + "|--archaic|--new-data)", + ); + assert_fails(&d, "scripts/setup.sh: flag --new-data has no row"); + } + + #[test] + fn a_new_optional_image_fails() { + let d = copy("image"); + edit( + &d, + "versions.env", + "TIDDIT_IMAGE=", + "NEWTOOL_IMAGE=\"quay.io/biocontainers/newtool:1.0--0\" # optional: step 99\nTIDDIT_IMAGE=", + ); + assert_fails(&d, "versions.env: optional image NEWTOOL_IMAGE has no row"); + } + + #[test] + fn a_new_run_all_switch_fails() { + let d = copy("switch"); + edit( + &d, + "scripts/run-all.sh", + "SWITCHES=(", + "SWITCHES=(NEWSWITCH ", + ); + assert_fails( + &d, + "scripts/run-all.sh: switch NEWSWITCH of SWITCHES has no row", + ); + } + + #[test] + fn a_stale_row_fails() { + let d = copy("stale"); + edit(&d, "main.nf", " 'mocha',\n", ""); + assert_fails(&d, "tool row 'mocha' names nothing the repository declares"); + } + + #[test] + fn the_profile_and_the_stub_run_must_follow_the_rows() { + let d = copy("profile"); + edit(&d, "conf/everything.config", "hla_qc", "hla_qc_off"); + assert_fails(&d, "conf/everything.config: does not set hla_qc = true"); + let d = copy("profile-tool"); + edit(&d, "conf/everything.config", ",locityper", ""); + assert_fails(&d, "conf/everything.config: tools lacks 'locityper'"); + let d = copy("test-all"); + edit(&d, "conf/test_all.config", ",locityper", ""); + assert_fails(&d, "conf/test_all.config: tools lacks 'locityper'"); + } -/// `og dev everything-check ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og dev everything-check: not ported yet"); - 2 + #[test] + fn a_bad_argument_is_refused() { + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!( + cli(&["--og-scaffold-probe".to_owned()], &mut out, &mut err), + 2 + ); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let missing = ["--repo".to_owned(), "/nonexistent-og-repo".to_owned()]; + assert_eq!(cli(&missing, &mut out, &mut err), 2); + } } From 9f9174d194ff22420ba77b66def7a81f0a60229f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:21:15 +0100 Subject: [PATCH 3/5] test(run): fake-docker case for EVERYTHING=true Every data folder stubbed: every tool row runs and every script step runs after the pipeline. Without Yleaf's tables it exits 2 before Nextflow and names setup.sh --yleaf-data. A female sample still runs with the Y steps skipped; a BAM that is not PacBio HiFi skips Paraphase. TOOLS, KIR=false or ARCHAIC_DIR=none with EVERYTHING=true are refused. Bead: D8 --- tests/fake-docker/run-all-everything.sh | 120 ++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/fake-docker/run-all-everything.sh diff --git a/tests/fake-docker/run-all-everything.sh b/tests/fake-docker/run-all-everything.sh new file mode 100644 index 000000000..f2fd8058e --- /dev/null +++ b/tests/fake-docker/run-all-everything.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# EVERYTHING=true runs every optional step of assets/everything.tsv. +# all data every data folder stubbed, a male BAM sample: every tool row +# but html_report (run-all.sh writes the report itself) is in +# --tools and listed as runs, with --kir true and --hla_qc +# true, and the scripts of TIDDIT, GRIDSS, IMPUTATION, SOMATIC, +# every EXTRA_CALLERS and BENCHMARK run after the pipeline +# no Yleaf exit 2 before Nextflow, naming setup.sh --yleaf-data +# female still runs; the two Y steps are skipped (female sample) +# not HiFi a BAM whose read groups are not PL:PACBIO skips Paraphase +# and the run goes on +# conflicts TOOLS, or a switch set to another value, stops it with exit 2 +# Without EVERYTHING the opt-in steps stay out, as before. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" + +export GENOME_DIR="${CASE_WORK}/genome" SKIP_VALIDATION=true +G=$GENOME_DIR +seed_reference "$G" +seed_clinvar "$G" +use_output_hook +seed_sample "$G" sample1 +rm -f "${G}/sample1/vcf/sample1.vcf.gz" "${G}/sample1/vcf/sample1.vcf.gz.tbi" # a BAM row: DeepVariant writes the gVCF +# shellcheck source=../../versions.env +. "${REPO_ROOT}/versions.env" +HLA=$(grep -E '^HLA_DB_RELEASE=' "${SCRIPTS}/lib/common.sh" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') +GC=$(sed -n 's/^GENCODE_RELEASE=\([0-9]*\)$/\1/p' "${SCRIPTS}/lib/common.sh") +[ -n "$HLA" ] && [ -n "$GC" ] || fail "cannot read HLA_DB_RELEASE and GENCODE_RELEASE from scripts/lib/common.sh" + +# Every data folder run-all.sh looks for, as setup.sh leaves it. +f() { local p; for p in "$@"; do mkdir -p "$(dirname "${G}/${p}")"; printf 'placeholder\n' > "${G}/${p}"; done; } +f "hla/IPD-IMGT-HLA_${HLA}/hla.dat" "reference/gencode.v${GC}.basic.genes.gtf" "kir/IPD-KIR_${KIR_DB_RELEASE}/kir.dat" \ + "reference/strchive-${STRCHIVE_VERSION}/catalog.json" \ + "vep_cache/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38/info.txt" "annotations/revel_grch38.tsv.gz" "annotations/revel_grch38.tsv.gz.tbi" \ + "vep_cache/homo_sapiens/${PCGR_VEP_CACHE_RELEASE}_GRCh38/info.txt" "pcgr_data/${PCGR_DATA_BUNDLE}/data/README" \ + "reference/cnvpytor/gc_hg38.pytor" "annotsv_annotations/Annotations_Human/Genes/GRCh38/README" "reference/pypgx-bundle/README" \ + "reference/somalier/sites.hg38.vcf.gz" \ + "reference/verifybamid2/1000g.phase3.100k.b38.vcf.gz.dat.UD" "reference/verifybamid2/1000g.phase3.100k.b38.vcf.gz.dat.mu" \ + "reference/verifybamid2/1000g.phase3.100k.b38.vcf.gz.dat.bed" \ + "reference/parascopy-${PARASCOPY_DATA_VERSION}/homology_table/GRCh38.bed.gz" "prs_scores/PGS000018.txt.gz" \ + "reference/pgsc_calc/${PGSC_PANEL}.tar.zst" "reference/pgsc_calc/${PGSC_PANEL}_GRCh38_sites.tsv" \ + "reference/ancestry_fine/${ANCESTRY_FINE_PANEL}/model/DONE" "reference/yleaf-${YLEAF_DATA_VERSION}/data/hg38/new_positions.txt" \ + "reference/consumer/23andme_v5_sites_dbsnp${CONSUMER_DBSNP_BUILD}.tsv.gz" \ + "reference/ensembl/Homo_sapiens.GRCh38.${ENSEMBL_GFF_RELEASE}.chr.gff3.gz" "reference/archaic/${ARCHAIC_DATA_VERSION}/DONE" \ + "reference/ancient/aadr_${AADR_VERSION}/DONE" "reference/viral-${VIRAL_REF_VERSION}/viral_genomes.fa" \ + "reference/megane/GRCh38_no_alt_analysis_set.mk" "reference/megane/GRCh38_no_alt_analysis_set.mi" \ + "reference/mocha-${MOCHA_PANEL_VERSION}/DONE" "reference/locityper-${LOCITYPER_PANEL_VERSION}/DONE" +# Cyrius counts as installed when its stamp names this lock file and image (setup.sh --cyrius). +if command -v sha256sum > /dev/null 2>&1; then LOCK=$(sha256sum "${SCRIPTS}/cyrius-constraints.txt"); else LOCK=$(shasum -a 256 "${SCRIPTS}/cyrius-constraints.txt"); fi +mkdir -p "${G}/tools/cyrius-${CYRIUS_VERSION}" +printf 'python=%s lock=%s\n' "$PYTHON_IMAGE" "${LOCK%% *}" > "${G}/tools/cyrius-${CYRIUS_VERSION}/INSTALLED" + +last() { grep '^nextflow :: ' "$FAKE_DOCKER_LOG" | tail -1; } +nfcalls() { grep -c '^nextflow :: ' "$FAKE_DOCKER_LOG" || true; } +has_args() { grep -qF -- "$(printf '%q ' "$@")" <<<"$(last)" || fail "nextflow lacks '$*': $(last)"; } +tools_arg() { grep -oE -- '--tools [^ ]+' <<<"$(last)" | cut -d' ' -f2 | tr -d '\134' | tr ',' '\n' | LC_ALL=C sort; } +STATUS="${G}/sample1/logs/run_status.tsv" +WANT=$(awk -F'\t' '$1 == "tool" && $2 != "html_report" {print $2}' "${REPO_ROOT}/assets/everything.tsv" | LC_ALL=C sort) +[ "$(grep -c . <<<"$WANT")" -ge 40 ] || fail "assets/everything.tsv has fewer than 40 tool rows: ${WANT}" + +# Every data folder there, a male BAM sample: every tool row runs, and every script-only step is queued. +run_rc all env EVERYTHING=true "${SCRIPTS}/run-all.sh" sample1 male +[ "$(nfcalls)" -eq 1 ] || fail "nextflow was not called once" +[ "$(tools_arg)" = "$WANT" ] || fail "--tools is not every tool row of assets/everything.tsv but html_report: +$(diff <(echo "$WANT") <(tools_arg))" +output_lacks all '^ [0-9]+b? .* skipped ' +has_args --kir true +has_args --hla_qc true +output_has all '^\[Everything\] EVERYTHING=true' +for s in 04a-tiddit 04b-gridss 14-imputation-prep 29-mutect2-somatic 03a-gatk-haplotypecaller 03b-freebayes \ + 03c-strelka2-germline 03d-octopus benchmark-variants; do + [ -f "${G}/sample1/logs/${s}.log" ] || fail "scripts/${s}.sh did not run after the pipeline" +done +grep -qE $'^step\t04a\t(ok|failed)$' "$STATUS" || fail "step 04a (TIDDIT=true) has no status line" +grep -qE $'^meta\tcommand\t.*EVERYTHING=true ' "$STATUS" || fail "the recorded command lacks EVERYTHING=true: $(grep command "$STATUS")" +if grep -qE $'^meta\tcommand\t.*TOOLS=' "$STATUS"; then fail "the recorded command names TOOLS: a replay would refuse it"; fi + +# Without Yleaf's tables: exit 2 before Nextflow, naming the install command. +mv "${G}/reference/yleaf-${YLEAF_DATA_VERSION}" "${CASE_WORK}/yleaf-away" +n=$(nfcalls) +run_expect 2 no-yleaf env EVERYTHING=true "${SCRIPTS}/run-all.sh" sample1 male +[ "$(nfcalls)" -eq "$n" ] || fail "nextflow started without Yleaf's tables" +output_has no-yleaf 'ERROR: EVERYTHING=true runs every optional step, and 1 cannot run' +output_has no-yleaf '^ 37 Y haplogroup \(Yleaf\): data not installed: reference/yleaf-.*; install: setup.sh --yleaf-data$' + +# A female sample still runs without them: the Y steps are skipped for the sample, not for want of data. +run_rc female env EVERYTHING=true IMPUTATION=true "${SCRIPTS}/run-all.sh" sample1 female +[ "$(nfcalls)" -eq $((n + 1)) ] || fail "nextflow did not run for a female sample" +output_has female '^ 37 Y haplogroup \(Yleaf\) +skipped +\(female sample: no Y chromosome\)$' +output_has female '^ 45 Y-STR markers \(HipSTR\) +skipped +\(female sample: no Y chromosome\)$' +if tools_arg | grep -qE '^y_(haplogroup|str)$'; then fail "a Y step reached --tools for a female sample: $(last)"; fi +mv "${CASE_WORK}/yleaf-away" "${G}/reference/yleaf-${YLEAF_DATA_VERSION}" + +# A BAM that is not PacBio HiFi: og paraphase check-bam exits 3, and step 35b is a skip, not a stop. +cat > "${CASE_WORK}/not-hifi" <<'HOOK' +#!/usr/bin/env bash +case "$*" in *"og paraphase check-bam "*) exit 3 ;; esac +exec "${CASE_WORK}/hook-outputs" "$@" +HOOK +chmod +x "${CASE_WORK}/not-hifi" +run_rc not-hifi env EVERYTHING=true FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/not-hifi" "${SCRIPTS}/run-all.sh" sample1 male +[ "$(nfcalls)" -eq $((n + 2)) ] || fail "nextflow did not run with a BAM that is not PacBio HiFi" +output_has not-hifi '^ 35b Paraphase \(PacBio HiFi\) +skipped +\(not PacBio HiFi: a read group of the BAM is not PL:PACBIO\)$' +if tools_arg | grep -qx paraphase; then fail "paraphase reached --tools for a BAM that is not PacBio HiFi"; fi +docker_log_has 'og paraphase check-bam /genome/sample1/aligned/sample1_sorted\.bam' "run-all.sh did not check the BAM with og paraphase check-bam" + +# EVERYTHING names the steps and sets the switches itself: another value is refused before anything runs. +run_expect 2 with-tools env EVERYTHING=true TOOLS=pharmcat "${SCRIPTS}/run-all.sh" sample1 male +output_has with-tools 'EVERYTHING=true names every step itself: unset TOOLS' +run_expect 2 kir-off env EVERYTHING=true KIR=false "${SCRIPTS}/run-all.sh" sample1 male +output_has kir-off 'EVERYTHING=true sets KIR=true: unset KIR' +run_expect 2 none env EVERYTHING=true ARCHAIC_DIR=none "${SCRIPTS}/run-all.sh" sample1 male +output_has none '^ 43 Archaic ancestry \(IBDmix\): ARCHAIC_DIR=none: unset ARCHAIC_DIR$' +[ "$(nfcalls)" -eq $((n + 2)) ] || fail "nextflow started on a refused EVERYTHING=true run" + +# Without EVERYTHING a default run leaves the opt-in steps out, as before. +run_expect 0 default "${SCRIPTS}/run-all.sh" sample1 male +output_has default '^ 37 Y haplogroup \(Yleaf\) +skipped +\(opt-in: add y_haplogroup to TOOLS\)$' +if grep -qE -- ' --kir true' <<<"$(last)"; then fail "a default run passed --kir true"; fi +echo "EVERYTHING=true runs every optional step, and stops before Nextflow when one lacks its data." From 87fc304648537e547ba90a74378f408698c56682 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:21:15 +0100 Subject: [PATCH 4/5] docs(run): say what EVERYTHING=true and -profile everything run Bead: D8 --- CHANGELOG.md | 1 + docs/getting-started.md | 17 ++++++++++++++++- docs/parameters.md | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb7a130e7..75c642f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- One switch now runs every optional step. Before, a run that should run everything had to name each opt-in step in `TOOLS` and set each switch by hand, and nothing said when a new opt-in was missing from that list. `EVERYTHING=true ./scripts/run-all.sh ` reads the new list `assets/everything.tsv`: it puts every `--tools` name in `TOOLS`, sets `KIR`, `HLA_QC`, `TIDDIT`, `GRIDSS`, `IMPUTATION`, `SOMATIC` and `BENCHMARK` to `true` and `EXTRA_CALLERS` to all four callers. A step whose data is not installed, or a panel set to `none`, stops the run with exit 2 before Nextflow, and the error names the setup command. `TOOLS`, or a switch set to another value, also stops it with exit 2. A skip that comes from the sample stays a skip: no BAM, no gVCF, a female sample for the Y steps, a BAM that is not PacBio HiFi for Paraphase. On the Nextflow path, `-profile everything` (`conf/everything.config`) sets the same tools and switches. `og dev everything-check`, in Guard, fails when `main.nf`, `nextflow.config`, `run-all.sh`, `setup.sh` or `versions.env` gains an opt-in with no row in the list. Without `EVERYTHING` a default run does not change, with three exceptions. `TIDDIT=true` is a new switch that runs `04a-tiddit.sh` after the pipeline. Step 37 (Yleaf) on a female sample is now listed as `skipped (female sample: no Y chromosome)`, as step 45 already was; the pipeline skipped it for her before, so no result changes. Step 35b (Paraphase) on a BAM that is not PacBio HiFi, or on a FASTQ row, is now `skipped (not PacBio HiFi: ...)` instead of a failed step: `run-all.sh` runs `og paraphase check-bam` on the BAM first. `logs/run_status.tsv` records `EVERYTHING` and `TIDDIT` with the other switches, and `og recheck --rerun` replays them. - A third-party outage no longer fails the Freshness check of a pull request. Since 2026-11-18 rest.ensembl.org has not answered, and the pull-request job's dry run of `og dev freshness` failed on every pull request that touched `versions.env` or `og-fresh`. `og dev freshness --unreachable-warns` now lists a lookup whose upstream did not answer (no connection, a timeout, HTTP 5xx or 429 after the retries, or HTTP 403) under "Upstreams that did not answer" as a warning, and its database row reads `NO ANSWER`; it does not change the exit. The 403 counts because rest.ensembl.org came back on 2026-11-19 answering 403 to GitHub's runners and 200 elsewhere, which still failed the pull-request job. A 404, an empty answer or an answer in an odd shape still exits 1. `freshness.yml` passes the flag only in the pull-request dry run. Without the flag the report and the exit are as before, and the monthly report on main still counts an outage as an error in the issue. - Three statements in the docs were wrong. Troubleshooting said exit code 137 confirms a kill for memory, and checked it with `docker ps -a --latest`. 137 is any SIGKILL, and the step scripts run their containers with `--rm`, so that command shows an older, unrelated container. The page now says how to confirm a kill for memory: `docker events --filter event=oom`, the kernel log, or `.State.OOMKilled` for a container kept without `--rm`. The run-state and troubleshooting pages said `bin/og` does not run on a Mac and told Mac users to build `og` with cargo or run it in a container. `bin/og` runs the native macOS build that `scripts/setup.sh --og` installs, and the pages now say so; on a Mac, `og run headroom` reports the machine's memory as unknown. Troubleshooting, the FAQ and the lessons-learned page said the step scripts run their containers as root, and lessons-learned gave `:latest` images and `chmod 777` as current fixes. The step scripts run their containers as you; those entries are now marked as past fixes and name what the pipeline does today. Documentation only: no option, output or behaviour changes. - Lint's ShellCheck job now checks every tracked shell script, not only `scripts/`: the e2e cases and their library, the fake-docker and parity tests, `tests/*.sh`, `bin/og` and the fake tools, 482 files in all, at every ShellCheck level. The 265 findings these files hold today are listed in `tests/shellcheck-baseline.txt` and still pass; any other finding fails lint. A listed finding is matched by its file, its code and the text of its line, so it keeps matching when lines above it move, and a new line with the same mistake still fails. Each run plants an unquoted `$1` in a new test script and on a new line of a listed script, and fails if either passes or if the file search misses the new script or `bin/og`. The `scripts/` check at warning level stays as it was. Part of #411 (plat-10). diff --git a/docs/getting-started.md b/docs/getting-started.md index 9e37ca022..0251fbd75 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -177,9 +177,10 @@ When the run starts from an existing VCF, the pipeline does not read a gVCF besi | `MANTA_CALL_REGIONS=file` | a bgzipped BED of the regions Manta calls | `--manta_call_regions` | | `ANCESTRY_PANEL=file`, `PGSC_CALC_DIR=dir` | another ancestry panel (step 26 and the percentiles of step 25), another pgsc_calc checkout; both are passed when `setup.sh` installed them, and `ANCESTRY_PANEL=none` scores without the panel | `--ancestry_ref`, `--pgsc_calc` | | `ANCESTRY_FINE_DIR=dir` | another fine-grained ancestry panel (step 26b); passed when `setup.sh --ancestry-fine-panel` installed it, and `ANCESTRY_FINE_DIR=none` skips the step | `--ancestry_fine_panel` | -| `GRIDSS=true`, `IMPUTATION=true`, `SOMATIC=true` | run steps 4b, 14 and 29 as scripts after the pipeline | | +| `TIDDIT=true`, `GRIDSS=true`, `IMPUTATION=true`, `SOMATIC=true` | run steps 4a, 4b, 14 and 29 as scripts after the pipeline | | | `EXTRA_CALLERS=gatk,freebayes,strelka2,octopus` | run the alternative callers 3a to 3d after the pipeline | | | `BENCHMARK=true` | runs `benchmark-variants.sh` after them; it needs `EXTRA_CALLERS` or a second caller's VCF from an earlier 3a-3d run, and is skipped without one | | +| `EVERYTHING=true` | runs every optional step: see [Every optional step](#every-optional-step) | `-profile everything` | GRIDSS is not part of the SV consensus (step 22): it reports every event as a pair of breakends, which never match the DEL, DUP and INV records of Manta, Delly and CNVpytor. @@ -219,6 +220,20 @@ The `work/` folder holds a copy of every intermediate file (from FASTQ, several **After a failure** run the same command again: `-resume` reuses every task that finished and runs the rest, including from the start the one that was running when the run stopped. **To rerun one step by hand**, run its script, for example `./scripts/06-clinvar-screen.sh your_name` after a ClinVar update, then `./scripts/24-html-report.sh your_name` and `./scripts/generate-report.sh your_name` to refresh the reports. A script reads the folders the scripts write; where the pipeline's folder differs (above), a script that reads another step's output may need that step run by hand first. +### Every optional step + +`EVERYTHING=true ./scripts/run-all.sh your_name male` runs every step the pipeline has, the opt-in ones too. The list is [`assets/everything.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/everything.tsv), one row per optional item: + +- every `--tools` name goes into `TOOLS`, so Parascopy, Paraphase, the Y steps, the raw-data export, blood groups, Lp(a), VNtyper, CHIP, viral DNA, ancient populations, MEGAnE, MoChA and Locityper run; +- `KIR`, `HLA_QC`, `TIDDIT`, `GRIDSS`, `IMPUTATION`, `SOMATIC` and `BENCHMARK` are `true`, and `EXTRA_CALLERS` is `gatk,freebayes,strelka2,octopus`; +- `TOOLS`, or one of these switches set to another value, stops the run with exit 2. + +A step whose data is not installed stops the run with exit 2 before Nextflow starts, and the error names the command that installs it, for example `setup.sh --yleaf-data`. So does `ANCESTRY_PANEL`, `ANCESTRY_FINE_DIR` or `ARCHAIC_DIR` set to `none`. Cyrius counts too: `setup.sh --cyrius` installs it, and you accept its non-commercial licence when you run that command. A skip that comes from the sample stays a skip: no BAM, no gVCF, a female sample for the two Y steps, a BAM that is not PacBio HiFi for Paraphase. + +Some things only you can give, so `EVERYTHING=true` leaves them to you: the symptoms for Exomiser (`HPO`), the families file (`FAMILIES`) and the drugs you take (`--medications`). The long-read scripts (2b, 3e, 4c) read ONT or PacBio HiFi reads, so you run them by hand on such data. One step stays out: `cram_archive` writes a CRAM copy of each BAM, which costs disk and adds no result. + +On the Nextflow path, `-profile docker,everything` sets the same tools and switches (`conf/everything.config`). Pass the database of each tool; `main.nf` stops before any task and names the option a tool lacks. CI fails when a new opt-in step, switch, setup flag or optional image has no row in the list (`og dev everything-check`). + ### Path A: I Have FASTQ Files (Raw Reads) The paths below run the steps one by one, with Docker alone. Most common if you downloaded data from Nebula, Dante Labs, Novogene, BGI, or any sequencing provider. diff --git a/docs/parameters.md b/docs/parameters.md index b4cbdec3b..ee07bd76f 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -38,7 +38,7 @@ Paths to reference files required by the pipeline. ## Tool selection -Control which analysis tools to run. +Control which analysis tools to run. `-profile everything` (`conf/everything.config`) sets `--tools` to every tool of [`assets/everything.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/everything.tsv), and `--kir`, `--hla_qc` and `--cpsr_secondary_findings` to `true`: every optional step, as `EVERYTHING=true` does on `run-all.sh` ([Every optional step](getting-started.md#every-optional-step)). It leaves out `exomiser`, which runs when `--hpo` is set, and `cram_archive`. | Parameter | Default | What it does | |---|---|---| From c3a3c78ea7442afa461b4a55bb3f53e43ad1a928 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:42:03 +0100 Subject: [PATCH 5/5] fix(run): EVERYTHING=true stops before Nextflow without GRIDSS's BWA index GRIDSS (script 04b, after the pipeline) needs the classic BWA index, which setup.sh does not build; an EVERYTHING=true run now refuses up front instead of failing hours in. A refused run puts the last run's run_status.tsv back and posts no run-finished event. pipeline-overview.md lists TIDDIT=true. --- CHANGELOG.md | 2 +- docs/getting-started.md | 2 +- docs/pipeline-overview.md | 2 +- scripts/run-all.sh | 13 +++++++++++-- tests/fake-docker/run-all-everything.sh | 21 +++++++++++++++++++-- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75c642f71..e04e8a426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- One switch now runs every optional step. Before, a run that should run everything had to name each opt-in step in `TOOLS` and set each switch by hand, and nothing said when a new opt-in was missing from that list. `EVERYTHING=true ./scripts/run-all.sh ` reads the new list `assets/everything.tsv`: it puts every `--tools` name in `TOOLS`, sets `KIR`, `HLA_QC`, `TIDDIT`, `GRIDSS`, `IMPUTATION`, `SOMATIC` and `BENCHMARK` to `true` and `EXTRA_CALLERS` to all four callers. A step whose data is not installed, or a panel set to `none`, stops the run with exit 2 before Nextflow, and the error names the setup command. `TOOLS`, or a switch set to another value, also stops it with exit 2. A skip that comes from the sample stays a skip: no BAM, no gVCF, a female sample for the Y steps, a BAM that is not PacBio HiFi for Paraphase. On the Nextflow path, `-profile everything` (`conf/everything.config`) sets the same tools and switches. `og dev everything-check`, in Guard, fails when `main.nf`, `nextflow.config`, `run-all.sh`, `setup.sh` or `versions.env` gains an opt-in with no row in the list. Without `EVERYTHING` a default run does not change, with three exceptions. `TIDDIT=true` is a new switch that runs `04a-tiddit.sh` after the pipeline. Step 37 (Yleaf) on a female sample is now listed as `skipped (female sample: no Y chromosome)`, as step 45 already was; the pipeline skipped it for her before, so no result changes. Step 35b (Paraphase) on a BAM that is not PacBio HiFi, or on a FASTQ row, is now `skipped (not PacBio HiFi: ...)` instead of a failed step: `run-all.sh` runs `og paraphase check-bam` on the BAM first. `logs/run_status.tsv` records `EVERYTHING` and `TIDDIT` with the other switches, and `og recheck --rerun` replays them. +- One switch now runs every optional step. Before, a run that should run everything had to name each opt-in step in `TOOLS` and set each switch by hand, and nothing said when a new opt-in was missing from that list. `EVERYTHING=true ./scripts/run-all.sh ` reads the new list `assets/everything.tsv`: it puts every `--tools` name in `TOOLS`, sets `KIR`, `HLA_QC`, `TIDDIT`, `GRIDSS`, `IMPUTATION`, `SOMATIC` and `BENCHMARK` to `true` and `EXTRA_CALLERS` to all four callers. A step whose data is not installed, or a panel set to `none`, stops the run with exit 2 before Nextflow, and the error names the setup command. That includes the classic BWA index that GRIDSS (step 4b) needs, which `setup.sh` does not build. A refused run leaves the last run's `logs/run_status.tsv` as it was. `TOOLS`, or a switch set to another value, also stops it with exit 2. A skip that comes from the sample stays a skip: no BAM, no gVCF, a female sample for the Y steps, a BAM that is not PacBio HiFi for Paraphase. On the Nextflow path, `-profile everything` (`conf/everything.config`) sets the same tools and switches. `og dev everything-check`, in Guard, fails when `main.nf`, `nextflow.config`, `run-all.sh`, `setup.sh` or `versions.env` gains an opt-in with no row in the list. Without `EVERYTHING` a default run does not change, with three exceptions. `TIDDIT=true` is a new switch that runs `04a-tiddit.sh` after the pipeline. Step 37 (Yleaf) on a female sample is now listed as `skipped (female sample: no Y chromosome)`, as step 45 already was; the pipeline skipped it for her before, so no result changes. Step 35b (Paraphase) on a BAM that is not PacBio HiFi, or on a FASTQ row, is now `skipped (not PacBio HiFi: ...)` instead of a failed step: `run-all.sh` runs `og paraphase check-bam` on the BAM first. `logs/run_status.tsv` records `EVERYTHING` and `TIDDIT` with the other switches, and `og recheck --rerun` replays them. - A third-party outage no longer fails the Freshness check of a pull request. Since 2026-11-18 rest.ensembl.org has not answered, and the pull-request job's dry run of `og dev freshness` failed on every pull request that touched `versions.env` or `og-fresh`. `og dev freshness --unreachable-warns` now lists a lookup whose upstream did not answer (no connection, a timeout, HTTP 5xx or 429 after the retries, or HTTP 403) under "Upstreams that did not answer" as a warning, and its database row reads `NO ANSWER`; it does not change the exit. The 403 counts because rest.ensembl.org came back on 2026-11-19 answering 403 to GitHub's runners and 200 elsewhere, which still failed the pull-request job. A 404, an empty answer or an answer in an odd shape still exits 1. `freshness.yml` passes the flag only in the pull-request dry run. Without the flag the report and the exit are as before, and the monthly report on main still counts an outage as an error in the issue. - Three statements in the docs were wrong. Troubleshooting said exit code 137 confirms a kill for memory, and checked it with `docker ps -a --latest`. 137 is any SIGKILL, and the step scripts run their containers with `--rm`, so that command shows an older, unrelated container. The page now says how to confirm a kill for memory: `docker events --filter event=oom`, the kernel log, or `.State.OOMKilled` for a container kept without `--rm`. The run-state and troubleshooting pages said `bin/og` does not run on a Mac and told Mac users to build `og` with cargo or run it in a container. `bin/og` runs the native macOS build that `scripts/setup.sh --og` installs, and the pages now say so; on a Mac, `og run headroom` reports the machine's memory as unknown. Troubleshooting, the FAQ and the lessons-learned page said the step scripts run their containers as root, and lessons-learned gave `:latest` images and `chmod 777` as current fixes. The step scripts run their containers as you; those entries are now marked as past fixes and name what the pipeline does today. Documentation only: no option, output or behaviour changes. - Lint's ShellCheck job now checks every tracked shell script, not only `scripts/`: the e2e cases and their library, the fake-docker and parity tests, `tests/*.sh`, `bin/og` and the fake tools, 482 files in all, at every ShellCheck level. The 265 findings these files hold today are listed in `tests/shellcheck-baseline.txt` and still pass; any other finding fails lint. A listed finding is matched by its file, its code and the text of its line, so it keeps matching when lines above it move, and a new line with the same mistake still fails. Each run plants an unquoted `$1` in a new test script and on a new line of a listed script, and fails if either passes or if the file search misses the new script or `bin/og`. The `scripts/` check at warning level stays as it was. Part of #411 (plat-10). diff --git a/docs/getting-started.md b/docs/getting-started.md index 0251fbd75..3c0585e61 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -228,7 +228,7 @@ The `work/` folder holds a copy of every intermediate file (from FASTQ, several - `KIR`, `HLA_QC`, `TIDDIT`, `GRIDSS`, `IMPUTATION`, `SOMATIC` and `BENCHMARK` are `true`, and `EXTRA_CALLERS` is `gatk,freebayes,strelka2,octopus`; - `TOOLS`, or one of these switches set to another value, stops the run with exit 2. -A step whose data is not installed stops the run with exit 2 before Nextflow starts, and the error names the command that installs it, for example `setup.sh --yleaf-data`. So does `ANCESTRY_PANEL`, `ANCESTRY_FINE_DIR` or `ARCHAIC_DIR` set to `none`. Cyrius counts too: `setup.sh --cyrius` installs it, and you accept its non-commercial licence when you run that command. A skip that comes from the sample stays a skip: no BAM, no gVCF, a female sample for the two Y steps, a BAM that is not PacBio HiFi for Paraphase. +A step whose data is not installed stops the run with exit 2 before Nextflow starts, and the error names the command that installs it, for example `setup.sh --yleaf-data`. So does `ANCESTRY_PANEL`, `ANCESTRY_FINE_DIR` or `ARCHAIC_DIR` set to `none`. The check covers the classic BWA index that GRIDSS (step 4b, after the pipeline) reads beside the reference: `setup.sh` does not build it, and the error names `bwa index` ([4b](04b-gridss.md)). A refused run leaves the last run's `logs/run_status.tsv` as it was. Cyrius counts too: `setup.sh --cyrius` installs it, and you accept its non-commercial licence when you run that command. A skip that comes from the sample stays a skip: no BAM, no gVCF, a female sample for the two Y steps, a BAM that is not PacBio HiFi for Paraphase. Some things only you can give, so `EVERYTHING=true` leaves them to you: the symptoms for Exomiser (`HPO`), the families file (`FAMILIES`) and the drugs you take (`--medications`). The long-read scripts (2b, 3e, 4c) read ONT or PacBio HiFi reads, so you run them by hand on such data. One step stays out: `cram_archive` writes a CRAM copy of each BAM, which costs disk and adds no result. diff --git a/docs/pipeline-overview.md b/docs/pipeline-overview.md index 8f28913c3..313c9de9e 100644 --- a/docs/pipeline-overview.md +++ b/docs/pipeline-overview.md @@ -178,7 +178,7 @@ These run after the core pipeline completes and combine outputs from earlier ste A default `./scripts/run-all.sh ` runs **37 numbered steps**: 1b and 2 (only when there is no BAM yet), 3 (only when there is no VCF yet), 4, 5, 6, 7, 8, 9, 9b, 10, 11, 12, 13, 15, 16, 16b, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 26b, 27, 28, 30, 31, 32, 33, 36, 42 and 43. Steps 5, 8, 13, 17, 18, 32 and 33 are reported as skipped when their data is not installed (for 33, somalier's sites and VerifyBamID2's panel, which `setup.sh` installs), 21 when Cyrius is not installed (`setup.sh --cyrius`; non-commercial licence), 25 when no score file is, 26 (inside the PRS run of step 25) when the ancestry panel is not, 26b when its panel (`setup.sh --ancestry-fine-panel`) is not, 43 when its table (`setup.sh --archaic`) is not, 23, 30 and 31 when step 13 did not run, 36 (inside the PharmCAT stage) runs when step 8 or 32 did, and 42 and 43 (43 once its table is installed) are reported as skipped for a sample that starts from a VCF, unless it has a BAM too and a gVCF beside the VCF (the gVCF `run-all.sh` passes on). It ends with the summary report (`generate-report.sh`). -Off unless you ask for them: 35 (`TOOLS=...,parascopy`, after `setup.sh --parascopy-data`), 35b (`TOOLS=...,paraphase`, on a PacBio HiFi BAM), KIR typing in step 8 (`KIR=true`, after `setup.sh --kir-data`), OptiType beside T1K in step 8 (`HLA_QC=true`), 4b (`GRIDSS=true`), 14 (`IMPUTATION=true`), 29 (`SOMATIC=true`), 37 (`TOOLS=...,y_haplogroup`, after `setup.sh --yleaf-data`, male samples), 40 (`TOOLS=...,consumer_export`, after `setup.sh --consumer-sites`), 41 (`TOOLS=...,bloodtype`), 44 (`TOOLS=...,ancient`, after `setup.sh --ancient-panel`), 45 (`TOOLS=...,y_str`, male samples), 46 (`TOOLS=...,lpa`), 47 (`TOOLS=...,viral`, after `setup.sh --viral-data`), 48 (`HPO=...` with the symptoms as HPO terms, after `setup.sh --exomiser-data`), 49 (`TOOLS=...,chip`, after `setup.sh --chip-data`), 50 (`TOOLS=...,mei`, after `setup.sh --megane-kmers`), 51 (`TOOLS=...,mocha`, after `setup.sh --mocha-data`), 52 (`TOOLS=...,vntyper`), 53 (`TOOLS=...,locityper`, after `setup.sh --locityper-data`), the alternative callers 3a to 3d (`EXTRA_CALLERS=gatk,freebayes,strelka2,octopus`) and the caller comparison (`BENCHMARK=true`). Step 1 (ORA input) and the other alternative scripts (2a, 2b, 3e, 4a, 4c) run only by hand. Step 33 stops the run when somalier's sex from the reads differs from the declared one, as step 16 does (`--sex_check warn` logs it and goes on). Step 34 runs by hand, or in the Nextflow pipeline with `cram_archive` in `--tools`. Steps 38 and 39 work on a family, not one sample. With `FAMILIES=`, `run-all.sh` runs them after the pipeline once the members they need are processed, and records `skipped (waiting: ...)` until then ([Running it](multi-sample.md#running-it)). Without `FAMILIES`, run `38-family.sh` and `39-couple.sh` by hand with a pedigree, or give the Nextflow pipeline `--pedigree` (step 39 then runs for two samples the pedigree marks as partners, when `clinvar` is in `--tools`). +Off unless you ask for them: 35 (`TOOLS=...,parascopy`, after `setup.sh --parascopy-data`), 35b (`TOOLS=...,paraphase`, on a PacBio HiFi BAM), KIR typing in step 8 (`KIR=true`, after `setup.sh --kir-data`), OptiType beside T1K in step 8 (`HLA_QC=true`), 4a (`TIDDIT=true`), 4b (`GRIDSS=true`), 14 (`IMPUTATION=true`), 29 (`SOMATIC=true`), 37 (`TOOLS=...,y_haplogroup`, after `setup.sh --yleaf-data`, male samples), 40 (`TOOLS=...,consumer_export`, after `setup.sh --consumer-sites`), 41 (`TOOLS=...,bloodtype`), 44 (`TOOLS=...,ancient`, after `setup.sh --ancient-panel`), 45 (`TOOLS=...,y_str`, male samples), 46 (`TOOLS=...,lpa`), 47 (`TOOLS=...,viral`, after `setup.sh --viral-data`), 48 (`HPO=...` with the symptoms as HPO terms, after `setup.sh --exomiser-data`), 49 (`TOOLS=...,chip`, after `setup.sh --chip-data`), 50 (`TOOLS=...,mei`, after `setup.sh --megane-kmers`), 51 (`TOOLS=...,mocha`, after `setup.sh --mocha-data`), 52 (`TOOLS=...,vntyper`), 53 (`TOOLS=...,locityper`, after `setup.sh --locityper-data`), the alternative callers 3a to 3d (`EXTRA_CALLERS=gatk,freebayes,strelka2,octopus`) and the caller comparison (`BENCHMARK=true`). `EVERYTHING=true` turns on all of these at once ([Every optional step](getting-started.md#every-optional-step)). Step 1 (ORA input) and the other alternative scripts (2a, 2b, 3e, 4c) run only by hand. Step 33 stops the run when somalier's sex from the reads differs from the declared one, as step 16 does (`--sex_check warn` logs it and goes on). Step 34 runs by hand, or in the Nextflow pipeline with `cram_archive` in `--tools`. Steps 38 and 39 work on a family, not one sample. With `FAMILIES=`, `run-all.sh` runs them after the pipeline once the members they need are processed, and records `skipped (waiting: ...)` until then ([Running it](multi-sample.md#running-it)). Without `FAMILIES`, run `38-family.sh` and `39-couple.sh` by hand with a pedigree, or give the Nextflow pipeline `--pedigree` (step 39 then runs for two samples the pedigree marks as partners, when `clinvar` is in `--tools`). The [Nextflow pipeline](nextflow.md) runs the same chain from a samplesheet, from FASTQ (steps 1b, 2, 16 and 3) to the report, with the steps above that have a module. diff --git a/scripts/run-all.sh b/scripts/run-all.sh index caa04ba62..dfa11e549 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -132,6 +132,8 @@ fi mkdir -p "${S}/nextflow" "$LOG_DIR" GENOME_DIR="$G" bash "${PGP_ROOT}/bin/write_manifest.sh" "$SAMPLE" run-all.sh "$SEX" || echo "WARNING: could not write ${S}/run_manifest.tsv" STATUS="${LOG_DIR}/run_status.tsv" # bin/collect_summary.py marks results older than this run stale +# EVERYTHING=true can still refuse before Nextflow: keep the last run's status to put back then. +[ -z "$EV" ] || [ ! -f "$STATUS" ] || cp "$STATUS" "${STATUS}.prev" printf '# run-all.sh: when this run started and how each step ended\nmeta\tstarted_epoch\t%s\nmeta\tstarted_utc\t%s\nmeta\tdeclared_sex\t%s\nmeta\tcommand\t%s\nmeta\tswitches\t%s\n' \ "$(date +%s)" "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" "$SEX" "$RERUN_CMD" "${SWITCHES[*]}" > "$STATUS" # As run-all.sh ends, however it ends: a step still 'planned' did not finish, and the time is recorded @@ -340,10 +342,17 @@ case "$LD" in /*) ;; *) LD="${G}/${LD}" ;; esac if [[ ",${T// /}," == *",locityper,"* ]]; then plan "53 Hard loci (Locityper)" locityper "$(if [ -z "$NOBAM" ]; then echo 'no BAM'; else need "${LD}/DONE"; fi)" \ && NF+=(--locityper_data "$LD") && { [ -z "${LOCITYPER_BG_REGION:-}" ] || NF+=(--locityper_bg_region "$LOCITYPER_BG_REGION"); } || true; else KNOWN+=" locityper"; fi for t in ${T//,/ }; do [[ "${KNOWN} " == *" ${t} "* ]] || { echo "ERROR: unknown step '${t}' in TOOLS. Known:${KNOWN}" >&2; exit 2; }; done +# Step 4b (GRIDSS=true) runs after the pipeline, and GRIDSS needs the classic BWA index beside the reference, which +# setup.sh does not build: check it now, so the run stops here and not after hours of pipeline. +if [ -n "$EV" ] && [ -n "$NOBAM" ]; then for e in amb ann bwt pac sa; do [ -f "${REF_FASTA}.${e}" ] || { + EVMISS+=("04b GRIDSS (GRIDSS=true): classic BWA index not installed: ${REF_FASTA#"$G"/}.${e}; install: bwa index (docs/04b-gridss.md)"); break; }; done; fi if [ "${#EVMISS[@]}" -gt 0 ]; then - { echo "ERROR: EVERYTHING=true runs every optional step, and ${#EVMISS[@]} cannot run (nothing started):"; printf ' %s\n' "${EVMISS[@]}" - echo "Install what is missing, or run without EVERYTHING=true to skip them."; } >&2; exit 2 + { echo "ERROR: EVERYTHING=true runs every optional step, and ${#EVMISS[@]} cannot run (Nextflow did not start):"; printf ' %s\n' "${EVMISS[@]}" + echo "Install what is missing, or run without EVERYTHING=true to skip them."; } >&2 + # Nothing ran: the last run's status stays as it was (or there is none), and no run-finished event goes out. + trap - EXIT; if [ -f "${STATUS}.prev" ]; then mv "${STATUS}.prev" "$STATUS"; else rm -f "$STATUS"; fi; exit 2 fi +rm -f "${STATUS}.prev" arg --cytoband "$(data_file cytoband || true)"; ! GEX=$(data_file gencode_exons) || NF+=(--gencode_exons "$GEX"); arg --delly_exclude "$(data_file delly_exclude || true)"; arg --manta_call_regions "${MANTA_CALL_REGIONS:-}" # Step 20's annotation of the chrM calls (MITO_VARIANTS_ANNOTATE): ClinVar with its index, and gnomAD's chrM sites. MC="${G}/clinvar/clinvar_chr.vcf.gz"; if [ -s "$MC" ] && [ -s "${MC}.tbi" ]; then NF+=(--mito_clinvar "$MC"); fi; arg --gnomad_mito "${G}/gnomad_mito/${GNOMAD_MITO_URL##*/}" diff --git a/tests/fake-docker/run-all-everything.sh b/tests/fake-docker/run-all-everything.sh index f2fd8058e..616a7ef05 100644 --- a/tests/fake-docker/run-all-everything.sh +++ b/tests/fake-docker/run-all-everything.sh @@ -5,7 +5,10 @@ # --tools and listed as runs, with --kir true and --hla_qc # true, and the scripts of TIDDIT, GRIDSS, IMPUTATION, SOMATIC, # every EXTRA_CALLERS and BENCHMARK run after the pipeline -# no Yleaf exit 2 before Nextflow, naming setup.sh --yleaf-data +# no Yleaf exit 2 before Nextflow, naming setup.sh --yleaf-data, and +# the last run's logs/run_status.tsv is left as it was +# no BWA index exit 2 before Nextflow, naming the classic BWA index that +# script 04b (GRIDSS) needs and bwa index # female still runs; the two Y steps are skipped (female sample) # not HiFi a BAM whose read groups are not PL:PACBIO skips Paraphase # and the run goes on @@ -45,6 +48,9 @@ f "hla/IPD-IMGT-HLA_${HLA}/hla.dat" "reference/gencode.v${GC}.basic.genes.gtf" " "reference/ancient/aadr_${AADR_VERSION}/DONE" "reference/viral-${VIRAL_REF_VERSION}/viral_genomes.fa" \ "reference/megane/GRCh38_no_alt_analysis_set.mk" "reference/megane/GRCh38_no_alt_analysis_set.mi" \ "reference/mocha-${MOCHA_PANEL_VERSION}/DONE" "reference/locityper-${LOCITYPER_PANEL_VERSION}/DONE" +# GRIDSS (script 04b) reads the classic BWA index beside the reference. +BWAX="amb ann bwt pac sa" +for e in $BWAX; do f "reference/GRCh38_no_alt_analysis_set.fasta.${e}"; done # Cyrius counts as installed when its stamp names this lock file and image (setup.sh --cyrius). if command -v sha256sum > /dev/null 2>&1; then LOCK=$(sha256sum "${SCRIPTS}/cyrius-constraints.txt"); else LOCK=$(shasum -a 256 "${SCRIPTS}/cyrius-constraints.txt"); fi mkdir -p "${G}/tools/cyrius-${CYRIUS_VERSION}" @@ -78,10 +84,14 @@ if grep -qE $'^meta\tcommand\t.*TOOLS=' "$STATUS"; then fail "the recorded comma # Without Yleaf's tables: exit 2 before Nextflow, naming the install command. mv "${G}/reference/yleaf-${YLEAF_DATA_VERSION}" "${CASE_WORK}/yleaf-away" n=$(nfcalls) +cp "$STATUS" "${CASE_WORK}/status-before" run_expect 2 no-yleaf env EVERYTHING=true "${SCRIPTS}/run-all.sh" sample1 male [ "$(nfcalls)" -eq "$n" ] || fail "nextflow started without Yleaf's tables" -output_has no-yleaf 'ERROR: EVERYTHING=true runs every optional step, and 1 cannot run' +output_has no-yleaf 'ERROR: EVERYTHING=true runs every optional step, and 1 cannot run \(Nextflow did not start\)' output_has no-yleaf '^ 37 Y haplogroup \(Yleaf\): data not installed: reference/yleaf-.*; install: setup.sh --yleaf-data$' +cmp -s "${CASE_WORK}/status-before" "$STATUS" || fail "a refused EVERYTHING=true run changed the last run's status: +$(diff "${CASE_WORK}/status-before" "$STATUS")" +[ ! -e "${STATUS}.prev" ] || fail "a refused EVERYTHING=true run left ${STATUS}.prev behind" # A female sample still runs without them: the Y steps are skipped for the sample, not for want of data. run_rc female env EVERYTHING=true IMPUTATION=true "${SCRIPTS}/run-all.sh" sample1 female @@ -91,6 +101,13 @@ output_has female '^ 45 Y-STR markers \(HipSTR\) +skipped +\(female sample: no if tools_arg | grep -qE '^y_(haplogroup|str)$'; then fail "a Y step reached --tools for a female sample: $(last)"; fi mv "${CASE_WORK}/yleaf-away" "${G}/reference/yleaf-${YLEAF_DATA_VERSION}" +# Without the classic BWA index: GRIDSS (script 04b, after the pipeline) would fail hours in, so the run stops first. +for e in $BWAX; do mv "${G}/reference/GRCh38_no_alt_analysis_set.fasta.${e}" "${CASE_WORK}/"; done +run_expect 2 no-bwa env EVERYTHING=true "${SCRIPTS}/run-all.sh" sample1 male +[ "$(nfcalls)" -eq $((n + 1)) ] || fail "nextflow started without the classic BWA index" +output_has no-bwa '^ 04b GRIDSS \(GRIDSS=true\): classic BWA index not installed: reference/GRCh38_no_alt_analysis_set\.fasta\.amb; install: bwa index \(docs/04b-gridss\.md\)$' +for e in $BWAX; do mv "${CASE_WORK}/GRCh38_no_alt_analysis_set.fasta.${e}" "${G}/reference/"; done + # A BAM that is not PacBio HiFi: og paraphase check-bam exits 3, and step 35b is a skip, not a stop. cat > "${CASE_WORK}/not-hifi" <<'HOOK' #!/usr/bin/env bash