From 0984c5d914df68b5c8e3188230b2dc1105d9134d Mon Sep 17 00:00:00 2001 From: Raymond Lim Date: Mon, 22 Jun 2026 12:25:58 -0400 Subject: [PATCH] add neural segmentation correctness validation --- .../neural_segmentation_correctness_pr136.md | 189 +++++++++++ ...n_neural_segmentation_correctness_panel.sh | 32 ++ .../neural_segmentation_correctness_panel.py | 312 ++++++++++++++++++ 3 files changed, 533 insertions(+) create mode 100644 docs/validation/neural_segmentation_correctness_pr136.md create mode 100755 tests/slurm/run_neural_segmentation_correctness_panel.sh create mode 100755 tests/validation/neural_segmentation_correctness_panel.py diff --git a/docs/validation/neural_segmentation_correctness_pr136.md b/docs/validation/neural_segmentation_correctness_pr136.md new file mode 100644 index 00000000..1949ee9d --- /dev/null +++ b/docs/validation/neural_segmentation_correctness_pr136.md @@ -0,0 +1,189 @@ +# Mussel Neural Segmentation Correctness Validation - PR #136 on main + +## Summary + +Validation target: fixed neural segmentation behavior on `main` after PR #136 (`fix circular utils import`) was merged. + +Conclusion: PASS. The neural segmentation path completed on a 10-slide panel spanning multiple formats, slide sizes, MPP values, and tissue burdens. All generated HDF5 outputs had valid coordinates, correct `seg_model` attributes, non-empty patch sets, and neural/classic patch-count ratios inside the configured sanity range. + +## Code Under Validation + +Repository: `pathology-data-mining/Mussel` + +Base branch: `origin/main` + +Validation branch: `codex/neural-segmentation-correctness-validation` + +Base commit: `5d6892e fix circular utils import (#136)` + +Validation commit: branch head at PR creation + +Relevant upstream fixes already present on `main`: + +- `5d6892e fix circular utils import (#136)` +- `fb35ace [codex] fix tessellate neural segmentation (#135)` +- `2230233 [codex] add batch mode to tessellate CLI (#134)` +- `86b2270 feat: add abmil_benchmark CLI for precision benchmarking (#124)` + +## Acceptance Criteria + +The validation fails if any slide has: + +- neural segmentation exception or `segment_tissue(..., seg_model="neural")` returns `None` +- zero neural patches +- missing or invalid HDF5 output +- HDF5 patch count mismatch against returned coordinates +- HDF5 `coords.attrs["seg_model"]` not equal to `"neural"` +- any generated coordinate outside slide bounds +- neural/classic patch-count ratio outside `[0.10, 10.0]` + +The ratio check is intentionally broad. It is not an equivalence test between classic HSV and neural segmentation; it is a bug-catcher for empty masks, pathological over-segmentation, wrong pyramid level selection, or broken rescaling. + +## Validation Harness + +Added reusable validation entrypoints: + +- [neural_segmentation_correctness_panel.py](/gpfs/mskmind_ess/limr/repos/Mussel-2/tests/validation/neural_segmentation_correctness_panel.py) +- [run_neural_segmentation_correctness_panel.sh](/gpfs/mskmind_ess/limr/repos/Mussel-2/tests/slurm/run_neural_segmentation_correctness_panel.sh) + +Panel command: + +```bash +sbatch --qos=premium tests/slurm/run_neural_segmentation_correctness_panel.sh +``` + +The SLURM wrapper runs: + +```bash +uv sync --extra torch-gpu +uv run python tests/validation/neural_segmentation_correctness_panel.py \ + --output-dir /gpfs/cdsi_ess/home/limr/logs/slurm/neural_seg_panel_${SLURM_JOB_ID} \ + --device cuda \ + --batch-size 8 +``` + +## Results + +### 10-slide neural segmentation panel + +SLURM job: `3563436` + +Node: `pllimsksparky3` + +State: `COMPLETED` + +Exit code: `0:0` + +Elapsed: `00:27:00` + +Max RSS: `6738208K` + +Log: + +`/gpfs/cdsi_ess/home/limr/logs/slurm/neural_seg_panel_3563436.out` + +Machine-readable outputs: + +- `/gpfs/cdsi_ess/home/limr/logs/slurm/neural_seg_panel_3563436/neural_segmentation_correctness_panel.csv` +- `/gpfs/cdsi_ess/home/limr/logs/slurm/neural_seg_panel_3563436/neural_segmentation_correctness_panel.json` + +Result: `10/10 passed` + +| # | Slide | Format/source | Dimensions | MPP | Classic patches | Neural patches | Ratio | Status | +|---:|---|---|---:|---:|---:|---:|---:|---| +| 1 | `948176` | SVS testdata | 85656 x 19917 | 0.5026 | 1515 | 1560 | 1.030 | PASS | +| 2 | `8471385` | SVS large local slide | 117749 x 77677 | 0.262882 | 9617 | 15038 | 1.564 | PASS | +| 3 | `1065626` | SVS small local slide | 11952 x 10249 | 0.5026 | 9 | 30 | 3.333 | PASS | +| 4 | `1005517` | SVS workflow slide | 89640 x 33960 | 0.5021 | 9491 | 18386 | 1.937 | PASS | +| 5 | `PROV-000-000001` | NDPI GigaPath sample | 145920 x 20736 | 0.23016 | 1015 | 1004 | 0.989 | PASS | +| 6 | `TCGA-AZ-4313` | TCGA SVS | 115109 x 62467 | 0.252 | 14916 | 15600 | 1.046 | PASS | +| 7 | `TCGA-86-8074` | TCGA SVS | 91128 x 71141 | 0.252 | 16863 | 13651 | 0.810 | PASS | +| 8 | `001d865e...` | PANDA TIFF | 28672 x 34560 | 0.452018 | 941 | 941 | 1.000 | PASS | +| 9 | `TCGA-RM-A68W` | TCGA SVS | 57767 x 50248 | 0.2527 | 5234 | 5231 | 0.999 | PASS | +| 10 | `1079807` | SVS local slide | 53784 x 39410 | 0.5026 | 4651 | 6649 | 1.430 | PASS | + +Observed ratio range: `0.810` to `3.333`. + +Observed neural patch-count range: `30` to `18386`. + +All failures fields were empty in the CSV summary. + +### Existing neural/artifact integration suite + +SLURM array job: `3563392` + +Tasks: `34-37` + +State: all `COMPLETED` + +Exit code: all `0:0` + +Covered tests: + +- `test_neural_segmentation_produces_valid_patches` +- `test_neural_segmentation_patch_count_close_to_hsv` +- `test_grandqc_artifact_remover_runs_on_real_slide` +- `test_grandqc_artifact_remover_integrated_with_segment_tissue` + +Logs: + +- `/gpfs/cdsi_ess/home/limr/logs/slurm/test_integration_3563392_34.out` +- `/gpfs/cdsi_ess/home/limr/logs/slurm/test_integration_3563392_35.out` +- `/gpfs/cdsi_ess/home/limr/logs/slurm/test_integration_3563392_36.out` +- `/gpfs/cdsi_ess/home/limr/logs/slurm/test_integration_3563392_37.out` + +### Tessellate/extract GPU E2E + +SLURM job: `3563398` + +State: `COMPLETED` + +Exit code: `0:0` + +Elapsed: `00:18:19` + +Max RSS: `4860184K` + +Log: + +`/gpfs/cdsi_ess/home/limr/logs/slurm/test_tef_e2e_pr136_3563398.out` + +Result: `6 passed, 11 deselected in 1088.87s` + +This specifically confirms that the prior DataLoader spawned-worker circular import failure no longer reproduces after PR #136 is present on `main`. + +### Import-order regression + +Command: + +```bash +uv run pytest tests/mussel/test_import_order.py -q +``` + +Result: `3 passed in 38.77s` + +## Risk Coverage + +This validation materially reduces the chance of bugs slipping through by covering: + +- 10 slides instead of a single fixture +- SVS, NDPI, and TIFF inputs +- slide widths from 11952 to 145920 pixels +- slide heights from 10249 to 77677 pixels +- MPP values from approximately 0.230 to 0.503 +- patch-count outputs from sparse tissue (`30` neural patches) to dense tissue (`18386` neural patches) +- neural segmentation HDF5 persistence and coordinate validity +- comparison against classic segmentation as a broad sanity check +- tessellate/extract E2E execution with DataLoader workers enabled + +## Limitations + +This is a correctness and regression validation, not a pathologist-reviewed segmentation-quality study. The classic-vs-neural ratio is used only as a broad anomaly detector. + +The panel includes local filesystem slides outside the repository. Re-running on another machine requires those paths or an equivalent manifest. + +## Final Assessment + +The fixed neural segmentation behavior passes the expanded correctness validation on current `main` with PR #136 included. + +Recommendation: retain the 10-slide panel as the broader regression check for future changes to segmentation level selection, neural mask rescaling, HDF5 coordinate output, or DataLoader/import behavior. diff --git a/tests/slurm/run_neural_segmentation_correctness_panel.sh b/tests/slurm/run_neural_segmentation_correctness_panel.sh new file mode 100755 index 00000000..828366be --- /dev/null +++ b/tests/slurm/run_neural_segmentation_correctness_panel.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# SLURM job: 10-slide neural segmentation correctness validation panel. +# +# Submit from the repo root: +# sbatch --qos=premium tests/slurm/run_neural_segmentation_correctness_panel.sh + +#SBATCH --job-name=mussel-neural-panel +#SBATCH --partition=hpc +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=48G +#SBATCH --time=3:00:00 +#SBATCH --output=/gpfs/cdsi_ess/home/limr/logs/slurm/neural_seg_panel_%j.out + +set -euo pipefail + +REPO_DIR="${SLURM_SUBMIT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +cd "$REPO_DIR" + +if [[ -f ~/.hf_cred.env ]]; then + source ~/.hf_cred.env +fi + +uv sync --extra torch-gpu + +OUT_DIR="/gpfs/cdsi_ess/home/limr/logs/slurm/neural_seg_panel_${SLURM_JOB_ID}" +mkdir -p "$OUT_DIR" + +uv run python tests/validation/neural_segmentation_correctness_panel.py \ + --output-dir "$OUT_DIR" \ + --device cuda \ + --batch-size 8 diff --git a/tests/validation/neural_segmentation_correctness_panel.py b/tests/validation/neural_segmentation_correctness_panel.py new file mode 100755 index 00000000..0112b6a3 --- /dev/null +++ b/tests/validation/neural_segmentation_correctness_panel.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python +"""Run neural segmentation correctness validation on a fixed multi-slide panel.""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +import time +import traceback +from pathlib import Path +from typing import Any + +import h5py +import numpy as np +import tiffslide + +from mussel.utils.neural_seg import NeuralTissueSegmenter +from mussel.utils.segment import segment_tissue + +DEFAULT_SLIDES = [ + "tests/testdata/948176.svs", + "docker_work/8471385.svs", + "/gpfs/mskmind_ess/limr/repos/Mussel-2.old/tests/testdata/1065626.svs", + "/gpfs/mskmind_ess/limr/repos/workflows2/1005517.svs", + "/gpfs/mskmind_ess/limr/repos/hf/prov-gigapath/sample_data/PROV-000-000001.ndpi", + "/gpfs/mskmind_ess/limr/gpu1_cache/mosaic/tcga_slides/d65c5d21-6333-4a9e-9a2a-139a122a3c8a/TCGA-AZ-4313-01Z-00-DX1.5e7ecf69-d1fd-4997-9dcc-ab8e9f10b423.svs", + "/gpfs/mskmind_ess/limr/gpu1_cache/mosaic/tcga_slides/6a0ea716-a5f2-47f3-880b-537a5cdc2324/TCGA-86-8074-01Z-00-DX1.0c34b434-8701-4060-a4ea-08a72371ee1e.svs", + "/gpfs/mskmind_ess/limr/repos/Mussel-3/panda_slides/train_images/001d865e65ef5d2579c190a0e0350d8f.tiff", + "/gpfs/mskmind_ess/limr/repos/Mussel-3/tcga_slides/TCGA-RM-A68W-01Z-00-DX1.4E62E4F4-415C-46EB-A6C8-45BA14E82708.svs", + "/gpfs/mskmind_ess/limr/repos/mussel-nf/tests/data/1079807.svs", +] + + +def _slide_metadata(slide_path: str) -> dict[str, Any]: + with tiffslide.TiffSlide(slide_path) as slide: + props = slide.properties + mpp_x = props.get("tiffslide.mpp-x") or props.get("openslide.mpp-x") + mpp_y = props.get("tiffslide.mpp-y") or props.get("openslide.mpp-y") + width, height = slide.dimensions + return { + "width": int(width), + "height": int(height), + "levels": len(slide.level_dimensions), + "mpp_x": float(mpp_x) if mpp_x is not None else None, + "mpp_y": float(mpp_y) if mpp_y is not None else None, + } + + +def _run_segmentation( + *, + slide_path: str, + slide_id: str, + seg_model: str, + output_dir: Path, + patch_size: int, + mpp: float, + tissue_area_threshold: float, + neural_segmenter: NeuralTissueSegmenter | None, +) -> dict[str, Any]: + h5_path = output_dir / f"{slide_id}.{seg_model}.h5" + start = time.monotonic() + result = segment_tissue( + slide_path=slide_path, + slide_id=slide_id, + patch_size=patch_size, + mpp=mpp, + tissue_area_threshold=tissue_area_threshold, + output_h5_path=str(h5_path), + seg_model=seg_model, + neural_segmenter=neural_segmenter, + ) + elapsed_sec = time.monotonic() - start + if result is None: + return { + "seg_model": seg_model, + "status": "failed", + "error": "segment_tissue returned None", + "elapsed_sec": elapsed_sec, + "h5_path": str(h5_path), + } + + _polygon, _grid, coords, attrs = result + with h5py.File(h5_path, "r") as h5: + h5_coords = h5["coords"][:] + h5_attrs = dict(h5["coords"].attrs) + + return { + "seg_model": seg_model, + "status": "passed", + "elapsed_sec": elapsed_sec, + "h5_path": str(h5_path), + "patch_count": int(len(coords)), + "h5_patch_count": int(h5_coords.shape[0]), + "coords": h5_coords, + "attrs": attrs, + "h5_attrs": h5_attrs, + } + + +def _validate_result( + *, + slide_meta: dict[str, Any], + seg_model: str, + result: dict[str, Any], +) -> list[str]: + failures: list[str] = [] + if result["status"] != "passed": + return [result.get("error", f"{seg_model} segmentation failed")] + + if result["patch_count"] <= 0: + failures.append(f"{seg_model} produced zero patches") + if result["h5_patch_count"] != result["patch_count"]: + failures.append( + f"{seg_model} HDF5 count mismatch: " + f"{result['h5_patch_count']} != {result['patch_count']}" + ) + if result["h5_attrs"].get("seg_model") != seg_model: + failures.append( + f"{seg_model} HDF5 seg_model attr is " + f"{result['h5_attrs'].get('seg_model')!r}" + ) + + coords = result["coords"] + if coords.size: + if coords.ndim != 2 or coords.shape[1] != 2: + failures.append(f"{seg_model} coords shape is {coords.shape}") + elif not ( + np.all(coords[:, 0] >= 0) + and np.all(coords[:, 1] >= 0) + and np.all(coords[:, 0] < slide_meta["width"]) + and np.all(coords[:, 1] < slide_meta["height"]) + ): + failures.append(f"{seg_model} produced out-of-bounds coordinates") + + return failures + + +def _summarize_for_json(row: dict[str, Any]) -> dict[str, Any]: + summarized = dict(row) + for key in ("classic", "neural"): + if key in summarized: + summarized[key] = { + k: v + for k, v in summarized[key].items() + if k not in {"coords", "attrs", "h5_attrs"} + } + return summarized + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", required=True) + parser.add_argument("--slides", nargs="*", default=DEFAULT_SLIDES) + parser.add_argument("--device", default="auto") + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--patch-size", type=int, default=256) + parser.add_argument("--mpp", type=float, default=0.5) + parser.add_argument("--tissue-area-threshold", type=float, default=1.0) + parser.add_argument("--ratio-min", type=float, default=0.10) + parser.add_argument("--ratio-max", type=float, default=10.0) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + neural_segmenter = NeuralTissueSegmenter( + device=args.device, + batch_size=args.batch_size, + ) + rows = [] + failures = [] + + for index, slide_path in enumerate(args.slides, start=1): + slide_id = f"{index:02d}_{Path(slide_path).stem}" + row: dict[str, Any] = { + "slide_index": index, + "slide_id": slide_id, + "slide_path": slide_path, + "exists": os.path.exists(slide_path), + } + print(f"[{index}/{len(args.slides)}] {slide_path}", flush=True) + + try: + if not row["exists"]: + raise FileNotFoundError(slide_path) + row.update(_slide_metadata(slide_path)) + row["classic"] = _run_segmentation( + slide_path=slide_path, + slide_id=slide_id, + seg_model="classic", + output_dir=output_dir, + patch_size=args.patch_size, + mpp=args.mpp, + tissue_area_threshold=args.tissue_area_threshold, + neural_segmenter=None, + ) + row["neural"] = _run_segmentation( + slide_path=slide_path, + slide_id=slide_id, + seg_model="neural", + output_dir=output_dir, + patch_size=args.patch_size, + mpp=args.mpp, + tissue_area_threshold=args.tissue_area_threshold, + neural_segmenter=neural_segmenter, + ) + + slide_failures = [] + slide_failures.extend( + _validate_result( + slide_meta=row, + seg_model="classic", + result=row["classic"], + ) + ) + slide_failures.extend( + _validate_result( + slide_meta=row, + seg_model="neural", + result=row["neural"], + ) + ) + + classic_count = row["classic"].get("patch_count", 0) + neural_count = row["neural"].get("patch_count", 0) + row["neural_classic_patch_ratio"] = ( + float(neural_count) / float(classic_count) if classic_count else None + ) + if row["neural_classic_patch_ratio"] is not None and not ( + args.ratio_min <= row["neural_classic_patch_ratio"] <= args.ratio_max + ): + slide_failures.append( + "neural/classic patch-count ratio " + f"{row['neural_classic_patch_ratio']:.3f} outside " + f"[{args.ratio_min}, {args.ratio_max}]" + ) + + row["status"] = "passed" if not slide_failures else "failed" + row["failures"] = slide_failures + except Exception as exc: + row["status"] = "failed" + row["failures"] = [f"{type(exc).__name__}: {exc}"] + row["traceback"] = traceback.format_exc() + + if row["status"] != "passed": + failures.append(row["slide_id"]) + rows.append(row) + summary = _summarize_for_json(row) + print(json.dumps(summary, indent=2), flush=True) + + json_path = output_dir / "neural_segmentation_correctness_panel.json" + csv_path = output_dir / "neural_segmentation_correctness_panel.csv" + with json_path.open("w", encoding="utf-8") as f: + json.dump([_summarize_for_json(row) for row in rows], f, indent=2) + + with csv_path.open("w", newline="", encoding="utf-8") as f: + fieldnames = [ + "slide_index", + "slide_id", + "status", + "width", + "height", + "levels", + "mpp_x", + "mpp_y", + "classic_patch_count", + "neural_patch_count", + "neural_classic_patch_ratio", + "classic_elapsed_sec", + "neural_elapsed_sec", + "failures", + "slide_path", + ] + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + writer.writerow( + { + "slide_index": row.get("slide_index"), + "slide_id": row.get("slide_id"), + "status": row.get("status"), + "width": row.get("width"), + "height": row.get("height"), + "levels": row.get("levels"), + "mpp_x": row.get("mpp_x"), + "mpp_y": row.get("mpp_y"), + "classic_patch_count": row.get("classic", {}).get("patch_count"), + "neural_patch_count": row.get("neural", {}).get("patch_count"), + "neural_classic_patch_ratio": row.get("neural_classic_patch_ratio"), + "classic_elapsed_sec": row.get("classic", {}).get("elapsed_sec"), + "neural_elapsed_sec": row.get("neural", {}).get("elapsed_sec"), + "failures": "; ".join(row.get("failures", [])), + "slide_path": row.get("slide_path"), + } + ) + + print(f"Wrote {json_path}", flush=True) + print(f"Wrote {csv_path}", flush=True) + if failures: + print(f"FAILED slides: {', '.join(failures)}", flush=True) + return 1 + print("All slides passed neural segmentation correctness checks.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())