diff --git a/tests/acceptance/cci_sample.py b/tests/acceptance/cci_sample.py index 9efa0d62..571f44ba 100644 --- a/tests/acceptance/cci_sample.py +++ b/tests/acceptance/cci_sample.py @@ -11,14 +11,7 @@ from anndata import AnnData from scipy import sparse -# 8 µm bins imaged at 0.2738 µm per pixel, hires image at 0.08 of full size. -SCALEFACTORS = { - "bin_size_um": 8.0, - "microns_per_pixel": 0.2738, - "spot_diameter_fullres": 8.0 / 0.2738, - "tissue_hires_scalef": 0.08, - "tissue_lowres_scalef": 0.024, -} +from tests.acceptance.visium_hd import barcodes, bin_grid, place_bins def make_sample( @@ -26,8 +19,8 @@ def make_sample( ) -> AnnData: """The sample, with `lr_genes` first among its genes.""" rng = np.random.default_rng(seed) - n_side = round(np.sqrt(n_bins)) - n_bins = n_side * n_side + _, row, col = bin_grid(n_bins) + n_bins = len(row) genes = lr_genes + list(map(lambda i: f"GENE{i}", range(n_genes - len(lr_genes)))) # Each gene's share of bins expressing it: mostly rare, a few common. @@ -44,26 +37,10 @@ def make_sample( (values, (rows, columns)), shape=(n_bins, n_genes), dtype=np.float32 ) - row, col = np.divmod(np.arange(n_bins), n_side) - barcodes = list( - map( - lambda rc: f"s_008um_{rc[0]:05d}_{rc[1]:05d}-1", - zip(row, col, strict=True), - ) + adata = AnnData( + X, obs=pd.DataFrame(index=barcodes(row, col)), var=pd.DataFrame(index=genes) ) - adata = AnnData(X, obs=pd.DataFrame(index=barcodes), var=pd.DataFrame(index=genes)) - spot = SCALEFACTORS["spot_diameter_fullres"] - hires = SCALEFACTORS["tissue_hires_scalef"] - adata.obsm["spatial"] = np.column_stack([col, row]) * spot + 1000 - adata.obs["imagecol"] = adata.obsm["spatial"][:, 0] * hires - adata.obs["imagerow"] = adata.obsm["spatial"][:, 1] * hires - adata.uns["spatial"] = { - "synthetic_hd": { - "images": {}, - "scalefactors": SCALEFACTORS, - "use_quality": "hires", - } - } + place_bins(adata, row, col) # Mostly one cell type per bin. proportions = pd.DataFrame( diff --git a/tests/acceptance/harness.py b/tests/acceptance/harness.py new file mode 100644 index 00000000..2e31cd95 --- /dev/null +++ b/tests/acceptance/harness.py @@ -0,0 +1,241 @@ +"""What the acceptance tests share: running a scenario's script in a fresh +process within time and memory limits, writing or comparing its baseline, and +keeping its time and memory in a history, a benchmark and a summary.""" + +import csv +import json +import shlex +import shutil +import signal +import subprocess +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import anndata +import psutil +import pytest + +BASELINES = Path(__file__).parent / "baselines" +ROOT = Path(__file__).resolve().parents[2] +GB = 1024**3 + + +def metrics(run: anndata.AnnData) -> dict: + """The script's measurements and settings for the run.""" + return json.loads(run.uns["metrics"]) + + +def times(ratio: float | None) -> str: + return "-" if ratio is None else f"{ratio:.2f}x" + + +def ratio(value: float, earlier: float | str | None) -> float | None: + """value / earlier, or None without an earlier value (CSV text is fine).""" + return None if earlier in (None, "") else value / float(earlier) + + +def sample_folder(request: pytest.FixtureRequest, tmp_path_factory, name: str) -> Path: + """Where the samples are kept: pytest's cache, or a temporary folder without + it.""" + if getattr(request.config, "cache", None) is not None: + return request.config.cache.mkdir(name) + return tmp_path_factory.mktemp(name) + + +@dataclass +class Run: + """How a script ended: its command and output, how long it ran, the most + memory seen, and why it was stopped, if it was.""" + + script: str + command: list[str] + finished: subprocess.CompletedProcess + seconds: float + memory_gb: float + stopped: str | None + + @property + def failed(self) -> bool: + return bool(self.stopped) or self.finished.returncode != 0 + + +def run_script( + script: str, + arguments: list[str], + max_minutes: float | None, + max_memory_gb: float | None, +) -> Run: + """Runs tests/acceptance/