Skip to content

Commit 1deafa9

Browse files
committed
Simplify.
1 parent e49ff12 commit 1deafa9

1 file changed

Lines changed: 128 additions & 126 deletions

File tree

‎tests/acceptance/test_cci_acceptance.py‎

Lines changed: 128 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,12 @@
2828
import datetime
2929
import json
3030
import os
31+
import shlex
3132
import shutil
33+
import signal
3234
import subprocess
3335
import sys
3436
from pathlib import Path
35-
from typing import Any
3637

3738
import anndata
3839
import numpy as np
@@ -50,41 +51,14 @@
5051
"lit": [],
5152
"cell_types": ["--use-label"],
5253
}
53-
HISTORY_COLUMNS = [
54-
"timestamp",
55-
"scenario",
56-
"outcome",
57-
"commit",
58-
"branch",
59-
"uncommitted_changes",
60-
"wall_seconds",
61-
"cpu_seconds",
62-
"cores_busy",
63-
"peak_memory_gb",
64-
"memory_added_gb",
65-
"warm_up_seconds",
66-
"baseline_wall_seconds",
67-
"baseline_peak_memory_gb",
68-
"time_vs_baseline",
69-
"memory_vs_baseline",
70-
"phase_neighbours_seconds",
71-
"phase_cell_type_counts_seconds",
72-
"phase_lr_scores_seconds",
73-
"phase_testing_seconds",
74-
"phase_testing_backgrounds_seconds",
75-
"bins",
76-
"genes",
77-
"lr_pairs_tested",
78-
"n_pairs",
79-
"platform",
80-
"processor",
81-
"cpus",
82-
"numba_threads",
83-
"python",
84-
"numpy",
85-
"numba",
86-
"stlearn",
87-
]
54+
# Phases of cci.run timed in history.csv: phase name: column.
55+
PHASES = {
56+
"neighbours": "phase_neighbours_seconds",
57+
"cell type counts": "phase_cell_type_counts_seconds",
58+
"LR scores": "phase_lr_scores_seconds",
59+
"testing": "phase_testing_seconds",
60+
"testing: backgrounds": "phase_testing_backgrounds_seconds",
61+
}
8862

8963

9064
@pytest.fixture(scope="session")
@@ -106,24 +80,19 @@ def sample(request: pytest.FixtureRequest, tmp_path_factory) -> Path:
10680
@pytest.fixture(scope="session")
10781
def report(request: pytest.FixtureRequest):
10882
"""Collects each scenario's row; writes benchmark.json and a summary at the end."""
109-
rows: list[dict[str, Any]] = []
83+
rows: list[dict] = []
11084
yield rows
11185
if not rows:
11286
return
11387
folder = Path(request.config.getoption("--acceptance-results"))
114-
benchmarks = []
115-
for row in rows:
116-
name = f"cci.run {row['scenario']}"
117-
benchmarks.append(
118-
{"name": f"{name}: time", "unit": "s", "value": row["wall_seconds"]}
119-
)
120-
benchmarks.append(
121-
{
122-
"name": f"{name}: peak memory",
123-
"unit": "GB",
124-
"value": row["peak_memory_gb"],
125-
}
126-
)
88+
benchmarks = [
89+
{"name": f"cci.run {row['scenario']}: {name}", "unit": unit, "value": row[key]}
90+
for row in rows
91+
for name, unit, key in [
92+
("time", "s", "wall_seconds"),
93+
("peak memory", "GB", "peak_memory_gb"),
94+
]
95+
]
12796
(folder / "benchmark.json").write_text(json.dumps(benchmarks, indent=2))
12897

12998
plugins = request.config.pluginmanager
@@ -141,52 +110,89 @@ def report(request: pytest.FixtureRequest):
141110
for row in rows:
142111
terminal.write_line(
143112
f"{row['scenario']:12s}{row['outcome']:>18s}"
144-
f"{row['wall_seconds']:11.1f}{_ratio(row, 'time_vs_baseline'):>9s}"
145-
f"{_ratio(row, 'time_vs_last'):>9s}{row['peak_memory_gb']:13.2f}"
146-
f"{_ratio(row, 'memory_vs_baseline'):>9s}"
147-
f"{_ratio(row, 'memory_vs_last'):>9s}"
113+
f"{row['wall_seconds']:11.1f}{_times(row['time_vs_baseline']):>9s}"
114+
f"{_times(row['time_vs_last']):>9s}{row['peak_memory_gb']:13.2f}"
115+
f"{_times(row['memory_vs_baseline']):>9s}"
116+
f"{_times(row['memory_vs_last']):>9s}"
148117
)
149118
terminal.write_line(f"History: {folder / 'history.csv'}")
150119

151120

152-
def _ratio(row: dict, key: str) -> str:
153-
value = row.get(key)
154-
return f"{value:.2f}x" if value not in (None, "") else "-"
121+
def _metrics(run: anndata.AnnData) -> dict:
122+
"""run_cci.py's measurements and settings for the run."""
123+
return json.loads(run.uns["metrics"])
124+
125+
126+
def _times(ratio: float | None) -> str:
127+
return "-" if ratio is None else f"{ratio:.2f}x"
155128

156129

157-
def run_scenario(sample: Path, result: Path, arguments: list[str], stlearn) -> dict:
158-
"""Runs run_cci.py in a fresh process; its measurements."""
130+
def _ratio(value: float, earlier: float | str | None) -> float | None:
131+
"""value / earlier, or None without an earlier value (CSV text is fine)."""
132+
return None if earlier in (None, "") else value / float(earlier)
133+
134+
135+
def run_scenario(sample: Path, result: Path, arguments: list[str], stlearn):
136+
"""Runs run_cci.py in a fresh process; its results, measurements included."""
159137
command = [sys.executable, "-m", "tests.acceptance.run_cci", str(sample)]
160138
command += [str(result), *arguments]
161139
if stlearn:
162140
command += ["--stlearn", str(Path(stlearn).resolve())]
163-
path = [str(ROOT), os.environ.get("PYTHONPATH", "")]
164-
environment = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, path))}
141+
path = os.pathsep.join(filter(None, [str(ROOT), os.environ.get("PYTHONPATH")]))
165142
finished = subprocess.run(
166-
command, cwd=ROOT, env=environment, capture_output=True, text=True
143+
command,
144+
cwd=ROOT,
145+
env={**os.environ, "PYTHONPATH": path},
146+
capture_output=True,
147+
text=True,
148+
errors="replace",
167149
)
168150
if finished.returncode != 0:
169-
pytest.fail(f"run_cci.py failed:\n{finished.stderr[-5000:]}")
170-
return json.loads(anndata.read_h5ad(result).uns["metrics"])
151+
pytest.fail(_failure_message(command, finished), pytrace=False)
152+
return anndata.read_h5ad(result)
153+
154+
155+
def _failure_message(command: list[str], finished: subprocess.CompletedProcess) -> str:
156+
"""How run_cci.py ended, how to rerun it, and what it printed."""
157+
code = finished.returncode
158+
if code >= 0:
159+
lines = [f"run_cci.py exited with code {code}"]
160+
else:
161+
try:
162+
how = signal.Signals(-code).name
163+
except ValueError:
164+
how = f"signal {-code}"
165+
lines = [f"run_cci.py was killed by {how}"]
166+
if how in ("SIGSEGV", "SIGBUS", "SIGABRT"):
167+
lines.append(
168+
"A crash in compiled code. For a readable traceback, rerun with "
169+
"NUMBA_NUM_THREADS=1 NUMBA_BOUNDSCHECK=1"
170+
)
171+
rerun = f"cd {shlex.quote(str(ROOT))} && PYTHONPATH=. {shlex.join(command)}"
172+
lines.append(f"Rerun: {rerun}")
173+
for stream, text in [("stdout", finished.stdout), ("stderr", finished.stderr)]:
174+
text = text.strip()
175+
lines.append(f"--- {stream} ---\n{text[-5000:] if text else '(empty)'}")
176+
return "\n".join(lines)
171177

172178

173179
def differences(baseline: anndata.AnnData, current: anndata.AnnData, atol: float):
174180
"""How the current results differ from the baseline, matching LRs by name."""
175-
lrs = baseline.uns["lr_summary"].index
176-
current_lrs = current.uns["lr_summary"].index
181+
if _metrics(baseline)["settings"] != _metrics(current)["settings"]:
182+
return ["the baseline used different settings: write a new one"]
177183
if not baseline.obs_names.equals(current.obs_names):
178184
return ["the samples' bins differ: write a new baseline"]
179-
if set(lrs) != set(current_lrs):
180-
only = set(lrs).symmetric_difference(current_lrs)
185+
lrs = baseline.uns["lr_summary"].index
186+
current_lrs = current.uns["lr_summary"].index
187+
if only := set(lrs) ^ set(current_lrs):
181188
return [f"different LR pairs tested ({len(only)} in only one run)"]
182189
order = current_lrs.get_indexer(lrs)
183190

184191
problems = []
185192
for key in RESULTS:
186193
difference = abs(baseline.obsm[key] - current.obsm[key][:, order])
187-
largest = difference.max() if difference.nnz else 0
188-
if largest > atol:
189-
n = (difference > atol).nnz
194+
if n := (difference > atol).nnz:
195+
largest = difference.max()
190196
problems.append(f"{key}: {n} values differ, by up to {largest:.3g}")
191197

192198
summary = current.uns["lr_summary"].loc[lrs]
@@ -197,33 +203,47 @@ def differences(baseline: anndata.AnnData, current: anndata.AnnData, atol: float
197203
if not np.allclose(features, current_features, rtol=0, atol=atol, equal_nan=True):
198204
problems.append("lrfeatures differ")
199205
for key in [*NEIGHBOURS, "cci_het"]:
200-
if key in baseline.obs or key in current.obs:
201-
if key not in baseline.obs or key not in current.obs:
202-
problems.append(f"{key}: in only one run")
203-
else:
204-
before = baseline.obs[key].astype(str).to_numpy()
205-
n = (before != current.obs[key].astype(str).to_numpy()).sum()
206-
if n:
207-
problems.append(f"{key}: {n} spots differ")
206+
in_baseline, in_current = key in baseline.obs, key in current.obs
207+
if in_baseline != in_current:
208+
problems.append(f"{key}: in only one run")
209+
elif in_baseline:
210+
before = baseline.obs[key].astype(str).to_numpy()
211+
if n := (before != current.obs[key].astype(str).to_numpy()).sum():
212+
problems.append(f"{key}: {n} spots differ")
208213
return problems
209214

210215

211216
def history_row(scenario: str, outcome: str, metrics: dict, baseline: dict | None):
217+
"""A row of history.csv. It has every column, even without a baseline, as
218+
a new history.csv takes its header from the first row."""
212219
machine = metrics["machine"]
213-
phases = metrics["phases"]
214-
row = {
220+
seconds = metrics["wall_seconds"]
221+
memory_gb = metrics["peak_memory_bytes"] / GB
222+
baseline_seconds = baseline_memory_gb = None
223+
if baseline is not None:
224+
baseline_seconds = baseline["wall_seconds"]
225+
baseline_memory_gb = baseline["peak_memory_bytes"] / GB
226+
return {
215227
"timestamp": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
216228
"scenario": scenario,
217229
"outcome": outcome,
218230
"commit": metrics["commit"],
219231
"branch": metrics["branch"],
220232
"uncommitted_changes": metrics["uncommitted_changes"],
221-
"wall_seconds": metrics["wall_seconds"],
233+
"wall_seconds": seconds,
222234
"cpu_seconds": metrics["cpu_seconds"],
223-
"cores_busy": metrics["cpu_seconds"] / metrics["wall_seconds"],
224-
"peak_memory_gb": metrics["peak_memory_bytes"] / GB,
235+
"cores_busy": metrics["cpu_seconds"] / seconds,
236+
"peak_memory_gb": memory_gb,
225237
"memory_added_gb": metrics["memory_added_by_run_bytes"] / GB,
226238
"warm_up_seconds": metrics["warm_up_seconds"],
239+
"baseline_wall_seconds": baseline_seconds,
240+
"baseline_peak_memory_gb": baseline_memory_gb,
241+
"time_vs_baseline": _ratio(seconds, baseline_seconds),
242+
"memory_vs_baseline": _ratio(memory_gb, baseline_memory_gb),
243+
**{
244+
column: metrics["phases"].get(phase, {}).get("wall_seconds")
245+
for phase, column in PHASES.items()
246+
},
227247
"bins": metrics["bins"],
228248
"genes": metrics["genes"],
229249
"lr_pairs_tested": metrics["lr_pairs_tested"],
@@ -237,40 +257,28 @@ def history_row(scenario: str, outcome: str, metrics: dict, baseline: dict | Non
237257
"numba": machine["versions"]["numba"],
238258
"stlearn": metrics["stlearn"],
239259
}
240-
for name, column in [
241-
("neighbours", "phase_neighbours_seconds"),
242-
("cell type counts", "phase_cell_type_counts_seconds"),
243-
("LR scores", "phase_lr_scores_seconds"),
244-
("testing", "phase_testing_seconds"),
245-
("testing: backgrounds", "phase_testing_backgrounds_seconds"),
246-
]:
247-
row[column] = phases.get(name, {}).get("wall_seconds", "")
248-
if baseline is not None:
249-
row["baseline_wall_seconds"] = baseline["wall_seconds"]
250-
row["baseline_peak_memory_gb"] = baseline["peak_memory_bytes"] / GB
251-
row["time_vs_baseline"] = metrics["wall_seconds"] / baseline["wall_seconds"]
252-
row["memory_vs_baseline"] = (
253-
metrics["peak_memory_bytes"] / baseline["peak_memory_bytes"]
254-
)
255-
return row
256260

257261

258-
def record_history(path: Path, row: dict) -> dict | None:
259-
"""Appends the row to the history; the previous run on this machine, if any."""
260-
previous = None
262+
def append_history(path: Path, row: dict) -> dict:
263+
"""Appends the row to history.csv; the scenario's previous run on this
264+
machine, or {} if there is none."""
265+
earlier, columns = [], None
261266
if path.exists():
262267
with open(path, newline="") as file:
263-
for earlier in csv.DictReader(file):
264-
same_machine = earlier["platform"] == row["platform"]
265-
if earlier["scenario"] == row["scenario"] and same_machine:
266-
previous = earlier
267-
new_file = not path.exists()
268+
reader = csv.DictReader(file)
269+
earlier, columns = list(reader), reader.fieldnames
268270
with open(path, "a", newline="") as file:
269-
writer = csv.DictWriter(file, fieldnames=HISTORY_COLUMNS, extrasaction="ignore")
270-
if new_file:
271+
# An existing file keeps its header, so the row lines up with it.
272+
writer = csv.DictWriter(file, columns or list(row), extrasaction="ignore")
273+
if columns is None:
271274
writer.writeheader()
272275
writer.writerow(row)
273-
return previous
276+
same = [
277+
run
278+
for run in earlier
279+
if run["scenario"] == row["scenario"] and run["platform"] == row["platform"]
280+
]
281+
return same[-1] if same else {}
274282

275283

276284
@pytest.mark.acceptance
@@ -283,10 +291,10 @@ def test_cci_run(scenario, sample, report, request, record_testsuite_property):
283291
result_path = folder / f"{scenario}.h5ad"
284292
baseline_path = BASELINES / f"{scenario}.h5ad"
285293

286-
metrics = run_scenario(
294+
current = run_scenario(
287295
sample, result_path, SCENARIOS[scenario], option("--acceptance-stlearn")
288296
)
289-
current = anndata.read_h5ad(result_path)
297+
metrics = _metrics(current)
290298

291299
problems, baseline_metrics = [], None
292300
if option("--acceptance-baseline") == "write":
@@ -301,20 +309,14 @@ def test_cci_run(scenario, sample, report, request, record_testsuite_property):
301309
)
302310
else:
303311
baseline = anndata.read_h5ad(baseline_path)
304-
baseline_metrics = json.loads(baseline.uns["metrics"])
305-
if baseline_metrics["settings"] != metrics["settings"]:
306-
problems.append("the baseline used different settings: write a new one")
307-
else:
308-
problems += differences(baseline, current, option("--acceptance-atol"))
312+
baseline_metrics = _metrics(baseline)
313+
problems = differences(baseline, current, option("--acceptance-atol"))
309314
outcome = "results differ" if problems else "results identical"
310315

311316
row = history_row(scenario, outcome, metrics, baseline_metrics)
312-
previous = record_history(folder / "history.csv", row)
313-
if previous is not None:
314-
row["time_vs_last"] = row["wall_seconds"] / float(previous["wall_seconds"])
315-
row["memory_vs_last"] = row["peak_memory_gb"] / float(
316-
previous["peak_memory_gb"]
317-
)
317+
last = append_history(folder / "history.csv", row)
318+
row["time_vs_last"] = _ratio(row["wall_seconds"], last.get("wall_seconds"))
319+
row["memory_vs_last"] = _ratio(row["peak_memory_gb"], last.get("peak_memory_gb"))
318320
report.append(row)
319321
for key in ["wall_seconds", "cpu_seconds", "peak_memory_gb", "lr_pairs_tested"]:
320322
record_testsuite_property(f"{scenario}.{key}", row[key])
@@ -323,6 +325,6 @@ def test_cci_run(scenario, sample, report, request, record_testsuite_property):
323325
("time_vs_baseline", option("--acceptance-max-time-ratio")),
324326
("memory_vs_baseline", option("--acceptance-max-memory-ratio")),
325327
]:
326-
if limit is not None and ratio in row and row[ratio] > limit:
328+
if None not in (limit, row[ratio]) and row[ratio] > limit:
327329
problems.append(f"{ratio} is {row[ratio]:.2f}, over the limit {limit}")
328-
assert not problems, "\n".join(problems)
330+
assert not problems, "\n".join(problems)

0 commit comments

Comments
 (0)