2828import datetime
2929import json
3030import os
31+ import shlex
3132import shutil
33+ import signal
3234import subprocess
3335import sys
3436from pathlib import Path
35- from typing import Any
3637
3738import anndata
3839import numpy as np
5051 "lit" : [],
5152 "cell_types" : ["--use-label" ],
5253}
53- HISTORY_COLUMNS = [
54- "timestamp" ,
55- "scenario" ,
56- "outcome" ,
57- "commit" ,
58- "branch" ,
59- "uncommitted_changes" ,
60- "wall_seconds" ,
61- "cpu_seconds" ,
62- "cores_busy" ,
63- "peak_memory_gb" ,
64- "memory_added_gb" ,
65- "warm_up_seconds" ,
66- "baseline_wall_seconds" ,
67- "baseline_peak_memory_gb" ,
68- "time_vs_baseline" ,
69- "memory_vs_baseline" ,
70- "phase_neighbours_seconds" ,
71- "phase_cell_type_counts_seconds" ,
72- "phase_lr_scores_seconds" ,
73- "phase_testing_seconds" ,
74- "phase_testing_backgrounds_seconds" ,
75- "bins" ,
76- "genes" ,
77- "lr_pairs_tested" ,
78- "n_pairs" ,
79- "platform" ,
80- "processor" ,
81- "cpus" ,
82- "numba_threads" ,
83- "python" ,
84- "numpy" ,
85- "numba" ,
86- "stlearn" ,
87- ]
54+ # Phases of cci.run timed in history.csv: phase name: column.
55+ PHASES = {
56+ "neighbours" : "phase_neighbours_seconds" ,
57+ "cell type counts" : "phase_cell_type_counts_seconds" ,
58+ "LR scores" : "phase_lr_scores_seconds" ,
59+ "testing" : "phase_testing_seconds" ,
60+ "testing: backgrounds" : "phase_testing_backgrounds_seconds" ,
61+ }
8862
8963
9064@pytest .fixture (scope = "session" )
@@ -106,24 +80,19 @@ def sample(request: pytest.FixtureRequest, tmp_path_factory) -> Path:
10680@pytest .fixture (scope = "session" )
10781def report (request : pytest .FixtureRequest ):
10882 """Collects each scenario's row; writes benchmark.json and a summary at the end."""
109- rows : list [dict [ str , Any ] ] = []
83+ rows : list [dict ] = []
11084 yield rows
11185 if not rows :
11286 return
11387 folder = Path (request .config .getoption ("--acceptance-results" ))
114- benchmarks = []
115- for row in rows :
116- name = f"cci.run { row ['scenario' ]} "
117- benchmarks .append (
118- {"name" : f"{ name } : time" , "unit" : "s" , "value" : row ["wall_seconds" ]}
119- )
120- benchmarks .append (
121- {
122- "name" : f"{ name } : peak memory" ,
123- "unit" : "GB" ,
124- "value" : row ["peak_memory_gb" ],
125- }
126- )
88+ benchmarks = [
89+ {"name" : f"cci.run { row ['scenario' ]} : { name } " , "unit" : unit , "value" : row [key ]}
90+ for row in rows
91+ for name , unit , key in [
92+ ("time" , "s" , "wall_seconds" ),
93+ ("peak memory" , "GB" , "peak_memory_gb" ),
94+ ]
95+ ]
12796 (folder / "benchmark.json" ).write_text (json .dumps (benchmarks , indent = 2 ))
12897
12998 plugins = request .config .pluginmanager
@@ -141,52 +110,89 @@ def report(request: pytest.FixtureRequest):
141110 for row in rows :
142111 terminal .write_line (
143112 f"{ row ['scenario' ]:12s} { row ['outcome' ]:>18s} "
144- f"{ row ['wall_seconds' ]:11.1f} { _ratio (row , 'time_vs_baseline' ):>9s} "
145- f"{ _ratio (row , 'time_vs_last' ):>9s} { row ['peak_memory_gb' ]:13.2f} "
146- f"{ _ratio (row , 'memory_vs_baseline' ):>9s} "
147- f"{ _ratio (row , 'memory_vs_last' ):>9s} "
113+ f"{ row ['wall_seconds' ]:11.1f} { _times (row [ 'time_vs_baseline' ] ):>9s} "
114+ f"{ _times (row [ 'time_vs_last' ] ):>9s} { row ['peak_memory_gb' ]:13.2f} "
115+ f"{ _times (row [ 'memory_vs_baseline' ] ):>9s} "
116+ f"{ _times (row [ 'memory_vs_last' ] ):>9s} "
148117 )
149118 terminal .write_line (f"History: { folder / 'history.csv' } " )
150119
151120
152- def _ratio (row : dict , key : str ) -> str :
153- value = row .get (key )
154- return f"{ value :.2f} x" if value not in (None , "" ) else "-"
121+ def _metrics (run : anndata .AnnData ) -> dict :
122+ """run_cci.py's measurements and settings for the run."""
123+ return json .loads (run .uns ["metrics" ])
124+
125+
126+ def _times (ratio : float | None ) -> str :
127+ return "-" if ratio is None else f"{ ratio :.2f} x"
155128
156129
157- def run_scenario (sample : Path , result : Path , arguments : list [str ], stlearn ) -> dict :
158- """Runs run_cci.py in a fresh process; its measurements."""
130+ def _ratio (value : float , earlier : float | str | None ) -> float | None :
131+ """value / earlier, or None without an earlier value (CSV text is fine)."""
132+ return None if earlier in (None , "" ) else value / float (earlier )
133+
134+
135+ def run_scenario (sample : Path , result : Path , arguments : list [str ], stlearn ):
136+ """Runs run_cci.py in a fresh process; its results, measurements included."""
159137 command = [sys .executable , "-m" , "tests.acceptance.run_cci" , str (sample )]
160138 command += [str (result ), * arguments ]
161139 if stlearn :
162140 command += ["--stlearn" , str (Path (stlearn ).resolve ())]
163- path = [str (ROOT ), os .environ .get ("PYTHONPATH" , "" )]
164- environment = {** os .environ , "PYTHONPATH" : os .pathsep .join (filter (None , path ))}
141+ path = os .pathsep .join (filter (None , [str (ROOT ), os .environ .get ("PYTHONPATH" )]))
165142 finished = subprocess .run (
166- command , cwd = ROOT , env = environment , capture_output = True , text = True
143+ command ,
144+ cwd = ROOT ,
145+ env = {** os .environ , "PYTHONPATH" : path },
146+ capture_output = True ,
147+ text = True ,
148+ errors = "replace" ,
167149 )
168150 if finished .returncode != 0 :
169- pytest .fail (f"run_cci.py failed:\n { finished .stderr [- 5000 :]} " )
170- return json .loads (anndata .read_h5ad (result ).uns ["metrics" ])
151+ pytest .fail (_failure_message (command , finished ), pytrace = False )
152+ return anndata .read_h5ad (result )
153+
154+
155+ def _failure_message (command : list [str ], finished : subprocess .CompletedProcess ) -> str :
156+ """How run_cci.py ended, how to rerun it, and what it printed."""
157+ code = finished .returncode
158+ if code >= 0 :
159+ lines = [f"run_cci.py exited with code { code } " ]
160+ else :
161+ try :
162+ how = signal .Signals (- code ).name
163+ except ValueError :
164+ how = f"signal { - code } "
165+ lines = [f"run_cci.py was killed by { how } " ]
166+ if how in ("SIGSEGV" , "SIGBUS" , "SIGABRT" ):
167+ lines .append (
168+ "A crash in compiled code. For a readable traceback, rerun with "
169+ "NUMBA_NUM_THREADS=1 NUMBA_BOUNDSCHECK=1"
170+ )
171+ rerun = f"cd { shlex .quote (str (ROOT ))} && PYTHONPATH=. { shlex .join (command )} "
172+ lines .append (f"Rerun: { rerun } " )
173+ for stream , text in [("stdout" , finished .stdout ), ("stderr" , finished .stderr )]:
174+ text = text .strip ()
175+ lines .append (f"--- { stream } ---\n { text [- 5000 :] if text else '(empty)' } " )
176+ return "\n " .join (lines )
171177
172178
173179def differences (baseline : anndata .AnnData , current : anndata .AnnData , atol : float ):
174180 """How the current results differ from the baseline, matching LRs by name."""
175- lrs = baseline . uns [ "lr_summary" ]. index
176- current_lrs = current . uns [ "lr_summary" ]. index
181+ if _metrics ( baseline )[ "settings" ] != _metrics ( current )[ "settings" ]:
182+ return [ "the baseline used different settings: write a new one" ]
177183 if not baseline .obs_names .equals (current .obs_names ):
178184 return ["the samples' bins differ: write a new baseline" ]
179- if set (lrs ) != set (current_lrs ):
180- only = set (lrs ).symmetric_difference (current_lrs )
185+ lrs = baseline .uns ["lr_summary" ].index
186+ current_lrs = current .uns ["lr_summary" ].index
187+ if only := set (lrs ) ^ set (current_lrs ):
181188 return [f"different LR pairs tested ({ len (only )} in only one run)" ]
182189 order = current_lrs .get_indexer (lrs )
183190
184191 problems = []
185192 for key in RESULTS :
186193 difference = abs (baseline .obsm [key ] - current .obsm [key ][:, order ])
187- largest = difference .max () if difference .nnz else 0
188- if largest > atol :
189- n = (difference > atol ).nnz
194+ if n := (difference > atol ).nnz :
195+ largest = difference .max ()
190196 problems .append (f"{ key } : { n } values differ, by up to { largest :.3g} " )
191197
192198 summary = current .uns ["lr_summary" ].loc [lrs ]
@@ -197,33 +203,47 @@ def differences(baseline: anndata.AnnData, current: anndata.AnnData, atol: float
197203 if not np .allclose (features , current_features , rtol = 0 , atol = atol , equal_nan = True ):
198204 problems .append ("lrfeatures differ" )
199205 for key in [* NEIGHBOURS , "cci_het" ]:
200- if key in baseline .obs or key in current .obs :
201- if key not in baseline .obs or key not in current .obs :
202- problems .append (f"{ key } : in only one run" )
203- else :
204- before = baseline .obs [key ].astype (str ).to_numpy ()
205- n = (before != current .obs [key ].astype (str ).to_numpy ()).sum ()
206- if n :
207- problems .append (f"{ key } : { n } spots differ" )
206+ in_baseline , in_current = key in baseline .obs , key in current .obs
207+ if in_baseline != in_current :
208+ problems .append (f"{ key } : in only one run" )
209+ elif in_baseline :
210+ before = baseline .obs [key ].astype (str ).to_numpy ()
211+ if n := (before != current .obs [key ].astype (str ).to_numpy ()).sum ():
212+ problems .append (f"{ key } : { n } spots differ" )
208213 return problems
209214
210215
211216def history_row (scenario : str , outcome : str , metrics : dict , baseline : dict | None ):
217+ """A row of history.csv. It has every column, even without a baseline, as
218+ a new history.csv takes its header from the first row."""
212219 machine = metrics ["machine" ]
213- phases = metrics ["phases" ]
214- row = {
220+ seconds = metrics ["wall_seconds" ]
221+ memory_gb = metrics ["peak_memory_bytes" ] / GB
222+ baseline_seconds = baseline_memory_gb = None
223+ if baseline is not None :
224+ baseline_seconds = baseline ["wall_seconds" ]
225+ baseline_memory_gb = baseline ["peak_memory_bytes" ] / GB
226+ return {
215227 "timestamp" : datetime .datetime .now (datetime .UTC ).isoformat (timespec = "seconds" ),
216228 "scenario" : scenario ,
217229 "outcome" : outcome ,
218230 "commit" : metrics ["commit" ],
219231 "branch" : metrics ["branch" ],
220232 "uncommitted_changes" : metrics ["uncommitted_changes" ],
221- "wall_seconds" : metrics [ "wall_seconds" ] ,
233+ "wall_seconds" : seconds ,
222234 "cpu_seconds" : metrics ["cpu_seconds" ],
223- "cores_busy" : metrics ["cpu_seconds" ] / metrics [ "wall_seconds" ] ,
224- "peak_memory_gb" : metrics [ "peak_memory_bytes" ] / GB ,
235+ "cores_busy" : metrics ["cpu_seconds" ] / seconds ,
236+ "peak_memory_gb" : memory_gb ,
225237 "memory_added_gb" : metrics ["memory_added_by_run_bytes" ] / GB ,
226238 "warm_up_seconds" : metrics ["warm_up_seconds" ],
239+ "baseline_wall_seconds" : baseline_seconds ,
240+ "baseline_peak_memory_gb" : baseline_memory_gb ,
241+ "time_vs_baseline" : _ratio (seconds , baseline_seconds ),
242+ "memory_vs_baseline" : _ratio (memory_gb , baseline_memory_gb ),
243+ ** {
244+ column : metrics ["phases" ].get (phase , {}).get ("wall_seconds" )
245+ for phase , column in PHASES .items ()
246+ },
227247 "bins" : metrics ["bins" ],
228248 "genes" : metrics ["genes" ],
229249 "lr_pairs_tested" : metrics ["lr_pairs_tested" ],
@@ -237,40 +257,28 @@ def history_row(scenario: str, outcome: str, metrics: dict, baseline: dict | Non
237257 "numba" : machine ["versions" ]["numba" ],
238258 "stlearn" : metrics ["stlearn" ],
239259 }
240- for name , column in [
241- ("neighbours" , "phase_neighbours_seconds" ),
242- ("cell type counts" , "phase_cell_type_counts_seconds" ),
243- ("LR scores" , "phase_lr_scores_seconds" ),
244- ("testing" , "phase_testing_seconds" ),
245- ("testing: backgrounds" , "phase_testing_backgrounds_seconds" ),
246- ]:
247- row [column ] = phases .get (name , {}).get ("wall_seconds" , "" )
248- if baseline is not None :
249- row ["baseline_wall_seconds" ] = baseline ["wall_seconds" ]
250- row ["baseline_peak_memory_gb" ] = baseline ["peak_memory_bytes" ] / GB
251- row ["time_vs_baseline" ] = metrics ["wall_seconds" ] / baseline ["wall_seconds" ]
252- row ["memory_vs_baseline" ] = (
253- metrics ["peak_memory_bytes" ] / baseline ["peak_memory_bytes" ]
254- )
255- return row
256260
257261
258- def record_history (path : Path , row : dict ) -> dict | None :
259- """Appends the row to the history; the previous run on this machine, if any."""
260- previous = None
262+ def append_history (path : Path , row : dict ) -> dict :
263+ """Appends the row to history.csv; the scenario's previous run on this
264+ machine, or {} if there is none."""
265+ earlier , columns = [], None
261266 if path .exists ():
262267 with open (path , newline = "" ) as file :
263- for earlier in csv .DictReader (file ):
264- same_machine = earlier ["platform" ] == row ["platform" ]
265- if earlier ["scenario" ] == row ["scenario" ] and same_machine :
266- previous = earlier
267- new_file = not path .exists ()
268+ reader = csv .DictReader (file )
269+ earlier , columns = list (reader ), reader .fieldnames
268270 with open (path , "a" , newline = "" ) as file :
269- writer = csv .DictWriter (file , fieldnames = HISTORY_COLUMNS , extrasaction = "ignore" )
270- if new_file :
271+ # An existing file keeps its header, so the row lines up with it.
272+ writer = csv .DictWriter (file , columns or list (row ), extrasaction = "ignore" )
273+ if columns is None :
271274 writer .writeheader ()
272275 writer .writerow (row )
273- return previous
276+ same = [
277+ run
278+ for run in earlier
279+ if run ["scenario" ] == row ["scenario" ] and run ["platform" ] == row ["platform" ]
280+ ]
281+ return same [- 1 ] if same else {}
274282
275283
276284@pytest .mark .acceptance
@@ -283,10 +291,10 @@ def test_cci_run(scenario, sample, report, request, record_testsuite_property):
283291 result_path = folder / f"{ scenario } .h5ad"
284292 baseline_path = BASELINES / f"{ scenario } .h5ad"
285293
286- metrics = run_scenario (
294+ current = run_scenario (
287295 sample , result_path , SCENARIOS [scenario ], option ("--acceptance-stlearn" )
288296 )
289- current = anndata . read_h5ad ( result_path )
297+ metrics = _metrics ( current )
290298
291299 problems , baseline_metrics = [], None
292300 if option ("--acceptance-baseline" ) == "write" :
@@ -301,20 +309,14 @@ def test_cci_run(scenario, sample, report, request, record_testsuite_property):
301309 )
302310 else :
303311 baseline = anndata .read_h5ad (baseline_path )
304- baseline_metrics = json .loads (baseline .uns ["metrics" ])
305- if baseline_metrics ["settings" ] != metrics ["settings" ]:
306- problems .append ("the baseline used different settings: write a new one" )
307- else :
308- problems += differences (baseline , current , option ("--acceptance-atol" ))
312+ baseline_metrics = _metrics (baseline )
313+ problems = differences (baseline , current , option ("--acceptance-atol" ))
309314 outcome = "results differ" if problems else "results identical"
310315
311316 row = history_row (scenario , outcome , metrics , baseline_metrics )
312- previous = record_history (folder / "history.csv" , row )
313- if previous is not None :
314- row ["time_vs_last" ] = row ["wall_seconds" ] / float (previous ["wall_seconds" ])
315- row ["memory_vs_last" ] = row ["peak_memory_gb" ] / float (
316- previous ["peak_memory_gb" ]
317- )
317+ last = append_history (folder / "history.csv" , row )
318+ row ["time_vs_last" ] = _ratio (row ["wall_seconds" ], last .get ("wall_seconds" ))
319+ row ["memory_vs_last" ] = _ratio (row ["peak_memory_gb" ], last .get ("peak_memory_gb" ))
318320 report .append (row )
319321 for key in ["wall_seconds" , "cpu_seconds" , "peak_memory_gb" , "lr_pairs_tested" ]:
320322 record_testsuite_property (f"{ scenario } .{ key } " , row [key ])
@@ -323,6 +325,6 @@ def test_cci_run(scenario, sample, report, request, record_testsuite_property):
323325 ("time_vs_baseline" , option ("--acceptance-max-time-ratio" )),
324326 ("memory_vs_baseline" , option ("--acceptance-max-memory-ratio" )),
325327 ]:
326- if limit is not None and ratio in row and row [ratio ] > limit :
328+ if None not in ( limit , row [ ratio ]) and row [ratio ] > limit :
327329 problems .append (f"{ ratio } is { row [ratio ]:.2f} , over the limit { limit } " )
328- assert not problems , "\n " .join (problems )
330+ assert not problems , "\n " .join (problems )
0 commit comments