From cdfa5c4288dce2fb05e15ed1b78555077db28c26 Mon Sep 17 00:00:00 2001 From: Aleksei Rechinskii Date: Mon, 31 Aug 2026 11:17:00 +0000 Subject: [PATCH] add bulkbench package --- bulkbench/.gitignore | 0 bulkbench/README.md | 323 ++++ .../03_pyt_IK_reorder/base_model.patch | 30 + .../example/04_pyt_IK_bucket/base_model.patch | 24 + .../05_pyt_IK_reorderbucket/base_model.patch | 37 + .../example/11_PR_only_meta/base_model.patch | 15 + .../12_PR_new_flags_no_meta/base_model.patch | 18 + .../base_model.patch | 19 + .../14_PR_legacy_flags/base_model.patch | 17 + bulkbench/example/README.md | 33 + bulkbench/example/configs.yaml | 18 + bulkbench/example/patches_PR.yaml | 26 + bulkbench/example/patches_pyt213.yaml | 31 + .../example/report/benchstats-fix-groups.html | 225 +++ bulkbench/example/report/run-to-run.html | 114 ++ bulkbench/pyproject.toml | 44 + bulkbench/src/bulkbench/__init__.py | 6 + bulkbench/src/bulkbench/__main__.py | 19 + .../src/bulkbench/benchmark_plan_loader.py | 553 ++++++ bulkbench/src/bulkbench/benchmark_sources.py | 193 ++ bulkbench/src/bulkbench/bulkbench.py | 732 ++++++++ bulkbench/src/bulkbench/cli_parser.py | 174 ++ bulkbench/src/bulkbench/parser_JSON.py | 173 ++ bulkbench/src/bulkbench/script_runner.py | 131 ++ bulkbench/tests/proj0/my_configs | 21 + bulkbench/tests/proj0/patches.yaml | 2 + bulkbench/tests/test_parser_json.py | 234 +++ bulkbench/tests/test_script_runner.py | 222 +++ bulkbench/tests/test_system.py | 1645 +++++++++++++++++ 29 files changed, 5079 insertions(+) create mode 100644 bulkbench/.gitignore create mode 100644 bulkbench/README.md create mode 100644 bulkbench/example/03_pyt_IK_reorder/base_model.patch create mode 100644 bulkbench/example/04_pyt_IK_bucket/base_model.patch create mode 100644 bulkbench/example/05_pyt_IK_reorderbucket/base_model.patch create mode 100644 bulkbench/example/11_PR_only_meta/base_model.patch create mode 100644 bulkbench/example/12_PR_new_flags_no_meta/base_model.patch create mode 100644 bulkbench/example/13_PR_new_flags_with_meta/base_model.patch create mode 100644 bulkbench/example/14_PR_legacy_flags/base_model.patch create mode 100644 bulkbench/example/README.md create mode 100644 bulkbench/example/configs.yaml create mode 100644 bulkbench/example/patches_PR.yaml create mode 100644 bulkbench/example/patches_pyt213.yaml create mode 100644 bulkbench/example/report/benchstats-fix-groups.html create mode 100644 bulkbench/example/report/run-to-run.html create mode 100644 bulkbench/pyproject.toml create mode 100644 bulkbench/src/bulkbench/__init__.py create mode 100644 bulkbench/src/bulkbench/__main__.py create mode 100644 bulkbench/src/bulkbench/benchmark_plan_loader.py create mode 100644 bulkbench/src/bulkbench/benchmark_sources.py create mode 100644 bulkbench/src/bulkbench/bulkbench.py create mode 100644 bulkbench/src/bulkbench/cli_parser.py create mode 100644 bulkbench/src/bulkbench/parser_JSON.py create mode 100644 bulkbench/src/bulkbench/script_runner.py create mode 100644 bulkbench/tests/proj0/my_configs create mode 100644 bulkbench/tests/proj0/patches.yaml create mode 100644 bulkbench/tests/test_parser_json.py create mode 100644 bulkbench/tests/test_script_runner.py create mode 100644 bulkbench/tests/test_system.py diff --git a/bulkbench/.gitignore b/bulkbench/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/bulkbench/README.md b/bulkbench/README.md new file mode 100644 index 0000000..43a924a --- /dev/null +++ b/bulkbench/README.md @@ -0,0 +1,323 @@ +# Bulk Benchmarking Driver with Statistical Results Analysis for xDiT + +`bulkbench` is a standalone CLI tool that: + +- runs a single-machine benchmarking project over arbitrary combinations of xDiT model + configurations and implementation changes expressed as standard patch files; +- analyzes the resulting latencies for statistical significance with the `benchstats` package. The + supplied `timings.json` parser also lets run that analysis separately, on any combination of + model output directories. + +## Installation + +There's no `pip` release yet, so install from a local checkout: + +```bash +pip install --upgrade . +``` + +or straight from the repo: + +```bash +pip install --upgrade git+https://github.com/AMD-AGI/diffusion-models-inference.git#subdirectory=bulkbench +``` + +Run `bulkbench -h/--help` for CLI help. + +## Typical Workflow Example + +Say you need to measure how several code changes affect performance in isolation: each has to be +tested on its own before you can tell which change set is fastest overall. By hand that's tedious, +error-prone, needs constant supervision and isn't reproducible - repeating the setup on another +machine means starting over. Across many models, code variants and machines it becomes +infeasible. + +`bulkbench` automates it: describe the models and code changesets once, launch the project +(possibly on several machines), and the tool does the rest, including a robust statistical +comparison of the results. + +The high-level workflow is: + +1. Pick a directory for the project files. It needs at least: + + - `configs.yaml`, describing the grouped model configurations to benchmark; + - `patches.yaml`, describing the patch sets to apply to any files on the system + - (this assumes patching is enough to change run-time behavior, so compiled code isn't + supported yet, though that's easy to add if needed); + - optionally, the `.patch` files with the exact changes. + + These 3 entities constitute your project. Keep it in git for sharing and version control - just + `.gitignore` the results, report and backup subdirectories, which by default also live in the + project dir. + +2. Run `bulkbench` in the project dir, or point it there with `--project_dir` (other CLI arguments + could override everything else). It validates the files, dry-runs patch application so it won't + fail later, and launches the `/app/.ci/run.py` runner for each model configuration under each + patch set. From here the session needs no attention. + +3. On completion you get a full console report and several new subdirectories in the project dir + (unless you overrode their location): + + - `results` - a nested tree of each model's output under each patch set: essentially standard + `/output` directories grouped under the respective patch_set/benchmark_group subdirectory. + - `report` - one or more `*.html` reports on the statistical significance of the measured + performance differences (also printed to the console). Examples: + - [benchstats-fix-groups.html](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/benchstats-fix-groups.html) + - [run-to-run.html](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/run-to-run.html) + - `_backups` - appears if something prevented `bulkbench` from reverting the patched files (a + killed process, say); holds the original copies plus their paths for manual restoration. + + Now you can inspect the generated media for artifacts and examine the performance report. + +4. Optionally fix the system after failures, or update `configs.yaml` or `patches.yaml`, and rerun. + `bulkbench` scans `results` and will NOT rerun configs that already have results, i.e. at least + one media file plus `timings.json` in the config directory. A rerun overwrites only the + comparison reports in `--report_dir`, which must account for the new data. + + - To force a rerun of a benchmark config, config group or patch set, delete its subdirectory + under `--results_dir` first. + - To regenerate everything the project produces on the current machine, use + `--regenerate_results` (`-r`). + +5. To run the whole project on another machine, copy the project files (`configs.yaml`, + `patches.yaml` and the patch files) there and run the tool. + +6. To compare latencies across platforms, say MI300 vs MI350, create a directory and copy or + symlink each platform's `--results_dir` directory into it as `MI300` and `MI350` respectively. Then run the `benchstats` comparison utility manually: + + ```bash + benchstats {directory} --files_parser=bulkbench.parser_JSON \ + --sample_stats 0 100 --always_show_pvalues + ``` + +7. A machine can't always be properly quiesced, so false positives happen and statistical analysis + is no silver bullet. It's therefore a very good idea to rerun the project on the same machine + into a different `--results_dir` and `--report_dir`, then compare the same benchmarks across the + two runs: this shows how noisy the results are, how much to trust them, and whether something + simply needs a rerun. The easiest way: + + ```bash + # first run, with results and report in separate dirs + bulkbench --results_dir results/run1 --report_dir report/run1 + # second run, with its own outputs + bulkbench --results_dir results/run2 --report_dir report/run2 + + # now compare the same benchmarks across the runs + benchstats results --files_parser=bulkbench.parser_JSON --always_show_pvalues --filter1=2 + ``` + + You'd get something like: + + ```text + │ flux.usp/00_pyt_baseline | run1/a vs run2/a │ 777.8ms > 772.7ms {-0.7%} p=0.00001 (27 vs 27) │ + ``` + + which says you should be suspicious of differences smaller than ~1% on that machine. + +## Details + +### Format of `--configs_file` + +A YAML file listing objects that describe benchmark config groups. Each group represents a single +invocation of the `/app/.ci/run.py` runner and accepts: + +- `name` (required) - the group name: unique within the file, matching + `[-a-zA-Z0-9_+={}., ~!()\[\]]+` after stripping, and neither `.`, `..`, nor starting with the + reserved `eager_` prefix. Prefer short names for groups holding only configs not used elsewhere. + +- `configs` (required) - a non-empty list of benchmark config names to execute, each passed as the + `--name` argument to the runner. + + Names follow the `.[.]` convention, where `` is the basename of the + yaml file in `/app/.ci/benchmark_configs/` describing all of the model's configurations. + `bulkbench` resolves every name to its yaml file, extracts the config definition, and checks the + current GPU architecture (autodetected on AMD, overridable with `--arch`) against the config's + tags; a config whose tags don't match is ignored with a warning. So, as long as the names + reference real configs, you may list configs for as many architectures as the project needs. + +- `override_args` (optional) - key-value overrides of specific settings for every config in the + group, such as `num_iterations: `. +- `enabled` (optional) - whether to use the group; defaults to `true`. Valid values are unquoted + YAML `true`/`false`, the standard `yes`/`no` and `on`/`off` aliases, integers 1/0, and the quoted + `"true"`, `"false"`, `"1"`, `"0"`. Disabled groups are omitted without validating their other + attributes. +- `only_in_patches` (optional) - restricts the group to the listed patch sets. Names absent from the + `--patches_file` are ignored with a warning; a present but empty list disables the group + completely, and an absent field means 'run on all patch sets'. +- `eager_in_patches` (optional) - additionally runs the group's configs in eager mode in the listed + patch sets; once unknown names are stripped, an empty list is equivalent to an absent field. + Internally it creates an extra config group, `eager_`, inheriting `configs` from its + parent (which affects the `--results_dir` layout). The eager group's `only_in_patches` is the + intersection of the parent's `only_in_patches` and `eager_in_patches` (an empty intersection is an + error); its `override_args` are the parent's with `num_iterations` set to 1 and + `use_torch_compile` to `false`. Performance results of eager groups are ignored by the statistical + analysis. + +### Format of `--patches_file` + +A YAML file with a non-empty list of patch sets. Each patch set requires: + +- `name` - the patch set name: unique within the file, matching + `[-a-zA-Z0-9_+={}., ~!()\[\]]+` after stripping, and not `.` or `..`, +- `patches` - a list of patch objects, each patching a single file. An object may occur only once in + its set, and the patch lists themselves must be unique across the file regardless of object order; + only one empty baseline is allowed. + +Each patch object has the following attributes: + +- `patch` (required) - a path to the patch file; relative paths are resolved under + `{--project_dir}/{patch set name}`, absolute ones are used as-is. Generate it with + `git diff > changes.patch`, `diff -u original_file modified_file > changes.patch` or similar. + **Using a patch file that modifies several files is undefined behavior (UB)!** +- `target` (required) - a path to the file the `patch` applies to; relative paths are resolved under + the `/app` directory, absolute ones are used as-is. Both files must exist. **Applying several + patches to the same target file is UB.** +- `enabled` (optional) - whether to apply the patch; defaults to `true`. Accepts the same values as + the config group's `enabled` above. Disabled patches are omitted without validating their other + attributes. + +### Benchmarks Statistical Testing + +Once the project finishes, `bulkbench` automatically runs pairwise comparisons of the measured +latencies for statistical significance. This section explains how to control that and +craft practically any comparison your project needs. If comparisons the tool make by default are +enough for you, you can stop reading here. + +The backbone of the testing and report generation is the `benchstats` CLI tool - essentially a fancy +wrapper around `scipy` statistical tests that takes two sets of numbers (each holding measured +runtime durations of some code) and tells whether they differ significantly. Such a set is +called a "benchmark" in `benchstats` terminology and is identified by a name. `benchstats` itself +knows nothing about how the `/app/.ci/run.py` runner stores latencies, or which latencies to compare +against which, but it offers a standard interface for data source parsers plus a few conventions +that, once satisfied, let you arrange a statistical comparison of any source data. + +The `bulkbench.parser_JSON` module implements such a parser. It handles both an individual +`timings.json` file generated by the runner for a single benchmark config, and an arbitrary nested +tree of such files, deriving benchmark names from the directory names in their paths. + +For example, a single run of a `bulkbench` project produces this hierarchy inside `--results_dir`: + + `///timings.json`. + +so the command + +```bash +benchstats {--results_dir} --files_parser=bulkbench.parser_JSON \ + --sample_stats 0 100 --always_show_pvalues --export_to=all_to_all.html +``` + +compares each `` with every other identically named config in the tree, across different ``s and ``s, and saves the report to `all_to_all.html` in +the current directory. For instance, this `--results_dir` tree: + +``` +. +├── baseline +│ ├── group1 +│ │ ├── modelA +│ │ └── modelB +│ └── group2 +│ └── modelA +└── code_patch + ├── group1 + │ ├── modelA + │ └── modelB + └── group2 + └── modelA +``` + +(each leaf directory holding its own `timings.json`) yields these comparisons: + +``` +modelA | baseline/group1 vs baseline/group2 +modelA | baseline/group1 vs code_patch/group1 +modelA | baseline/group1 vs code_patch/group2 +modelA | baseline/group2 vs code_patch/group1 +modelA | baseline/group2 vs code_patch/group2 +modelA | code_patch/group1 vs code_patch/group2 +modelB | baseline/group1 vs code_patch/group1 +``` + +`bulkbench` runs this comparison into `{--report_dir}/benchstats-all-to-all.html` when it detects +the same `` under different `` parent directories; otherwise it +skips the comparison entirely. + +That's quite a few comparisons. Sometimes that's exactly what you need, but sometimes comparisons +within the same patch set may be uninteresting. `benchstats` passes its `--filter1` value verbatim +to the parser, and `bulkbench.parser_JSON` reads it as a comma-separated list of directory indices, +counted from `timings.json`, to include in the benchmark name. The default (and always implied) +value is `0`, the directory holding `timings.json`, i.e. ``. `--filter1=1` +additionally "freezes" the next hierarchy level, ``, which is what we need here: + +``` +modelA/group1 | baseline vs code_patch +modelA/group2 | baseline vs code_patch +modelB/group1 | baseline vs code_patch +``` + +`bulkbench` always saves this comparison into `{--report_dir}/benchstats-fix-groups.html` +([example](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/benchstats-fix-groups.html)). + +Now suppose we have results from different GPUs for the same project and want all the statistics in +one report. We just create a dedicated top-level directory and copy or symlink the `--results_dir` +directories of each `bulkbench` execution into it: + +``` +. +├── GPU1 +│ ├── baseline +│ │ ├── group1 +│ │ │ ├── modelA +│ │ │ └── modelB +│ │ └── group2 +│ │ └── modelA +│ └── code_patch +│ ├── group1 +│ │ ├── modelA +│ │ └── modelB +│ └── group2 +│ └── modelA +└── GPU2 + ├── baseline + │ ├── group1 + │ │ ├── modelA + │ │ └── modelB + │ └── group2 + │ └── modelA + └── code_patch + ├── group1 + │ ├── modelA + │ └── modelB + └── group2 + └── modelA +``` + +The GPU identifier now sits 3 levels above each `timings.json`, so `--filter1=1,3` gives this neat +comparison: + +``` +modelA/group1/GPU1 | baseline vs code_patch +modelA/group2/GPU1 | baseline vs code_patch +modelB/group1/GPU1 | baseline vs code_patch + +modelA/group1/GPU2 | baseline vs code_patch +modelA/group2/GPU2 | baseline vs code_patch +modelB/group1/GPU2 | baseline vs code_patch +``` + +`--filter1=1,2` instead "freezes" the patch set names and "unfreezes" the GPU identifiers, giving +comparisons across GPUs: + +``` +modelA/group1/baseline | GPU1 vs GPU2 +modelA/group2/baseline | GPU1 vs GPU2 +modelA/group1/code_patch | GPU1 vs GPU2 +modelA/group2/code_patch | GPU1 vs GPU2 +modelB/group1/baseline | GPU1 vs GPU2 +modelB/group1/code_patch | GPU1 vs GPU2 +``` + +As mentioned above, the same filter is useful for sanity checking of benchmarking results: you can run the same +project twice or more times on the same machine and then compare these runs one to the other to identify systematically +biased results (this happens when the machine was busy doing something else when the benchmark was run, - you can't +typically notice that in another way). See [run-to-run.html](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/run-to-run.html) for an example. + diff --git a/bulkbench/example/03_pyt_IK_reorder/base_model.patch b/bulkbench/example/03_pyt_IK_reorder/base_model.patch new file mode 100644 index 0000000..10553b0 --- /dev/null +++ b/bulkbench/example/03_pyt_IK_reorder/base_model.patch @@ -0,0 +1,30 @@ +diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py +index 9eb76e3..410ae87 100644 +--- a/xfuser/model_executor/models/runner_models/base_model.py ++++ b/xfuser/model_executor/models/runner_models/base_model.py +@@ -1,5 +1,6 @@ + import abc + import torch ++import torch._inductor.config_comms + import copy + import argparse + import json +@@ -470,6 +471,18 @@ class xFuserModel(abc.ABC): + + torch._inductor.config.reorder_for_compute_comm_overlap = True + ++ torch._inductor.config.runtime_estimations_mms_benchmark = True ++ torch._inductor.config.reorder_for_compute_comm_overlap_passes = [ ++ "reorder_communication_preserving_peak_memory", ++ "sink_waits_iterative", ++ "reorder_communication_preserving_peak_memory", ++ ] ++ torch._inductor.config_comms.reorder_iterative_peak_memory_budget = 0.2 # 0.2 is default?, if you have more spare memory - you can increase it ++ torch._inductor.config_comms.sink_iterative_peak_memory_budget = 0.2 # 0.2 is default?, increase if spare memory ++ torch._inductor.config_comms.reorder_iterative_use_runtime_estimations = True # default False ++ torch._inductor.config_comms.sink_iterative_use_runtime_estimations = True # default False ++ torch._inductor.config_comms.runtime_estimations_align_across_all_distributed_ranks = True ++ + # torch >= ~2.13: enabling the overlap machinery activates an SPMD + # graph-consistency check that issues a WORLD-group all_gather_object at + # compile time. Pipeline parallelism is non-SPMD (stages compile different diff --git a/bulkbench/example/04_pyt_IK_bucket/base_model.patch b/bulkbench/example/04_pyt_IK_bucket/base_model.patch new file mode 100644 index 0000000..909a445 --- /dev/null +++ b/bulkbench/example/04_pyt_IK_bucket/base_model.patch @@ -0,0 +1,24 @@ +diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py +index 9eb76e3..e9216b8 100644 +--- a/xfuser/model_executor/models/runner_models/base_model.py ++++ b/xfuser/model_executor/models/runner_models/base_model.py +@@ -1,5 +1,6 @@ + import abc + import torch ++import torch._inductor.config as ic + import copy + import argparse + import json +@@ -470,6 +471,12 @@ class xFuserModel(abc.ABC): + + torch._inductor.config.reorder_for_compute_comm_overlap = True + ++ torch._inductor.config.bucket_all_gathers_fx = "all" ++ torch._inductor.config.bucket_all_gathers_fx_bucket_size_determinator = lambda bucket_id: 100.0 if bucket_id < 2 else 1000.0 ++ ++ torch._inductor.config.bucket_reduce_scatters_fx = "all" ++ torch._inductor.config.bucket_reduce_scatters_fx_bucket_size_determinator = lambda bucket_id: 1000.0 ++ + # torch >= ~2.13: enabling the overlap machinery activates an SPMD + # graph-consistency check that issues a WORLD-group all_gather_object at + # compile time. Pipeline parallelism is non-SPMD (stages compile different diff --git a/bulkbench/example/05_pyt_IK_reorderbucket/base_model.patch b/bulkbench/example/05_pyt_IK_reorderbucket/base_model.patch new file mode 100644 index 0000000..1e8e8db --- /dev/null +++ b/bulkbench/example/05_pyt_IK_reorderbucket/base_model.patch @@ -0,0 +1,37 @@ +diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py +index 9eb76e3..c605873 100644 +--- a/xfuser/model_executor/models/runner_models/base_model.py ++++ b/xfuser/model_executor/models/runner_models/base_model.py +@@ -1,5 +1,7 @@ + import abc + import torch ++import torch._inductor.config_comms ++import torch._inductor.config as ic + import copy + import argparse + import json +@@ -470,6 +472,24 @@ class xFuserModel(abc.ABC): + + torch._inductor.config.reorder_for_compute_comm_overlap = True + ++ torch._inductor.config.runtime_estimations_mms_benchmark = True ++ torch._inductor.config.reorder_for_compute_comm_overlap_passes = [ ++ "reorder_communication_preserving_peak_memory", ++ "sink_waits_iterative", ++ "reorder_communication_preserving_peak_memory", ++ ] ++ torch._inductor.config_comms.reorder_iterative_peak_memory_budget = 0.2 # 0.2 is default?, if you have more spare memory - you can increase it ++ torch._inductor.config_comms.sink_iterative_peak_memory_budget = 0.2 # 0.2 is default?, increase if spare memory ++ torch._inductor.config_comms.reorder_iterative_use_runtime_estimations = True # default False ++ torch._inductor.config_comms.sink_iterative_use_runtime_estimations = True # default False ++ torch._inductor.config_comms.runtime_estimations_align_across_all_distributed_ranks = True ++ ++ torch._inductor.config.bucket_all_gathers_fx = "all" ++ torch._inductor.config.bucket_all_gathers_fx_bucket_size_determinator = lambda bucket_id: 100.0 if bucket_id < 2 else 1000.0 ++ ++ torch._inductor.config.bucket_reduce_scatters_fx = "all" ++ torch._inductor.config.bucket_reduce_scatters_fx_bucket_size_determinator = lambda bucket_id: 1000.0 ++ + # torch >= ~2.13: enabling the overlap machinery activates an SPMD + # graph-consistency check that issues a WORLD-group all_gather_object at + # compile time. Pipeline parallelism is non-SPMD (stages compile different diff --git a/bulkbench/example/11_PR_only_meta/base_model.patch b/bulkbench/example/11_PR_only_meta/base_model.patch new file mode 100644 index 0000000..98d9071 --- /dev/null +++ b/bulkbench/example/11_PR_only_meta/base_model.patch @@ -0,0 +1,15 @@ +diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py +index 4a1c68a..20be7d5 100644 +--- a/xfuser/model_executor/models/runner_models/base_model.py ++++ b/xfuser/model_executor/models/runner_models/base_model.py +@@ -468,6 +468,10 @@ class xFuserModel(abc.ABC): + + torch._inductor.config.reorder_for_compute_comm_overlap = True + ++ aten_opts = getattr(torch._inductor.config, "aten_distributed_optimizations", None) ++ assert aten_opts is not None ++ aten_opts.insert_overlap_deps_impl = "meta" ++ + # torch >= ~2.13: enabling the overlap machinery activates an SPMD + # graph-consistency check that issues a WORLD-group all_gather_object at + # compile time. Pipeline parallelism is non-SPMD (stages compile different diff --git a/bulkbench/example/12_PR_new_flags_no_meta/base_model.patch b/bulkbench/example/12_PR_new_flags_no_meta/base_model.patch new file mode 100644 index 0000000..7efd2cb --- /dev/null +++ b/bulkbench/example/12_PR_new_flags_no_meta/base_model.patch @@ -0,0 +1,18 @@ +diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py +index 4a1c68a..2bfb6b9 100644 +--- a/xfuser/model_executor/models/runner_models/base_model.py ++++ b/xfuser/model_executor/models/runner_models/base_model.py +@@ -468,6 +468,13 @@ class xFuserModel(abc.ABC): + + torch._inductor.config.reorder_for_compute_comm_overlap = True + ++ aten_opts = getattr(torch._inductor.config, "aten_distributed_optimizations", None) ++ assert aten_opts is not None ++ aten_opts.enable_overlap_scheduling = True ++ aten_opts.collective_bucketing = True ++ aten_opts.insert_overlap_deps = True ++ aten_opts.collective_estimator = "benchmark" ++ + # torch >= ~2.13: enabling the overlap machinery activates an SPMD + # graph-consistency check that issues a WORLD-group all_gather_object at + # compile time. Pipeline parallelism is non-SPMD (stages compile different diff --git a/bulkbench/example/13_PR_new_flags_with_meta/base_model.patch b/bulkbench/example/13_PR_new_flags_with_meta/base_model.patch new file mode 100644 index 0000000..1b30c14 --- /dev/null +++ b/bulkbench/example/13_PR_new_flags_with_meta/base_model.patch @@ -0,0 +1,19 @@ +diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py +index 4a1c68a..dd9eb05 100644 +--- a/xfuser/model_executor/models/runner_models/base_model.py ++++ b/xfuser/model_executor/models/runner_models/base_model.py +@@ -468,6 +468,14 @@ class xFuserModel(abc.ABC): + + torch._inductor.config.reorder_for_compute_comm_overlap = True + ++ aten_opts = getattr(torch._inductor.config, "aten_distributed_optimizations", None) ++ assert aten_opts is not None ++ aten_opts.enable_overlap_scheduling = True ++ aten_opts.collective_bucketing = True ++ aten_opts.insert_overlap_deps = True ++ aten_opts.collective_estimator = "benchmark" ++ aten_opts.insert_overlap_deps_impl = "meta" ++ + # torch >= ~2.13: enabling the overlap machinery activates an SPMD + # graph-consistency check that issues a WORLD-group all_gather_object at + # compile time. Pipeline parallelism is non-SPMD (stages compile different diff --git a/bulkbench/example/14_PR_legacy_flags/base_model.patch b/bulkbench/example/14_PR_legacy_flags/base_model.patch new file mode 100644 index 0000000..7bd45a3 --- /dev/null +++ b/bulkbench/example/14_PR_legacy_flags/base_model.patch @@ -0,0 +1,17 @@ +diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py +index 4a1c68a..00e2355 100644 +--- a/xfuser/model_executor/models/runner_models/base_model.py ++++ b/xfuser/model_executor/models/runner_models/base_model.py +@@ -468,6 +468,12 @@ class xFuserModel(abc.ABC): + + torch._inductor.config.reorder_for_compute_comm_overlap = True + ++ torch._inductor.config.reorder_for_compute_comm_overlap_passes = [ ++ "reorder_compute_for_overlap", ++ "sink_waits", ++ "raise_comms", ++ ] ++ + # torch >= ~2.13: enabling the overlap machinery activates an SPMD + # graph-consistency check that issues a WORLD-group all_gather_object at + # compile time. Pipeline parallelism is non-SPMD (stages compile different diff --git a/bulkbench/example/README.md b/bulkbench/example/README.md new file mode 100644 index 0000000..b114681 --- /dev/null +++ b/bulkbench/example/README.md @@ -0,0 +1,33 @@ +# Example `bulkbench` project + +This directory provides an example of how a `bulkbench` project could look like and is based on a +real-world investigation on how a certain PyTorch pull request (PR) behaves under different set of +PyTorch flags compared to raw PyTorch, on `gfx942` and `gfx950` GPUs. + +The project has [`configs.yaml`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/configs.yaml) +defining 3 model configurations in 2 groups (each group could have own `override_args`): + +- `flux.usp` +- `flux2.quantgemm` +- `wan2_2.quantgemm_fp8attn` + +Raw PyTorch instance and the instance with the applied PR expected to live in separate +docker containers with the project directory mounted into each of them. Depending on a +container (base code instance), one could select which patch sets to apply with `--patches_file` argument. + +Two patch files are defined: [`patches_PR.yaml`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/patches_PR.yaml) +with 5 patch variants for the PR instance, and [`patches_py213.yaml`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/patches_py213.yaml) +with 6 patch variants for the raw PyTorch instance. + +[`report`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report) directory shows two reports +made from obtained results: + +- [`benchstats-fix-groups.html`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/benchstats-fix-groups.html) was +generated automatically by `bulkbench` and shows relative performance of models for `gfx950`, +- [`run-to-run.html`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/run-to-run.html) was generated manually +on results from 2 runs of the same project on the machine to estimate machine's noise level with command: + +``` +benchstats . --files_parser=bulkbench.parser_JSON --sample_stats 0 100 --always_show_pvalues \ + --filter1=1,2 --export_to=./run-to-run.html +``` diff --git a/bulkbench/example/configs.yaml b/bulkbench/example/configs.yaml new file mode 100644 index 0000000..ac2dfad --- /dev/null +++ b/bulkbench/example/configs.yaml @@ -0,0 +1,18 @@ +- name: a + configs: + - flux.usp + # - flux2.default + - flux2.quantgemm.gfx942 + - flux2.quantgemm.gfx950 + eager_in_patches: [00_pyt_baseline, 10_PR_baseline, 11_PR_only_meta] + override_args: + num_iterations: 30 + +- name: b + # enabled: false + configs: + - wan2_2.quantgemm_fp8attn.gfx950 + - wan2_2.quantgemm_fp8attn.gfx942 + eager_in_patches: [00_pyt_baseline, 10_PR_baseline, 11_PR_only_meta] + override_args: + num_iterations: 28 diff --git a/bulkbench/example/patches_PR.yaml b/bulkbench/example/patches_PR.yaml new file mode 100644 index 0000000..1983b06 --- /dev/null +++ b/bulkbench/example/patches_PR.yaml @@ -0,0 +1,26 @@ +# assumes run in a container with changeset #188404 applied on top of pytorch=2.13 +# run with `bulkbench --patches_file ./patches_PR.yaml` + +- name: 10_PR_baseline + patches: [] + +- name: 11_PR_only_meta + patches: + # relative path names are resolved using the patch set name, `11_PR_only_meta` in this case + - patch: "base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" + +- name: 12_PR_new_flags_no_meta + patches: + - patch: "base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" + +- name: 13_PR_new_flags_with_meta + patches: + - patch: "base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" + +- name: 14_PR_legacy_flags + patches: + - patch: "base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" diff --git a/bulkbench/example/patches_pyt213.yaml b/bulkbench/example/patches_pyt213.yaml new file mode 100644 index 0000000..95509a7 --- /dev/null +++ b/bulkbench/example/patches_pyt213.yaml @@ -0,0 +1,31 @@ +# assumes run in a container with a clean pytorch=2.13 +# run with `bulkbench --patches_file ./patches_pyt213.yaml` + +- name: 00_pyt_baseline + patches: [] + +- name: 01_pyt_new_flags + patches: + # relative path names are resolved using the patch set name, `01_pyt_new_flags` in this case + - patch: "../12_PR_new_flags_no_meta/base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" + +- name: 02_pyt_legacy_flags + patches: + - patch: "../14_PR_legacy_flags/base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" + +- name: 03_pyt_IK_reorder + patches: + - patch: "base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" + +- name: 04_pyt_IK_bucket + patches: + - patch: "base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" + +- name: 05_pyt_IK_reorderbucket + patches: + - patch: "base_model.patch" + target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py" diff --git a/bulkbench/example/report/benchstats-fix-groups.html b/bulkbench/example/report/benchstats-fix-groups.html new file mode 100644 index 0000000..ee43888 --- /dev/null +++ b/bulkbench/example/report/benchstats-fix-groups.html @@ -0,0 +1,225 @@ + + + + + + + + + + + + 
[warn]  The following directories don't contain an immediate or nested timings.json, and are ignored:
+- ./03_pyt_IK_reorder/b
+- ./03_pyt_IK_reorder/b/wan2_2.quantgemm_fp8attn.gfx950
+- ./05_pyt_IK_reorderbucket/b
+- ./05_pyt_IK_reorderbucket/b/wan2_2.quantgemm_fp8attn.gfx950
+                                                              Benchmark comparison results (Brunner Munzel test, alpha=0.00100)                                                              
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                        Benchmark                                         ┃                                 real_time (means), [0%, 100%]                                  ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+│ flux.usp/a | 00_pyt_baseline vs 01_pyt_new_flags                                         │ 777.8ms < 801.9ms {+3.1%} [769.9m,785.2m] < [797.1m,824.6m] {+3.5%,+5.0%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 02_pyt_legacy_flags                                      │ 777.8ms < 792.7ms {+1.9%} [769.9m,785.2m] < [788.5m,799.2m] {+2.4%,+1.8%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 03_pyt_IK_reorder                                        │ 777.8ms ~ 775.6ms {-0.3%} [769.9m,785.2m] ~ [771.2m,780.7m] {+0.2%,-0.6%} p=0.02968 (27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 04_pyt_IK_bucket                                         │ 777.8ms ~ 778.9ms {+0.1%} [769.9m,785.2m] ~ [772.0m,800.4m] {+0.3%,+1.9%} p=0.49670 (27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 05_pyt_IK_reorderbucket                                  │ 777.8ms ~ 774.3ms {-0.5%} [769.9m,785.2m] ~ [767.6m,790.4m] {-0.3%,+0.7%} p=0.00114 (27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 10_PR_baseline                                           │ 777.8ms ~ 777.8ms {-0.0%} [769.9m,785.2m] ~ [769.3m,784.6m] {-0.1%,-0.1%} p=0.44963 (27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 11_PR_only_meta                                          │ 777.8ms > 772.5ms {-0.7%} [769.9m,785.2m] > [768.0m,778.0m] {-0.3%,-0.9%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 12_PR_new_flags_no_meta                                  │ 777.8ms < 823.9ms {+5.9%} [769.9m,785.2m] < [816.0m,842.4m] {+6.0%,+7.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 13_PR_new_flags_with_meta                                │ 777.8ms < 794.2ms {+2.1%} [769.9m,785.2m] < [789.0m,809.2m] {+2.5%,+3.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 00_pyt_baseline vs 14_PR_legacy_flags                                       │ 777.8ms < 795.6ms {+2.3%} [769.9m,785.2m] < [782.4m,816.5m] {+1.6%,+4.0%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 02_pyt_legacy_flags                                     │ 801.9ms > 792.7ms {-1.1%} [797.1m,824.6m] > [788.5m,799.2m] {-1.1%,-3.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 03_pyt_IK_reorder                                       │ 801.9ms > 775.6ms {-3.3%} [797.1m,824.6m] > [771.2m,780.7m] {-3.2%,-5.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 04_pyt_IK_bucket                                        │ 801.9ms > 778.9ms {-2.9%} [797.1m,824.6m] > [772.0m,800.4m] {-3.1%,-2.9%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 05_pyt_IK_reorderbucket                                 │ 801.9ms > 774.3ms {-3.4%} [797.1m,824.6m] > [767.6m,790.4m] {-3.7%,-4.2%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 10_PR_baseline                                          │ 801.9ms > 777.8ms {-3.0%} [797.1m,824.6m] > [769.3m,784.6m] {-3.5%,-4.9%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 11_PR_only_meta                                         │ 801.9ms > 772.5ms {-3.7%} [797.1m,824.6m] > [768.0m,778.0m] {-3.7%,-5.7%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 12_PR_new_flags_no_meta                                 │ 801.9ms < 823.9ms {+2.7%} [797.1m,824.6m] < [816.0m,842.4m] {+2.4%,+2.2%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 13_PR_new_flags_with_meta                               │ 801.9ms > 794.2ms {-1.0%} [797.1m,824.6m] > [789.0m,809.2m] {-1.0%,-1.9%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 01_pyt_new_flags vs 14_PR_legacy_flags                                      │ 801.9ms > 795.6ms {-0.8%} [797.1m,824.6m] > [782.4m,816.5m] {-1.8%,-1.0%} p=0.00076 (27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 03_pyt_IK_reorder                                    │ 792.7ms > 775.6ms {-2.2%} [788.5m,799.2m] > [771.2m,780.7m] {-2.2%,-2.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 04_pyt_IK_bucket                                     │ 792.7ms > 778.9ms {-1.8%} [788.5m,799.2m] > [772.0m,800.4m] {-2.1%,+0.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 05_pyt_IK_reorderbucket                              │ 792.7ms > 774.3ms {-2.3%} [788.5m,799.2m] > [767.6m,790.4m] {-2.6%,-1.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 10_PR_baseline                                       │ 792.7ms > 777.8ms {-1.9%} [788.5m,799.2m] > [769.3m,784.6m] {-2.4%,-1.8%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 11_PR_only_meta                                      │ 792.7ms > 772.5ms {-2.6%} [788.5m,799.2m] > [768.0m,778.0m] {-2.6%,-2.7%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 12_PR_new_flags_no_meta                              │ 792.7ms < 823.9ms {+3.9%} [788.5m,799.2m] < [816.0m,842.4m] {+3.5%,+5.4%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 13_PR_new_flags_with_meta                            │ 792.7ms ~ 794.2ms {+0.2%} [788.5m,799.2m] ~ [789.0m,809.2m] {+0.1%,+1.2%} p=0.08305 (27 vs 27) │
+│ flux.usp/a | 02_pyt_legacy_flags vs 14_PR_legacy_flags                                   │ 792.7ms ~ 795.6ms {+0.4%} [788.5m,799.2m] ~ [782.4m,816.5m] {-0.8%,+2.2%} p=0.10976 (27 vs 27) │
+│ flux.usp/a | 03_pyt_IK_reorder vs 04_pyt_IK_bucket                                       │ 775.6ms < 778.9ms {+0.4%} [771.2m,780.7m] < [772.0m,800.4m] {+0.1%,+2.5%} p=0.00017 (27 vs 27) │
+│ flux.usp/a | 03_pyt_IK_reorder vs 05_pyt_IK_reorderbucket                                │ 775.6ms ~ 774.3ms {-0.2%} [771.2m,780.7m] ~ [767.6m,790.4m] {-0.5%,+1.2%} p=0.01120 (27 vs 27) │
+│ flux.usp/a | 03_pyt_IK_reorder vs 10_PR_baseline                                         │ 775.6ms ~ 777.8ms {+0.3%} [771.2m,780.7m] ~ [769.3m,784.6m] {-0.2%,+0.5%} p=0.05730 (27 vs 27) │
+│ flux.usp/a | 03_pyt_IK_reorder vs 11_PR_only_meta                                        │ 775.6ms > 772.5ms {-0.4%} [771.2m,780.7m] > [768.0m,778.0m] {-0.4%,-0.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 03_pyt_IK_reorder vs 12_PR_new_flags_no_meta                                │ 775.6ms < 823.9ms {+6.2%} [771.2m,780.7m] < [816.0m,842.4m] {+5.8%,+7.9%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 03_pyt_IK_reorder vs 13_PR_new_flags_with_meta                              │ 775.6ms < 794.2ms {+2.4%} [771.2m,780.7m] < [789.0m,809.2m] {+2.3%,+3.6%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 03_pyt_IK_reorder vs 14_PR_legacy_flags                                     │ 775.6ms < 795.6ms {+2.6%} [771.2m,780.7m] < [782.4m,816.5m] {+1.5%,+4.6%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 04_pyt_IK_bucket vs 05_pyt_IK_reorderbucket                                 │ 778.9ms > 774.3ms {-0.6%} [772.0m,800.4m] > [767.6m,790.4m] {-0.6%,-1.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 04_pyt_IK_bucket vs 10_PR_baseline                                          │ 778.9ms ~ 777.8ms {-0.1%} [772.0m,800.4m] ~ [769.3m,784.6m] {-0.4%,-2.0%} p=0.42006 (27 vs 27) │
+│ flux.usp/a | 04_pyt_IK_bucket vs 11_PR_only_meta                                         │ 778.9ms > 772.5ms {-0.8%} [772.0m,800.4m] > [768.0m,778.0m] {-0.5%,-2.8%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 04_pyt_IK_bucket vs 12_PR_new_flags_no_meta                                 │ 778.9ms < 823.9ms {+5.8%} [772.0m,800.4m] < [816.0m,842.4m] {+5.7%,+5.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 04_pyt_IK_bucket vs 13_PR_new_flags_with_meta                               │ 778.9ms < 794.2ms {+2.0%} [772.0m,800.4m] < [789.0m,809.2m] {+2.2%,+1.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 04_pyt_IK_bucket vs 14_PR_legacy_flags                                      │ 778.9ms < 795.6ms {+2.1%} [772.0m,800.4m] < [782.4m,816.5m] {+1.3%,+2.0%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 05_pyt_IK_reorderbucket vs 10_PR_baseline                                   │ 774.3ms < 777.8ms {+0.5%} [767.6m,790.4m] < [769.3m,784.6m] {+0.2%,-0.7%} p=0.00094 (27 vs 27) │
+│ flux.usp/a | 05_pyt_IK_reorderbucket vs 11_PR_only_meta                                  │ 774.3ms ~ 772.5ms {-0.2%} [767.6m,790.4m] ~ [768.0m,778.0m] {+0.0%,-1.6%} p=0.03753 (27 vs 27) │
+│ flux.usp/a | 05_pyt_IK_reorderbucket vs 12_PR_new_flags_no_meta                          │ 774.3ms < 823.9ms {+6.4%} [767.6m,790.4m] < [816.0m,842.4m] {+6.3%,+6.6%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 05_pyt_IK_reorderbucket vs 13_PR_new_flags_with_meta                        │ 774.3ms < 794.2ms {+2.6%} [767.6m,790.4m] < [789.0m,809.2m] {+2.8%,+2.4%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 05_pyt_IK_reorderbucket vs 14_PR_legacy_flags                               │ 774.3ms < 795.6ms {+2.7%} [767.6m,790.4m] < [782.4m,816.5m] {+1.9%,+3.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 10_PR_baseline vs 11_PR_only_meta                                           │ 777.8ms > 772.5ms {-0.7%} [769.3m,784.6m] > [768.0m,778.0m] {-0.2%,-0.8%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 10_PR_baseline vs 12_PR_new_flags_no_meta                                   │ 777.8ms < 823.9ms {+5.9%} [769.3m,784.6m] < [816.0m,842.4m] {+6.1%,+7.4%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 10_PR_baseline vs 13_PR_new_flags_with_meta                                 │ 777.8ms < 794.2ms {+2.1%} [769.3m,784.6m] < [789.0m,809.2m] {+2.6%,+3.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 10_PR_baseline vs 14_PR_legacy_flags                                        │ 777.8ms < 795.6ms {+2.3%} [769.3m,784.6m] < [782.4m,816.5m] {+1.7%,+4.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 11_PR_only_meta vs 12_PR_new_flags_no_meta                                  │ 772.5ms < 823.9ms {+6.7%} [768.0m,778.0m] < [816.0m,842.4m] {+6.3%,+8.3%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 11_PR_only_meta vs 13_PR_new_flags_with_meta                                │ 772.5ms < 794.2ms {+2.8%} [768.0m,778.0m] < [789.0m,809.2m] {+2.7%,+4.0%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 11_PR_only_meta vs 14_PR_legacy_flags                                       │ 772.5ms < 795.6ms {+3.0%} [768.0m,778.0m] < [782.4m,816.5m] {+1.9%,+4.9%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 12_PR_new_flags_no_meta vs 13_PR_new_flags_with_meta                        │ 823.9ms > 794.2ms {-3.6%} [816.0m,842.4m] > [789.0m,809.2m] {-3.3%,-4.0%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 12_PR_new_flags_no_meta vs 14_PR_legacy_flags                               │ 823.9ms > 795.6ms {-3.4%} [816.0m,842.4m] > [782.4m,816.5m] {-4.1%,-3.1%} p=0.00000+(27 vs 27) │
+│ flux.usp/a | 13_PR_new_flags_with_meta vs 14_PR_legacy_flags                             │ 794.2ms ~ 795.6ms {+0.2%} [789.0m,809.2m] ~ [782.4m,816.5m] {-0.8%,+0.9%} p=0.25237 (27 vs 27) │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 01_pyt_new_flags                           │ 2.272s < 2.349s {+3.4%} [2.260,2.281] < [2.333,2.365] {+3.2%,+3.7%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 02_pyt_legacy_flags                        │ 2.272s ~ 2.272s {-0.0%} [2.260,2.281] ~ [2.259,2.292] {-0.0%,+0.5%} p=0.38322 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 03_pyt_IK_reorder                          │ 2.272s ~ 2.268s {-0.2%} [2.260,2.281] ~ [2.255,2.280] {-0.2%,-0.1%} p=0.00843 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 04_pyt_IK_bucket                           │ 2.272s ~ 2.274s {+0.1%} [2.260,2.281] ~ [2.263,2.303] {+0.2%,+0.9%} p=0.26098 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 05_pyt_IK_reorderbucket                    │ 2.272s > 2.262s {-0.4%} [2.260,2.281] > [2.249,2.280] {-0.5%,-0.1%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 10_PR_baseline                             │ 2.272s ~ 2.272s {+0.0%} [2.260,2.281] ~ [2.260,2.288] {-0.0%,+0.3%} p=0.49002 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 11_PR_only_meta                            │ 2.272s ~ 2.271s {-0.0%} [2.260,2.281] ~ [2.259,2.284] {-0.0%,+0.1%} p=0.32689 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 12_PR_new_flags_no_meta                    │ 2.272s < 2.334s {+2.8%} [2.260,2.281] < [2.324,2.356] {+2.9%,+3.3%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 13_PR_new_flags_with_meta                  │ 2.272s < 2.284s {+0.5%} [2.260,2.281] < [2.267,2.306] {+0.3%,+1.1%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 00_pyt_baseline vs 14_PR_legacy_flags                         │ 2.272s ~ 2.268s {-0.2%} [2.260,2.281] ~ [2.256,2.280] {-0.2%,-0.0%} p=0.02789 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 02_pyt_legacy_flags                       │ 2.349s > 2.272s {-3.3%} [2.333,2.365] > [2.259,2.292] {-3.2%,-3.1%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 03_pyt_IK_reorder                         │ 2.349s > 2.268s {-3.4%} [2.333,2.365] > [2.255,2.280] {-3.3%,-3.6%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 04_pyt_IK_bucket                          │ 2.349s > 2.274s {-3.2%} [2.333,2.365] > [2.263,2.303] {-3.0%,-2.6%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 05_pyt_IK_reorderbucket                   │ 2.349s > 2.262s {-3.7%} [2.333,2.365] > [2.249,2.280] {-3.6%,-3.6%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 10_PR_baseline                            │ 2.349s > 2.272s {-3.3%} [2.333,2.365] > [2.260,2.288] {-3.1%,-3.3%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 11_PR_only_meta                           │ 2.349s > 2.271s {-3.3%} [2.333,2.365] > [2.259,2.284] {-3.2%,-3.4%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 12_PR_new_flags_no_meta                   │ 2.349s > 2.334s {-0.6%} [2.333,2.365] > [2.324,2.356] {-0.4%,-0.4%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 13_PR_new_flags_with_meta                 │ 2.349s > 2.284s {-2.8%} [2.333,2.365] > [2.267,2.306] {-2.8%,-2.5%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 01_pyt_new_flags vs 14_PR_legacy_flags                        │ 2.349s > 2.268s {-3.4%} [2.333,2.365] > [2.256,2.280] {-3.3%,-3.6%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 03_pyt_IK_reorder                      │ 2.272s ~ 2.268s {-0.2%} [2.259,2.292] ~ [2.255,2.280] {-0.2%,-0.5%} p=0.01873 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 04_pyt_IK_bucket                       │ 2.272s ~ 2.274s {+0.1%} [2.259,2.292] ~ [2.263,2.303] {+0.2%,+0.4%} p=0.21477 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 05_pyt_IK_reorderbucket                │ 2.272s > 2.262s {-0.4%} [2.259,2.292] > [2.249,2.280] {-0.4%,-0.5%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 10_PR_baseline                         │ 2.272s ~ 2.272s {+0.0%} [2.259,2.292] ~ [2.260,2.288] {+0.0%,-0.2%} p=0.40667 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 11_PR_only_meta                        │ 2.272s ~ 2.271s {-0.0%} [2.259,2.292] ~ [2.259,2.284] {+0.0%,-0.4%} p=0.48333 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 12_PR_new_flags_no_meta                │ 2.272s < 2.334s {+2.8%} [2.259,2.292] < [2.324,2.356] {+2.9%,+2.8%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 13_PR_new_flags_with_meta              │ 2.272s < 2.284s {+0.5%} [2.259,2.292] < [2.267,2.306] {+0.4%,+0.6%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 02_pyt_legacy_flags vs 14_PR_legacy_flags                     │ 2.272s ~ 2.268s {-0.1%} [2.259,2.292] ~ [2.256,2.280] {-0.1%,-0.5%} p=0.05876 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 03_pyt_IK_reorder vs 04_pyt_IK_bucket                         │ 2.268s ~ 2.274s {+0.3%} [2.255,2.280] ~ [2.263,2.303] {+0.4%,+1.0%} p=0.00234 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 03_pyt_IK_reorder vs 05_pyt_IK_reorderbucket                  │ 2.268s > 2.262s {-0.3%} [2.255,2.280] > [2.249,2.280] {-0.3%,+0.0%} p=0.00002 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 03_pyt_IK_reorder vs 10_PR_baseline                           │ 2.268s ~ 2.272s {+0.2%} [2.255,2.280] ~ [2.260,2.288] {+0.2%,+0.3%} p=0.02788 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 03_pyt_IK_reorder vs 11_PR_only_meta                          │ 2.268s ~ 2.271s {+0.1%} [2.255,2.280] ~ [2.259,2.284] {+0.2%,+0.2%} p=0.04006 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 03_pyt_IK_reorder vs 12_PR_new_flags_no_meta                  │ 2.268s < 2.334s {+2.9%} [2.255,2.280] < [2.324,2.356] {+3.1%,+3.3%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 03_pyt_IK_reorder vs 13_PR_new_flags_with_meta                │ 2.268s < 2.284s {+0.7%} [2.255,2.280] < [2.267,2.306] {+0.5%,+1.2%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 03_pyt_IK_reorder vs 14_PR_legacy_flags                       │ 2.268s ~ 2.268s {+0.0%} [2.255,2.280] ~ [2.256,2.280] {+0.0%,+0.0%} p=0.47659 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 04_pyt_IK_bucket vs 05_pyt_IK_reorderbucket                   │ 2.274s > 2.262s {-0.5%} [2.263,2.303] > [2.249,2.280] {-0.6%,-1.0%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 04_pyt_IK_bucket vs 10_PR_baseline                            │ 2.274s ~ 2.272s {-0.1%} [2.263,2.303] ~ [2.260,2.288] {-0.2%,-0.6%} p=0.29173 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 04_pyt_IK_bucket vs 11_PR_only_meta                           │ 2.274s ~ 2.271s {-0.1%} [2.263,2.303] ~ [2.259,2.284] {-0.2%,-0.8%} p=0.11920 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 04_pyt_IK_bucket vs 12_PR_new_flags_no_meta                   │ 2.274s < 2.334s {+2.6%} [2.263,2.303] < [2.324,2.356] {+2.7%,+2.3%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 04_pyt_IK_bucket vs 13_PR_new_flags_with_meta                 │ 2.274s < 2.284s {+0.4%} [2.263,2.303] < [2.267,2.306] {+0.2%,+0.2%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 04_pyt_IK_bucket vs 14_PR_legacy_flags                        │ 2.274s ~ 2.268s {-0.3%} [2.263,2.303] ~ [2.256,2.280] {-0.3%,-1.0%} p=0.00655 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 05_pyt_IK_reorderbucket vs 10_PR_baseline                     │ 2.262s < 2.272s {+0.5%} [2.249,2.280] < [2.260,2.288] {+0.5%,+0.3%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 05_pyt_IK_reorderbucket vs 11_PR_only_meta                    │ 2.262s < 2.271s {+0.4%} [2.249,2.280] < [2.259,2.284] {+0.4%,+0.2%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 05_pyt_IK_reorderbucket vs 12_PR_new_flags_no_meta            │ 2.262s < 2.334s {+3.2%} [2.249,2.280] < [2.324,2.356] {+3.3%,+3.3%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 05_pyt_IK_reorderbucket vs 13_PR_new_flags_with_meta          │ 2.262s < 2.284s {+1.0%} [2.249,2.280] < [2.267,2.306] {+0.8%,+1.2%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 05_pyt_IK_reorderbucket vs 14_PR_legacy_flags                 │ 2.262s < 2.268s {+0.3%} [2.249,2.280] < [2.256,2.280] {+0.3%,+0.0%} p=0.00015 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 10_PR_baseline vs 11_PR_only_meta                             │ 2.272s ~ 2.271s {-0.0%} [2.260,2.288] ~ [2.259,2.284] {-0.0%,-0.2%} p=0.37099 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 10_PR_baseline vs 12_PR_new_flags_no_meta                     │ 2.272s < 2.334s {+2.7%} [2.260,2.288] < [2.324,2.356] {+2.9%,+3.0%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 10_PR_baseline vs 13_PR_new_flags_with_meta                   │ 2.272s < 2.284s {+0.5%} [2.260,2.288] < [2.267,2.306] {+0.4%,+0.8%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 10_PR_baseline vs 14_PR_legacy_flags                          │ 2.272s ~ 2.268s {-0.2%} [2.260,2.288] ~ [2.256,2.280] {-0.1%,-0.3%} p=0.03955 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 11_PR_only_meta vs 12_PR_new_flags_no_meta                    │ 2.271s < 2.334s {+2.8%} [2.259,2.284] < [2.324,2.356] {+2.9%,+3.1%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 11_PR_only_meta vs 13_PR_new_flags_with_meta                  │ 2.271s < 2.284s {+0.6%} [2.259,2.284] < [2.267,2.306] {+0.4%,+1.0%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 11_PR_only_meta vs 14_PR_legacy_flags                         │ 2.271s ~ 2.268s {-0.1%} [2.259,2.284] ~ [2.256,2.280] {-0.1%,-0.2%} p=0.06611 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 12_PR_new_flags_no_meta vs 13_PR_new_flags_with_meta          │ 2.334s > 2.284s {-2.2%} [2.324,2.356] > [2.267,2.306] {-2.4%,-2.1%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 12_PR_new_flags_no_meta vs 14_PR_legacy_flags                 │ 2.334s > 2.268s {-2.8%} [2.324,2.356] > [2.256,2.280] {-2.9%,-3.2%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a | 13_PR_new_flags_with_meta vs 14_PR_legacy_flags               │ 2.284s > 2.268s {-0.7%} [2.267,2.306] > [2.256,2.280] {-0.5%,-1.1%} p=0.00000+(27 vs 27)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 01_pyt_new_flags                  │ 34.32s < 35.90s {+4.6%} [34.23,34.44] < [35.78,36.01] {+4.5%,+4.6%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 02_pyt_legacy_flags               │ 34.32s > 33.97s {-1.0%} [34.23,34.44] > [33.85,34.10] {-1.1%,-1.0%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 04_pyt_IK_bucket                  │ 34.32s > 34.27s {-0.1%} [34.23,34.44] > [34.19,34.38] {-0.1%,-0.2%} p=0.00003 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 10_PR_baseline                    │ 34.32s ~ 34.30s {-0.1%} [34.23,34.44] ~ [34.15,34.40] {-0.2%,-0.1%} p=0.03236 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 11_PR_only_meta                   │ 34.32s ~ 34.27s {-0.1%} [34.23,34.44] ~ [34.14,34.38] {-0.3%,-0.2%} p=0.00104 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 12_PR_new_flags_no_meta           │ 34.32s < 35.89s {+4.6%} [34.23,34.44] < [35.78,36.00] {+4.5%,+4.5%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 13_PR_new_flags_with_meta         │ 34.32s < 35.17s {+2.5%} [34.23,34.44] < [35.07,35.32] {+2.5%,+2.6%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 00_pyt_baseline vs 14_PR_legacy_flags                │ 34.32s > 33.96s {-1.0%} [34.23,34.44] > [33.87,34.11] {-1.1%,-1.0%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 01_pyt_new_flags vs 02_pyt_legacy_flags              │ 35.90s > 33.97s {-5.4%} [35.78,36.01] > [33.85,34.10] {-5.4%,-5.3%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 01_pyt_new_flags vs 04_pyt_IK_bucket                 │ 35.90s > 34.27s {-4.5%} [35.78,36.01] > [34.19,34.38] {-4.4%,-4.5%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 01_pyt_new_flags vs 10_PR_baseline                   │ 35.90s > 34.30s {-4.5%} [35.78,36.01] > [34.15,34.40] {-4.6%,-4.5%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 01_pyt_new_flags vs 11_PR_only_meta                  │ 35.90s > 34.27s {-4.5%} [35.78,36.01] > [34.14,34.38] {-4.6%,-4.5%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 01_pyt_new_flags vs 12_PR_new_flags_no_meta          │ 35.90s ~ 35.89s {-0.0%} [35.78,36.01] ~ [35.78,36.00] {-0.0%,-0.0%} p=0.22786 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 01_pyt_new_flags vs 13_PR_new_flags_with_meta        │ 35.90s > 35.17s {-2.0%} [35.78,36.01] > [35.07,35.32] {-2.0%,-1.9%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 01_pyt_new_flags vs 14_PR_legacy_flags               │ 35.90s > 33.96s {-5.4%} [35.78,36.01] > [33.87,34.11] {-5.3%,-5.3%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 02_pyt_legacy_flags vs 04_pyt_IK_bucket              │ 33.97s < 34.27s {+0.9%} [33.85,34.10] < [34.19,34.38] {+1.0%,+0.8%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 02_pyt_legacy_flags vs 10_PR_baseline                │ 33.97s < 34.30s {+1.0%} [33.85,34.10] < [34.15,34.40] {+0.9%,+0.9%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 02_pyt_legacy_flags vs 11_PR_only_meta               │ 33.97s < 34.27s {+0.9%} [33.85,34.10] < [34.14,34.38] {+0.8%,+0.8%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 02_pyt_legacy_flags vs 12_PR_new_flags_no_meta       │ 33.97s < 35.89s {+5.6%} [33.85,34.10] < [35.78,36.00] {+5.7%,+5.6%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 02_pyt_legacy_flags vs 13_PR_new_flags_with_meta     │ 33.97s < 35.17s {+3.5%} [33.85,34.10] < [35.07,35.32] {+3.6%,+3.6%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 02_pyt_legacy_flags vs 14_PR_legacy_flags            │ 33.97s ~ 33.96s {-0.0%} [33.85,34.10] ~ [33.87,34.11] {+0.1%,+0.0%} p=0.37695 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 04_pyt_IK_bucket vs 10_PR_baseline                   │ 34.27s ~ 34.30s {+0.1%} [34.19,34.38] ~ [34.15,34.40] {-0.1%,+0.1%} p=0.08041 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 04_pyt_IK_bucket vs 11_PR_only_meta                  │ 34.27s ~ 34.27s {-0.0%} [34.19,34.38] ~ [34.14,34.38] {-0.2%,+0.0%} p=0.48129 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 04_pyt_IK_bucket vs 12_PR_new_flags_no_meta          │ 34.27s < 35.89s {+4.7%} [34.19,34.38] < [35.78,36.00] {+4.6%,+4.7%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 04_pyt_IK_bucket vs 13_PR_new_flags_with_meta        │ 34.27s < 35.17s {+2.6%} [34.19,34.38] < [35.07,35.32] {+2.6%,+2.8%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 04_pyt_IK_bucket vs 14_PR_legacy_flags               │ 34.27s > 33.96s {-0.9%} [34.19,34.38] > [33.87,34.11] {-0.9%,-0.8%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 10_PR_baseline vs 11_PR_only_meta                    │ 34.30s ~ 34.27s {-0.1%} [34.15,34.40] ~ [34.14,34.38] {-0.0%,-0.1%} p=0.16533 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 10_PR_baseline vs 12_PR_new_flags_no_meta            │ 34.30s < 35.89s {+4.6%} [34.15,34.40] < [35.78,36.00] {+4.8%,+4.7%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 10_PR_baseline vs 13_PR_new_flags_with_meta          │ 34.30s < 35.17s {+2.6%} [34.15,34.40] < [35.07,35.32] {+2.7%,+2.7%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 10_PR_baseline vs 14_PR_legacy_flags                 │ 34.30s > 33.96s {-1.0%} [34.15,34.40] > [33.87,34.11] {-0.8%,-0.8%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 11_PR_only_meta vs 12_PR_new_flags_no_meta           │ 34.27s < 35.89s {+4.7%} [34.14,34.38] < [35.78,36.00] {+4.8%,+4.7%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 11_PR_only_meta vs 13_PR_new_flags_with_meta         │ 34.27s < 35.17s {+2.6%} [34.14,34.38] < [35.07,35.32] {+2.7%,+2.7%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 11_PR_only_meta vs 14_PR_legacy_flags                │ 34.27s > 33.96s {-0.9%} [34.14,34.38] > [33.87,34.11] {-0.8%,-0.8%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 12_PR_new_flags_no_meta vs 13_PR_new_flags_with_meta │ 35.89s > 35.17s {-2.0%} [35.78,36.00] > [35.07,35.32] {-2.0%,-1.9%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 12_PR_new_flags_no_meta vs 14_PR_legacy_flags        │ 35.89s > 33.96s {-5.4%} [35.78,36.00] > [33.87,34.11] {-5.3%,-5.2%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b | 13_PR_new_flags_with_meta vs 14_PR_legacy_flags      │ 35.17s > 33.96s {-3.4%} [35.07,35.32] > [33.87,34.11] {-3.4%,-3.4%} p=0.00000+(25 vs 25)       │
+└──────────────────────────────────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────┘
+[warn]  At least one significant difference in main metrics was detected. Returning with exit(1).
+
+ + + + + + diff --git a/bulkbench/example/report/run-to-run.html b/bulkbench/example/report/run-to-run.html new file mode 100644 index 0000000..10fa932 --- /dev/null +++ b/bulkbench/example/report/run-to-run.html @@ -0,0 +1,114 @@ + + + + + + + + + + + + 
[warn]  The following directories don't contain an immediate or nested timings.json, and are ignored:
+- ./run1/03_pyt_IK_reorder/b
+- ./run1/03_pyt_IK_reorder/b/wan2_2.quantgemm_fp8attn.gfx950
+- ./run1/05_pyt_IK_reorderbucket/b
+- ./run1/05_pyt_IK_reorderbucket/b/wan2_2.quantgemm_fp8attn.gfx950
+- ./run2/03_pyt_IK_reorder/b
+- ./run2/03_pyt_IK_reorder/b/wan2_2.quantgemm_fp8attn.gfx950
+- ./run2/05_pyt_IK_reorderbucket/b
+- ./run2/05_pyt_IK_reorderbucket/b/wan2_2.quantgemm_fp8attn.gfx950
+                                                       Benchmark comparison results (Brunner Munzel test, alpha=0.00100)                                                       
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                 Benchmark                                  ┃                                 real_time (means), [0%, 100%]                                  ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+│ flux.usp/a/00_pyt_baseline | run1 vs run2                                  │ 777.8ms > 772.7ms {-0.7%} [769.9m,785.2m] > [764.7m,777.4m] {-0.7%,-1.0%} p=0.00001 (27 vs 27) │
+│ flux.usp/a/01_pyt_new_flags | run1 vs run2                                 │ 801.9ms ~ 802.9ms {+0.1%} [797.1m,824.6m] ~ [792.7m,833.6m] {-0.5%,+1.1%} p=0.42689 (27 vs 27) │
+│ flux.usp/a/02_pyt_legacy_flags | run1 vs run2                              │ 792.7ms ~ 791.2ms {-0.2%} [788.5m,799.2m] ~ [783.7m,812.7m] {-0.6%,+1.7%} p=0.00693 (27 vs 27) │
+│ flux.usp/a/03_pyt_IK_reorder | run1 vs run2                                │ 775.6ms ~ 776.6ms {+0.1%} [771.2m,780.7m] ~ [772.0m,781.6m] {+0.1%,+0.1%} p=0.05611 (27 vs 27) │
+│ flux.usp/a/04_pyt_IK_bucket | run1 vs run2                                 │ 778.9ms < 784.0ms {+0.7%} [772.0m,800.4m] < [778.0m,789.2m] {+0.8%,-1.4%} p=0.00000+(27 vs 27) │
+│ flux.usp/a/05_pyt_IK_reorderbucket | run1 vs run2                          │ 774.3ms ~ 773.6ms {-0.1%} [767.6m,790.4m] ~ [766.0m,794.1m] {-0.2%,+0.5%} p=0.16445 (27 vs 27) │
+│ flux.usp/a/10_PR_baseline | run1 vs run2                                   │ 777.8ms > 772.3ms {-0.7%} [769.3m,784.6m] > [767.6m,777.0m] {-0.2%,-1.0%} p=0.00000+(27 vs 27) │
+│ flux.usp/a/11_PR_only_meta | run1 vs run2                                  │ 772.5ms < 775.5ms {+0.4%} [768.0m,778.0m] < [770.2m,779.4m] {+0.3%,+0.2%} p=0.00000+(27 vs 27) │
+│ flux.usp/a/12_PR_new_flags_no_meta | run1 vs run2                          │ 823.9ms ~ 825.2ms {+0.2%} [816.0m,842.4m] ~ [818.6m,847.9m] {+0.3%,+0.7%} p=0.15810 (27 vs 27) │
+│ flux.usp/a/13_PR_new_flags_with_meta | run1 vs run2                        │ 794.2ms ~ 794.3ms {+0.0%} [789.0m,809.2m] ~ [788.5m,798.8m] {-0.1%,-1.3%} p=0.29075 (27 vs 27) │
+│ flux.usp/a/14_PR_legacy_flags | run1 vs run2                               │ 795.6ms > 784.0ms {-1.5%} [782.4m,816.5m] > [779.8m,799.6m] {-0.3%,-2.1%} p=0.00000+(27 vs 27) │
+│ flux2.quantgemm.gfx950/a/00_pyt_baseline | run1 vs run2                    │ 2.272s ~ 2.273s {+0.0%} [2.260,2.281] ~ [2.255,2.293] {-0.2%,+0.5%} p=0.33377 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/01_pyt_new_flags | run1 vs run2                   │ 2.349s > 2.330s {-0.8%} [2.333,2.365] > [2.321,2.354] {-0.5%,-0.5%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/02_pyt_legacy_flags | run1 vs run2                │ 2.272s ~ 2.275s {+0.1%} [2.259,2.292] ~ [2.265,2.288] {+0.3%,-0.2%} p=0.04836 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/03_pyt_IK_reorder | run1 vs run2                  │ 2.268s ~ 2.270s {+0.1%} [2.255,2.280] ~ [2.257,2.297] {+0.1%,+0.8%} p=0.29175 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/04_pyt_IK_bucket | run1 vs run2                   │ 2.274s ~ 2.274s {-0.0%} [2.263,2.303] ~ [2.258,2.301] {-0.2%,-0.1%} p=0.41048 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/05_pyt_IK_reorderbucket | run1 vs run2            │ 2.262s < 2.268s {+0.3%} [2.249,2.280] < [2.256,2.283] {+0.3%,+0.1%} p=0.00004 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/10_PR_baseline | run1 vs run2                     │ 2.272s ~ 2.277s {+0.2%} [2.260,2.288] ~ [2.261,2.302] {+0.1%,+0.6%} p=0.02500 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/11_PR_only_meta | run1 vs run2                    │ 2.271s ~ 2.272s {+0.0%} [2.259,2.284] ~ [2.258,2.294] {-0.0%,+0.5%} p=0.43609 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/12_PR_new_flags_no_meta | run1 vs run2            │ 2.334s < 2.345s {+0.5%} [2.324,2.356] < [2.332,2.367] {+0.3%,+0.5%} p=0.00000+(27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/13_PR_new_flags_with_meta | run1 vs run2          │ 2.284s ~ 2.282s {-0.1%} [2.267,2.306] ~ [2.269,2.300] {+0.1%,-0.3%} p=0.14248 (27 vs 27)       │
+│ flux2.quantgemm.gfx950/a/14_PR_legacy_flags | run1 vs run2                 │ 2.268s ~ 2.263s {-0.2%} [2.256,2.280] ~ [2.253,2.286] {-0.1%,+0.2%} p=0.00251 (27 vs 27)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/00_pyt_baseline | run1 vs run2           │ 34.32s > 34.28s {-0.1%} [34.23,34.44] > [34.21,34.35] {-0.0%,-0.3%} p=0.00069 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/01_pyt_new_flags | run1 vs run2          │ 35.90s > 35.75s {-0.4%} [35.78,36.01] > [35.66,35.88] {-0.3%,-0.4%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/02_pyt_legacy_flags | run1 vs run2       │ 33.97s ~ 33.97s {+0.0%} [33.85,34.10] ~ [33.89,34.05] {+0.1%,-0.2%} p=0.38500 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/04_pyt_IK_bucket | run1 vs run2          │ 34.27s ~ 34.30s {+0.1%} [34.19,34.38] ~ [34.19,34.42] {+0.0%,+0.1%} p=0.02572 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/10_PR_baseline | run1 vs run2            │ 34.30s > 34.24s {-0.1%} [34.15,34.40] > [34.13,34.35] {-0.0%,-0.1%} p=0.00007 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/11_PR_only_meta | run1 vs run2           │ 34.27s ~ 34.28s {+0.0%} [34.14,34.38] ~ [34.16,34.40] {+0.1%,+0.0%} p=0.34806 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/12_PR_new_flags_no_meta | run1 vs run2   │ 35.89s > 35.80s {-0.2%} [35.78,36.00] > [35.73,35.88] {-0.1%,-0.3%} p=0.00000+(25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/13_PR_new_flags_with_meta | run1 vs run2 │ 35.17s > 35.12s {-0.2%} [35.07,35.32] > [35.07,35.18] {+0.0%,-0.4%} p=0.00047 (25 vs 25)       │
+│ wan2_2.quantgemm_fp8attn.gfx950/b/14_PR_legacy_flags | run1 vs run2        │ 33.96s ~ 33.92s {-0.1%} [33.87,34.11] ~ [33.84,34.08] {-0.1%,-0.1%} p=0.00304 (25 vs 25)       │
+└────────────────────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────┘
+[warn]  At least one significant difference in main metrics was detected. Returning with exit(1).
+
+ + + + + + diff --git a/bulkbench/pyproject.toml b/bulkbench/pyproject.toml new file mode 100644 index 0000000..3e5b737 --- /dev/null +++ b/bulkbench/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "bulkbench" +description = "Bulk benchmarking driver for xDiT with QA & statistical analysis of the results" +requires-python = ">=3.11" +dynamic = ["version", "readme"] +dependencies = [ + "benchstats==3.6.3", + "PyYAML", + "rich", +] + +[project.scripts] +bulkbench = "bulkbench.__main__:main" + +[tool.setuptools.dynamic] +version = {attr = "bulkbench.__version__"} +readme = {file = ["README.md"], content-type = "text/markdown"} + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.black] +line-length = 100 +target-version = ['py311'] + +[tool.ruff] +preview = true +exclude = [ + ".git", + "build", + "__pycache__", +] +line-length = 100 +indent-width = 4 +target-version = "py311" + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 70 +line-ending = "lf" diff --git a/bulkbench/src/bulkbench/__init__.py b/bulkbench/src/bulkbench/__init__.py new file mode 100644 index 0000000..5415ac8 --- /dev/null +++ b/bulkbench/src/bulkbench/__init__.py @@ -0,0 +1,6 @@ +from .bulkbench import BulkBench, GroupFailureCapture, GroupRunError +from .cli_parser import makeParser + +__version__ = "0.1.0" + +__all__ = ["BulkBench", "GroupFailureCapture", "GroupRunError", "__version__", "makeParser"] diff --git a/bulkbench/src/bulkbench/__main__.py b/bulkbench/src/bulkbench/__main__.py new file mode 100644 index 0000000..078c270 --- /dev/null +++ b/bulkbench/src/bulkbench/__main__.py @@ -0,0 +1,19 @@ +"""Entry point of the `bulkbench` tool.""" + +import sys + +from .bulkbench import BulkBench +from .cli_parser import makeParser + + +def main() -> int: + parser = makeParser() + args = parser.parse_args() + try: + return BulkBench(args).run() + except ValueError as exc: + parser.error(str(exc)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bulkbench/src/bulkbench/benchmark_plan_loader.py b/bulkbench/src/bulkbench/benchmark_plan_loader.py new file mode 100644 index 0000000..b482742 --- /dev/null +++ b/bulkbench/src/bulkbench/benchmark_plan_loader.py @@ -0,0 +1,553 @@ +"""Loading and validation of benchmark plans.""" + +import json +import os +import re +from pathlib import Path +from typing import Any, TypedDict + +import yaml # pyright: ignore[reportMissingModuleSource] +from benchstats.common import LoggingConsole +from yaml.constructor import ConstructorError # pyright: ignore[reportMissingModuleSource] +from yaml.nodes import MappingNode # pyright: ignore[reportMissingModuleSource] + +DEFAULT_CONFIGS_FILE = "configs.yaml" +DEFAULT_PATCHES_FILE = "patches.yaml" +EAGER_GROUP_PREFIX = "eager_" +VALID_NAME_PATTERN = r"[-a-zA-Z0-9_+={}., ~!()\[\]]+" + +_VALID_NAME_RE = re.compile(VALID_NAME_PATTERN) +_PATCH_TARGETS_BASE_DIR = Path("/app") + +StrPath = str | os.PathLike[str] + + +class ConfigGroup(TypedDict): + """Validated benchmark config group loaded from the configs YAML file.""" + + name: str + configs: list[str] + override_args: str | None + only_in_patches: frozenset[str] | None + + +class PatchData(TypedDict): + """Validated patch description loaded from the patches YAML file.""" + + patch: Path + target: Path + + +class PatchSet(TypedDict): + """Validated named patch set loaded from the patches YAML file.""" + + name: str + patches: list[PatchData] + + +class _UniqueKeySafeLoader(yaml.SafeLoader): + """Safe YAML loader that rejects duplicate mapping keys.""" + + def construct_mapping(self, node: MappingNode, deep: bool = False) -> dict[Any, Any]: + self.flatten_mapping(node) + keys: set[Any] = set() + for key_node, _ in node.value: + key = self.construct_object(key_node, deep=deep) + try: + duplicate = key in keys + keys.add(key) + except TypeError: + # The base constructor emits a contextual error for unhashable keys. + continue + if duplicate: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + return super().construct_mapping(node, deep=deep) + + +def benchmarkConfigPath( + benchmark_configs_dir: Path, config_name: str, config_context: str +) -> Path: + """Builds a benchmark YAML path from a validated config name.""" + stem = config_name.partition(".")[0] + if not stem or stem in (".", "..") or _VALID_NAME_RE.fullmatch(stem) is None: + raise ValueError( + f"{config_context} prefix before the first dot must match " + f"{VALID_NAME_PATTERN!r} and must not be '.' or '..'" + ) + return benchmark_configs_dir / f"{stem}.yaml" + + +class BenchmarkPlanLoader: + """Loads and validates patch sets and benchmark config groups.""" + + def __init__( + self, + project_dir: Path, + arch: str, + console: LoggingConsole, + benchmark_configs_dir: Path, + ) -> None: + self.project_dir = project_dir + self.arch = arch + self.console = console + self.benchmark_configs_dir = benchmark_configs_dir + + def _resolvedPath(self, value: StrPath | None, default: str) -> Path: + """Resolves a path, taking a relative path relative to the project directory.""" + path = Path(default if value is None else value).expanduser() + return (path if path.is_absolute() else self.project_dir / path).resolve() + + def _validatedConfigsFile(self, value: StrPath | None) -> Path: + """Resolves `value` and makes sure it points to an existing configs file.""" + path = self._resolvedPath(value, DEFAULT_CONFIGS_FILE) + if not path.is_file(): + raise ValueError(f"configs_file '{path}' doesn't exist or isn't a file") + return path + + def _validatedPatchesFile(self, value: StrPath | None) -> Path: + """Resolves `value` and makes sure it points to an existing patches file.""" + path = self._resolvedPath(value, DEFAULT_PATCHES_FILE) + if not path.is_file(): + raise ValueError(f"patches_file '{path}' doesn't exist or isn't a file") + return path + + @staticmethod + def _validateJsonMappingKeys( + value: Any, location: str, active_containers: set[int] | None = None + ) -> None: + """Makes sure every mapping nested in `value` has string keys and no cycles.""" + if not isinstance(value, (dict, list)): + return + + active_containers = active_containers if active_containers is not None else set() + container_id = id(value) + if container_id in active_containers: + raise ValueError(f"{location} contains a circular reference") + + active_containers.add(container_id) + try: + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError( # noqa: TRY004 - invalid public config value + f"{location} contains non-string mapping key {key!r}" + ) + BenchmarkPlanLoader._validateJsonMappingKeys( + item, f"{location}.{key}", active_containers + ) + else: + for index, item in enumerate(value): + BenchmarkPlanLoader._validateJsonMappingKeys( + item, f"{location}[{index}]", active_containers + ) + finally: + active_containers.remove(container_id) + + @staticmethod + def _validatedEnabled(value: Any, object_context: str) -> bool: + """Validates and normalizes an object's `enabled` value.""" + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + if value in ("true", "1"): + return True + if value in ("false", "0"): + return False + raise ValueError( + f"{object_context} attribute 'enabled' must be a YAML boolean, integer 1 or 0, " + 'or one of the strings "true", "false", "1", "0"' + ) + + @staticmethod + def _validatedName(value: Any, object_context: str) -> str: + """Strips and validates a config-group or patch-set name.""" + if not isinstance(value, str) or not (name := value.strip()): + raise ValueError(f"{object_context} attribute 'name' must be a non-empty string") + if name in (".", "..") or _VALID_NAME_RE.fullmatch(name) is None: + raise ValueError( + f"{object_context} attribute 'name' must match " + f"{VALID_NAME_PATTERN!r} and must not be '.' or '..'" + ) + return name + + def _benchmarkConfigTags( + self, + config_name: str, + config_context: str, + benchmark_configs_cache: dict[Path, list[Any]], + ) -> Any: + """Checks that a benchmark config exists and returns its tags.""" + path = benchmarkConfigPath( + self.benchmark_configs_dir, + config_name, + config_context, + ) + if not path.is_file(): + raise ValueError( + f"{config_context} requires benchmark config file '{path}', " + "which doesn't exist or isn't a file" + ) + + if path not in benchmark_configs_cache: + try: + with path.open("r", encoding="utf-8") as file: + raw_benchmark_configs = yaml.safe_load(file) + except (OSError, UnicodeError, yaml.YAMLError) as exc: + raise ValueError(f"failed to read benchmark config file '{path}': {exc}") from exc + if not isinstance(raw_benchmark_configs, list): + raise ValueError(f"benchmark config file '{path}' must contain a YAML list") + benchmark_configs_cache[path] = raw_benchmark_configs + + for benchmark_config in benchmark_configs_cache[path]: + if isinstance(benchmark_config, dict) and benchmark_config.get("name") == config_name: + return benchmark_config.get("tags") + + raise ValueError(f"{config_context} config {config_name!r} wasn't found in '{path}'") + + def _validatedConfigPatchNames( + self, + value: Any, + attribute_name: str, + group_context: str, + patch_set_names: set[str], + ) -> frozenset[str]: + """Validates, deduplicates, and resolves a config group's patch-set references.""" + if value is None: + raise ValueError(f"{attribute_name} field is set but empty") + if not isinstance(value, list): + raise ValueError( # noqa: TRY004 - public API reports invalid config values + f"{group_context} attribute '{attribute_name}' must be a list" + ) + + referenced_names: set[str] = set() + seen_names: set[str] = set() + for patch_index, patch_name in enumerate(value, start=1): + patch_context = f"{group_context} attribute '{attribute_name}', item {patch_index}" + if not isinstance(patch_name, str) or not (patch_name := patch_name.strip()): + raise ValueError(f"{patch_context} must be a non-empty string") + if patch_name in seen_names: + continue + seen_names.add(patch_name) + if patch_name not in patch_set_names: + self.console.warning( + f"Patch set '{patch_name}' referenced in {group_context} attribute " + f"'{attribute_name}' isn't defined. Ignoring it." + ) + continue + referenced_names.add(patch_name) + + return frozenset(referenced_names) + + def readConfigs( + self, configs_file_value: StrPath | None, patch_set_names: set[str] + ) -> dict[str, ConfigGroup]: + """Reads and validates benchmark config groups from a YAML file.""" + configs_file = self._validatedConfigsFile(configs_file_value) + try: + with configs_file.open("r", encoding="utf-8") as file: + raw_groups = yaml.load(file, Loader=_UniqueKeySafeLoader) + except (OSError, UnicodeError, yaml.YAMLError) as exc: + raise ValueError(f"failed to read configs_file '{configs_file}': {exc}") from exc + + error_prefix = f"configs_file '{configs_file}'" + if not isinstance(raw_groups, list): + raise ValueError( # noqa: TRY004 - public API reports invalid config values + f"{error_prefix} must contain a YAML list" + ) + + allowed_attributes = { + "configs", + "eager_in_patches", + "enabled", + "name", + "only_in_patches", + "override_args", + } + benchmark_configs_cache: dict[Path, list[Any]] = {} + enabled_group_names: set[str] = set() + applicable_group_names: set[str] = set() + groups: dict[str, ConfigGroup] = {} + for group_index, raw_group in enumerate(raw_groups, start=1): + group_context = f"{error_prefix}, group {group_index}" + if not isinstance(raw_group, dict): + raise ValueError( # noqa: TRY004 - public API reports invalid config values + f"{group_context} must be an object" + ) + + if not self._validatedEnabled(raw_group.get("enabled", True), group_context): + continue + + non_string_attributes = [key for key in raw_group if not isinstance(key, str)] + if non_string_attributes: + raise ValueError( + f"{group_context} contains non-string attribute {non_string_attributes[0]!r}" + ) + + unknown_attributes = set(raw_group) - allowed_attributes + if unknown_attributes: + unknown = ", ".join(sorted(unknown_attributes)) + raise ValueError(f"{group_context} contains unknown attribute(s): {unknown}") + + if "name" not in raw_group: + raise ValueError(f"{group_context} is missing required attribute 'name'") + name = self._validatedName(raw_group["name"], group_context) + if name.startswith(EAGER_GROUP_PREFIX): + raise ValueError( + f"{group_context} attribute 'name' must not start with reserved prefix " + f"'{EAGER_GROUP_PREFIX}'" + ) + if name in enabled_group_names: + raise ValueError(f"{error_prefix} contains duplicate group name {name!r}") + enabled_group_names.add(name) + + if "configs" not in raw_group: + raise ValueError(f"{group_context} is missing required attribute 'configs'") + group_configs = raw_group["configs"] + if not isinstance(group_configs, list) or not group_configs: + raise ValueError(f"{group_context} attribute 'configs' must be a non-empty list") + + seen_configs: set[str] = set() + supported_configs: list[str] = [] + for config_index, config in enumerate(group_configs, start=1): + config_context = f"{group_context} attribute 'configs', item {config_index}" + if not isinstance(config, str) or not (config := config.strip()): + raise ValueError(f"{config_context} must be a non-empty string") + tags = self._benchmarkConfigTags(config, config_context, benchmark_configs_cache) + if config in seen_configs: + raise ValueError(f"{group_context} contains duplicate config name {config!r}") + seen_configs.add(config) + if self.arch and tags and self.arch not in tags: + self.console.warning( + f"Config '{config}' is not supported for the current architecture " + f"(--tag={self.arch} mismatch)" + ) + continue + supported_configs.append(config) + + override_args = raw_group.get("override_args") + serialized_override_args: str | None = None + if override_args is not None: + if not isinstance(override_args, dict): + raise ValueError( + f"{group_context} attribute 'override_args' must be an object or null" + ) + try: + self._validateJsonMappingKeys( + override_args, f"{group_context} attribute 'override_args'" + ) + serialized_override_args = json.dumps( + override_args, allow_nan=False, separators=(",", ":") + ) + except (TypeError, ValueError) as exc: + raise ValueError( + f"{group_context} attribute 'override_args' isn't valid JSON: {exc}" + ) from exc + + only_in_patches = ( + self._validatedConfigPatchNames( + raw_group["only_in_patches"], + "only_in_patches", + group_context, + patch_set_names, + ) + if "only_in_patches" in raw_group + else None + ) + if only_in_patches is not None and not only_in_patches: + self.console.warning( + f"Group '{name}' was fully omitted; attribute 'only_in_patches' " + "contains no defined patch set names" + ) + continue + applicable_group_names.add(name) + + eager_in_patches = ( + self._validatedConfigPatchNames( + raw_group["eager_in_patches"], + "eager_in_patches", + group_context, + patch_set_names, + ) + if "eager_in_patches" in raw_group + else None + ) + eager_only_in_patches: frozenset[str] | None = None + if eager_in_patches: + if only_in_patches is None: + eager_only_in_patches = eager_in_patches + else: + eager_only_in_patches = only_in_patches & eager_in_patches + if not eager_only_in_patches: + self.console.warning( + f"Eager group '{EAGER_GROUP_PREFIX + name}' was fully omitted; " + f"{group_context} attributes 'only_in_patches' and " + "'eager_in_patches' have an empty intersection" + ) + eager_only_in_patches = None + + if not supported_configs: + self.console.warning( + f"Group '{name}' was fully omitted; " + f"no config matches the architecture (--tag={self.arch} mismatch)" + ) + continue + + groups[name] = { + "name": name, + "configs": supported_configs, + "override_args": serialized_override_args, + "only_in_patches": only_in_patches, + } + if eager_only_in_patches is not None: + eager_override_args = dict(override_args or {}) + eager_override_args["num_iterations"] = 1 + eager_override_args["use_torch_compile"] = False + eager_name = EAGER_GROUP_PREFIX + name + groups[eager_name] = { + "name": eager_name, + "configs": supported_configs.copy(), + "override_args": json.dumps( + eager_override_args, allow_nan=False, separators=(",", ":") + ), + "only_in_patches": eager_only_in_patches, + } + + if not applicable_group_names: + raise ValueError(f"{error_prefix} must contain at least one enabled config group") + + self.console.debug(f"Read {len(groups)} config groups from {configs_file}: ", groups) + return groups + + @staticmethod + def _validatedPatchPath(value: Any, base_dir: Path, attribute: str, context: str) -> Path: + """Resolves and validates a patch or target path.""" + if not isinstance(value, str) or not (value := value.strip()): + raise ValueError(f"{context} attribute '{attribute}' must be a non-empty string") + + path = Path(value).expanduser() + path = (path if path.is_absolute() else base_dir / path).resolve() + if not path.is_file(): + raise ValueError( + f"{context} attribute '{attribute}' path '{path}' doesn't exist or isn't a file" + ) + return path + + def readPatches(self, patches_file_value: StrPath | None) -> tuple[Path, list[PatchSet]]: + """Reads and validates named patch sets from a YAML file.""" + patches_file = self._validatedPatchesFile(patches_file_value) + try: + with patches_file.open("r", encoding="utf-8") as file: + raw_patch_sets = yaml.load(file, Loader=_UniqueKeySafeLoader) + except (OSError, UnicodeError, yaml.YAMLError) as exc: + raise ValueError(f"failed to read patches_file '{patches_file}': {exc}") from exc + + error_prefix = f"patches_file '{patches_file}'" + if not isinstance(raw_patch_sets, list): + raise ValueError( # noqa: TRY004 - public API reports invalid patch values + f"{error_prefix} must contain a YAML list" + ) + if not raw_patch_sets: + raise ValueError(f"{error_prefix} must contain at least one patch set") + + patch_set_attributes = {"name", "patches"} + patch_attributes = {"enabled", "patch", "target"} + patch_set_names: set[str] = set() + seen_patch_sets: dict[frozenset[tuple[Path, Path]], str] = {} + loaded_patch_sets: list[PatchSet] = [] + + for patch_set_index, raw_patch_set in enumerate(raw_patch_sets, start=1): + patch_set_context = f"{error_prefix}, patch set {patch_set_index}" + if not isinstance(raw_patch_set, dict): + raise ValueError( # noqa: TRY004 - public API reports invalid patch values + f"{patch_set_context} must be an object" + ) + + non_string_attributes = [key for key in raw_patch_set if not isinstance(key, str)] + if non_string_attributes: + raise ValueError( + f"{patch_set_context} contains non-string attribute " + f"{non_string_attributes[0]!r}" + ) + + unknown_attributes = set(raw_patch_set) - patch_set_attributes + if unknown_attributes: + unknown = ", ".join(sorted(unknown_attributes)) + raise ValueError(f"{patch_set_context} contains unknown attribute(s): {unknown}") + + if "name" not in raw_patch_set: + raise ValueError(f"{patch_set_context} is missing required attribute 'name'") + name = self._validatedName(raw_patch_set["name"], patch_set_context) + if name in patch_set_names: + raise ValueError(f"{error_prefix} contains duplicate patch set name {name!r}") + patch_set_names.add(name) + + if "patches" not in raw_patch_set: + raise ValueError(f"{patch_set_context} is missing required attribute 'patches'") + raw_patches = raw_patch_set["patches"] + if not isinstance(raw_patches, list): + raise ValueError( # noqa: TRY004 - public API reports invalid patch values + f"{patch_set_context} attribute 'patches' must be a list" + ) + + patches: list[PatchData] = [] + patch_keys: set[tuple[Path, Path]] = set() + for patch_index, raw_patch in enumerate(raw_patches, start=1): + patch_context = f"{patch_set_context}, patch {patch_index}" + if not isinstance(raw_patch, dict): + raise ValueError( # noqa: TRY004 - public API reports invalid patch values + f"{patch_context} must be an object" + ) + + if not self._validatedEnabled(raw_patch.get("enabled", True), patch_context): + continue + + non_string_attributes = [key for key in raw_patch if not isinstance(key, str)] + if non_string_attributes: + raise ValueError( + f"{patch_context} contains non-string attribute " + f"{non_string_attributes[0]!r}" + ) + + unknown_attributes = set(raw_patch) - patch_attributes + if unknown_attributes: + unknown = ", ".join(sorted(unknown_attributes)) + raise ValueError(f"{patch_context} contains unknown attribute(s): {unknown}") + + if "patch" not in raw_patch: + raise ValueError(f"{patch_context} is missing required attribute 'patch'") + if "target" not in raw_patch: + raise ValueError(f"{patch_context} is missing required attribute 'target'") + + patch_path = self._validatedPatchPath( + raw_patch["patch"], self.project_dir / name, "patch", patch_context + ) + target_path = self._validatedPatchPath( + raw_patch["target"], _PATCH_TARGETS_BASE_DIR, "target", patch_context + ) + + patch_key = (patch_path, target_path) + if patch_key in patch_keys: + raise ValueError( + f"{patch_set_context} contains duplicate patch object " + f"({patch_path}, {target_path})" + ) + patch_keys.add(patch_key) + patches.append({"patch": patch_path, "target": target_path}) + + patch_set_key = frozenset(patch_keys) + if duplicate_patch_set := seen_patch_sets.get(patch_set_key): + raise ValueError( + f"{error_prefix} patch sets {duplicate_patch_set!r} and {name!r} " + "contain duplicate patch sets" + ) + seen_patch_sets[patch_set_key] = name + loaded_patch_sets.append({"name": name, "patches": patches}) + + return patches_file, loaded_patch_sets diff --git a/bulkbench/src/bulkbench/benchmark_sources.py b/bulkbench/src/bulkbench/benchmark_sources.py new file mode 100644 index 0000000..c63d595 --- /dev/null +++ b/bulkbench/src/bulkbench/benchmark_sources.py @@ -0,0 +1,193 @@ +"""Discover benchmark result sources from files and directory trees.""" + +import os + +from .bulkbench import EAGER_GROUP_PREFIX + +_TIMINGS_FILENAME = "timings.json" +_ALT_DELIMITER = "|" + + +def _warn(message, debug_log=None) -> None: + if debug_log: + debug_log.warning(message) + else: + print(message) + + +def parse_filter(filter) -> set[int] | None: + if filter is None: + return None + if not isinstance(filter, str): + raise TypeError("Filter must be a comma-separated string of non-negative integers") + + filter = filter.strip() + if not filter: + return None + + indices: set[int] = {0} + for value in filter.split(","): + value = value.strip() + if not value or not value.isdecimal(): + raise ValueError( + f"Invalid filter {filter!r}; expected comma-separated non-negative integers" + ) + indices.add(int(value)) + return indices + + +def _format_warning(message: str, paths: list[str]) -> str: + return message + "\n" + "\n".join(f"- {path}" for path in sorted(paths)) + + +def _walk_directories_following_symlinks(fpath: str): + """Yield directories bottom-up while following symlinks without entering cycles.""" + entries = [] + ancestors_by_path: dict[str, frozenset[tuple[int, int]]] = {fpath: frozenset()} + + for current_dir, child_dirs, files in os.walk(fpath, topdown=True, followlinks=True): + ancestors = ancestors_by_path.get(current_dir, frozenset()) + stat = os.stat(current_dir) + current_identity = (stat.st_dev, stat.st_ino) + current_ancestors = ancestors | {current_identity} + + traversable_children = [] + for child_dir in child_dirs: + child_path = os.path.join(current_dir, child_dir) + try: + child_stat = os.stat(child_path) + except OSError: + continue + child_identity = (child_stat.st_dev, child_stat.st_ino) + if child_identity in current_ancestors: + continue + ancestors_by_path[child_path] = current_ancestors + traversable_children.append(child_dir) + + child_dirs[:] = traversable_children + entries.append((current_dir, traversable_children, files)) + + yield from reversed(entries) + + +def get_benchmark_sources( + fpath: str, + filter_indices: set[int] | None, + debug_log=None, + ignore_eager: bool = True, +) -> set[tuple[str, str]]: + """Return ``(benchmark name, result directory)`` pairs for the directories under ``fpath``. + + When ``ignore_eager`` is true, omit nested result directories whose immediate + parent name starts with ``eager_`` (``bulkbench.EAGER_GROUP_PREFIX``). + """ + fpath = os.fspath(fpath) + + if os.path.isfile(fpath): + if filter_indices is not None: + raise ValueError("A filter can only be applied when fpath is a directory") + if os.path.splitext(fpath)[1].lower() != ".json": + raise ValueError(f"Expected a JSON file, got {fpath!r}") + result_dir = os.path.dirname(fpath) or os.curdir + return {(os.path.basename(os.path.abspath(result_dir)), result_dir)} + + if not os.path.isdir(fpath): + raise ValueError(f"fpath is neither a file nor a directory: {fpath!r}") + + benchmarks: set[tuple[str, str]] = set() + directories_without_timings: list[str] = [] + directories_rejected_by_filter: list[str] = [] + subtree_timings: dict[str, list[str]] = {} + subtrees_with_ignored_timings: set[str] = set() + + selected_indices = filter_indices if filter_indices is not None else {0} + + for current_dir, child_dirs, files in _walk_directories_following_symlinks(fpath): + relative_dir = os.path.relpath(current_dir, fpath) + path_parts = [] if relative_dir == os.curdir else relative_dir.split(os.sep) + current_timings = os.path.join(current_dir, _TIMINGS_FILENAME) + has_immediate_timings = _TIMINGS_FILENAME in files and os.path.isfile(current_timings) + ignored_immediate_timings = ( + has_immediate_timings + and ignore_eager + and len(path_parts) >= 2 + and path_parts[-2].startswith(EAGER_GROUP_PREFIX) + ) + has_immediate_timings = has_immediate_timings and not ignored_immediate_timings + nested_timings = sorted( + timing + for child_dir in child_dirs + for timing in subtree_timings.get(os.path.join(current_dir, child_dir), []) + ) + has_ignored_nested_timings = any( + os.path.join(current_dir, child_dir) in subtrees_with_ignored_timings + for child_dir in child_dirs + ) + + if has_immediate_timings and nested_timings: + if len(nested_timings) == 1: + raise ValueError( + "The directory is malformed and contains 2 timings.json files, " + f"an immediate and a nested in {nested_timings[0]}" + ) + raise ValueError( + f"The directory is malformed and contains {len(nested_timings) + 1} " + "timings.json files, an immediate and nested in " + ", ".join(nested_timings) + ) + + subtree_timings[current_dir] = ( + [current_timings] if has_immediate_timings else [] + ) + nested_timings + if ignored_immediate_timings or has_ignored_nested_timings: + subtrees_with_ignored_timings.add(current_dir) + + if not subtree_timings[current_dir] and current_dir not in subtrees_with_ignored_timings: + directories_without_timings.append(current_dir) + + if not has_immediate_timings: + continue + + if not path_parts: + if filter_indices is not None: + directories_rejected_by_filter.append(current_dir) + continue + + if max(selected_indices) >= len(path_parts): + directories_rejected_by_filter.append(current_dir) + continue + + entity_parts = [path_parts[-index - 1] for index in sorted(selected_indices)] + alternative_parts = [ + part + for position, part in enumerate(path_parts) + if len(path_parts) - position - 1 not in selected_indices + ] + benchmark_name = "/".join(entity_parts) + _ALT_DELIMITER + "/".join(alternative_parts) + benchmarks.add((benchmark_name, current_dir)) + + if directories_without_timings: + _warn( + _format_warning( + "The following directories don't contain an immediate or nested timings.json, and are ignored:", + directories_without_timings, + ), + debug_log, + ) + + if directories_rejected_by_filter: + _warn( + _format_warning( + "These paths containing timings.json can't have the given " + f"filter '{filter_indices}' be applied to them, and are ignored:", + directories_rejected_by_filter, + ), + debug_log, + ) + + if filter_indices is None and os.path.isfile(os.path.join(fpath, _TIMINGS_FILENAME)): + return {(os.path.basename(os.path.abspath(fpath)), fpath)} + + if not benchmarks: + raise ValueError(f"No benchmarks were found under {fpath!r}") + + return benchmarks diff --git a/bulkbench/src/bulkbench/bulkbench.py b/bulkbench/src/bulkbench/bulkbench.py new file mode 100644 index 0000000..bd6c667 --- /dev/null +++ b/bulkbench/src/bulkbench/bulkbench.py @@ -0,0 +1,732 @@ +"""Implementation of the `bulkbench` tool.""" + +import os +import shutil +import subprocess +import traceback +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from time import monotonic +from typing import Any, TypedDict + +from benchstats.common import LoggingConsole + +from .benchmark_plan_loader import ( + DEFAULT_CONFIGS_FILE as _PLAN_DEFAULT_CONFIGS_FILE, +) +from .benchmark_plan_loader import ( + EAGER_GROUP_PREFIX, + BenchmarkPlanLoader, + ConfigGroup, + PatchData, + PatchSet, + StrPath, + benchmarkConfigPath, +) +from .benchmark_plan_loader import ( + VALID_NAME_PATTERN as _PLAN_VALID_NAME_PATTERN, +) +from .script_runner import run_with_script + +DEFAULT_CONSOLE_LOG_LEVEL = LoggingConsole.LogLevel.Info +DEFAULT_RESULTS_SUBDIR = "results" +DEFAULT_REPORT_SUBDIR = "report" +DEFAULT_BACKUP_SUBDIR = "_backups" +DEFAULT_CONFIGS_FILE = _PLAN_DEFAULT_CONFIGS_FILE +VALID_NAME_PATTERN = _PLAN_VALID_NAME_PATTERN + +_APP_DIR = Path("/app") +_BENCHMARK_CONFIGS_DIR = _APP_DIR / ".ci" / "benchmark_configs" +_BENCHMARK_RUNNER = _APP_DIR / ".ci" / "run.py" +_RESULT_IMAGE_SUFFIXES = {".jpg", ".png"} +_RESULT_VIDEO_SUFFIXES = {".mp4"} +_RESULT_MEDIA_SUFFIXES = _RESULT_IMAGE_SUFFIXES | _RESULT_VIDEO_SUFFIXES + + +def configMightHaveRunSuccessfully(workdir: Path, config_name: str | None = None) -> bool: + """Checks whether a config's direct result files indicate a successful prior run.""" + config_dir = workdir if config_name is None else workdir / config_name + return (config_dir / "timings.json").is_file() and any( + child.is_file() and child.suffix in _RESULT_MEDIA_SUFFIXES for child in config_dir.iterdir() + ) + + +class TargetBackup(TypedDict): + """Files needed to restore one patch target.""" + + backup: Path + path_file: Path + target: Path + + +@dataclass(frozen=True) +class GroupFailureCapture: + """Captured outcome of one benchmark config-group process.""" + + group_name: str + configs_to_run: list[str] + output: str + returncode: int | None + + +class GroupRunError(RuntimeError): + """Raised when a benchmark config-group process fails.""" + + def __init__(self, result: GroupFailureCapture) -> None: + self.result = result + status = ( + f"exit status {result.returncode}" + if result.returncode is not None + else "process start failure" + ) + super().__init__(f"benchmark config group {result.group_name!r} failed: {status}") + + +def _formatDuration(duration_seconds: float) -> str: + """Formats a nonnegative duration as unbounded hours, minutes, and seconds.""" + total_tenths = int(duration_seconds * 10 + 0.5) + hours, remaining_tenths = divmod(total_tenths, 60 * 60 * 10) + minutes, remaining_tenths = divmod(remaining_tenths, 60 * 10) + seconds = remaining_tenths / 10 + return f"{hours:02d}:{minutes:02d}:{seconds:04.1f}" + + +_gArch = None + + +def get_amd_gpu_arch_rocminfo(): + global _gArch + if _gArch is not None: + return _gArch + + try: + output = subprocess.check_output(["rocminfo"], text=True) + for line in output.splitlines(): + if "Name:" in line and "gfx" in line: + # Extract the architecture string (e.g., gfx950) + _gArch = line.split()[-1].strip() + break + except (subprocess.SubprocessError, FileNotFoundError): + pass + if _gArch is None: + raise ValueError("Failed to get AMD GPU architecture from rocminfo") + return _gArch + + +def _validatedConsole(Con: Any | None, console_log_level: int | None) -> LoggingConsole: + if Con is None: + Con = LoggingConsole( + log_level=LoggingConsole.LogLevel( + console_log_level + if isinstance(console_log_level, int) and (0 <= console_log_level <= 5) + else DEFAULT_CONSOLE_LOG_LEVEL.value + ) + ) + else: + assert isinstance(Con, LoggingConsole), "console must be a LoggingConsole" + return Con + + +class BulkBench: + """Runs a set of benchmarks of a project and analyzes their results statistically. + + Validates the arguments it's given, so it's equally safe to use from the `bulkbench` + CLI and from other Python programs. Raises `ValueError` on an invalid argument. + """ + + def __init__(self, *args, **kwargs) -> None: + """`args` and `kwargs` are used to initialize the object. `args` if set, must + be a single object containing attributes obtainable by the CLI parser. `kwargs` + takes precedence over `args`. + + A missing attribute is treated as `None`, i.e. the default value of the + corresponding argument is used. + """ + + assert len(args) <= 1, "Only one positional argument is allowed" + args = args[0] if args else {} # type: ignore + + def _get_arg(name: str, default: Any = None) -> Any: + return kwargs.get(name, getattr(args, name, default)) + + self.Con = _validatedConsole(_get_arg("console"), _get_arg("console_log_level")) + self.Con.trace(f"Console log level: {self.Con.log_level}") + + self.regenerate_results = _get_arg("regenerate_results", False) + assert isinstance(self.regenerate_results, bool), "regenerate_results must be a boolean" + + self.arch: str = _get_arg("arch") + if self.arch is None: + self.arch = get_amd_gpu_arch_rocminfo() + assert isinstance(self.arch, str), "arch must be a string" + self.Con.debug( + f"Using arch tag = {self.arch}" if self.arch else "Arch is not set, tags won't be used" + ) + + self.successful_runs: dict[str, list[tuple[str, float, list[str]]]] = {} + self.unsuccessful_runs: dict[str, list[tuple[GroupFailureCapture, float]]] = {} + + self.project_dir: Path = self._validatedProjectDir(_get_arg("project_dir")) + plan_loader = BenchmarkPlanLoader( + project_dir=self.project_dir, + arch=self.arch, + console=self.Con, + benchmark_configs_dir=_BENCHMARK_CONFIGS_DIR, + ) + patches_file_value = _get_arg("patches_file") + patches_file, self.patches = plan_loader.readPatches(patches_file_value) + # Validate applicability early to prevent failures during long-running work. + for patch_set in self.patches: + self._dryRunPatches(patch_set) + self.Con.debug( + f"Read {len(self.patches)} patch sets from {patches_file}:", + self.patches, + ) + self.configs: dict[str, ConfigGroup] = plan_loader.readConfigs( + _get_arg("configs_file"), + {patch_set["name"] for patch_set in self.patches}, + ) + self.results_dir: Path = self._validatedOutputDir( + _get_arg("results_dir"), DEFAULT_RESULTS_SUBDIR, "results_dir" + ) + if not self.results_dir.exists(): + self.results_dir.mkdir(parents=True, exist_ok=True) + + self.report_dir: Path = self._validatedOutputDir( + _get_arg("report_dir"), DEFAULT_REPORT_SUBDIR, "report_dir" + ) # create it on demand when needed + + self.backup_dir: Path = self._validatedOutputDir( + _get_arg("backup_dir"), DEFAULT_BACKUP_SUBDIR, "backup_dir" + ) + if not self.backup_dir.exists(): + self.backup_dir.mkdir(parents=True, exist_ok=True) + if any(self.backup_dir.iterdir()): + raise ValueError(f"--backup_dir directory '{self.backup_dir}' isn't empty") + self._validateBackupDirIsDisjoint() + + def _Con_begin(self, level: LoggingConsole.LogLevel = LoggingConsole.LogLevel.Critical) -> None: + if self.Con.will_log(level): + self.Con.print("[bold bright_blue]==== BulkBench >>>>>>>>[/bold bright_blue]") + + def _Con_end(self, level: LoggingConsole.LogLevel = LoggingConsole.LogLevel.Critical) -> None: + if self.Con.will_log(level): + self.Con.print("[bold bright_blue]<<<<<<<< BulkBench ====[/bold bright_blue]") + + def _cleanBackupDir(self) -> None: + if self.backup_dir.exists(): + if any(self.backup_dir.iterdir()): + self.Con.warning( + f"--backup_dir '{self.backup_dir}' is not empty, will NOT delete it.\n" + "Inspect it manually and ensure all patched files are properly reverted." + ) + else: + self.Con.debug(f"Deleting --backup_dir '{self.backup_dir}'") + try: + self.backup_dir.rmdir() + except Exception as exc: # ruff: ignore[blind-except] + self.Con.error(f"Failed to delete --backup_dir '{self.backup_dir}':", exc) + + @staticmethod + def _validatedProjectDir(value: StrPath | None) -> Path: + """Resolves `value` (defaults to the current working directory) and makes sure it + points to an existing directory.""" + path = (Path.cwd() if value is None else Path(value).expanduser()).resolve() + if not path.is_dir(): + raise ValueError(f"project_dir '{path}' doesn't exist or isn't a directory") + return path + + def _resolvedPath(self, value: StrPath | None, default: str) -> Path: + """Turns `value` (or `default`, if `value` is `None`) into a resolved absolute + path, taking a relative one relative to `self.project_dir`. Note that `.resolve()` + can't be applied earlier, as it anchors a relative path to the current working + directory.""" + path = Path(default if value is None else value).expanduser() + return (path if path.is_absolute() else self.project_dir / path).resolve() + + def _validatedOutputDir( + self, value: StrPath | None, default_subdir: str, arg_name: str + ) -> Path: + """Resolves `value` (defaults to `default_subdir`) and makes sure it points to a + non-existing, or to an existing but empty directory.""" + path = self._resolvedPath(value, default_subdir) + if path.exists() and not path.is_dir(): + raise ValueError(f"--{arg_name} '{path}' exists and isn't a directory") + return path + + @staticmethod + def _pathsOverlap(first: Path, second: Path) -> bool: + return first == second or first in second.parents or second in first.parents + + def _validateBackupDirIsDisjoint(self) -> None: + for arg_name, path in ( + ("results_dir", self.results_dir), + ("report_dir", self.report_dir), + ): + if self._pathsOverlap(self.backup_dir, path): + raise ValueError( + f"--backup_dir '{self.backup_dir}' must not overlap --{arg_name} '{path}'" + ) + + @staticmethod + def _benchmarkConfigPath(config_name: str, config_context: str) -> Path: + return benchmarkConfigPath(_BENCHMARK_CONFIGS_DIR, config_name, config_context) + + def _removeBackupArtifacts(self, backups: list[TargetBackup]) -> list[BaseException]: + errors: list[BaseException] = [] + for target_backup in reversed(backups): + try: + # Keep the path metadata when deleting the backup data fails. + target_backup["backup"].unlink(missing_ok=True) + target_backup["path_file"].unlink(missing_ok=True) + except BaseException as exc: # noqa: BLE001 - continue all cleanup attempts + errors.append(exc) + return errors + + def _snapshotTargets(self, patch_set: PatchSet) -> list[TargetBackup]: + """Copies every patch target to an indexed backup file before any mutation.""" + if any(self.backup_dir.iterdir()): + raise RuntimeError(f"backup_dir '{self.backup_dir}' must be empty before snapshotting") + + backups: list[TargetBackup] = [] + try: + for patch_index, patch_data in enumerate(patch_set["patches"]): + backup = self.backup_dir / f"{patch_index:05d}" + path_file = self.backup_dir / f"{patch_index:05d}.path" + target_backup: TargetBackup = { + "backup": backup, + "path_file": path_file, + "target": patch_data["target"], + } + backups.append(target_backup) + + shutil.copy2(patch_data["target"], backup) + path_file.write_text(str(patch_data["target"]), encoding="utf-8") + except BaseException as primary_error: + cleanup_errors = self._removeBackupArtifacts(backups) + if cleanup_errors: + raise BaseExceptionGroup( + "target snapshot and backup cleanup both failed", + [primary_error, *cleanup_errors], + ) from None + raise + + return backups + + def _restoreAllTargets(self, backups: list[TargetBackup]) -> list[BaseException]: + """Restores all targets and returns failures after attempting every restoration.""" + self.Con.debug(f"Restoring all {len(backups)} targets from backups") + restore_errors: list[BaseException] = [] + restored_backups: list[TargetBackup] = [] + + for target_backup in reversed(backups): + try: + shutil.copy2(target_backup["backup"], target_backup["target"]) + except BaseException as exc: # noqa: BLE001 - continue all restore attempts + restore_errors.append(exc) + else: + restored_backups.append(target_backup) + + restore_errors.extend(self._removeBackupArtifacts(restored_backups)) + return restore_errors + + def _runPatchCommand(self, patch_set_name: str, patch_data: PatchData, dry_run: bool) -> None: + phase = "dry-run" if dry_run else "application" + command = ["patch", "--batch"] + if dry_run: + command.append("--dry-run") + command.extend((str(patch_data["target"]), str(patch_data["patch"]))) + + try: + subprocess.run( + command, + capture_output=True, + check=True, + shell=False, + text=True, + ) + except subprocess.CalledProcessError as exc: + raise ValueError( + f"patch {phase} failed for patch set '{patch_set_name!r}', " + f"patch '{patch_data['patch']}', target '{patch_data['target']}', " + f"exit status {exc.returncode}; stdout={exc.stdout!r}; stderr={exc.stderr!r}" + ) from exc + except OSError as exc: + raise ValueError( + f"patch {phase} failed for patch set '{patch_set_name!r}', " + f"patch '{patch_data['patch']}', target '{patch_data['target']}': {exc}" + ) from exc + + def _dryRunPatches(self, patch_set: PatchSet) -> None: + """Checks that every patch can be applied before any target is backed up.""" + for patch_data in patch_set["patches"]: + self._runPatchCommand(patch_set["name"], patch_data, dry_run=True) + + def _applyPatches(self, patch_set: PatchSet) -> None: + """Applies every patch in a patch set in list order.""" + self.Con.debug(f"Applying patch set '{patch_set['name']}'") + for patch_data in patch_set["patches"]: + self._runPatchCommand(patch_set["name"], patch_data, dry_run=False) + + @contextmanager + def _appliedPatchSet(self, patch_set: PatchSet) -> Iterator[None]: + self._dryRunPatches(patch_set) + backups = self._snapshotTargets(patch_set) + try: + self._applyPatches(patch_set) + yield + except BaseException as primary_error: + restore_errors = self._restoreAllTargets(backups) + if restore_errors: + raise BaseExceptionGroup( + "patch-set execution and target restoration both failed", + [primary_error, *restore_errors], + ) from None + raise + else: + restore_errors = self._restoreAllTargets(backups) + if restore_errors: + raise BaseExceptionGroup("target restoration failed", restore_errors) + + def run(self) -> int: + """Executes the whole benchmarking pipeline. Returns the process exit code.""" + self.successful_runs: dict[str, list[tuple[str, float, list[str]]]] = {} + self.unsuccessful_runs: dict[str, list[tuple[GroupFailureCapture, float]]] = {} + for patch_set in self.patches: + with self._appliedPatchSet(patch_set): + self._runAllConfigs(patch_set["name"]) + + self._makeReport() + + success = self._printQuickStats() + if success: + self.Con.info("BulkBench session completed successfully") + else: + self.Con.error( + "BulkBench session completed with some failures. Inspect the report and " + "the console output for details." + ) + + self._cleanBackupDir() # in the end to make important errors visible if they present + return 0 if success else 1 + + def _makeReport(self) -> None: + self.report_dir.mkdir(parents=True, exist_ok=True) + + # each subsequent run of the tool will overwrite the previous report, which should be + # fine, as we typically are only interested in the final report that combines everything. + + def _makeBSFile(basename: str) -> Path: + bs_path = self.report_dir / basename + if bs_path.exists(): + self.Con.warning( + "Benchstats report '", str(bs_path), "' already exists and will be overwritten." + ) + return bs_path + + disjoint = self._areGroupsDisjoint() + if disjoint: + # since each benchmark config is defined in exactly one group, in benchmark comparisons + # we could make the group name part of benchmarking entity id from the very beginning, + # i.e. run only a single comparison with --filter1=1 only. + self.Con.info( + "All benchmark config groups are disjoint. Comparison with groups fixed is enough." + ) + bs_file = None + else: + self.Con.info( + "Found a non-disjoint benchmark config group. Will run a global all-to-all " + "comparison, and a fixed-groups comparison." + ) + bs_file = _makeBSFile("benchstats-all-to-all.html") + bs_filter1_file = _makeBSFile("benchstats-fix-groups.html") + + if bs_file: + self.Con.info(f"Running global all-to-all comparison into {bs_file}") + self._compareBenchmarks(export_to=str(bs_file)) + if bs_filter1_file: + self.Con.info(f"Running per-group comparison into {bs_filter1_file}") + self._compareBenchmarks(filter_val="1", export_to=str(bs_filter1_file)) + + def _printQuickStats(self) -> bool: + self.Con.info("Session statistics:") + n_successful_groups = sum(len(r) for r in self.successful_runs.values()) + n_configs_run = sum(len(c) for r in self.successful_runs.values() for (_, _, c) in r) + if n_configs_run: + # dict[str, list[tuple[str, float, list[str]]]] + self.Con.info( + n_successful_groups, + "groups,", + n_configs_run, + "individual configs in total ran successfully. Details by patch set:", + ) + for patch_set_name, data in self.successful_runs.items(): + self.Con.info( + f" {patch_set_name}, {len(data)} config groups: ", + ", ".join( + f"'{gn}' ({_formatDuration(d)}, run configs: {', '.join(cr)})" + if cr + else f"'{gn}' (cached)" + for (gn, d, cr) in data + ), + ) + else: + self.Con.info("All", n_successful_groups, "groups didn't actually run due to results already present.") + + n_unsuccessful_groups = sum(len(r) for r in self.unsuccessful_runs.values()) + if n_unsuccessful_groups: + # dict[str, list[tuple[GroupFailureCapture, float]]] + n_configs_run = sum( + len(gc.configs_to_run) for pd in self.unsuccessful_runs.values() for (gc, _) in pd + ) + self.Con.error( + "Total number of config groups run unsuccessfully is", + n_unsuccessful_groups, + "groups, up to", + n_configs_run, + "individual configs in total failed. Details by patch set:", + ) + for patch_set_name, data in self.unsuccessful_runs.items(): + if len(data) > 0: + self.Con.warning( + f" in patch set '{patch_set_name}' these", len(data), "config groups failed:" + ) + for (run_result, duration) in data: + assert isinstance(run_result, GroupFailureCapture) + self.Con.error( + f" '{run_result.group_name}' ({_formatDuration(duration)}, " + f"tried to run: {', '.join(run_result.configs_to_run)}). " + f"Got return code {run_result.returncode}", + ) + self.Con.debug(f"Registered output:\n{run_result.output}") + return n_unsuccessful_groups == 0 # all groups ran successfully + + def _areGroupsDisjoint(self) -> bool: + """Infers from directory structure if each config belong to exactly one group. + + Using ground truth from the filesystem enables support for multiple runs of the tool + over arbitrary set of configs and patches into the same --results_dir. + """ + configs_by_group: dict[str, set[str]] = {} + for directory, subdirectories, filenames in os.walk(self.results_dir): + relative_parts = Path(directory).relative_to(self.results_dir).parts + if len(relative_parts) == 1: + subdirectories[:] = [ + name for name in subdirectories if not name.startswith(EAGER_GROUP_PREFIX) + ] + + if "timings.json" not in filenames: + continue + if len(relative_parts) != 3: + self.Con.warning( + f"Found unexpectedly nested timings.json in {directory}. " + "Cancelling '--filter1' argument estimation" + ) + return False + + _, group_name, config_name = relative_parts + configs_by_group.setdefault(group_name, set()).add(config_name) + + seen_configs: set[str] = set() + for configs in configs_by_group.values(): + if not seen_configs.isdisjoint(configs): + return False + seen_configs.update(configs) + return True + + def _compareBenchmarks( + self, filter_val: str | None = None, export_to: str | None = None + ) -> None: + """Compares benchmark results in an interactive terminal.""" + args = [ + "benchstats", + str(self.results_dir), + "--files_parser=bulkbench.parser_JSON", + "--sample_stats", + "0", + "100", + "--always_show_pvalues", + ] + if self.Con.will_log(LoggingConsole.LogLevel.Debug): + args.append("--show_debug") + + if filter_val: + args.append(f"--filter1={filter_val}") + if export_to: + args.append(f"--export_to={export_to}") + + self.Con.info("Running command: '", " ".join(args), "'") + + try: + self._Con_end() + _ = run_with_script(args, cwd=self.results_dir, ignore_output=True) + except KeyboardInterrupt: + self._Con_begin() + self.Con.warning( + "Caught KeyboardInterrupt. If you want to abort BulkBench too, hit Ctrl-C again." + ) + except OSError as exc: + self._Con_begin() + self.Con.error(f"Error running command: {exc}") + else: + self._Con_begin() + + def _logConfigRunError( + self, + patch_set_name: str, + fc: GroupFailureCapture, + duration: float, + err_pfx: str = "", + ) -> None: + self.Con.error( + f"{err_pfx}Config group '{fc.group_name}' " + f"(run configs:{', '.join(fc.configs_to_run)}) " + f"on patch set '{patch_set_name}' failed in {_formatDuration(duration)}." + ) + self.Con.debug(f"Return code: {fc.returncode}") + + def _runAllConfigs(self, patch_set_name: str) -> None: + """Runs every config group and records its outcome for the patch set.""" + self.Con.info(f"Running all config groups for patch set '{patch_set_name}'") + successful: list[tuple[str, float, list[str]]] = [] + unsuccessful: list[tuple[GroupFailureCapture, float]] = [] + for cfg in self.configs.values(): + group_name = cfg["name"] + only_in_patches = cfg["only_in_patches"] + if only_in_patches is not None and patch_set_name not in only_in_patches: + self.Con.info( + f"Config group '{group_name}' is disabled for patch set '{patch_set_name}'. Ignoring it." + ) + continue + configs_to_run: list[str] = [] + started = monotonic() + try: + try: + workdir = self.results_dir / patch_set_name / group_name + for config_name in cfg["configs"]: + if not self.regenerate_results and configMightHaveRunSuccessfully( + workdir, config_name + ): + self.Con.info( + f"Skipping config '{config_name}' in config group " + f"'{group_name}' for patch set '{patch_set_name}': " + "its existing results might be successful" + ) + else: + configs_to_run.append(config_name) + + if configs_to_run: + self._runConfig( + patch_set_name, + workdir, + group_name, + configs_to_run, + cfg["override_args"], + ) + else: + self.Con.info( + f"All configs in config group '{group_name}' for patch set " + f"'{patch_set_name}' might already have run successfully; " + "treating the config group as successful" + ) + finally: + duration = monotonic() - started + except GroupRunError as exc: + unsuccessful.append((exc.result, duration)) + self._logConfigRunError(patch_set_name, exc.result, duration) + + except Exception as exc: # noqa: BLE001 - one config must not stop the remaining runs + exc_result = GroupFailureCapture( + group_name=group_name, + configs_to_run=configs_to_run, + output="".join(traceback.format_exception(exc)), + returncode=None, + ) + unsuccessful.append((exc_result, duration)) + self._logConfigRunError( + patch_set_name, + exc_result, + duration, + "[UNEXPECTED ERROR] ", + ) + else: + successful.append((group_name, duration, configs_to_run)) + self.Con.info( + f"Config '{group_name}' (run configs:{', '.join(configs_to_run)}) " + f"succeeded in {_formatDuration(duration)}" + ) + + self.successful_runs[patch_set_name] = successful + self.unsuccessful_runs[patch_set_name] = unsuccessful + + def _runConfig( + self, + patch_set_name: str, + workdir: Path, + group_name: str, + configs_to_run: list[str], + cfg_override_args: str | None, + ) -> None: + """Runs one config group and raises GroupRunError on process failure.""" + self.Con.info( + f"Running '{group_name}' config group ({', '.join(configs_to_run)}) " + f"for patch set '{patch_set_name}'" + ) + workdir.mkdir(parents=True, exist_ok=True) + + args = ["python", str(_BENCHMARK_RUNNER)] + for config_name in configs_to_run: + args.extend(("--name", config_name)) + if cfg_override_args is not None: + args.extend(("--override-args-json", cfg_override_args)) + args.extend(("--results-directory", str(workdir))) + + benchmark_configs = dict.fromkeys( + self._benchmarkConfigPath( + config_name, + f"config group {group_name!r}, config {config_name!r}", + ) + for config_name in configs_to_run + ) + args.extend(str(path) for path in benchmark_configs) + self.Con.trace("Running command: '", " ".join(args), "'") + + try: + self._Con_end() + completed = run_with_script(args, cwd=_APP_DIR) + except KeyboardInterrupt as exc: + self._Con_begin() + self.Con.warning( + "Caught KeyboardInterrupt. If you want to abort BulkBench too, hit Ctrl-C again." + ) + raise GroupRunError( + GroupFailureCapture( + group_name=group_name, + configs_to_run=configs_to_run, + output="User interrupted!", + returncode=None, + ) + ) from exc + except OSError as exc: + self._Con_begin() + result = GroupFailureCapture( + group_name=group_name, + configs_to_run=configs_to_run, + output=str(exc), + returncode=None, + ) + raise GroupRunError(result) from exc + else: + self._Con_begin() + + if completed.returncode != 0: + raise GroupRunError( + GroupFailureCapture( + group_name=group_name, + configs_to_run=configs_to_run, + output=completed.output, + returncode=completed.returncode, + ) + ) diff --git a/bulkbench/src/bulkbench/cli_parser.py b/bulkbench/src/bulkbench/cli_parser.py new file mode 100644 index 0000000..5104f69 --- /dev/null +++ b/bulkbench/src/bulkbench/cli_parser.py @@ -0,0 +1,174 @@ +"""Command line interface of the `bulkbench` tool.""" + +import argparse + +from benchstats.common import LoggingConsole + +from .benchmark_plan_loader import DEFAULT_CONFIGS_FILE, DEFAULT_PATCHES_FILE, VALID_NAME_PATTERN +from .bulkbench import ( + DEFAULT_BACKUP_SUBDIR, + DEFAULT_CONSOLE_LOG_LEVEL, + DEFAULT_REPORT_SUBDIR, + DEFAULT_RESULTS_SUBDIR, +) + + +# WARNING: argparse doesn't guarantee its private API stability. WTH?! +class BetterHelpFormatter(argparse.HelpFormatter): + kMaxWidth = 100 + + @staticmethod + def _useWidth(width): + return min(BetterHelpFormatter.kMaxWidth, width) + + def _fill_text(self, text, width, indent): + width = self._useWidth(width) + return "\n".join( + indent + ("" if s == "%" else s) + for line in text.splitlines() + for s in argparse.HelpFormatter._split_lines(self, line or "%", width) + ) + + def _split_lines(self, text, width): + width = self._useWidth(width) + return [ + "" if s == "%" else s + for line in text.splitlines() + for s in argparse.HelpFormatter._split_lines(self, line or "%", width) + ] + + +def makeParser() -> argparse.ArgumentParser: + """Makes a parser of the `bulkbench` command line arguments. The parser only collects + the values, `BulkBench` validates them.""" + parser = argparse.ArgumentParser( + prog="bulkbench", + description="Runs a set of benchmarks and analyzes their results statistically.", + # formatter_class=argparse.RawTextHelpFormatter, + formatter_class=BetterHelpFormatter, + ) + parser.add_argument( + "--console_log_level", + default=DEFAULT_CONSOLE_LOG_LEVEL.value, + choices=[level.value for level in LoggingConsole.LogLevel], + type=int, + help="Set the logging level for the console output verbosity (as an integer).\nValid values are: " + + ", ".join([f"{level.value} ({level.name})" for level in LoggingConsole.LogLevel]) + + ".\nDefaults to `%(default)s`.", + ) + parser.add_argument( + "--project_dir", + default=None, + help="Path to an existing directory describing a benchmark project. " + "Defaults to the current working directory.", + ) + parser.add_argument( + "--configs_file", + default=DEFAULT_CONFIGS_FILE, + help="Override yaml file describing benchmark configs to execute. Relative paths " + "are resolved under --project_dir. The file must exist.\n" + "The file must be a valid YAML file containing a list of objects describing benchmark " + "config groups with attributes:\n" + "- name (required) - name of the benchmark config group (must be unique within the file, " + f"match `{VALID_NAME_PATTERN}` after stripping, and not be `.` `..`, or starting with an " + "`eager_` literal). " + "It's recommended to keep the name short whenever the group contain only unique configs " + "not used in other groups.\n" + "- configs (required) - a non empty list of strings naming benchmark configs to execute " + "(these are passed as `--name` argument to the `/app/.ci/run.py` script),\n" + "- override_args (optional) - an optional key-value object to override specific settings " + "for all the configs in the group, such as setting `num_iterations: ` or similar.\n" + "- enabled (optional) - a boolean flag indicating whether the group should be used. " + "Valid values are unquoted YAML `true`/`false`, standard aliases " + "(`yes`/`no` and `on`/`off`), integers 1/0, quoted values " + '"true", "false", "1", and "0". Defaults to `true`. Disabled groups are omitted; ' + "their other attributes aren't validated.\n" + "- only_in_patches (optional) - an optional list of patch names, that when " + "present, enables execution of the group configs only in these patch sets. " + "The list can also contain patch set names not present in the --patches_file (these are " + "ignored with a warning). If the list is present but empty, it disables the group completely. " + "Absence of the field is an equivalent to 'run on all patch sets'.\n" + "- eager_in_patches (optional) - an optional list of patch names, that when present, " + "additionally enables execution of the group configs in eager mode in these patch sets " + "Empty list (after stripping unknown patch set names) is an equivalent of " + "absence of the field. " + "Internally (which affects --results_dir layout) it creates an additional config group " + "named `eager_` inheriting `configs` from this group. " + "Eager group's `only_in_patches` is set to an intersection of the parent's `only_in_patches` " + "and `eager_in_patches` lists (it's an error if the intersection is empty). " + "Parent group's `override_args` are copied and their `num_iterations` " + "are set to 1, and `use_torch_compile` is set to `false`.\nEager groups performance results " + "are ignored by the statistical analysis.", + ) + parser.add_argument( + "--patches_file", + default=DEFAULT_PATCHES_FILE, + help="Override yaml file describing which code needs to be patched for each set of " + "benchmark runs. Relative paths are resolved under --project_dir. The file must exist.\n" + "Every enabled patch must pass `patch --batch --dry-run` validation.\n" + "The file must be a valid YAML file containing a non-empty list of patch sets. " + "A patch set object has the following required attributes:\n" + "- name - name of the patch set (must be unique within the file, " + f"match `{VALID_NAME_PATTERN}` after stripping, and not be `.` or `..`),\n" + "- patches - a list of patch objects, each describing a patch to apply to a single file. " + "Patch lists must be unique regardless of object order; only one empty baseline is allowed. " + "A patch object may occur only once in its set. Each patch object has the following attributes:\n" + "- patch (required) - a path to a file containing the patch to apply. Relative paths are " + "resolved under --project_dir/; absolute paths are used as-is.\n" + " The file must be generated with `diff -u original_file modified_file > changes.patch` " + "command or similar. Using a patch file that modifies several files is UB.\n" + "- target (required) - a path to a file to apply the patch to. Relative paths are resolved under " + "the `/app` directory; absolute paths are used as-is. Both patch and target files must exist. " + "Applying several patches to the same target file is UB.\n" + "- enabled (optional) - a boolean flag indicating whether the patch should be applied. " + "Valid values are unquoted YAML `true`/`false`, standard aliases " + "(`yes`/`no` and `on`/`off`), integers 1/0, quoted values " + '"true", "false", "1", and "0". Defaults to `true`. Disabled patches are omitted; ' + "their other attributes aren't validated.\n", + ) + parser.add_argument( + "--backup_dir", + default=DEFAULT_BACKUP_SUBDIR, + help="Override the directory used to back up target files subjected to patches. Relative paths " + "are resolved under --project_dir. The directory must either not exist, or be empty, " + "and must not overlap with --results_dir or --report_dir.", + ) + parser.add_argument( + "--results_dir", + default=DEFAULT_RESULTS_SUBDIR, + help="Override a directory to store benchmarking results in. Relative paths are " + "resolved under --project_dir.\n" + "The directory may contain results from previous " + "runs, they will be overwritten if their paths coincide with new runs, or they " + "will be used for the statistical analysis.\n" + "The directory has the following nested structure:\n" + "///\n" + "where:\n" + "- is the name of respective code patch (patch directory name in --project_dir),\n" + "- and are the names of respective benchmark " + "group and config from the --configs_file,\n" + "- are the files containing the results of the benchmark run, suchs " + "as generated images/videos and timings.json file used for statistics analysis.", + ) + parser.add_argument( + "-r", + "--regenerate_results", + action="store_true", + help="Regenerate all config results, including results that might already be successful. " + "By default, a config is skipped when its result directory directly contains " + "timings.json and at least one .jpg, .mp4, or .png file.", + ) + parser.add_argument( + "--report_dir", + default=DEFAULT_REPORT_SUBDIR, + help="Override a directory to store the report in. Relative paths are resolved " + "under --project_dir.", + ) + parser.add_argument( + "--arch", + default=None, + help="String identifying the GPU architecture. More specifically a value for a single " + "--tag argument of the `/app/.ci/run.py` script to filter benchmark configs. By default " + "tries to get value from `rocminfo`. Passing empty string disables `--tag` use.", + ) + return parser diff --git a/bulkbench/src/bulkbench/parser_JSON.py b/bulkbench/src/bulkbench/parser_JSON.py new file mode 100644 index 0000000..88bc676 --- /dev/null +++ b/bulkbench/src/bulkbench/parser_JSON.py @@ -0,0 +1,173 @@ +# ruff: noqa: N999 # invalid-module-name +""" +This is a module to feed the results of xDiT benchmarking to `benchstats` CLI utility +for statistical analysis of the results. + +Typical use on a bulkbench --results_dir directory is: + +```bash +benchstats . --files_parser=bulkbench.parser_JSON \ + --sample_stats 0 100 --always_show_pvalues --filter1=1 +``` + +But there's much more. For the usage details see the documentation in the parser_JSON class +docstring below. + +""" + +import json +import os + +import numpy as np +from benchstats.common import ParserBase + +from .benchmark_sources import ( + _ALT_DELIMITER, + _TIMINGS_FILENAME, + get_benchmark_sources, + parse_filter, +) + + +class parser_JSON(ParserBase): + def __init__(self, fpath, filter, metrics, debug_log=None) -> None: + r"""Initialization of the parser. + + A short recap of use context first: + - xDiT's /app/.ci/run.py script produces an outputs directory with subdirectories for each + model, each containing a `timings.json` file with the results of the benchmarking. + - `benchstats` is a wrapper around a statistical test method that compares two sets of numbers + (each set contains measured runtime durations of the same code) and tells if results are + significantly different. Each such a set is called a "benchmark" in `benchstats` + terminology and is identified by a name. + - `file1` and `--filter1` arguments of `benchstats` are passed verbatim to + `fpath` and `filter` parameters of a parser constructor respectively. When a two-source + mode is used, another parser instance is created with `file2` and `--filter2` arguments + passed to it. + - `benchstats` has two modes to find benchmarks to compare one against the other among + all benchmarks it sees: + 1. find same benchmark names in two different sources (two-source mode) + 2. pool all benchmarks into several disjoint sets, find matching benchmarks in each set + and compare them pairwise (single-source mode). This enables N-way comparisons, but + a single source (parser/loader object) must return all benchmarks at once. This work + thanks to a convention that each benchmark name returned by a parser/loader is composed + of two parts with a configurable separator (typically a pipe `|` symbol) between them: + - an identifier of an entity under a benchmark (such as a model name) + - an identifier of a benchmark configuration (such an identifier of a set of flags + used to get particular benchmark result, or software stack versions, or anything + you'd like to vary and compare how it influences the results) + For example, this set of benchmark names: + {"bm1|var1", "bm1|var2", "bm2|opt1", "bm2|opt2", "bm2|opt3"} + Makes `benchstats` to do the following 4 comparisons: + - bm1|var1 vs bm1|var2 (shown as bm1 | var1 vs var2) + - bm2|opt1 vs bm2|opt2 (shown as bm2 | opt1 vs opt2) + - bm2|opt1 vs bm2|opt3 (shown as bm2 | opt1 vs opt3) + - bm2|opt2 vs bm2|opt3 (shown as bm2 | opt2 vs opt3) + So by choosing how you name the benchmarks the parser returns, you control what + `benchstats` will compare against what. + - `bulkbench` runner structures model results in a directory hierarchy inside --results_dir + directory like this: + `///timings.json`. + - This parser accepts an arbitrary directory as `fpath` (not necessary produced by a + `bulkbench` runner) assuming it contains `/timings.json` files nested + somewhere deep inside. Then it constructs benchmark names based on the directory + structure and the value of the `filter` argument, enabling different kinds of N-way + comparisons. + + Comparison modes: + A. when `filter` argument isn't set or empty: + A.1. When `fpath` is a single `.json` file, or it's a directory having an immediate child + `timings.json`, the parser enables in a two-source mode, i.e. user has to supply two such + sources to benchstats CLI, and two such parser objects will create a single benchmark each, + and benchstats will compare them against each other. + A.2 Otherwise, the `fpath` must be a directory having subdirectories, with `timings.json` + file being nested somewhere deep inside `path/to/model1/timings.json`, the parser enables a + single-source mode and for each `timings.json` it finds, it creates a single benchmark named + `model1|path/to`. There'll be as many comparisons as there are same named directories having + `timings.json` as an immediate child. + + B. when `filter` argument is set, it must be a comma separated set of numbers each of which + denoting an index of a nested subdirectory to include in the identifier of the entity under + benchmark name (`fpath` must be a directory having subdirectories). Index counts from the + most nested directory (in xDiT it's a benchmark config name) upwards. + For example, bulkbench runner typically produces the following directory structure for a + project: `results////timings.json`, so + assuming one pass `results` as `fpath`, the following filter value will make the following + comparisons: + - --filter1=0 is exactly the A.2 case: benchmarks will be named like + `|/` which will compare + configs across all combinations of patches and groups. I.e. for a given model, + it'll compare all combinations of groups and patches to each other. + - --filter1=1 adds `` to the benchmark name (with `` + already being there by default, like it was --filter1=0,1), naming benchmarks like + `/|` comparing the same + model+group_name combination across patches. + - --filter1=2 adds `` to the benchmark name (with `` + already being there by default, like it was --filter1=0,2), naming benchmarks like + `/|` comparing the same models+patch + combination across groups (not useful if a single config isn't a member of + multiple groups). + - --filter1=1,2 (or --filter1=0,1,2) makes everything count as an entifier of an + entity under a benchmark, and that will break the name pooling, because a single + benchmark won't have an alternative to compare against. However, if the `fpath` + refers to a dir with the following structure: + `////timings.json`, this + filter value will allow to compare results of the same combinations of + // across platforms. (Remember that + `benchstats` supports independent modification of read benchmark names using regular + expressions in --from, --to arguments. This could be handy if one wants to + compare results of different configs: for example, `flux2.quantgemm.gfx942` vs + `flux2.quantgemm.gfx950` can't be compared directly since names differ, but we + can simply strip `gfx..` suffixes with `--from \\.gfx\\d+ --to ""` arguments to enable + the comparison.) + + Since, bulkbench runner produce eager mode results in a separate group directory starting + with "eager_", such results are ignored by the statistical analysis. + + Note that directory symlinks are supported, so you can narrow the scope of comparisons by + crafting a top-level directory with symlinks to the actually interesting results directories + and analyzing such a directory instead. + """ + + assert metrics == ["real_time"], ( + "Only default metrics are supported for xDiT SingleModel parser" + ) + + filter_indices = parse_filter(filter) + fpath = os.fspath(fpath) + is_direct_file = os.path.isfile(fpath) + has_immediate_timings = os.path.isfile(os.path.join(fpath, _TIMINGS_FILENAME)) + self.alt_delimiter = ( + None + if filter_indices is None and (is_direct_file or has_immediate_timings) + else _ALT_DELIMITER + ) + + sources = get_benchmark_sources(fpath, filter_indices, debug_log) + if debug_log: + debug_log.debug("parser_JSON: reading the following benchmarks:", sources) + + self.stats = {} + warned = False + for bmname, result_dir in sources: + json_path = fpath if is_direct_file else os.path.join(result_dir, _TIMINGS_FILENAME) + with open(json_path, "r") as file: + data = json.load(file) + if len(data) > 2: + if not warned: + warned = True + lgr = debug_log.debug if debug_log else print + lgr( + "parser_JSON: Dropping 2 first elements of each latency set " + "to ignore first 3 iterations (in total) as a warmup." + ) + lgr = None + data = data[2:] + assert bmname not in self.stats, f"Benchmark {bmname} already exists" # sanity check + self.stats[bmname] = {"real_time": data} + + def getStats(self) -> dict[str, dict[str, np.ndarray]]: + return self.stats + + def getAltDelimiter(self) -> str | None: + return self.alt_delimiter diff --git a/bulkbench/src/bulkbench/script_runner.py b/bulkbench/src/bulkbench/script_runner.py new file mode 100644 index 0000000..c933ae3 --- /dev/null +++ b/bulkbench/src/bulkbench/script_runner.py @@ -0,0 +1,131 @@ +"""Run a command in a recorded pseudo-terminal.""" + +import math +import os +import shlex +import subprocess +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory + +_SCRIPT_EXECUTABLE = "script" +_TERMINATION_TIMEOUT_SECONDS = 5 + +StrPath = str | os.PathLike[str] + + +@dataclass(frozen=True) +class ScriptRunResult: + """Result of a command recorded through util-linux `script`.""" + + args: tuple[str, ...] + returncode: int + output: str + + +def _terminate(process: subprocess.Popen[bytes]) -> None: + """Terminates `script` and gives it time to clean up its PTY child.""" + if process.poll() is not None: + return + + process.terminate() + try: + process.wait(timeout=_TERMINATION_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def _read_output(transcript: Path, timing: Path) -> str: + """Reads child output without `script`'s transcript metadata.""" + if not timing.is_file(): + raise ValueError(f"script timing '{timing}' doesn't exist or isn't a file") + if not transcript.is_file(): + raise ValueError(f"script transcript '{transcript}' doesn't exist or isn't a file") + + try: + timing_lines = timing.read_text(encoding="ascii").splitlines() + except UnicodeDecodeError as exc: + raise ValueError("malformed script timing data: file is not ASCII") from exc + + output_size = 0 + for line_number, line in enumerate(timing_lines, start=1): + fields = line.split() + if len(fields) != 2: + raise ValueError(f"malformed script timing data at line {line_number}: {line!r}") + try: + elapsed = float(fields[0]) + byte_count = int(fields[1]) + except ValueError as exc: + raise ValueError( + f"malformed script timing data at line {line_number}: {line!r}" + ) from exc + if ( + not math.isfinite(elapsed) + or elapsed < 0 + or not fields[1].isascii() + or not fields[1].isdigit() + or byte_count < 0 + ): + raise ValueError(f"malformed script timing data at line {line_number}: {line!r}") + output_size += byte_count + + transcript_data = transcript.read_bytes() + metadata_end = transcript_data.find(b"\n") + if metadata_end < 0: + raise ValueError("script transcript is missing its metadata header") + output_start = metadata_end + 1 + output_end = output_start + output_size + if output_end > len(transcript_data): + raise ValueError( + f"script transcript contains fewer than the recorded {output_size} output bytes" + ) + trailer = transcript_data[output_end:] + if not trailer.startswith(b"\n") or not trailer.endswith(b"]\n"): + raise ValueError("script transcript is missing or has malformed trailing metadata") + + output_data = transcript_data[output_start:output_end] + return output_data.decode("utf-8", errors="replace").replace("\r\n", "\n") + + +def run_with_script( + args: Sequence[str], *, cwd: StrPath | None = None, ignore_output: bool = False +) -> ScriptRunResult: + """Runs a command in a PTY, mirrors it live, and captures its combined output.""" + command = tuple(args) + if not command: + raise ValueError("args must contain at least one item") + if not all(isinstance(arg, str) for arg in command): + raise TypeError("every args item must be a string") + + with TemporaryDirectory(prefix="bulkbench-script-") as temp_dir: + transcript = Path(temp_dir) / "output.log" + timing = Path(temp_dir) / "timing.log" + script_args = [ + _SCRIPT_EXECUTABLE, + "--quiet", + "--return", + "--flush", + "--log-out", + str(transcript), + "--log-timing", + str(timing), + "--logging-format", + "classic", + "--command", + f"exec {shlex.join(command)}", + ] + process = subprocess.Popen( + script_args, + cwd=os.fspath(cwd) if cwd is not None else None, + shell=False, + ) + try: + returncode = process.wait() + except BaseException: + _terminate(process) + raise + + output = _read_output(transcript, timing) if not ignore_output else None + return ScriptRunResult(args=command, returncode=returncode, output=output) diff --git a/bulkbench/tests/proj0/my_configs b/bulkbench/tests/proj0/my_configs new file mode 100644 index 0000000..dfed26c --- /dev/null +++ b/bulkbench/tests/proj0/my_configs @@ -0,0 +1,21 @@ +- name: group1 + configs: + - cfg1 + - cfg2 + override_args: + num_iterations: 5 + use_cfg_parallel: true + nested: + values: [1, null] + +- name: group2 + configs: [cfg3] + override_args: null + enabled: yes + +- name: group3 + configs: [cfg4] + enabled: "1" + +- enabled: false + unknown: this disabled group is ignored diff --git a/bulkbench/tests/proj0/patches.yaml b/bulkbench/tests/proj0/patches.yaml new file mode 100644 index 0000000..6d78499 --- /dev/null +++ b/bulkbench/tests/proj0/patches.yaml @@ -0,0 +1,2 @@ +- name: baseline + patches: [] diff --git a/bulkbench/tests/test_parser_json.py b/bulkbench/tests/test_parser_json.py new file mode 100644 index 0000000..dac0523 --- /dev/null +++ b/bulkbench/tests/test_parser_json.py @@ -0,0 +1,234 @@ +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from bulkbench.benchmark_sources import get_benchmark_sources +from bulkbench.parser_JSON import parser_JSON + + +class TestBenchmarkSourceDiscovery(unittest.TestCase): + @staticmethod + def _write_timings(directory: Path, data=None) -> None: + directory.mkdir(parents=True, exist_ok=True) + (directory / "timings.json").write_text( + json.dumps([1, 2, 3] if data is None else data), + encoding="utf-8", + ) + + def test_direct_json_file_uses_two_source_mode_name(self): + with TemporaryDirectory() as temp_dir: + result_dir = Path(temp_dir) / "model" + result_dir.mkdir() + json_path = result_dir / "custom.json" + json_path.write_text("[1, 2]", encoding="utf-8") + + self.assertEqual( + get_benchmark_sources(json_path, None), + {("model", str(result_dir))}, + ) + + def test_rejects_immediate_and_nested_timings_files(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) / "model" + self._write_timings(root) + self._write_timings(root / "nested") + + with self.assertRaises(ValueError) as context: + get_benchmark_sources(root, None) + + self.assertEqual( + str(context.exception), + "The directory is malformed and contains 2 timings.json files, " + f"an immediate and a nested in {root / 'nested' / 'timings.json'}", + ) + + def test_rejects_immediate_and_nested_timings_files_deep_in_tree(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + malformed_dir = root / "patch" / "group" + nested_dir = malformed_dir / "config" + self._write_timings(malformed_dir) + self._write_timings(nested_dir) + + with self.assertRaises(ValueError) as context: + get_benchmark_sources(root, None) + + self.assertEqual( + str(context.exception), + "The directory is malformed and contains 2 timings.json files, " + f"an immediate and a nested in {nested_dir / 'timings.json'}", + ) + + def test_rejects_timings_file_nested_three_layers_below_immediate_file(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + malformed_dir = root / "patch" + nested_dir = malformed_dir / "one" / "two" / "three" + self._write_timings(malformed_dir) + self._write_timings(nested_dir) + + with self.assertRaises(ValueError) as context: + get_benchmark_sources(root, None) + + self.assertEqual( + str(context.exception), + "The directory is malformed and contains 2 timings.json files, " + f"an immediate and a nested in {nested_dir / 'timings.json'}", + ) + + def test_accepts_a_single_immediate_timings_file(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) / "model" + self._write_timings(root) + + self.assertEqual( + get_benchmark_sources(root, None), + {("model", str(root))}, + ) + + def test_discovers_nested_timings_and_warns_about_barren_directories(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + first = root / "patch1" / "group" / "model" + second = root / "patch2" / "group" / "model" + barren = root / "unused" / "leaf" + self._write_timings(first) + self._write_timings(second) + barren.mkdir(parents=True) + debug_log = Mock() + + sources = get_benchmark_sources(root, None, debug_log) + + self.assertEqual( + sources, + { + ("model|patch1/group", str(first)), + ("model|patch2/group", str(second)), + }, + ) + debug_log.warning.assert_called_once() + warning = debug_log.warning.call_args.args[0] + self.assertIn(str(root / "unused"), warning) + self.assertIn(str(barren), warning) + self.assertNotIn(str(root / "patch1"), warning) + + def test_discovers_timings_below_directory_symlink(self): + with TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + root = temp_path / "results" + target = temp_path / "external" + result_dir = target / "config" + root.mkdir() + self._write_timings(result_dir) + (root / "linked").symlink_to(target, target_is_directory=True) + + self.assertEqual( + get_benchmark_sources(root, None), + {("config|linked", str(root / "linked" / "config"))}, + ) + + def test_filter_selects_indexed_directories_and_implicitly_selects_zero(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + result_dir = root / "patch" / "group" / "config" + self._write_timings(result_dir) + + self.assertEqual( + get_benchmark_sources(root, {0, 1}), + {("config/group|patch", str(result_dir))}, + ) + self.assertEqual( + get_benchmark_sources(root, {0, 2}), + {("config/patch|group", str(result_dir))}, + ) + self.assertEqual( + get_benchmark_sources(root, {0, 1, 2}), + {("config/group/patch|", str(result_dir))}, + ) + + def test_rejects_invalid_filter_values(self): + with TemporaryDirectory() as temp_dir: + for invalid_filter in ("one", "-1", "1,,2", "1,"): + with self.subTest(filter=invalid_filter), self.assertRaises(ValueError): + get_benchmark_sources(temp_dir, {invalid_filter}) + + def test_warns_and_skips_paths_too_shallow_for_filter(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + shallow = root / "group" / "config" + deep = root / "platform" / "patch" / "group" / "config" + self._write_timings(shallow) + self._write_timings(deep) + debug_log = Mock() + + sources = get_benchmark_sources(root, {0, 3}, debug_log) + + self.assertEqual( + sources, + {("config/platform|patch/group", str(deep))}, + ) + debug_log.warning.assert_called_once() + warning = debug_log.warning.call_args.args[0] + self.assertIn("filter '{0, 3}'", warning) + self.assertIn(str(shallow), warning) + self.assertNotIn(str(deep), warning) + + def test_warns_and_raises_when_no_benchmark_is_found(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + leaf = root / "empty" + leaf.mkdir() + debug_log = Mock() + + with self.assertRaisesRegex(ValueError, "No benchmarks"): + get_benchmark_sources(root, None, debug_log) + + debug_log.warning.assert_called_once() + warning = debug_log.warning.call_args.args[0] + self.assertIn(str(root), warning) + self.assertIn(str(leaf), warning) + + +class TestParserJSON(unittest.TestCase): + @staticmethod + def _write_timings(directory: Path, data) -> None: + directory.mkdir(parents=True, exist_ok=True) + (directory / "timings.json").write_text(json.dumps(data), encoding="utf-8") + + def test_loads_all_nested_benchmarks_and_enables_single_source_mode(self): + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + first = root / "patch1" / "group" / "model" + second = root / "patch2" / "group" / "model" + self._write_timings(first, [1, 2, 3, 4]) + self._write_timings(second, [5, 6]) + + with patch("builtins.print"): + parser = parser_JSON(root, None, ["real_time"]) + + self.assertEqual( + parser.getStats(), + { + "model|patch1/group": {"real_time": [3, 4]}, + "model|patch2/group": {"real_time": [5, 6]}, + }, + ) + self.assertEqual(parser.getAltDelimiter(), "|") + + def test_loads_direct_json_file_and_uses_two_source_mode(self): + with TemporaryDirectory() as temp_dir: + result_dir = Path(temp_dir) / "model" + result_dir.mkdir() + json_path = result_dir / "results.json" + json_path.write_text("[10, 11]", encoding="utf-8") + + parser = parser_JSON(json_path, None, ["real_time"]) + + self.assertEqual(parser.getStats(), {"model": {"real_time": [10, 11]}}) + self.assertIsNone(parser.getAltDelimiter()) + + +if __name__ == "__main__": + unittest.main() diff --git a/bulkbench/tests/test_script_runner.py b/bulkbench/tests/test_script_runner.py new file mode 100644 index 0000000..20be278 --- /dev/null +++ b/bulkbench/tests/test_script_runner.py @@ -0,0 +1,222 @@ +import signal +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from bulkbench.script_runner import ( + _TERMINATION_TIMEOUT_SECONDS, + _read_output, + _terminate, + run_with_script, +) + + +class TestScriptRunner(unittest.TestCase): + def test_mirrors_output_to_the_current_console(self): + helper = ( + "import sys\n" + "from bulkbench.script_runner import run_with_script\n" + "run_with_script([sys.executable, '-c', " + "\"print('live output', flush=True)\"])\n" + ) + + completed = subprocess.run( + [sys.executable, "-c", helper], + capture_output=True, + check=False, + text=True, + ) + + self.assertEqual(completed.returncode, 0) + self.assertEqual(completed.stdout, "live output\n") + self.assertEqual(completed.stderr, "") + + def test_runs_in_pty_and_captures_ordered_combined_output(self): + code = ( + "import os, sys\n" + "print(f'tty={os.isatty(1)}', flush=True)\n" + "print('stdout-1', flush=True)\n" + "print('stderr-1', file=sys.stderr, flush=True)\n" + "print('stdout-2', flush=True)\n" + ) + + result = run_with_script([sys.executable, "-c", code]) + + self.assertEqual(result.returncode, 0) + self.assertEqual(result.args, (sys.executable, "-c", code)) + self.assertEqual( + result.output, + "tty=True\nstdout-1\nstderr-1\nstdout-2\n", + ) + + def test_preserves_arguments_and_working_directory(self): + with TemporaryDirectory() as cwd: + argument = "value with spaces; $(not-a-command)" + code = ( + "import os, sys\nprint(os.getcwd(), flush=True)\nprint(sys.argv[1], flush=True)\n" + ) + + result = run_with_script( + [sys.executable, "-c", code, argument], + cwd=cwd, + ) + + self.assertEqual(result.returncode, 0) + self.assertEqual( + result.output, + f"{Path(cwd).resolve()}\n{argument}\n", + ) + + def test_returns_signal_exit_status(self): + code = ( + "import os, signal\n" + "print('before signal', flush=True)\n" + "os.kill(os.getpid(), signal.SIGTERM)\n" + ) + + result = run_with_script([sys.executable, "-c", code]) + + self.assertEqual(result.returncode, 128 + signal.SIGTERM) + self.assertEqual(result.output, "before signal\n") + + def test_existing_empty_timing_data_means_empty_output(self): + result = run_with_script([sys.executable, "-c", ""]) + + self.assertEqual(result.returncode, 0) + self.assertEqual(result.output, "") + + def test_rejects_malformed_timing_data(self): + malformed_values = ( + "invalid\n", + "nan 1\n", + "0.1 -1\n", + "0.1 invalid\n", + "0.1 1 extra\n", + ) + with TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + transcript = temp_path / "output.log" + timing = temp_path / "timing.log" + transcript.write_bytes(b"metadata\nx\ntrailer]\n") + + for value in malformed_values: + with self.subTest(value=value): + timing.write_text(value, encoding="ascii") + with self.assertRaisesRegex(ValueError, "malformed script timing data"): + _read_output(transcript, timing) + timing.write_bytes(b"\xff") + with self.assertRaisesRegex(ValueError, "malformed script timing data"): + _read_output(transcript, timing) + + def test_rejects_malformed_transcript(self): + cases = ( + (None, "doesn't exist or isn't a file"), + (b"metadata without newline", "missing its metadata header"), + (b"metadata\nx", "fewer than the recorded"), + (b"metadata\nxyzbad trailer", "malformed trailing metadata"), + ) + with TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + transcript = temp_path / "output.log" + timing = temp_path / "timing.log" + timing.write_text("0.1 3\n", encoding="ascii") + + for transcript_data, expected_message in cases: + with self.subTest(expected_message=expected_message): + transcript.unlink(missing_ok=True) + if transcript_data is not None: + transcript.write_bytes(transcript_data) + with self.assertRaisesRegex(ValueError, expected_message): + _read_output(transcript, timing) + + def test_removes_intermediate_files_after_success(self): + with TemporaryDirectory() as parent_dir: + created_paths = [] + + def tracked_temporary_directory(*args, **kwargs): + temporary_directory = TemporaryDirectory( + *args, + dir=parent_dir, + **kwargs, + ) + created_paths.append(Path(temporary_directory.name)) + return temporary_directory + + with patch( + "bulkbench.script_runner.TemporaryDirectory", + side_effect=tracked_temporary_directory, + ): + result = run_with_script([sys.executable, "-c", "print('output')"]) + + self.assertEqual(result.output, "output\n") + self.assertTrue(created_paths) + self.assertTrue(all(not path.exists() for path in created_paths)) + self.assertEqual(list(Path(parent_dir).iterdir()), []) + + def test_terminates_script_when_parent_wait_is_interrupted(self): + process = Mock() + process.wait.side_effect = [KeyboardInterrupt, 0] + process.poll.return_value = None + + with TemporaryDirectory() as parent_dir: + created_paths = [] + + def tracked_temporary_directory(*args, **kwargs): + temporary_directory = TemporaryDirectory( + *args, + dir=parent_dir, + **kwargs, + ) + created_paths.append(Path(temporary_directory.name)) + return temporary_directory + + with ( + patch( + "bulkbench.script_runner.subprocess.Popen", + return_value=process, + ) as process_manager, + patch( + "bulkbench.script_runner.TemporaryDirectory", + side_effect=tracked_temporary_directory, + ), + self.assertRaises(KeyboardInterrupt), + ): + run_with_script(["command"]) + + self.assertTrue(created_paths) + self.assertTrue(all(not path.exists() for path in created_paths)) + self.assertEqual(list(Path(parent_dir).iterdir()), []) + + process.terminate.assert_called_once_with() + script_args = process_manager.call_args.args[0] + self.assertIn("--logging-format", script_args) + self.assertEqual( + script_args[script_args.index("--logging-format") + 1], + "classic", + ) + self.assertEqual( + process.wait.call_args_list[1].kwargs, + {"timeout": _TERMINATION_TIMEOUT_SECONDS}, + ) + process.kill.assert_not_called() + + def test_kills_script_if_graceful_termination_times_out(self): + process = Mock() + process.poll.return_value = None + process.wait.side_effect = [ + subprocess.TimeoutExpired(["script"], _TERMINATION_TIMEOUT_SECONDS), + 0, + ] + + _terminate(process) + + process.terminate.assert_called_once_with() + process.kill.assert_called_once_with() + self.assertEqual(process.wait.call_count, 2) + + +if __name__ == "__main__": + sys.exit(unittest.main()) diff --git a/bulkbench/tests/test_system.py b/bulkbench/tests/test_system.py new file mode 100644 index 0000000..3cceb22 --- /dev/null +++ b/bulkbench/tests/test_system.py @@ -0,0 +1,1645 @@ +import difflib +import shutil +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, call, patch + +import pytest + +from bulkbench import BulkBench, GroupFailureCapture, GroupRunError, makeParser +from bulkbench.bulkbench import configMightHaveRunSuccessfully +from bulkbench.script_runner import ScriptRunResult + + +def _makeBulkBenchNoReport(*args, **kwargs): + bb = BulkBench(*args, **kwargs) + bb._makeReport = Mock() + return bb + + +class TestSystem(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._benchmark_configs_temp = TemporaryDirectory() + cls.benchmark_configs_dir = Path(cls._benchmark_configs_temp.name) + (cls.benchmark_configs_dir / "cfg.yaml").write_text( + """ +- name: cfg + tags: [gfx942] +- name: cfg.variant + tags: [gfx950] +- name: cfg.first +- name: cfg.second + tags: [] +""", + encoding="utf-8", + ) + for stem in ("cfg1", "cfg2", "cfg3", "cfg4"): + (cls.benchmark_configs_dir / f"{stem}.yaml").write_text( + f"- name: {stem}\n tags: [gfx942]\n", + encoding="utf-8", + ) + cls._benchmark_configs_patch = patch( + "bulkbench.bulkbench._BENCHMARK_CONFIGS_DIR", + cls.benchmark_configs_dir, + ) + cls._benchmark_configs_patch.start() + + @classmethod + def tearDownClass(cls): + cls._benchmark_configs_patch.stop() + cls._benchmark_configs_temp.cleanup() + + def test_configs_file(self): + project_dir = Path(__file__).parent / "proj0" + with TemporaryDirectory() as output_dir: + bulk_bench = _makeBulkBenchNoReport( + project_dir=project_dir, + backup_dir=Path(output_dir) / "backups", + configs_file="my_configs", + results_dir=Path(output_dir) / "results", + report_dir=Path(output_dir) / "report", + # Specify arch to avoid a rocminfo call. + arch="", + ) + + self.assertEqual( + bulk_bench.configs, + { + "group1": { + "name": "group1", + "configs": ["cfg1", "cfg2"], + "override_args": ( + '{"num_iterations":5,"use_cfg_parallel":true,"nested":{"values":[1,null]}}' + ), + "only_in_patches": None, + }, + "group2": { + "name": "group2", + "configs": ["cfg3"], + "override_args": None, + "only_in_patches": None, + }, + "group3": { + "name": "group3", + "configs": ["cfg4"], + "override_args": None, + "only_in_patches": None, + }, + }, + ) + + def test_regenerate_results_cli_argument(self): + parser = makeParser() + + self.assertFalse(parser.parse_args([]).regenerate_results) + self.assertTrue(parser.parse_args(["-r"]).regenerate_results) + self.assertTrue(parser.parse_args(["--regenerate_results"]).regenerate_results) + + def _read_configs(self, contents: str, *, arch: str = "") -> BulkBench: + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + (project_dir / "configs.yaml").write_text(contents, encoding="utf-8") + (project_dir / "patches.yaml").write_text( + "- name: baseline\n patches: []\n", encoding="utf-8" + ) + return _makeBulkBenchNoReport(project_dir=project_dir, arch=arch) + + def assertInvalidConfigs(self, contents: str, expected_message: str) -> None: + with self.assertRaises(ValueError) as context: + self._read_configs(contents) + self.assertIn(expected_message, str(context.exception)) + + def test_configs_file_schema_errors(self): + cases = ( + ("", "must contain a YAML list"), + ("[]", "must contain at least one enabled config group"), + ("{}", "must contain a YAML list"), + ("- configs: [cfg]", "missing required attribute 'name'"), + ("- name: group\n configs: [cfg]\n extra: true", "unknown attribute(s): extra"), + ("- name: ' '\n configs: [cfg]", "'name' must be a non-empty string"), + ("- name: group", "missing required attribute 'configs'"), + ("- name: group\n configs: []", "'configs' must be a non-empty list"), + ("- name: group\n configs: [cfg, 1]", "item 2 must be a non-empty string"), + ("- name: group\n configs: [cfg, cfg]", "duplicate config name 'cfg'"), + ( + "- name: group\n configs: [cfg]\n- name: group\n configs: [other]", + "duplicate group name 'group'", + ), + ) + for contents, expected_message in cases: + with self.subTest(expected_message=expected_message): + self.assertInvalidConfigs(contents, expected_message) + + def test_enabled_values_are_normalized_and_disabled_groups_are_ignored(self): + bulk_bench = self._read_configs( + """ +- name: bool_true + configs: [cfg] + enabled: true +- name: int_one + configs: [cfg] + enabled: 1 +- name: string_true + configs: [cfg] + enabled: "true" +- name: string_one + configs: [cfg] + enabled: "1" +- name: alias_yes + configs: [cfg] + enabled: yes +- name: alias_on + configs: [cfg] + enabled: on +- enabled: false + unknown: ignored +- enabled: 0 +- enabled: "false" +- enabled: "0" +- enabled: no +- enabled: off +""" + ) + + self.assertEqual( + set(bulk_bench.configs), + {"bool_true", "int_one", "string_true", "string_one", "alias_yes", "alias_on"}, + ) + self.assertTrue( + all("enabled" not in config_group for config_group in bulk_bench.configs.values()) + ) + + def test_invalid_enabled_values_are_rejected(self): + for value in ("2", "-1", "null", "[]", "{}", '"True"', '"yes"', '""'): + with self.subTest(value=value): + self.assertInvalidConfigs( + f"- name: group\n configs: [cfg]\n enabled: {value}", + "attribute 'enabled' must be a YAML boolean", + ) + + def test_all_groups_disabled_is_rejected(self): + self.assertInvalidConfigs( + '- enabled: false\n- enabled: 0\n- enabled: "false"\n- enabled: "0"', + "must contain at least one enabled config group", + ) + + def test_override_args_errors(self): + cases = ( + ( + "- name: group\n configs: [cfg]\n override_args: [value]", + "'override_args' must be an object or null", + ), + ( + "- name: group\n configs: [cfg]\n override_args:\n 1: value", + "contains non-string mapping key 1", + ), + ( + ("- name: group\n configs: [cfg]\n override_args:\n generated_at: 2026-08-14"), + "isn't valid JSON", + ), + ( + "- name: group\n configs: [cfg]\n override_args:\n value: .nan", + "isn't valid JSON", + ), + ( + ("- name: group\n configs: [cfg]\n override_args: &args\n self: *args"), + "contains a circular reference", + ), + ) + for contents, expected_message in cases: + with self.subTest(expected_message=expected_message): + self.assertInvalidConfigs(contents, expected_message) + + def test_config_patch_fields_reject_invalid_values(self): + cases = ( + ("only_in_patches: null", "only_in_patches field is set but empty"), + ("only_in_patches: baseline", "'only_in_patches' must be a list"), + ("only_in_patches: [' ']", "item 1 must be a non-empty string"), + ("only_in_patches: [1]", "item 1 must be a non-empty string"), + ("eager_in_patches: null", "eager_in_patches field is set but empty"), + ("eager_in_patches: baseline", "'eager_in_patches' must be a list"), + ("eager_in_patches: [' ']", "item 1 must be a non-empty string"), + ("eager_in_patches: [1]", "item 1 must be a non-empty string"), + ) + for field, expected_message in cases: + with self.subTest(field=field): + self.assertInvalidConfigs( + f"- name: group\n configs: [cfg]\n {field}", + expected_message, + ) + + def test_empty_only_in_patches_omits_group(self): + for restriction in ("[]", "[missing]"): + with self.subTest(restriction=restriction): + with patch("bulkbench.bulkbench.LoggingConsole.warning") as warning: + bulk_bench = self._read_configs( + f""" +- name: omitted + configs: [cfg] + only_in_patches: {restriction} + eager_in_patches: invalid-but-not-validated +- name: retained + configs: [cfg] +""" + ) + + self.assertEqual(set(bulk_bench.configs), {"retained"}) + self.assertTrue( + any( + "Group 'omitted' was fully omitted" in warning_call.args[0] + for warning_call in warning.call_args_list + ) + ) + + def test_all_groups_omitted_by_only_in_patches_are_rejected(self): + self.assertInvalidConfigs( + "- name: group\n configs: [cfg]\n only_in_patches: []", + "must contain at least one enabled config group", + ) + + def test_empty_eager_in_patches_does_not_create_eager_group(self): + for restriction in ("[]", "[missing]"): + with self.subTest(restriction=restriction): + bulk_bench = self._read_configs( + f""" +- name: group + configs: [cfg] + eager_in_patches: {restriction} +""" + ) + + self.assertEqual(set(bulk_bench.configs), {"group"}) + + def test_config_patch_fields_deduplicate_and_ignore_unknown_patch_sets(self): + with patch("bulkbench.bulkbench.LoggingConsole.warning") as warning: + bulk_bench = self._read_configs( + """ +- name: group + configs: [cfg] + only_in_patches: [baseline, " baseline ", missing, missing] + eager_in_patches: [baseline, " baseline ", missing, missing] +""" + ) + + self.assertEqual(bulk_bench.configs["group"]["only_in_patches"], frozenset({"baseline"})) + eager_group = bulk_bench.configs["eager_group"] + self.assertEqual(eager_group["configs"], ["cfg"]) + self.assertEqual(eager_group["only_in_patches"], frozenset({"baseline"})) + self.assertEqual( + eager_group["override_args"], + '{"num_iterations":1,"use_torch_compile":false}', + ) + self.assertEqual(warning.call_count, 2) + for warning_call, attribute_name in zip( + warning.call_args_list, + ("only_in_patches", "eager_in_patches"), + strict=True, + ): + message = warning_call.args[0] + self.assertIn(f"attribute '{attribute_name}' isn't defined", message) + self.assertIn("'missing'", message) + + def test_eager_group_intersects_restrictions_and_overrides_eager_arguments(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + for name in ("changes", "quality"): + patch_path = project_dir / name / f"{name}.patch" + patch_path.parent.mkdir() + patch_path.touch() + (project_dir / f"{name}.py").touch() + (project_dir / "patches.yaml").write_text( + f""" +- name: baseline + patches: [] +- name: changes + patches: + - patch: changes.patch + target: {project_dir / "changes.py"} +- name: quality + patches: + - patch: quality.patch + target: {project_dir / "quality.py"} +""", + encoding="utf-8", + ) + (project_dir / "configs.yaml").write_text( + """ +- name: group + configs: [cfg] + only_in_patches: [baseline, changes] + eager_in_patches: [quality, changes] + override_args: + num_iterations: 17 + use_torch_compile: true + nested: {value: original} +""", + encoding="utf-8", + ) + with patch.object(BulkBench, "_dryRunPatches"): + bulk_bench = _makeBulkBenchNoReport(project_dir=project_dir, arch="") + + self.assertEqual( + bulk_bench.configs, + { + "group": { + "name": "group", + "configs": ["cfg"], + "override_args": ( + '{"num_iterations":17,"use_torch_compile":true,' + '"nested":{"value":"original"}}' + ), + "only_in_patches": frozenset({"baseline", "changes"}), + }, + "eager_group": { + "name": "eager_group", + "configs": ["cfg"], + "override_args": ( + '{"num_iterations":1,"use_torch_compile":false,' + '"nested":{"value":"original"}}' + ), + "only_in_patches": frozenset({"changes"}), + }, + }, + ) + + def test_eager_group_with_empty_restriction_intersection_is_omitted(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + (project_dir / "changes").mkdir() + (project_dir / "changes" / "change.patch").touch() + (project_dir / "target.py").touch() + (project_dir / "patches.yaml").write_text( + f""" +- name: baseline + patches: [] +- name: changes + patches: + - patch: change.patch + target: {project_dir / "target.py"} +""", + encoding="utf-8", + ) + (project_dir / "configs.yaml").write_text( + """ +- name: group + configs: [cfg] + only_in_patches: [baseline] + eager_in_patches: [changes] +""", + encoding="utf-8", + ) + with ( + patch.object(BulkBench, "_dryRunPatches"), + patch("bulkbench.bulkbench.LoggingConsole.warning") as warning, + ): + bulk_bench = _makeBulkBenchNoReport(project_dir=project_dir, arch="") + + self.assertEqual(set(bulk_bench.configs), {"group"}) + self.assertTrue( + any( + "Eager group 'eager_group' was fully omitted" in warning_call.args[0] + and "have an empty intersection" in warning_call.args[0] + for warning_call in warning.call_args_list + ) + ) + + def test_config_group_name_cannot_use_reserved_eager_prefix(self): + self.assertInvalidConfigs( + "- name: eager_group\n configs: [cfg]", + "attribute 'name' must not start with reserved prefix 'eager_'", + ) + + def test_duplicate_yaml_keys_are_rejected(self): + self.assertInvalidConfigs( + "- enabled: false\n name: group\n name: other\n" + "- name: enabled_group\n configs: [cfg]", + "found duplicate key 'name'", + ) + + def test_malformed_yaml_is_reported_as_value_error(self): + self.assertInvalidConfigs("- name: [", "failed to read configs_file") + + def test_config_prefix_selects_an_existing_benchmark_yaml(self): + bulk_bench = self._read_configs("- name: group\n configs: [cfg.variant, cfg2]\n") + self.assertEqual( + bulk_bench.configs["group"]["configs"], + ["cfg.variant", "cfg2"], + ) + + def test_benchmark_config_name_must_exist_in_selected_yaml(self): + self.assertInvalidConfigs( + "- name: group\n configs: [cfg.unknown]\n", + "config 'cfg.unknown' wasn't found", + ) + + def test_arch_filters_configs_and_omits_empty_groups(self): + with patch("bulkbench.bulkbench.LoggingConsole.warning") as warning: + bulk_bench = self._read_configs( + """ +- name: partly_supported + configs: [cfg, cfg.variant, cfg.first, cfg.second] +- name: unsupported + configs: [cfg.variant] +""", + arch="gfx942", + ) + + self.assertEqual( + bulk_bench.configs, + { + "partly_supported": { + "name": "partly_supported", + "configs": ["cfg", "cfg.first", "cfg.second"], + "override_args": None, + "only_in_patches": None, + } + }, + ) + self.assertEqual( + warning.call_args_list, + [ + call( + "Config 'cfg.variant' is not supported for the current architecture " + "(--tag=gfx942 mismatch)" + ), + call( + "Config 'cfg.variant' is not supported for the current architecture " + "(--tag=gfx942 mismatch)" + ), + call( + "Group 'unsupported' was fully omitted; " + "no config matches the architecture (--tag=gfx942 mismatch)" + ), + ], + ) + + def test_invalid_config_prefix_is_rejected(self): + for config_name in (".variant", "has/slash.variant", "has:colon.variant"): + with self.subTest(config_name=config_name): + self.assertInvalidConfigs( + f"- name: group\n configs: [{config_name!r}]\n", + "prefix before the first dot must match", + ) + + def test_benchmark_yaml_must_be_a_file(self): + (self.benchmark_configs_dir / "directory.yaml").mkdir() + for config_name in ("missing.variant", "directory.variant"): + with self.subTest(config_name=config_name): + self.assertInvalidConfigs( + f"- name: group\n configs: [{config_name}]\n", + "doesn't exist or isn't a file", + ) + + def _read_patches(self, contents: str, files: tuple[str, ...] = ()) -> BulkBench: + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + (project_dir / "configs.yaml").write_text( + "- name: configs\n configs: [cfg]\n", encoding="utf-8" + ) + for relative_path in files: + path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + (project_dir / "patches.yaml").write_text( + contents.replace("$PROJECT", str(project_dir)), encoding="utf-8" + ) + with patch.object(BulkBench, "_dryRunPatches"): + return _makeBulkBenchNoReport(project_dir=project_dir, arch="") + + def assertInvalidPatches(self, contents: str, expected_message: str, *files: str) -> None: + with self.assertRaises(ValueError) as context: + self._read_patches(contents, files) + self.assertIn(expected_message, str(context.exception)) + + def test_config_group_and_patch_set_name_validation(self): + valid_name = "aZ09-., ~!()[]_+={}" + bulk_bench = self._read_configs(f"- name: ' {valid_name} '\n configs: [cfg]") + self.assertEqual(list(bulk_bench.configs), [valid_name]) + + bulk_bench = self._read_patches(f"- name: ' {valid_name} '\n patches: []") + self.assertEqual( + [patch_set["name"] for patch_set in bulk_bench.patches], + [valid_name], + ) + + for invalid_name in (".", "..", "has/slash", "has:colon", "has?question", "has\\backslash"): + with self.subTest(invalid_name=invalid_name, object_type="config group"): + self.assertInvalidConfigs( + f"- name: '{invalid_name}'\n configs: [cfg]", + "attribute 'name' must match", + ) + with self.subTest(invalid_name=invalid_name, object_type="patch set"): + self.assertInvalidPatches( + f"- name: '{invalid_name}'\n patches: []", + "attribute 'name' must match", + ) + + def test_patches_file(self): + bulk_bench = self._read_patches( + """ +- name: baseline + patches: [] +- name: " changes " + patches: + - patch: " change.patch " + target: " $PROJECT/target.py " + - enabled: false + unknown: ignored +""", + ("changes/change.patch", "target.py"), + ) + + self.assertEqual( + bulk_bench.patches, + [ + {"name": "baseline", "patches": []}, + { + "name": "changes", + "patches": [ + { + "patch": ( + bulk_bench.project_dir / "changes" / "change.patch" + ).resolve(), + "target": (bulk_bench.project_dir / "target.py").resolve(), + } + ], + }, + ], + ) + + def test_patches_file_schema_errors(self): + cases = ( + ("", "must contain a YAML list"), + ("[]", "must contain at least one patch set"), + ("{}", "must contain a YAML list"), + ("- patches: []", "missing required attribute 'name'"), + ("- name: baseline\n patches: []\n extra: true", "unknown attribute(s): extra"), + ("- name: ' '\n patches: []", "'name' must be a non-empty string"), + ("- name: baseline", "missing required attribute 'patches'"), + ("- name: baseline\n patches: {}", "'patches' must be a list"), + ("- name: baseline\n patches: [value]", "patch 1 must be an object"), + ( + "- name: baseline\n patches:\n - patch: missing.patch", + "missing required attribute 'target'", + ), + ( + ( + "- name: baseline\n patches:\n" + " - patch: missing.patch\n target: missing.py\n extra: true" + ), + "unknown attribute(s): extra", + ), + ( + ( + "- name: baseline\n patches:\n" + " - patch: missing.patch\n target: missing.py\n enabled: 2" + ), + "attribute 'enabled' must be a YAML boolean", + ), + ) + for contents, expected_message in cases: + with self.subTest(expected_message=expected_message): + self.assertInvalidPatches(contents, expected_message) + + def test_patch_and_target_must_be_existing_files(self): + self.assertInvalidPatches( + "- name: changes\n patches:\n" + " - patch: missing.patch\n target: $PROJECT/target.py", + "attribute 'patch' path", + "target.py", + ) + self.assertInvalidPatches( + "- name: changes\n patches:\n" + " - patch: change.patch\n target: $PROJECT/missing.py", + "attribute 'target' path", + "changes/change.patch", + ) + + def test_duplicate_patch_objects_are_rejected(self): + self.assertInvalidPatches( + """ +- name: changes + patches: + - patch: change.patch + target: $PROJECT/target.py + - patch: ./change.patch + target: $PROJECT/./target.py +""", + "contains duplicate patch object", + "changes/change.patch", + "target.py", + ) + + def test_duplicate_patch_sets_are_order_independent(self): + self.assertInvalidPatches( + """ +- name: first + patches: + - patch: $PROJECT/a.patch + target: $PROJECT/a.py + - patch: $PROJECT/b.patch + target: $PROJECT/b.py +- name: second + patches: + - patch: $PROJECT/b.patch + target: $PROJECT/b.py + - patch: $PROJECT/a.patch + target: $PROJECT/a.py +""", + "patch sets 'first' and 'second' contain duplicate patch sets", + "a.patch", + "a.py", + "b.patch", + "b.py", + ) + + def test_patch_object_can_be_shared_by_distinct_patch_sets(self): + bulk_bench = self._read_patches( + """ +- name: first + patches: + - patch: $PROJECT/shared.patch + target: $PROJECT/shared.py + - patch: a.patch + target: $PROJECT/a.py +- name: second + patches: + - patch: $PROJECT/shared.patch + target: $PROJECT/shared.py + - patch: b.patch + target: $PROJECT/b.py +""", + ( + "shared.patch", + "shared.py", + "first/a.patch", + "a.py", + "second/b.patch", + "b.py", + ), + ) + + self.assertEqual( + [patch_set["name"] for patch_set in bulk_bench.patches], + ["first", "second"], + ) + self.assertEqual( + bulk_bench.patches[0]["patches"][0], + bulk_bench.patches[1]["patches"][0], + ) + self.assertEqual( + bulk_bench.patches[0]["patches"][0]["patch"], + (bulk_bench.project_dir / "shared.patch").resolve(), + ) + + def test_only_one_empty_patch_set_is_allowed(self): + self.assertInvalidPatches( + """ +- name: baseline + patches: [] +- name: disabled + patches: + - enabled: false + unknown: ignored +""", + "patch sets 'baseline' and 'disabled' contain duplicate patch sets", + ) + + def test_patch_set_names_are_unique(self): + self.assertInvalidPatches( + "- name: group\n patches: []\n- name: ' group '\n patches: []", + "contains duplicate patch set name 'group'", + ) + + def _make_config_runner_bulk_bench( + self, + project_dir: Path, + configs: str, + *, + arch: str = "gfx942", + regenerate_results: bool = False, + ) -> tuple[BulkBench, Mock]: + (project_dir / "configs.yaml").write_text(configs, encoding="utf-8") + (project_dir / "patches.yaml").write_text( + "- name: baseline\n patches: []\n", + encoding="utf-8", + ) + bulk_bench = _makeBulkBenchNoReport( + project_dir=project_dir, + results_dir=project_dir / "results", + report_dir=project_dir / "report", + backup_dir=project_dir / "backups", + arch=arch, + regenerate_results=regenerate_results, + ) + console = Mock() + bulk_bench.Con = console + return bulk_bench, console + + def test_config_might_have_run_successfully_checks_direct_result_files(self): + with TemporaryDirectory() as workdir_value: + workdir = Path(workdir_value) + + for suffix in (".jpg", ".mp4", ".png"): + config_dir = workdir / f"valid-{suffix[1:]}" + config_dir.mkdir() + (config_dir / "timings.json").touch() + (config_dir / f"result{suffix}").touch() + self.assertTrue(configMightHaveRunSuccessfully(workdir, config_dir.name)) + + uppercase_dir = workdir / "uppercase" + uppercase_dir.mkdir() + (uppercase_dir / "timings.json").touch() + (uppercase_dir / "result.PNG").touch() + self.assertFalse(configMightHaveRunSuccessfully(workdir, "uppercase")) + + nested_dir = workdir / "nested" + (nested_dir / "output").mkdir(parents=True) + (nested_dir / "timings.json").touch() + (nested_dir / "output" / "result.png").touch() + self.assertFalse(configMightHaveRunSuccessfully(workdir, "nested")) + + media_directory = workdir / "media-directory" + media_directory.mkdir() + (media_directory / "timings.json").touch() + (media_directory / "result.jpg").mkdir() + self.assertFalse(configMightHaveRunSuccessfully(workdir, "media-directory")) + + timings_directory = workdir / "timings-directory" + timings_directory.mkdir() + (timings_directory / "timings.json").mkdir() + (timings_directory / "result.mp4").touch() + self.assertFalse(configMightHaveRunSuccessfully(workdir, "timings-directory")) + + self.assertFalse(configMightHaveRunSuccessfully(workdir, "missing")) + + def test_run_config_builds_command_and_logs_output(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + bulk_bench, console = self._make_config_runner_bulk_bench( + project_dir, + ( + "- name: group\n" + " configs: [cfg.first, cfg.second, cfg2]\n" + " override_args:\n" + " iterations: 5\n" + ), + ) + completed = ScriptRunResult( + args=(), + returncode=0, + output="benchmark output\nbenchmark warning", + ) + workdir = project_dir / "results" / "changes" / "group" + + with patch( + "bulkbench.bulkbench.run_with_script", + return_value=completed, + ) as run_process: + cfg = bulk_bench.configs["group"] + bulk_bench._runConfig( + "changes", + workdir, + cfg["name"], + cfg["configs"], + cfg["override_args"], + ) + + self.assertTrue(workdir.is_dir()) + run_process.assert_called_once_with( + [ + "python", + "/app/.ci/run.py", + "--name", + "cfg.first", + "--name", + "cfg.second", + "--name", + "cfg2", + "--override-args-json", + '{"iterations":5}', + "--results-directory", + str(workdir), + str(self.benchmark_configs_dir / "cfg.yaml"), + str(self.benchmark_configs_dir / "cfg2.yaml"), + ], + cwd=Path("/app"), + ) + self.assertFalse( + any( + "benchmark output" in trace_call.args[0] + for trace_call in console.trace.call_args_list + ) + ) + + def test_run_config_skips_completed_configs_by_default(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + bulk_bench, console = self._make_config_runner_bulk_bench( + project_dir, + "- name: group\n configs: [cfg.first, cfg.second, cfg2]\n", + ) + workdir = project_dir / "results" / "changes" / "group" + for config_name, media_name in ( + ("cfg.first", "result.jpg"), + ("cfg2", "result.mp4"), + ): + config_dir = workdir / config_name + config_dir.mkdir(parents=True) + (config_dir / "timings.json").touch() + (config_dir / media_name).touch() + + completed = ScriptRunResult(args=(), returncode=0, output="") + with patch( + "bulkbench.bulkbench.run_with_script", + return_value=completed, + ) as run_process: + bulk_bench._runAllConfigs("changes") + + run_process.assert_called_once_with( + [ + "python", + "/app/.ci/run.py", + "--name", + "cfg.second", + "--results-directory", + str(workdir), + str(self.benchmark_configs_dir / "cfg.yaml"), + ], + cwd=Path("/app"), + ) + console.info.assert_any_call( + "Skipping config 'cfg.first' in config group 'group' for patch set " + "'changes': its existing results might be successful" + ) + console.info.assert_any_call( + "Skipping config 'cfg2' in config group 'group' for patch set " + "'changes': its existing results might be successful" + ) + self.assertTrue( + any( + info_call.args[0].startswith( + "Config 'group' (run configs:cfg.second) succeeded in " + ) + for info_call in console.info.call_args_list + ) + ) + + def test_run_config_regenerate_results_runs_completed_configs(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + bulk_bench, _ = self._make_config_runner_bulk_bench( + project_dir, + "- name: group\n configs: [cfg]\n", + regenerate_results=True, + ) + workdir = project_dir / "results" / "baseline" / "group" + config_dir = workdir / "cfg" + config_dir.mkdir(parents=True) + (config_dir / "timings.json").touch() + (config_dir / "result.png").touch() + bulk_bench._runConfig = Mock() + + bulk_bench._runAllConfigs("baseline") + + bulk_bench._runConfig.assert_called_once_with( + "baseline", + workdir, + "group", + ["cfg"], + None, + ) + + def test_run_config_treats_fully_skipped_group_as_successful(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + bulk_bench, console = self._make_config_runner_bulk_bench( + project_dir, + "- name: group\n configs: [cfg]\n", + ) + config_dir = project_dir / "results" / "baseline" / "group" / "cfg" + config_dir.mkdir(parents=True) + (config_dir / "timings.json").touch() + (config_dir / "result.png").touch() + + with patch("bulkbench.bulkbench.run_with_script") as run_process: + bulk_bench._runAllConfigs("baseline") + + run_process.assert_not_called() + console.info.assert_any_call( + "Skipping config 'cfg' in config group 'group' for patch set " + "'baseline': its existing results might be successful" + ) + console.info.assert_any_call( + "All configs in config group 'group' for patch set 'baseline' might " + "already have run successfully; treating the config group as successful" + ) + + def test_run_config_raises_with_captured_nonzero_result(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + bulk_bench, _ = self._make_config_runner_bulk_bench( + project_dir, + "- name: group\n configs: [cfg]\n", + arch="", + ) + completed = ScriptRunResult( + args=(), + returncode=7, + output="partial output\nrunner failed", + ) + + with ( + patch( + "bulkbench.bulkbench.run_with_script", + return_value=completed, + ) as run_process, + self.assertRaises(GroupRunError) as context, + ): + bulk_bench._runConfig( + "baseline", + project_dir / "results" / "baseline" / "group", + "group", + ["cfg"], + None, + ) + + self.assertEqual( + context.exception.result, + GroupFailureCapture( + group_name="group", + configs_to_run=["cfg"], + output="partial output\nrunner failed", + returncode=7, + ), + ) + command = run_process.call_args.args[0] + self.assertNotIn("--tag", command) + self.assertNotIn("--override-args-json", command) + + def test_run_config_raises_with_process_start_failure(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + bulk_bench, _ = self._make_config_runner_bulk_bench( + project_dir, + "- name: group\n configs: [cfg]\n", + ) + failure = OSError("python isn't executable") + + with ( + patch( + "bulkbench.bulkbench.run_with_script", + side_effect=failure, + ), + self.assertRaises(GroupRunError) as context, + ): + bulk_bench._runConfig( + "baseline", + project_dir / "results" / "baseline" / "group", + "group", + ["cfg"], + None, + ) + + self.assertIs(context.exception.__cause__, failure) + self.assertEqual( + context.exception.result, + GroupFailureCapture( + group_name="group", + configs_to_run=["cfg"], + output="python isn't executable", + returncode=None, + ), + ) + + def test_run_all_configs_preserves_group_order(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, _ = self._make_config_runner_bulk_bench( + Path(project_dir_value), + ("- name: first\n configs: [cfg]\n- name: second\n configs: [cfg2]\n"), + ) + bulk_bench._runConfig = Mock() + bulk_bench.successful_runs = {} + bulk_bench.unsuccessful_runs = {} + + with patch( + "bulkbench.bulkbench.monotonic", + side_effect=(1.0, 2.0, 3.0, 5.0), + ): + bulk_bench._runAllConfigs("changes") + + self.assertEqual( + bulk_bench._runConfig.call_args_list, + [ + call( + "changes", + bulk_bench.results_dir / "changes" / "first", + "first", + ["cfg"], + None, + ), + call( + "changes", + bulk_bench.results_dir / "changes" / "second", + "second", + ["cfg2"], + None, + ), + ], + ) + self.assertEqual( + bulk_bench.successful_runs, + {"changes": [("first", 1.0, ["cfg"]), ("second", 2.0, ["cfg2"])]}, + ) + self.assertEqual( + bulk_bench.unsuccessful_runs, + {"changes": []}, + ) + + def test_run_all_configs_skips_groups_not_enabled_for_patch_set(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, _ = self._make_config_runner_bulk_bench( + Path(project_dir_value), + """ +- name: restricted + configs: [cfg] + only_in_patches: [baseline] +- name: eager + configs: [cfg2] + only_in_patches: [baseline] + eager_in_patches: [baseline] +""", + ) + bulk_bench._runConfig = Mock() + bulk_bench.successful_runs = {} + bulk_bench.unsuccessful_runs = {} + + bulk_bench._runAllConfigs("changes") + + bulk_bench._runConfig.assert_not_called() + self.assertEqual(bulk_bench.successful_runs, {"changes": []}) + self.assertEqual(bulk_bench.unsuccessful_runs, {"changes": []}) + + bulk_bench._runAllConfigs("baseline") + + self.assertEqual( + [config_call.args[2] for config_call in bulk_bench._runConfig.call_args_list], + ["restricted", "eager", "eager_eager"], + ) + + def test_run_all_configs_records_expected_and_unexpected_failures(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + bulk_bench, console = self._make_config_runner_bulk_bench( + project_dir, + ( + "- name: successful\n" + " configs: [cfg]\n" + "- name: process_failure\n" + " configs: [cfg.first, cfg2]\n" + "- name: unexpected_failure\n" + " configs: [cfg3]\n" + ), + ) + completed_config_dir = ( + project_dir / "results" / "changes" / "process_failure" / "cfg.first" + ) + completed_config_dir.mkdir(parents=True) + (completed_config_dir / "timings.json").touch() + (completed_config_dir / "result.png").touch() + process_result = GroupFailureCapture( + group_name="process_failure", + configs_to_run=["cfg2"], + output="partial output\nrunner failed", + returncode=3, + ) + bulk_bench._runConfig = Mock( + side_effect=( + None, + GroupRunError(process_result), + ValueError("invalid runtime state"), + ) + ) + bulk_bench.successful_runs = {} + bulk_bench.unsuccessful_runs = {} + + with patch( + "bulkbench.bulkbench.monotonic", + side_effect=( + 0.0, + 1.25, + 10.0, + 176_533.4, + 176_540.0, + 176_540.1, + ), + ): + bulk_bench._runAllConfigs("changes") + + self.assertEqual( + bulk_bench.successful_runs, + {"changes": [("successful", 1.25, ["cfg"])]}, + ) + failures = bulk_bench.unsuccessful_runs["changes"] + stored_process_result, process_duration = failures[0] + self.assertIs(stored_process_result, process_result) + self.assertAlmostEqual(process_duration, 49 * 60 * 60 + 2 * 60 + 3.4) + unexpected_result, unexpected_duration = failures[1] + self.assertEqual(unexpected_result.group_name, "unexpected_failure") + self.assertEqual(unexpected_result.configs_to_run, ["cfg3"]) + self.assertIsNone(unexpected_result.returncode) + self.assertIn("ValueError: invalid runtime state", unexpected_result.output) + self.assertAlmostEqual(unexpected_duration, 0.1) + console.info.assert_any_call( + "Config 'successful' (run configs:cfg) succeeded in 00:00:01.3" + ) + console.error.assert_any_call( + "Config group 'process_failure' (run configs:cfg2) " + "on patch set 'changes' failed in 49:02:03.4." + ) + console.error.assert_any_call( + "[UNEXPECTED ERROR] Config group 'unexpected_failure' " + "(run configs:cfg3) on patch set 'changes' failed in 00:00:00.1." + ) + self.assertEqual(bulk_bench._runConfig.call_count, 3) + + def test_run_all_configs_does_not_catch_keyboard_interrupt(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, _ = self._make_config_runner_bulk_bench( + Path(project_dir_value), + "- name: group\n configs: [cfg]\n", + ) + bulk_bench._runConfig = Mock(side_effect=KeyboardInterrupt) + bulk_bench.successful_runs = {} + bulk_bench.unsuccessful_runs = {} + + with self.assertRaises(KeyboardInterrupt): + bulk_bench._runAllConfigs("changes") + + self.assertNotIn("changes", bulk_bench.successful_runs) + self.assertNotIn("changes", bulk_bench.unsuccessful_runs) + + def test_run_reinitializes_result_dictionaries(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, _ = self._make_config_runner_bulk_bench( + Path(project_dir_value), + "- name: group\n configs: [cfg]\n", + ) + bulk_bench.successful_runs = {"old": ["group"]} + bulk_bench.unsuccessful_runs = { + "old": [ + ( + GroupFailureCapture( + group_name="group", + configs_to_run=["cfg"], + output="old failure", + returncode=1, + ), + 1.0, + ) + ] + } + + def verify_reset(_patch_set_name): + self.assertEqual(bulk_bench.successful_runs, {}) + self.assertEqual(bulk_bench.unsuccessful_runs, {}) + + bulk_bench._runAllConfigs = Mock(side_effect=verify_reset) + + self.assertEqual(bulk_bench.run(), 0) + bulk_bench._runAllConfigs.assert_called_once_with("baseline") + + def _make_runnable_bulk_bench( + self, + project_dir: Path, + mock_patch_commands: bool = True, + mock_constructor_dry_run: bool = True, + ) -> tuple[BulkBench, tuple[Path, Path]]: + (project_dir / "configs.yaml").write_text( + "- name: configs\n configs: [cfg]\n", encoding="utf-8" + ) + targets = (project_dir / "first.py", project_dir / "second.py") + patch_dir = project_dir / "changes" + patch_dir.mkdir() + for index, target in enumerate(targets): + target.write_text(f"original {index}", encoding="utf-8") + (patch_dir / f"{index}.patch").write_text(f"patch {index}", encoding="utf-8") + (project_dir / "patches.yaml").write_text( + ( + "- name: changes\n" + " patches:\n" + f" - patch: 0.patch\n target: {targets[0]}\n" + f" - patch: 1.patch\n target: {targets[1]}\n" + ), + encoding="utf-8", + ) + if mock_constructor_dry_run: + with patch.object(BulkBench, "_dryRunPatches"): + bulk_bench = _makeBulkBenchNoReport(project_dir=project_dir, arch="") + else: + bulk_bench = _makeBulkBenchNoReport(project_dir=project_dir, arch="") + if mock_patch_commands: + bulk_bench._dryRunPatches = Mock() + bulk_bench._applyPatches = Mock() + return bulk_bench, targets + + def _make_patch_integration_project( + self, project_dir: Path + ) -> tuple[BulkBench, tuple[Path, Path], tuple[str, str], tuple[str, str]]: + (project_dir / "configs.yaml").write_text( + "- name: configs\n configs: [cfg]\n", encoding="utf-8" + ) + + targets = (project_dir / "first.py", project_dir / "second.py") + original_contents = ("alpha\ncommon\n", "one\ntwo\n") + patched_contents = ("patched alpha\ncommon\n", "one\npatched two\n") + for index, (target, original, patched_content) in enumerate( + zip(targets, original_contents, patched_contents, strict=True) + ): + target.write_text(original, encoding="utf-8") + patch_contents = "".join( + difflib.unified_diff( + original.splitlines(keepends=True), + patched_content.splitlines(keepends=True), + fromfile=str(target), + tofile=str(target), + ) + ) + patch_set_names = ("first", "second") if index == 0 else ("second",) + for patch_set_name in patch_set_names: + patch_dir = project_dir / patch_set_name + patch_dir.mkdir(exist_ok=True) + (patch_dir / f"{index}.patch").write_text(patch_contents, encoding="utf-8") + + (project_dir / "patches.yaml").write_text( + ( + "- name: first\n" + " patches:\n" + f" - patch: 0.patch\n target: {targets[0]}\n" + "- name: second\n" + " patches:\n" + f" - patch: 0.patch\n target: {targets[0]}\n" + f" - patch: 1.patch\n target: {targets[1]}\n" + ), + encoding="utf-8", + ) + + bb = _makeBulkBenchNoReport(project_dir=project_dir, arch="") + return (bb, targets, original_contents, patched_contents) + + def _assert_real_patch_lifecycle(self, failure: BaseException | None) -> None: + with TemporaryDirectory() as project_dir_value: + bulk_bench, targets, originals, patched = self._make_patch_integration_project( + Path(project_dir_value) + ) + invocation = 0 + + def run_all_configs(_patch_set_name): + nonlocal invocation + expected = (patched[0], originals[1]) if invocation == 0 else patched + self.assertEqual( + tuple(target.read_text(encoding="utf-8") for target in targets), + expected, + ) + invocation += 1 + if failure is not None and invocation == 2: + raise failure + + bulk_bench._runAllConfigs = Mock(side_effect=run_all_configs) + + if failure is None: + self.assertEqual(bulk_bench.run(), 0) + else: + with self.assertRaises(type(failure)) as context: + bulk_bench.run() + self.assertIs(context.exception, failure) + + self.assertEqual(bulk_bench._runAllConfigs.call_count, 2) + self.assertEqual( + tuple(target.read_text(encoding="utf-8") for target in targets), + originals, + ) + # we must re-create object to ensure we aren't using stale fs handles + backup_dir = Path(str(bulk_bench.backup_dir)) + if failure is None: + self.assertFalse(backup_dir.exists()) + else: + self.assertTrue(backup_dir.exists()) + self.assertEqual(list(backup_dir.iterdir()), []) + + def test_real_patches_are_visible_to_benchmarks_and_always_reverted(self): + for failure in ( + None, + RuntimeError("benchmark failed"), + AssertionError("benchmark assertion failed"), + ): + with self.subTest(failure_type=type(failure).__name__): + self._assert_real_patch_lifecycle(failure) + + def test_backup_dir_must_be_empty(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + (project_dir / "configs.yaml").write_text( + "- name: configs\n configs: [cfg]\n", encoding="utf-8" + ) + (project_dir / "patches.yaml").write_text( + "- name: baseline\n patches: []\n", encoding="utf-8" + ) + backup_dir = project_dir / "backups" + backup_dir.mkdir() + (backup_dir / "existing").touch() + + with self.assertRaisesRegex(ValueError, "--backup_dir directory .* isn't empty"): + _makeBulkBenchNoReport(project_dir=project_dir, backup_dir=backup_dir, arch="") + + def test_backup_dir_must_not_overlap_output_dirs(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + (project_dir / "configs.yaml").write_text( + "- name: configs\n configs: [cfg]\n", encoding="utf-8" + ) + (project_dir / "patches.yaml").write_text( + "- name: baseline\n patches: []\n", encoding="utf-8" + ) + shared_dir = project_dir / "shared" + + with self.assertRaisesRegex(ValueError, "must not overlap --results_dir"): + BulkBench( + project_dir=project_dir, + backup_dir=shared_dir, + results_dir=shared_dir, + arch="", + ) + + def test_constructor_dry_runs_every_loaded_patch(self): + with TemporaryDirectory() as project_dir_value: + commands = [] + + def run_patch(command, **_kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, "", "") + + with patch("bulkbench.bulkbench.subprocess.run", side_effect=run_patch): + bulk_bench, targets = self._make_runnable_bulk_bench( + Path(project_dir_value), + mock_patch_commands=False, + mock_constructor_dry_run=False, + ) + + self.assertEqual( + commands, + [ + [ + "patch", + "--batch", + "--dry-run", + str(targets[0]), + str(bulk_bench.project_dir / "changes" / "0.patch"), + ], + [ + "patch", + "--batch", + "--dry-run", + str(targets[1]), + str(bulk_bench.project_dir / "changes" / "1.patch"), + ], + ], + ) + + def test_constructor_dry_run_failure_prevents_output_directory_creation(self): + with TemporaryDirectory() as project_dir_value: + project_dir = Path(project_dir_value) + patch_error = subprocess.CalledProcessError( + 2, + ["patch"], + output="dry-run stdout", + stderr="dry-run stderr", + ) + + with ( + patch( + "bulkbench.bulkbench.subprocess.run", + side_effect=patch_error, + ), + self.assertRaises(ValueError) as context, + ): + self._make_runnable_bulk_bench( + project_dir, + mock_patch_commands=False, + mock_constructor_dry_run=False, + ) + + self.assertIs(context.exception.__cause__, patch_error) + self.assertFalse((project_dir / "results").exists()) + self.assertFalse((project_dir / "report").exists()) + self.assertFalse((project_dir / "__backups").exists()) + + def test_run_dry_runs_all_patches_before_snapshot_and_applies_in_order(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, targets = self._make_runnable_bulk_bench( + Path(project_dir_value), mock_patch_commands=False + ) + commands = [] + + def run_patch(command, **kwargs): + commands.append(command) + self.assertEqual( + kwargs, + { + "capture_output": True, + "check": True, + "shell": False, + "text": True, + }, + ) + if "--dry-run" in command: + self.assertEqual(list(bulk_bench.backup_dir.iterdir()), []) + else: + self.assertEqual( + {path.name for path in bulk_bench.backup_dir.iterdir()}, + {"00000", "00000.path", "00001", "00001.path"}, + ) + return subprocess.CompletedProcess(command, 0, "", "") + + bulk_bench._runAllConfigs = Mock() + with patch("bulkbench.bulkbench.subprocess.run", side_effect=run_patch): + self.assertEqual(bulk_bench.run(), 0) + + patch_paths = ( + bulk_bench.project_dir / "changes" / "0.patch", + bulk_bench.project_dir / "changes" / "1.patch", + ) + self.assertEqual( + commands, + [ + [ + "patch", + "--batch", + "--dry-run", + str(targets[0]), + str(patch_paths[0]), + ], + [ + "patch", + "--batch", + "--dry-run", + str(targets[1]), + str(patch_paths[1]), + ], + ["patch", "--batch", str(targets[0]), str(patch_paths[0])], + ["patch", "--batch", str(targets[1]), str(patch_paths[1])], + ], + ) + bulk_bench._runAllConfigs.assert_called_once_with("changes") + self.assertFalse(Path(str(bulk_bench.backup_dir)).exists()) + + def test_dry_run_failure_prevents_backups_patching_and_benchmarks(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, targets = self._make_runnable_bulk_bench( + Path(project_dir_value), mock_patch_commands=False + ) + patch_error = subprocess.CalledProcessError( + 2, + ["patch"], + output="dry-run stdout", + stderr="dry-run stderr", + ) + bulk_bench._runAllConfigs = Mock() + + with ( + patch( + "bulkbench.bulkbench.subprocess.run", + side_effect=patch_error, + ), + self.assertRaises(ValueError) as context, + ): + bulk_bench.run() + + message = str(context.exception) + self.assertIn("patch dry-run failed for patch set ''changes''", message) + self.assertIn( + str(bulk_bench.project_dir / "changes" / "0.patch"), + message, + ) + self.assertIn(str(targets[0]), message) + self.assertIn("exit status 2", message) + self.assertIn("dry-run stdout", message) + self.assertIn("dry-run stderr", message) + self.assertIs(context.exception.__cause__, patch_error) + bulk_bench._runAllConfigs.assert_not_called() + self.assertEqual(list(bulk_bench.backup_dir.iterdir()), []) + + def test_run_snapshots_targets_by_patch_index_and_restores_them(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, targets = self._make_runnable_bulk_bench(Path(project_dir_value)) + original_contents = [target.read_text(encoding="utf-8") for target in targets] + + def run_all_configs(_patch_set_name): + self.assertEqual( + {path.name for path in bulk_bench.backup_dir.iterdir()}, + {"00000", "00000.path", "00001", "00001.path"}, + ) + for index, target in enumerate(targets): + self.assertEqual( + (bulk_bench.backup_dir / f"{index:05d}.path").read_text(encoding="utf-8"), + str(target.resolve()), + ) + target.write_text(f"modified {index}", encoding="utf-8") + return 37 + + bulk_bench._runAllConfigs = Mock(side_effect=run_all_configs) + + self.assertEqual(bulk_bench.run(), 0) + self.assertEqual(bulk_bench._runAllConfigs.call_count, 1) + self.assertEqual( + [target.read_text(encoding="utf-8") for target in targets], + original_contents, + ) + self.assertFalse(Path(str(bulk_bench.backup_dir)).exists()) + + def test_run_restores_targets_when_run_all_configs_raises(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, targets = self._make_runnable_bulk_bench(Path(project_dir_value)) + original_contents = [target.read_text(encoding="utf-8") for target in targets] + primary_error = RuntimeError("run failed") + + def run_all_configs(_patch_set_name): + targets[0].write_text("modified", encoding="utf-8") + raise primary_error + + bulk_bench._runAllConfigs = Mock(side_effect=run_all_configs) + + with self.assertRaises(RuntimeError) as context: + bulk_bench.run() + self.assertIs(context.exception, primary_error) + self.assertEqual( + [target.read_text(encoding="utf-8") for target in targets], + original_contents, + ) + self.assertEqual(list(bulk_bench.backup_dir.iterdir()), []) + + def test_run_restores_targets_when_patch_application_raises(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, targets = self._make_runnable_bulk_bench(Path(project_dir_value)) + original_contents = [target.read_text(encoding="utf-8") for target in targets] + primary_error = RuntimeError("apply failed") + + def apply_patches(_patch_set): + targets[0].write_text("partially patched", encoding="utf-8") + raise primary_error + + bulk_bench._applyPatches = Mock(side_effect=apply_patches) + bulk_bench._runAllConfigs = Mock() + + with self.assertRaises(RuntimeError) as context: + bulk_bench.run() + self.assertIs(context.exception, primary_error) + bulk_bench._runAllConfigs.assert_not_called() + self.assertEqual( + [target.read_text(encoding="utf-8") for target in targets], + original_contents, + ) + self.assertEqual(list(bulk_bench.backup_dir.iterdir()), []) + + def test_run_groups_primary_and_restoration_failures(self): + with TemporaryDirectory() as project_dir_value: + bulk_bench, targets = self._make_runnable_bulk_bench(Path(project_dir_value)) + primary_error = RuntimeError("run failed") + + def run_all_configs(_patch_set_name): + targets[0].write_text("modified", encoding="utf-8") + raise primary_error + + real_copy2 = shutil.copy2 + copy_count = 0 + + def fail_restoration(source, destination): + nonlocal copy_count + copy_count += 1 + if copy_count <= len(targets): + return real_copy2(source, destination) + raise OSError(f"can't restore {destination}") + + bulk_bench._runAllConfigs = Mock(side_effect=run_all_configs) + with ( + patch( + "bulkbench.bulkbench.shutil.copy2", + side_effect=fail_restoration, + ), + self.assertRaises(BaseExceptionGroup) as context, + ): + bulk_bench.run() + + self.assertIs(context.exception.exceptions[0], primary_error) + self.assertEqual( + {path.name for path in bulk_bench.backup_dir.iterdir()}, + {"00000", "00000.path", "00001", "00001.path"}, + ) + + +if __name__ == "__main__": + sys.exit(pytest.main(sys.argv))