add bulkbench tool

bulkbench/example/03_pyt_IK_reorder/base_model.patch

@@ -0,0 +1,30 @@
+diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py
+index 9eb76e3..410ae87 100644
+--- a/xfuser/model_executor/models/runner_models/base_model.py
++++ b/xfuser/model_executor/models/runner_models/base_model.py
+@@ -1,5 +1,6 @@
+ import abc
+ import torch
++import torch._inductor.config_comms
+ import copy
+ import argparse
+ import json
+@@ -470,6 +471,18 @@ class xFuserModel(abc.ABC):
+
+         torch._inductor.config.reorder_for_compute_comm_overlap = True
+
++        torch._inductor.config.runtime_estimations_mms_benchmark  = True
++        torch._inductor.config.reorder_for_compute_comm_overlap_passes = [
++            "reorder_communication_preserving_peak_memory",
++            "sink_waits_iterative",
++            "reorder_communication_preserving_peak_memory",
++        ]
++        torch._inductor.config_comms.reorder_iterative_peak_memory_budget = 0.2      # 0.2 is default?, if you have more spare memory - you can increase it
++        torch._inductor.config_comms.sink_iterative_peak_memory_budget    = 0.2     # 0.2 is default?, increase if spare memory
++        torch._inductor.config_comms.reorder_iterative_use_runtime_estimations = True   # default False
++        torch._inductor.config_comms.sink_iterative_use_runtime_estimations    = True   # default False
++        torch._inductor.config_comms.runtime_estimations_align_across_all_distributed_ranks = True
++
+         # torch >= ~2.13: enabling the overlap machinery activates an SPMD
+         # graph-consistency check that issues a WORLD-group all_gather_object at
+         # compile time. Pipeline parallelism is non-SPMD (stages compile different

bulkbench/example/04_pyt_IK_bucket/base_model.patch

@@ -0,0 +1,24 @@
+diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py
+index 9eb76e3..e9216b8 100644
+--- a/xfuser/model_executor/models/runner_models/base_model.py
++++ b/xfuser/model_executor/models/runner_models/base_model.py
+@@ -1,5 +1,6 @@
+ import abc
+ import torch
++import torch._inductor.config as ic
+ import copy
+ import argparse
+ import json
+@@ -470,6 +471,12 @@ class xFuserModel(abc.ABC):
+
+         torch._inductor.config.reorder_for_compute_comm_overlap = True
+
++        torch._inductor.config.bucket_all_gathers_fx = "all"
++        torch._inductor.config.bucket_all_gathers_fx_bucket_size_determinator = lambda bucket_id: 100.0 if bucket_id < 2 else 1000.0
++
++        torch._inductor.config.bucket_reduce_scatters_fx = "all"
++        torch._inductor.config.bucket_reduce_scatters_fx_bucket_size_determinator = lambda bucket_id: 1000.0
++
+         # torch >= ~2.13: enabling the overlap machinery activates an SPMD
+         # graph-consistency check that issues a WORLD-group all_gather_object at
+         # compile time. Pipeline parallelism is non-SPMD (stages compile different

bulkbench/example/05_pyt_IK_reorderbucket/base_model.patch

@@ -0,0 +1,37 @@
+diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py
+index 9eb76e3..c605873 100644
+--- a/xfuser/model_executor/models/runner_models/base_model.py
++++ b/xfuser/model_executor/models/runner_models/base_model.py
+@@ -1,5 +1,7 @@
+ import abc
+ import torch
++import torch._inductor.config_comms
++import torch._inductor.config as ic
+ import copy
+ import argparse
+ import json
+@@ -470,6 +472,24 @@ class xFuserModel(abc.ABC):
+
+         torch._inductor.config.reorder_for_compute_comm_overlap = True
+
++        torch._inductor.config.runtime_estimations_mms_benchmark  = True
++        torch._inductor.config.reorder_for_compute_comm_overlap_passes = [
++            "reorder_communication_preserving_peak_memory",
++            "sink_waits_iterative",
++            "reorder_communication_preserving_peak_memory",
++        ]
++        torch._inductor.config_comms.reorder_iterative_peak_memory_budget = 0.2      # 0.2 is default?, if you have more spare memory - you can increase it
++        torch._inductor.config_comms.sink_iterative_peak_memory_budget    = 0.2     # 0.2 is default?, increase if spare memory
++        torch._inductor.config_comms.reorder_iterative_use_runtime_estimations = True   # default False
++        torch._inductor.config_comms.sink_iterative_use_runtime_estimations    = True   # default False
++        torch._inductor.config_comms.runtime_estimations_align_across_all_distributed_ranks = True
++
++        torch._inductor.config.bucket_all_gathers_fx = "all"
++        torch._inductor.config.bucket_all_gathers_fx_bucket_size_determinator = lambda bucket_id: 100.0 if bucket_id < 2 else 1000.0
++
++        torch._inductor.config.bucket_reduce_scatters_fx = "all"
++        torch._inductor.config.bucket_reduce_scatters_fx_bucket_size_determinator = lambda bucket_id: 1000.0
++
+         # torch >= ~2.13: enabling the overlap machinery activates an SPMD
+         # graph-consistency check that issues a WORLD-group all_gather_object at
+         # compile time. Pipeline parallelism is non-SPMD (stages compile different

bulkbench/example/11_PR_only_meta/base_model.patch

@@ -0,0 +1,15 @@
+diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py
+index 4a1c68a..20be7d5 100644
+--- a/xfuser/model_executor/models/runner_models/base_model.py
++++ b/xfuser/model_executor/models/runner_models/base_model.py
+@@ -468,6 +468,10 @@ class xFuserModel(abc.ABC):
+
+         torch._inductor.config.reorder_for_compute_comm_overlap = True
+
++        aten_opts = getattr(torch._inductor.config, "aten_distributed_optimizations", None)
++        assert aten_opts is not None
++        aten_opts.insert_overlap_deps_impl = "meta"
++
+         # torch >= ~2.13: enabling the overlap machinery activates an SPMD
+         # graph-consistency check that issues a WORLD-group all_gather_object at
+         # compile time. Pipeline parallelism is non-SPMD (stages compile different

bulkbench/example/12_PR_new_flags_no_meta/base_model.patch

@@ -0,0 +1,18 @@
+diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py
+index 4a1c68a..2bfb6b9 100644
+--- a/xfuser/model_executor/models/runner_models/base_model.py
++++ b/xfuser/model_executor/models/runner_models/base_model.py
+@@ -468,6 +468,13 @@ class xFuserModel(abc.ABC):
+
+         torch._inductor.config.reorder_for_compute_comm_overlap = True
+
++        aten_opts = getattr(torch._inductor.config, "aten_distributed_optimizations", None)
++        assert aten_opts is not None
++        aten_opts.enable_overlap_scheduling = True
++        aten_opts.collective_bucketing = True
++        aten_opts.insert_overlap_deps = True
++        aten_opts.collective_estimator = "benchmark"
++
+         # torch >= ~2.13: enabling the overlap machinery activates an SPMD
+         # graph-consistency check that issues a WORLD-group all_gather_object at
+         # compile time. Pipeline parallelism is non-SPMD (stages compile different

bulkbench/example/13_PR_new_flags_with_meta/base_model.patch

@@ -0,0 +1,19 @@
+diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py
+index 4a1c68a..dd9eb05 100644
+--- a/xfuser/model_executor/models/runner_models/base_model.py
++++ b/xfuser/model_executor/models/runner_models/base_model.py
+@@ -468,6 +468,14 @@ class xFuserModel(abc.ABC):
+
+         torch._inductor.config.reorder_for_compute_comm_overlap = True
+
++        aten_opts = getattr(torch._inductor.config, "aten_distributed_optimizations", None)
++        assert aten_opts is not None
++        aten_opts.enable_overlap_scheduling = True
++        aten_opts.collective_bucketing = True
++        aten_opts.insert_overlap_deps = True
++        aten_opts.collective_estimator = "benchmark"
++        aten_opts.insert_overlap_deps_impl = "meta"
++
+         # torch >= ~2.13: enabling the overlap machinery activates an SPMD
+         # graph-consistency check that issues a WORLD-group all_gather_object at
+         # compile time. Pipeline parallelism is non-SPMD (stages compile different

bulkbench/example/14_PR_legacy_flags/base_model.patch

@@ -0,0 +1,17 @@
+diff --git a/xfuser/model_executor/models/runner_models/base_model.py b/xfuser/model_executor/models/runner_models/base_model.py
+index 4a1c68a..00e2355 100644
+--- a/xfuser/model_executor/models/runner_models/base_model.py
++++ b/xfuser/model_executor/models/runner_models/base_model.py
+@@ -468,6 +468,12 @@ class xFuserModel(abc.ABC):
+
+         torch._inductor.config.reorder_for_compute_comm_overlap = True
+
++        torch._inductor.config.reorder_for_compute_comm_overlap_passes = [
++            "reorder_compute_for_overlap",
++            "sink_waits",
++            "raise_comms",
++        ]
++
+         # torch >= ~2.13: enabling the overlap machinery activates an SPMD
+         # graph-consistency check that issues a WORLD-group all_gather_object at
+         # compile time. Pipeline parallelism is non-SPMD (stages compile different

bulkbench/example/README.md

@@ -0,0 +1,33 @@
+# Example `bulkbench` project
+
+This directory provides an example of how a `bulkbench` project could look like and is based on a
+real-world investigation on how a certain PyTorch pull request (PR) behaves under different set of
+PyTorch flags compared to raw PyTorch, on `gfx942` and `gfx950` GPUs.
+
+The project has [`configs.yaml`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/configs.yaml)
+defining 3 model configurations in 2 groups (each group could have own `override_args`):
+
+- `flux.usp`
+- `flux2.quantgemm`
+- `wan2_2.quantgemm_fp8attn`
+
+Raw PyTorch instance and the instance with the applied PR expected to live in separate
+docker containers with the project directory mounted into each of them. Depending on a
+container (base code instance), one could select which patch sets to apply with `--patches_file` argument.
+
+Two patch files are defined: [`patches_PR.yaml`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/patches_PR.yaml)
+with 5 patch variants for the PR instance, and [`patches_py213.yaml`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/patches_py213.yaml)
+with 6 patch variants for the raw PyTorch instance.
+
+[`report`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report) directory shows two reports
+made from obtained results:
+
+- [`benchstats-fix-groups.html`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/benchstats-fix-groups.html) was
+generated automatically by `bulkbench` and shows relative performance of models for `gfx950`,
+- [`run-to-run.html`](https://github.com/AMD-AGI/diffusion-models-inference/tree/main/bulkbench/example/report/run-to-run.html) was generated manually
+on results from 2 runs of the same project on the machine to estimate machine's noise level with command:
+
+```
+benchstats . --files_parser=bulkbench.parser_JSON --sample_stats 0 100 --always_show_pvalues \
+    --filter1=1,2 --export_to=./run-to-run.html
+```

bulkbench/example/configs.yaml

@@ -0,0 +1,18 @@
+- name: a
+  configs:
+    - flux.usp
+    # - flux2.default
+    - flux2.quantgemm.gfx942
+    - flux2.quantgemm.gfx950
+  eager_in_patches: [00_pyt_baseline, 10_PR_baseline, 11_PR_only_meta]
+  override_args:
+    num_iterations: 30
+
+- name: b
+  # enabled: false
+  configs:
+    - wan2_2.quantgemm_fp8attn.gfx950
+    - wan2_2.quantgemm_fp8attn.gfx942
+  eager_in_patches: [00_pyt_baseline, 10_PR_baseline, 11_PR_only_meta]
+  override_args:
+    num_iterations: 28

bulkbench/example/patches_PR.yaml

@@ -0,0 +1,26 @@
+# assumes run in a container with changeset #188404 applied on top of pytorch=2.13
+# run with `bulkbench --patches_file ./patches_PR.yaml`
+
+- name: 10_PR_baseline
+  patches: []
+
+- name: 11_PR_only_meta
+  patches:
+    # relative path names are resolved using the patch set name, `11_PR_only_meta` in this case
+    - patch: "base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"
+
+- name: 12_PR_new_flags_no_meta
+  patches:
+    - patch: "base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"
+
+- name: 13_PR_new_flags_with_meta
+  patches:
+    - patch: "base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"
+
+- name: 14_PR_legacy_flags
+  patches:
+    - patch: "base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"

bulkbench/example/patches_pyt213.yaml

@@ -0,0 +1,31 @@
+# assumes run in a container with a clean pytorch=2.13
+# run with `bulkbench --patches_file ./patches_pyt213.yaml`
+
+- name: 00_pyt_baseline
+  patches: []
+
+- name: 01_pyt_new_flags
+  patches:
+    # relative path names are resolved using the patch set name, `01_pyt_new_flags` in this case
+    - patch: "../12_PR_new_flags_no_meta/base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"
+
+- name: 02_pyt_legacy_flags
+  patches:
+    - patch: "../14_PR_legacy_flags/base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"
+
+- name: 03_pyt_IK_reorder
+  patches:
+    - patch: "base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"
+
+- name: 04_pyt_IK_bucket
+  patches:
+    - patch: "base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"
+
+- name: 05_pyt_IK_reorderbucket
+  patches:
+    - patch: "base_model.patch"
+      target: "/app/xDiT/xfuser/model_executor/models/runner_models/base_model.py"