From b40bb6d1a070c63019c495da23fb03349d036e59 Mon Sep 17 00:00:00 2001 From: shsaihdsaiudh <196440533+shsaihdsaiudh@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:14:41 +0800 Subject: [PATCH] feat(issue10): topology-aware Ring/Tree/NVLS algorithm auto-selector --- src/code/issue10/README-2026.md | 28 ++ .../mixed_workload_evaluation.json | 152 ++++++++++ .../issue10/results-2026/selection_table.json | 161 ++++++++++ .../results-2026/synthetic_calibration.csv | 190 ++++++++++++ src/code/issue10/selector_2026.py | 278 ++++++++++++++++++ src/code/issue10/test_selector_2026.py | 25 ++ 6 files changed, 834 insertions(+) create mode 100644 src/code/issue10/README-2026.md create mode 100644 src/code/issue10/results-2026/mixed_workload_evaluation.json create mode 100644 src/code/issue10/results-2026/selection_table.json create mode 100644 src/code/issue10/results-2026/synthetic_calibration.csv create mode 100644 src/code/issue10/selector_2026.py create mode 100644 src/code/issue10/test_selector_2026.py diff --git a/src/code/issue10/README-2026.md b/src/code/issue10/README-2026.md new file mode 100644 index 0000000..575b795 --- /dev/null +++ b/src/code/issue10/README-2026.md @@ -0,0 +1,28 @@ +# Issue 10 单机可完成部分：离线标定与在线选择器 + +`selector_2026.py` 读取如下 CSV： + +```text +collective,topology,world_size,message_bytes,algorithm,protocol,latency_us +``` + +它会在每个 `collective × topology × world_size × message size` 点选择最快的 +算法/协议，把连续相同选择压缩为消息区间，并生成可在线查询的 JSON 选择表。 + +本机先用明确标记的合成数据验证 Tree（小消息）、Ring（中消息）、NVLS（大消息） +三段切换和混合负载统计： + +```bash +python -m pytest src/code/issue10/test_selector_2026.py +python src/code/issue10/selector_2026.py --synthetic-demo +``` + +真实验收时，把 `nccl-tests` 在目标 NVSwitch 拓扑上采集的数据整理成上述 CSV，再运行： + +```bash +python src/code/issue10/selector_2026.py --calibration path/to/nccl_calibration.csv +``` + +合成结果只验证选择器逻辑，不构成性能结论。最终报告必须使用独立于标定集的混合消息 +trace 做验证，避免用训练数据自证收益。 + diff --git a/src/code/issue10/results-2026/mixed_workload_evaluation.json b/src/code/issue10/results-2026/mixed_workload_evaluation.json new file mode 100644 index 0000000..9cfe77e --- /dev/null +++ b/src/code/issue10/results-2026/mixed_workload_evaluation.json @@ -0,0 +1,152 @@ +[ + { + "collective": "all_gather", + "topology": "synthetic-nvswitch-8gpu", + "world_size": 8, + "message_sizes": [ + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072, + 262144, + 524288, + 1048576, + 2097152, + 4194304, + 8388608, + 16777216, + 33554432, + 67108864, + 134217728, + 268435456, + 536870912, + 1073741824 + ], + "dynamic_total_us": 6930.745453714286, + "fixed_choices": [ + { + "algorithm": "NVLS", + "protocol": "Simple", + "total_us": 7165.213380266667, + "dynamic_improvement": 0.03272309059184708 + }, + { + "algorithm": "Ring", + "protocol": "Simple", + "total_us": 14343.868672000002, + "dynamic_improvement": 0.5168147720674917 + }, + { + "algorithm": "Tree", + "protocol": "LL", + "total_us": 98861.48070400002, + "dynamic_improvement": 0.9298943794452609 + } + ], + "warning": "in-sample selector self-check; collect a separate validation trace for acceptance" + }, + { + "collective": "all_reduce", + "topology": "synthetic-nvswitch-8gpu", + "world_size": 8, + "message_sizes": [ + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072, + 262144, + 524288, + 1048576, + 2097152, + 4194304, + 8388608, + 16777216, + 33554432, + 67108864, + 134217728, + 268435456, + 536870912, + 1073741824 + ], + "dynamic_total_us": 7533.418971428571, + "fixed_choices": [ + { + "algorithm": "NVLS", + "protocol": "Simple", + "total_us": 7788.275413333334, + "dynamic_improvement": 0.03272309059184719 + }, + { + "algorithm": "Ring", + "protocol": "Simple", + "total_us": 15591.161600000001, + "dynamic_improvement": 0.5168147720674917 + }, + { + "algorithm": "Tree", + "protocol": "LL", + "total_us": 107458.1312, + "dynamic_improvement": 0.9298943794452609 + } + ], + "warning": "in-sample selector self-check; collect a separate validation trace for acceptance" + }, + { + "collective": "reduce_scatter", + "topology": "synthetic-nvswitch-8gpu", + "world_size": 8, + "message_sizes": [ + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072, + 262144, + 524288, + 1048576, + 2097152, + 4194304, + 8388608, + 16777216, + 33554432, + 67108864, + 134217728, + 268435456, + 536870912, + 1073741824 + ], + "dynamic_total_us": 7156.748022857142, + "fixed_choices": [ + { + "algorithm": "NVLS", + "protocol": "Simple", + "total_us": 7398.861642666667, + "dynamic_improvement": 0.03272309059184719 + }, + { + "algorithm": "Ring", + "protocol": "Simple", + "total_us": 14811.60352, + "dynamic_improvement": 0.5168147720674917 + }, + { + "algorithm": "Tree", + "protocol": "LL", + "total_us": 102085.22464, + "dynamic_improvement": 0.9298943794452609 + } + ], + "warning": "in-sample selector self-check; collect a separate validation trace for acceptance" + } +] diff --git a/src/code/issue10/results-2026/selection_table.json b/src/code/issue10/results-2026/selection_table.json new file mode 100644 index 0000000..2cc571f --- /dev/null +++ b/src/code/issue10/results-2026/selection_table.json @@ -0,0 +1,161 @@ +{ + "schema_version": 1, + "tables": [ + { + "collective": "all_gather", + "topology": "synthetic-nvswitch-8gpu", + "world_size": 8, + "intervals": [ + { + "min_message_bytes": 0, + "max_message_bytes": 185363, + "algorithm": "Tree", + "protocol": "LL", + "calibrated_sizes": [ + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072 + ] + }, + { + "min_message_bytes": 185364, + "max_message_bytes": 5931641, + "algorithm": "Ring", + "protocol": "Simple", + "calibrated_sizes": [ + 262144, + 524288, + 1048576, + 2097152, + 4194304 + ] + }, + { + "min_message_bytes": 5931642, + "max_message_bytes": null, + "algorithm": "NVLS", + "protocol": "Simple", + "calibrated_sizes": [ + 8388608, + 16777216, + 33554432, + 67108864, + 134217728, + 268435456, + 536870912, + 1073741824 + ] + } + ] + }, + { + "collective": "all_reduce", + "topology": "synthetic-nvswitch-8gpu", + "world_size": 8, + "intervals": [ + { + "min_message_bytes": 0, + "max_message_bytes": 185363, + "algorithm": "Tree", + "protocol": "LL", + "calibrated_sizes": [ + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072 + ] + }, + { + "min_message_bytes": 185364, + "max_message_bytes": 5931641, + "algorithm": "Ring", + "protocol": "Simple", + "calibrated_sizes": [ + 262144, + 524288, + 1048576, + 2097152, + 4194304 + ] + }, + { + "min_message_bytes": 5931642, + "max_message_bytes": null, + "algorithm": "NVLS", + "protocol": "Simple", + "calibrated_sizes": [ + 8388608, + 16777216, + 33554432, + 67108864, + 134217728, + 268435456, + 536870912, + 1073741824 + ] + } + ] + }, + { + "collective": "reduce_scatter", + "topology": "synthetic-nvswitch-8gpu", + "world_size": 8, + "intervals": [ + { + "min_message_bytes": 0, + "max_message_bytes": 185363, + "algorithm": "Tree", + "protocol": "LL", + "calibrated_sizes": [ + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072 + ] + }, + { + "min_message_bytes": 185364, + "max_message_bytes": 5931641, + "algorithm": "Ring", + "protocol": "Simple", + "calibrated_sizes": [ + 262144, + 524288, + 1048576, + 2097152, + 4194304 + ] + }, + { + "min_message_bytes": 5931642, + "max_message_bytes": null, + "algorithm": "NVLS", + "protocol": "Simple", + "calibrated_sizes": [ + 8388608, + 16777216, + 33554432, + 67108864, + 134217728, + 268435456, + 536870912, + 1073741824 + ] + } + ] + } + ] +} diff --git a/src/code/issue10/results-2026/synthetic_calibration.csv b/src/code/issue10/results-2026/synthetic_calibration.csv new file mode 100644 index 0000000..3b7f2f3 --- /dev/null +++ b/src/code/issue10/results-2026/synthetic_calibration.csv @@ -0,0 +1,190 @@ +collective,topology,world_size,message_bytes,algorithm,protocol,latency_us +all_reduce,synthetic-nvswitch-8gpu,8,1024,Tree,LL,4.0512 +all_reduce,synthetic-nvswitch-8gpu,8,1024,Ring,Simple,12.007314285714285 +all_reduce,synthetic-nvswitch-8gpu,8,1024,NVLS,Simple,30.003413333333334 +all_reduce,synthetic-nvswitch-8gpu,8,2048,Tree,LL,4.1024 +all_reduce,synthetic-nvswitch-8gpu,8,2048,Ring,Simple,12.014628571428572 +all_reduce,synthetic-nvswitch-8gpu,8,2048,NVLS,Simple,30.006826666666665 +all_reduce,synthetic-nvswitch-8gpu,8,4096,Tree,LL,4.2048 +all_reduce,synthetic-nvswitch-8gpu,8,4096,Ring,Simple,12.029257142857142 +all_reduce,synthetic-nvswitch-8gpu,8,4096,NVLS,Simple,30.013653333333334 +all_reduce,synthetic-nvswitch-8gpu,8,8192,Tree,LL,4.4096 +all_reduce,synthetic-nvswitch-8gpu,8,8192,Ring,Simple,12.058514285714285 +all_reduce,synthetic-nvswitch-8gpu,8,8192,NVLS,Simple,30.027306666666668 +all_reduce,synthetic-nvswitch-8gpu,8,16384,Tree,LL,4.8192 +all_reduce,synthetic-nvswitch-8gpu,8,16384,Ring,Simple,12.117028571428571 +all_reduce,synthetic-nvswitch-8gpu,8,16384,NVLS,Simple,30.054613333333332 +all_reduce,synthetic-nvswitch-8gpu,8,32768,Tree,LL,5.6384 +all_reduce,synthetic-nvswitch-8gpu,8,32768,Ring,Simple,12.234057142857143 +all_reduce,synthetic-nvswitch-8gpu,8,32768,NVLS,Simple,30.109226666666668 +all_reduce,synthetic-nvswitch-8gpu,8,65536,Tree,LL,7.2768 +all_reduce,synthetic-nvswitch-8gpu,8,65536,Ring,Simple,12.468114285714286 +all_reduce,synthetic-nvswitch-8gpu,8,65536,NVLS,Simple,30.218453333333333 +all_reduce,synthetic-nvswitch-8gpu,8,131072,Tree,LL,10.5536 +all_reduce,synthetic-nvswitch-8gpu,8,131072,Ring,Simple,12.936228571428572 +all_reduce,synthetic-nvswitch-8gpu,8,131072,NVLS,Simple,30.436906666666665 +all_reduce,synthetic-nvswitch-8gpu,8,262144,Tree,LL,17.1072 +all_reduce,synthetic-nvswitch-8gpu,8,262144,Ring,Simple,13.872457142857144 +all_reduce,synthetic-nvswitch-8gpu,8,262144,NVLS,Simple,30.873813333333334 +all_reduce,synthetic-nvswitch-8gpu,8,524288,Tree,LL,30.2144 +all_reduce,synthetic-nvswitch-8gpu,8,524288,Ring,Simple,15.744914285714286 +all_reduce,synthetic-nvswitch-8gpu,8,524288,NVLS,Simple,31.747626666666665 +all_reduce,synthetic-nvswitch-8gpu,8,1048576,Tree,LL,56.4288 +all_reduce,synthetic-nvswitch-8gpu,8,1048576,Ring,Simple,19.48982857142857 +all_reduce,synthetic-nvswitch-8gpu,8,1048576,NVLS,Simple,33.49525333333333 +all_reduce,synthetic-nvswitch-8gpu,8,2097152,Tree,LL,108.8576 +all_reduce,synthetic-nvswitch-8gpu,8,2097152,Ring,Simple,26.979657142857143 +all_reduce,synthetic-nvswitch-8gpu,8,2097152,NVLS,Simple,36.99050666666667 +all_reduce,synthetic-nvswitch-8gpu,8,4194304,Tree,LL,213.7152 +all_reduce,synthetic-nvswitch-8gpu,8,4194304,Ring,Simple,41.959314285714285 +all_reduce,synthetic-nvswitch-8gpu,8,4194304,NVLS,Simple,43.98101333333334 +all_reduce,synthetic-nvswitch-8gpu,8,8388608,Tree,LL,423.4304 +all_reduce,synthetic-nvswitch-8gpu,8,8388608,Ring,Simple,71.91862857142857 +all_reduce,synthetic-nvswitch-8gpu,8,8388608,NVLS,Simple,57.96202666666667 +all_reduce,synthetic-nvswitch-8gpu,8,16777216,Tree,LL,842.8608 +all_reduce,synthetic-nvswitch-8gpu,8,16777216,Ring,Simple,131.83725714285714 +all_reduce,synthetic-nvswitch-8gpu,8,16777216,NVLS,Simple,85.92405333333333 +all_reduce,synthetic-nvswitch-8gpu,8,33554432,Tree,LL,1681.7216 +all_reduce,synthetic-nvswitch-8gpu,8,33554432,Ring,Simple,251.6745142857143 +all_reduce,synthetic-nvswitch-8gpu,8,33554432,NVLS,Simple,141.84810666666667 +all_reduce,synthetic-nvswitch-8gpu,8,67108864,Tree,LL,3359.4432 +all_reduce,synthetic-nvswitch-8gpu,8,67108864,Ring,Simple,491.3490285714286 +all_reduce,synthetic-nvswitch-8gpu,8,67108864,NVLS,Simple,253.69621333333333 +all_reduce,synthetic-nvswitch-8gpu,8,134217728,Tree,LL,6714.8864 +all_reduce,synthetic-nvswitch-8gpu,8,134217728,Ring,Simple,970.6980571428572 +all_reduce,synthetic-nvswitch-8gpu,8,134217728,NVLS,Simple,477.39242666666667 +all_reduce,synthetic-nvswitch-8gpu,8,268435456,Tree,LL,13425.7728 +all_reduce,synthetic-nvswitch-8gpu,8,268435456,Ring,Simple,1929.3961142857145 +all_reduce,synthetic-nvswitch-8gpu,8,268435456,NVLS,Simple,924.7848533333333 +all_reduce,synthetic-nvswitch-8gpu,8,536870912,Tree,LL,26847.5456 +all_reduce,synthetic-nvswitch-8gpu,8,536870912,Ring,Simple,3846.792228571429 +all_reduce,synthetic-nvswitch-8gpu,8,536870912,NVLS,Simple,1819.5697066666667 +all_reduce,synthetic-nvswitch-8gpu,8,1073741824,Tree,LL,53691.0912 +all_reduce,synthetic-nvswitch-8gpu,8,1073741824,Ring,Simple,7681.584457142858 +all_reduce,synthetic-nvswitch-8gpu,8,1073741824,NVLS,Simple,3609.1394133333333 +all_gather,synthetic-nvswitch-8gpu,8,1024,Tree,LL,3.7271039999999998 +all_gather,synthetic-nvswitch-8gpu,8,1024,Ring,Simple,11.046729142857142 +all_gather,synthetic-nvswitch-8gpu,8,1024,NVLS,Simple,27.603140266666667 +all_gather,synthetic-nvswitch-8gpu,8,2048,Tree,LL,3.7742080000000002 +all_gather,synthetic-nvswitch-8gpu,8,2048,Ring,Simple,11.053458285714287 +all_gather,synthetic-nvswitch-8gpu,8,2048,NVLS,Simple,27.606280533333333 +all_gather,synthetic-nvswitch-8gpu,8,4096,Tree,LL,3.868416 +all_gather,synthetic-nvswitch-8gpu,8,4096,Ring,Simple,11.066916571428571 +all_gather,synthetic-nvswitch-8gpu,8,4096,NVLS,Simple,27.612561066666668 +all_gather,synthetic-nvswitch-8gpu,8,8192,Tree,LL,4.056832 +all_gather,synthetic-nvswitch-8gpu,8,8192,Ring,Simple,11.093833142857143 +all_gather,synthetic-nvswitch-8gpu,8,8192,NVLS,Simple,27.625122133333335 +all_gather,synthetic-nvswitch-8gpu,8,16384,Tree,LL,4.433664 +all_gather,synthetic-nvswitch-8gpu,8,16384,Ring,Simple,11.147666285714287 +all_gather,synthetic-nvswitch-8gpu,8,16384,NVLS,Simple,27.650244266666668 +all_gather,synthetic-nvswitch-8gpu,8,32768,Tree,LL,5.187328 +all_gather,synthetic-nvswitch-8gpu,8,32768,Ring,Simple,11.255332571428571 +all_gather,synthetic-nvswitch-8gpu,8,32768,NVLS,Simple,27.700488533333335 +all_gather,synthetic-nvswitch-8gpu,8,65536,Tree,LL,6.694656 +all_gather,synthetic-nvswitch-8gpu,8,65536,Ring,Simple,11.470665142857143 +all_gather,synthetic-nvswitch-8gpu,8,65536,NVLS,Simple,27.80097706666667 +all_gather,synthetic-nvswitch-8gpu,8,131072,Tree,LL,9.709312 +all_gather,synthetic-nvswitch-8gpu,8,131072,Ring,Simple,11.901330285714286 +all_gather,synthetic-nvswitch-8gpu,8,131072,NVLS,Simple,28.001954133333335 +all_gather,synthetic-nvswitch-8gpu,8,262144,Tree,LL,15.738624 +all_gather,synthetic-nvswitch-8gpu,8,262144,Ring,Simple,12.762660571428572 +all_gather,synthetic-nvswitch-8gpu,8,262144,NVLS,Simple,28.40390826666667 +all_gather,synthetic-nvswitch-8gpu,8,524288,Tree,LL,27.797248000000003 +all_gather,synthetic-nvswitch-8gpu,8,524288,Ring,Simple,14.485321142857144 +all_gather,synthetic-nvswitch-8gpu,8,524288,NVLS,Simple,29.207816533333332 +all_gather,synthetic-nvswitch-8gpu,8,1048576,Tree,LL,51.91449600000001 +all_gather,synthetic-nvswitch-8gpu,8,1048576,Ring,Simple,17.930642285714285 +all_gather,synthetic-nvswitch-8gpu,8,1048576,NVLS,Simple,30.815633066666667 +all_gather,synthetic-nvswitch-8gpu,8,2097152,Tree,LL,100.148992 +all_gather,synthetic-nvswitch-8gpu,8,2097152,Ring,Simple,24.82128457142857 +all_gather,synthetic-nvswitch-8gpu,8,2097152,NVLS,Simple,34.03126613333334 +all_gather,synthetic-nvswitch-8gpu,8,4194304,Tree,LL,196.617984 +all_gather,synthetic-nvswitch-8gpu,8,4194304,Ring,Simple,38.60256914285714 +all_gather,synthetic-nvswitch-8gpu,8,4194304,NVLS,Simple,40.46253226666667 +all_gather,synthetic-nvswitch-8gpu,8,8388608,Tree,LL,389.555968 +all_gather,synthetic-nvswitch-8gpu,8,8388608,Ring,Simple,66.16513828571429 +all_gather,synthetic-nvswitch-8gpu,8,8388608,NVLS,Simple,53.32506453333333 +all_gather,synthetic-nvswitch-8gpu,8,16777216,Tree,LL,775.4319360000001 +all_gather,synthetic-nvswitch-8gpu,8,16777216,Ring,Simple,121.29027657142858 +all_gather,synthetic-nvswitch-8gpu,8,16777216,NVLS,Simple,79.05012906666667 +all_gather,synthetic-nvswitch-8gpu,8,33554432,Tree,LL,1547.183872 +all_gather,synthetic-nvswitch-8gpu,8,33554432,Ring,Simple,231.54055314285716 +all_gather,synthetic-nvswitch-8gpu,8,33554432,NVLS,Simple,130.50025813333335 +all_gather,synthetic-nvswitch-8gpu,8,67108864,Tree,LL,3090.6877440000003 +all_gather,synthetic-nvswitch-8gpu,8,67108864,Ring,Simple,452.04110628571436 +all_gather,synthetic-nvswitch-8gpu,8,67108864,NVLS,Simple,233.40051626666667 +all_gather,synthetic-nvswitch-8gpu,8,134217728,Tree,LL,6177.695488 +all_gather,synthetic-nvswitch-8gpu,8,134217728,Ring,Simple,893.0422125714286 +all_gather,synthetic-nvswitch-8gpu,8,134217728,NVLS,Simple,439.2010325333334 +all_gather,synthetic-nvswitch-8gpu,8,268435456,Tree,LL,12351.710976 +all_gather,synthetic-nvswitch-8gpu,8,268435456,Ring,Simple,1775.0444251428573 +all_gather,synthetic-nvswitch-8gpu,8,268435456,NVLS,Simple,850.8020650666667 +all_gather,synthetic-nvswitch-8gpu,8,536870912,Tree,LL,24699.741952000004 +all_gather,synthetic-nvswitch-8gpu,8,536870912,Ring,Simple,3539.0488502857147 +all_gather,synthetic-nvswitch-8gpu,8,536870912,NVLS,Simple,1674.0041301333333 +all_gather,synthetic-nvswitch-8gpu,8,1073741824,Tree,LL,49395.80390400001 +all_gather,synthetic-nvswitch-8gpu,8,1073741824,Ring,Simple,7067.057700571429 +all_gather,synthetic-nvswitch-8gpu,8,1073741824,NVLS,Simple,3320.4082602666667 +reduce_scatter,synthetic-nvswitch-8gpu,8,1024,Tree,LL,3.8486399999999996 +reduce_scatter,synthetic-nvswitch-8gpu,8,1024,Ring,Simple,11.40694857142857 +reduce_scatter,synthetic-nvswitch-8gpu,8,1024,NVLS,Simple,28.503242666666665 +reduce_scatter,synthetic-nvswitch-8gpu,8,2048,Tree,LL,3.89728 +reduce_scatter,synthetic-nvswitch-8gpu,8,2048,Ring,Simple,11.413897142857143 +reduce_scatter,synthetic-nvswitch-8gpu,8,2048,NVLS,Simple,28.50648533333333 +reduce_scatter,synthetic-nvswitch-8gpu,8,4096,Tree,LL,3.9945599999999994 +reduce_scatter,synthetic-nvswitch-8gpu,8,4096,Ring,Simple,11.427794285714285 +reduce_scatter,synthetic-nvswitch-8gpu,8,4096,NVLS,Simple,28.512970666666664 +reduce_scatter,synthetic-nvswitch-8gpu,8,8192,Tree,LL,4.18912 +reduce_scatter,synthetic-nvswitch-8gpu,8,8192,Ring,Simple,11.45558857142857 +reduce_scatter,synthetic-nvswitch-8gpu,8,8192,NVLS,Simple,28.525941333333332 +reduce_scatter,synthetic-nvswitch-8gpu,8,16384,Tree,LL,4.57824 +reduce_scatter,synthetic-nvswitch-8gpu,8,16384,Ring,Simple,11.511177142857143 +reduce_scatter,synthetic-nvswitch-8gpu,8,16384,NVLS,Simple,28.551882666666664 +reduce_scatter,synthetic-nvswitch-8gpu,8,32768,Tree,LL,5.3564799999999995 +reduce_scatter,synthetic-nvswitch-8gpu,8,32768,Ring,Simple,11.622354285714286 +reduce_scatter,synthetic-nvswitch-8gpu,8,32768,NVLS,Simple,28.60376533333333 +reduce_scatter,synthetic-nvswitch-8gpu,8,65536,Tree,LL,6.912959999999999 +reduce_scatter,synthetic-nvswitch-8gpu,8,65536,Ring,Simple,11.844708571428571 +reduce_scatter,synthetic-nvswitch-8gpu,8,65536,NVLS,Simple,28.707530666666663 +reduce_scatter,synthetic-nvswitch-8gpu,8,131072,Tree,LL,10.02592 +reduce_scatter,synthetic-nvswitch-8gpu,8,131072,Ring,Simple,12.289417142857143 +reduce_scatter,synthetic-nvswitch-8gpu,8,131072,NVLS,Simple,28.91506133333333 +reduce_scatter,synthetic-nvswitch-8gpu,8,262144,Tree,LL,16.251839999999998 +reduce_scatter,synthetic-nvswitch-8gpu,8,262144,Ring,Simple,13.178834285714286 +reduce_scatter,synthetic-nvswitch-8gpu,8,262144,NVLS,Simple,29.330122666666668 +reduce_scatter,synthetic-nvswitch-8gpu,8,524288,Tree,LL,28.70368 +reduce_scatter,synthetic-nvswitch-8gpu,8,524288,Ring,Simple,14.95766857142857 +reduce_scatter,synthetic-nvswitch-8gpu,8,524288,NVLS,Simple,30.160245333333332 +reduce_scatter,synthetic-nvswitch-8gpu,8,1048576,Tree,LL,53.60736 +reduce_scatter,synthetic-nvswitch-8gpu,8,1048576,Ring,Simple,18.515337142857142 +reduce_scatter,synthetic-nvswitch-8gpu,8,1048576,NVLS,Simple,31.82049066666666 +reduce_scatter,synthetic-nvswitch-8gpu,8,2097152,Tree,LL,103.41472 +reduce_scatter,synthetic-nvswitch-8gpu,8,2097152,Ring,Simple,25.630674285714285 +reduce_scatter,synthetic-nvswitch-8gpu,8,2097152,NVLS,Simple,35.140981333333336 +reduce_scatter,synthetic-nvswitch-8gpu,8,4194304,Tree,LL,203.02944 +reduce_scatter,synthetic-nvswitch-8gpu,8,4194304,Ring,Simple,39.86134857142857 +reduce_scatter,synthetic-nvswitch-8gpu,8,4194304,NVLS,Simple,41.781962666666665 +reduce_scatter,synthetic-nvswitch-8gpu,8,8388608,Tree,LL,402.25888 +reduce_scatter,synthetic-nvswitch-8gpu,8,8388608,Ring,Simple,68.32269714285714 +reduce_scatter,synthetic-nvswitch-8gpu,8,8388608,NVLS,Simple,55.06392533333333 +reduce_scatter,synthetic-nvswitch-8gpu,8,16777216,Tree,LL,800.71776 +reduce_scatter,synthetic-nvswitch-8gpu,8,16777216,Ring,Simple,125.24539428571428 +reduce_scatter,synthetic-nvswitch-8gpu,8,16777216,NVLS,Simple,81.62785066666666 +reduce_scatter,synthetic-nvswitch-8gpu,8,33554432,Tree,LL,1597.63552 +reduce_scatter,synthetic-nvswitch-8gpu,8,33554432,Ring,Simple,239.0907885714286 +reduce_scatter,synthetic-nvswitch-8gpu,8,33554432,NVLS,Simple,134.75570133333332 +reduce_scatter,synthetic-nvswitch-8gpu,8,67108864,Tree,LL,3191.47104 +reduce_scatter,synthetic-nvswitch-8gpu,8,67108864,Ring,Simple,466.78157714285715 +reduce_scatter,synthetic-nvswitch-8gpu,8,67108864,NVLS,Simple,241.01140266666667 +reduce_scatter,synthetic-nvswitch-8gpu,8,134217728,Tree,LL,6379.14208 +reduce_scatter,synthetic-nvswitch-8gpu,8,134217728,Ring,Simple,922.1631542857143 +reduce_scatter,synthetic-nvswitch-8gpu,8,134217728,NVLS,Simple,453.52280533333334 +reduce_scatter,synthetic-nvswitch-8gpu,8,268435456,Tree,LL,12754.48416 +reduce_scatter,synthetic-nvswitch-8gpu,8,268435456,Ring,Simple,1832.9263085714288 +reduce_scatter,synthetic-nvswitch-8gpu,8,268435456,NVLS,Simple,878.5456106666667 +reduce_scatter,synthetic-nvswitch-8gpu,8,536870912,Tree,LL,25505.16832 +reduce_scatter,synthetic-nvswitch-8gpu,8,536870912,Ring,Simple,3654.4526171428574 +reduce_scatter,synthetic-nvswitch-8gpu,8,536870912,NVLS,Simple,1728.5912213333334 +reduce_scatter,synthetic-nvswitch-8gpu,8,1073741824,Tree,LL,51006.53664 +reduce_scatter,synthetic-nvswitch-8gpu,8,1073741824,Ring,Simple,7297.505234285714 +reduce_scatter,synthetic-nvswitch-8gpu,8,1073741824,NVLS,Simple,3428.6824426666667 diff --git a/src/code/issue10/selector_2026.py b/src/code/issue10/selector_2026.py new file mode 100644 index 0000000..2dc08c6 --- /dev/null +++ b/src/code/issue10/selector_2026.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Offline calibration and runtime lookup table for collective algorithms.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +from collections import defaultdict +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable + + +@dataclass(frozen=True) +class CalibrationRecord: + collective: str + topology: str + world_size: int + message_bytes: int + algorithm: str + protocol: str + latency_us: float + + def __post_init__(self) -> None: + if not self.collective or not self.topology or not self.algorithm or not self.protocol: + raise ValueError("categorical calibration fields cannot be empty") + if self.world_size <= 0 or self.message_bytes <= 0 or self.latency_us <= 0: + raise ValueError("world size, message size, and latency must be positive") + + @property + def group_key(self) -> tuple[str, str, int]: + return self.collective, self.topology, self.world_size + + @property + def choice(self) -> tuple[str, str]: + return self.algorithm, self.protocol + + +def load_calibration(path: Path) -> list[CalibrationRecord]: + required = { + "collective", + "topology", + "world_size", + "message_bytes", + "algorithm", + "protocol", + "latency_us", + } + with path.open("r", encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + missing = required - set(reader.fieldnames or []) + if missing: + raise ValueError(f"calibration CSV is missing columns: {sorted(missing)}") + records = [ + CalibrationRecord( + collective=row["collective"], + topology=row["topology"], + world_size=int(row["world_size"]), + message_bytes=int(row["message_bytes"]), + algorithm=row["algorithm"], + protocol=row["protocol"], + latency_us=float(row["latency_us"]), + ) + for row in reader + ] + if not records: + raise ValueError("calibration CSV is empty") + return records + + +def _boundary(left_size: int, right_size: int) -> int: + """Log-space midpoint between adjacent calibrated message sizes.""" + + return max(left_size + 1, int(round(math.sqrt(left_size * right_size)))) + + +def build_selection_table(records: Iterable[CalibrationRecord]) -> dict[str, Any]: + grouped: dict[tuple[str, str, int], list[CalibrationRecord]] = defaultdict(list) + for record in records: + grouped[record.group_key].append(record) + + tables = [] + for key, group_records in sorted(grouped.items()): + by_size: dict[int, list[CalibrationRecord]] = defaultdict(list) + for record in group_records: + by_size[record.message_bytes].append(record) + sizes = sorted(by_size) + winners = [min(by_size[size], key=lambda item: item.latency_us) for size in sizes] + + intervals = [] + start = 0 + while start < len(sizes): + end = start + while end + 1 < len(sizes) and winners[end + 1].choice == winners[start].choice: + end += 1 + minimum = 0 if start == 0 else _boundary(sizes[start - 1], sizes[start]) + maximum = ( + None + if end == len(sizes) - 1 + else _boundary(sizes[end], sizes[end + 1]) - 1 + ) + intervals.append( + { + "min_message_bytes": minimum, + "max_message_bytes": maximum, + "algorithm": winners[start].algorithm, + "protocol": winners[start].protocol, + "calibrated_sizes": sizes[start : end + 1], + } + ) + start = end + 1 + + collective, topology, world_size = key + tables.append( + { + "collective": collective, + "topology": topology, + "world_size": world_size, + "intervals": intervals, + } + ) + return {"schema_version": 1, "tables": tables} + + +def select( + table: dict[str, Any], collective: str, topology: str, world_size: int, message_bytes: int +) -> dict[str, str]: + if message_bytes <= 0: + raise ValueError("message_bytes must be positive") + candidates = [ + item + for item in table["tables"] + if item["collective"] == collective + and item["topology"] == topology + and item["world_size"] == world_size + ] + if not candidates: + raise KeyError(f"no table for {(collective, topology, world_size)}") + for interval in candidates[0]["intervals"]: + maximum = interval["max_message_bytes"] + if message_bytes >= interval["min_message_bytes"] and ( + maximum is None or message_bytes <= maximum + ): + return {"algorithm": interval["algorithm"], "protocol": interval["protocol"]} + raise RuntimeError("selection table contains a gap") + + +def evaluate_mixed_workload( + records: Iterable[CalibrationRecord], table: dict[str, Any] +) -> list[dict[str, Any]]: + """Evaluate oracle-table time against every complete fixed choice. + + Each calibrated message size receives equal weight. Because the same data + builds and evaluates the table, this is a selector self-check, not an + independent performance validation. + """ + + grouped: dict[tuple[str, str, int], list[CalibrationRecord]] = defaultdict(list) + for record in records: + grouped[record.group_key].append(record) + + reports = [] + for key, group_records in sorted(grouped.items()): + by_size_choice = { + (record.message_bytes, record.choice): record.latency_us for record in group_records + } + sizes = sorted({record.message_bytes for record in group_records}) + all_choices = sorted({record.choice for record in group_records}) + collective, topology, world_size = key + dynamic_us = 0.0 + for size in sizes: + picked = select(table, collective, topology, world_size, size) + dynamic_us += by_size_choice[(size, (picked["algorithm"], picked["protocol"]))] + + fixed = [] + for choice in all_choices: + if all((size, choice) in by_size_choice for size in sizes): + total = sum(by_size_choice[(size, choice)] for size in sizes) + fixed.append( + { + "algorithm": choice[0], + "protocol": choice[1], + "total_us": total, + "dynamic_improvement": 1.0 - dynamic_us / total, + } + ) + reports.append( + { + "collective": collective, + "topology": topology, + "world_size": world_size, + "message_sizes": sizes, + "dynamic_total_us": dynamic_us, + "fixed_choices": fixed, + "warning": "in-sample selector self-check; collect a separate validation trace for acceptance", + } + ) + return reports + + +def synthetic_records() -> list[CalibrationRecord]: + records = [] + sizes = [2**power for power in range(10, 31)] + models = [ + ("Tree", "LL", 4.0, 20.0), + ("Ring", "Simple", 12.0, 140.0), + ("NVLS", "Simple", 30.0, 300.0), + ] + for collective in ("all_reduce", "all_gather", "reduce_scatter"): + collective_factor = {"all_reduce": 1.0, "all_gather": 0.92, "reduce_scatter": 0.95}[ + collective + ] + for size in sizes: + for algorithm, protocol, base_us, bandwidth_gbytes_s in models: + latency = (base_us + size / (bandwidth_gbytes_s * 1e9) * 1e6) * collective_factor + records.append( + CalibrationRecord( + collective, + "synthetic-nvswitch-8gpu", + 8, + size, + algorithm, + protocol, + latency, + ) + ) + return records + + +def write_calibration(records: Iterable[CalibrationRecord], path: Path) -> None: + materialized = list(records) + if not materialized: + raise ValueError("cannot write an empty calibration set") + fields = list(asdict(materialized[0]).keys()) + with path.open("w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter(handle, fields) + writer.writeheader() + for record in materialized: + writer.writerow(asdict(record)) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Build a topology-aware NCCL algorithm selector") + parser.add_argument("--calibration", type=Path) + parser.add_argument("--synthetic-demo", action="store_true") + parser.add_argument("--output-dir", type=Path, default=Path("src/code/issue10/results-2026")) + return parser + + +def main() -> int: + args = build_parser().parse_args() + if bool(args.calibration) == bool(args.synthetic_demo): + raise SystemExit("choose exactly one of --calibration or --synthetic-demo") + args.output_dir.mkdir(parents=True, exist_ok=True) + if args.synthetic_demo: + records = synthetic_records() + calibration_path = args.output_dir / "synthetic_calibration.csv" + write_calibration(records, calibration_path) + else: + records = load_calibration(args.calibration) + + table = build_selection_table(records) + evaluation = evaluate_mixed_workload(records, table) + table_path = args.output_dir / "selection_table.json" + evaluation_path = args.output_dir / "mixed_workload_evaluation.json" + table_path.write_text(json.dumps(table, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + evaluation_path.write_text( + json.dumps(evaluation, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + print(f"Wrote {table_path}") + print(f"Wrote {evaluation_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/code/issue10/test_selector_2026.py b/src/code/issue10/test_selector_2026.py new file mode 100644 index 0000000..4538207 --- /dev/null +++ b/src/code/issue10/test_selector_2026.py @@ -0,0 +1,25 @@ +from selector_2026 import build_selection_table, evaluate_mixed_workload, select, synthetic_records + + +def test_selector_covers_small_medium_and_large_messages(): + records = synthetic_records() + table = build_selection_table(records) + + small = select(table, "all_reduce", "synthetic-nvswitch-8gpu", 8, 1024) + medium = select(table, "all_reduce", "synthetic-nvswitch-8gpu", 8, 1 << 20) + large = select(table, "all_reduce", "synthetic-nvswitch-8gpu", 8, 1 << 30) + + assert small == {"algorithm": "Tree", "protocol": "LL"} + assert medium == {"algorithm": "Ring", "protocol": "Simple"} + assert large == {"algorithm": "NVLS", "protocol": "Simple"} + + +def test_dynamic_selector_is_no_slower_than_complete_fixed_choices_in_sample(): + records = synthetic_records() + table = build_selection_table(records) + reports = evaluate_mixed_workload(records, table) + + for report in reports: + for fixed in report["fixed_choices"]: + assert report["dynamic_total_us"] <= fixed["total_us"] +