diff --git a/artifacts/tech_report_v1/section_4_leaderboards/out/byproducts/newly_reached_targets_relaxed.csv b/artifacts/tech_report_v1/section_4_leaderboards/out/byproducts/newly_reached_targets_relaxed.csv new file mode 100644 index 00000000..ee35afa6 --- /dev/null +++ b/artifacts/tech_report_v1/section_4_leaderboards/out/byproducts/newly_reached_targets_relaxed.csv @@ -0,0 +1,15 @@ +Submission,criteo1tb,fastmri,finewebedu_lm,imagenet_resnet,imagenet_vit,librispeech_conformer,librispeech_deepspeech,ogbg,wmt +AdEMAMix (PyTorch),False,False,False,False,False,False,True,False,False +Cautious NAdamW (JAX),True,False,False,True,True,False,False,False,False +Lion (PyTorch),True,False,False,False,False,False,True,False,False +Muon (JAX),True,False,False,True,False,False,False,True,True +Muon (PyTorch),False,True,False,False,False,True,False,False,False +NAdamW (JAX),False,False,False,True,False,False,False,True,True +NAdamW (Baseline AlgoPerf v0.5) (JAX),False,False,False,True,True,False,False,False,False +NAdamW (Tuned for ResNet) (JAX),True,False,False,True,True,False,False,False,True +Schedule-Free AdamW (PyTorch),False,False,False,False,False,True,False,False,False +Schedule-Free AdamW (JAX),True,False,False,False,False,False,False,False,False +Schedule-Free AdamW v2 (JAX),False,False,False,False,False,False,False,False,True +Schedule-Free AdamW v2 (PyTorch),False,False,True,False,False,False,False,False,False +Single Worker DiLoCo (JAX),True,False,False,True,True,False,False,True,True +Single Worker DiLoCo v2 (JAX),True,False,False,True,False,False,False,True,True diff --git a/artifacts/tech_report_v1/section_4_leaderboards/out/byproducts/scores_standard_vs_relaxed.csv b/artifacts/tech_report_v1/section_4_leaderboards/out/byproducts/scores_standard_vs_relaxed.csv new file mode 100644 index 00000000..6c1c7000 --- /dev/null +++ b/artifacts/tech_report_v1/section_4_leaderboards/out/byproducts/scores_standard_vs_relaxed.csv @@ -0,0 +1,15 @@ +submission,standard_score,relaxed_score,standard_rank,relaxed_rank,rank_shift +Schedule-Free AdamW v2 (PyTorch),0.5679012345679016,0.607182940516274,1,1,0 +NAdamW (Baseline AlgoPerf v0.5) (JAX),0.4506172839506181,0.6010101010101013,4,2,2 +Muon (PyTorch),0.4231200897867556,0.5993265993265987,6,3,3 +Schedule-Free AdamW (PyTorch),0.466891133557801,0.5785634118967448,3,4,-1 +Schedule-Free AdamW v2 (JAX),0.4405162738496074,0.5740740740740736,5,5,0 +Schedule-Free AdamW (JAX),0.4107744107744111,0.5561167227833891,7,6,1 +Muon (JAX),0.2845117845117842,0.5521885521885517,10,7,3 +AdEMAMix (PyTorch),0.5190796857463527,0.5213243546576883,2,8,-6 +Cautious NAdamW (JAX),0.31425364758698043,0.5089786756453424,9,9,0 +NAdamW (JAX),0.384399551066217,0.4814814814814808,8,10,-2 +Lion (PyTorch),0.27833894500561146,0.4438832772166114,11,11,0 +NAdamW (Tuned for ResNet) (JAX),0.18574635241301896,0.4343434343434347,12,12,0 +Single Worker DiLoCo (JAX),0.1369248035914701,0.42592592592592626,13,13,0 +Single Worker DiLoCo v2 (JAX),0.1301907968574634,0.3232323232323228,14,14,0 diff --git a/artifacts/tech_report_v1/section_4_leaderboards/out/results/scores_relaxed_table.tex b/artifacts/tech_report_v1/section_4_leaderboards/out/results/scores_relaxed_table.tex index b5918444..1ffc8692 100644 --- a/artifacts/tech_report_v1/section_4_leaderboards/out/results/scores_relaxed_table.tex +++ b/artifacts/tech_report_v1/section_4_leaderboards/out/results/scores_relaxed_table.tex @@ -1,25 +1,27 @@ -\begin{table}[h] +\begin{table}[htbp] \centering - \caption{AlgoPerf Self-Tuning Leaderboard (targets relaxed 10%)} + \caption{Standard vs.\ 10\% relaxed-target AlgoPerf self-tuning leaderboard. $\Delta$ is the rank change after relaxing targets.} \label{tab:scores_relaxed} - \begin{tabular}{rlr} + \begin{tabular}{lrrrrc} \toprule - Rank & Submission & Score \\ + & \multicolumn{2}{c}{Standard} & \multicolumn{2}{c}{10\% relaxed} & \\ + \cmidrule(lr){2-3}\cmidrule(lr){4-5} + Submission & Score & Rank & Score & Rank & $\Delta$ \\ \midrule - 1 & \sfadamwii & \textbf{0.6072} \\ - 2 & \nadamwbase & 0.6010 \\ - 3 & \muonpt & 0.5993 \\ - 4 & \sfadamw & 0.5786 \\ - 5 & \sfadamwjaxii & 0.5741 \\ - 6 & \sfadamwjax & 0.5561 \\ - 7 & \muonjax & 0.5522 \\ - 8 & \ademamix & 0.5213 \\ - 9 & \cautiousnadamw & 0.5090 \\ - 10 & \nadamw & 0.4815 \\ - 11 & \lion & 0.4439 \\ - 12 & \nadamwresnet & 0.4343 \\ - 13 & \dilocosw & 0.4259 \\ - 14 & \dilocoswii & 0.3232 \\ + \sfadamwii & \textbf{0.5679} & 1 & \textbf{0.6072} & 1 & -- \\ + \nadamwbase & 0.4506 & 4 & 0.6010 & 2 & $\uparrow$2 \\ + \muonpt & 0.4231 & 6 & 0.5993 & 3 & $\uparrow$3 \\ + \sfadamw & 0.4669 & 3 & 0.5786 & 4 & $\downarrow$1 \\ + \sfadamwjaxii & 0.4405 & 5 & 0.5741 & 5 & -- \\ + \sfadamwjax & 0.4108 & 7 & 0.5561 & 6 & $\uparrow$1 \\ + \muonjax & 0.2845 & 10 & 0.5522 & 7 & $\uparrow$3 \\ + \ademamix & 0.5191 & 2 & 0.5213 & 8 & $\downarrow$6 \\ + \cautiousnadamw & 0.3143 & 9 & 0.5090 & 9 & -- \\ + \nadamw & 0.3844 & 8 & 0.4815 & 10 & $\downarrow$2 \\ + \lion & 0.2783 & 11 & 0.4439 & 11 & -- \\ + \nadamwresnet & 0.1857 & 12 & 0.4343 & 12 & -- \\ + \dilocosw & 0.1369 & 13 & 0.4259 & 13 & -- \\ + \dilocoswii & 0.1302 & 14 & 0.3232 & 14 & -- \\ \bottomrule \end{tabular} \end{table} \ No newline at end of file diff --git a/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table.tex b/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table.tex index a9cea48d..446e542d 100644 --- a/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table.tex +++ b/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table.tex @@ -1,6 +1,6 @@ \begin{table}[htbp] \centering - \caption{Time to target as a fraction of each workload's self-tuning-ruleset time budget (1.5\times the external-tuning maximum allowed runtime). \textemdash{} = target not reached.} + \caption{Time to target as a fraction of each workload's self-tuning-ruleset time budget ($1.5\times$ the external-tuning maximum allowed runtime). \textemdash{} = target not reached.} \label{tab:time_to_target} \resizebox{\textwidth}{!}{% \begin{tabular}{lrrrrrrrrr} diff --git a/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table_relaxed.tex b/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table_relaxed.tex index ffd26667..b813d0b1 100644 --- a/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table_relaxed.tex +++ b/artifacts/tech_report_v1/section_4_leaderboards/out/results/time_to_target_table_relaxed.tex @@ -1,26 +1,26 @@ \begin{table}[htbp] \centering - \caption{Time to target with targets relaxed 10%, as a fraction of each workload's self-tuning-ruleset time budget (1.5\times the external-tuning maximum allowed runtime). \textemdash{} = target not reached.} + \caption{Time to target with targets relaxed 10\%, as a fraction of each workload's self-tuning-ruleset time budget ($1.5\times$ the external-tuning maximum allowed runtime). \textbf{Bold} values mark targets reached only after relaxation; \textemdash{} = target not reached.} \label{tab:time_to_target_relaxed} \resizebox{\textwidth}{!}{% \begin{tabular}{lrrrrrrrrr} \toprule Submission & \criteo & \fastmri & \finewebedu & \resnet & \vit & \conformer & \deepspeech & \ogbg & \wmt \\ \midrule - \sfadamwii & 0.03 & 0.06 & 0.59 & \textemdash{} & \textemdash{} & 0.61 & 0.42 & 0.16 & 0.06 \\ - \nadamwbase & 0.03 & 0.07 & 0.32 & 0.43 & 0.56 & \textemdash{} & \textemdash{} & 0.13 & 0.11 \\ - \muonpt & 0.03 & 0.08 & 0.22 & \textemdash{} & \textemdash{} & 0.72 & \textemdash{} & 0.08 & 0.08 \\ - \sfadamw & 0.03 & 0.08 & \textemdash{} & \textemdash{} & \textemdash{} & 0.72 & 0.42 & 0.13 & 0.08 \\ - \sfadamwjaxii & 0.03 & 0.07 & 0.21 & \textemdash{} & 0.27 & \textemdash{} & \textemdash{} & 0.13 & 0.08 \\ - \sfadamwjax & 0.03 & 0.09 & 0.16 & \textemdash{} & 0.27 & \textemdash{} & \textemdash{} & 0.16 & 0.08 \\ - \muonjax & 0.05 & 0.08 & 0.32 & 0.40 & 0.48 & \textemdash{} & \textemdash{} & 0.05 & 0.27 \\ - \ademamix & 0.03 & 0.08 & 0.32 & \textemdash{} & \textemdash{} & 0.67 & 0.47 & 0.19 & 0.24 \\ - \cautiousnadamw & 0.03 & 0.07 & 0.48 & 0.43 & 0.59 & \textemdash{} & \textemdash{} & 0.16 & 0.16 \\ - \nadamw & 0.03 & 0.06 & 0.48 & 0.45 & 0.51 & \textemdash{} & \textemdash{} & 0.16 & 0.27 \\ - \lion & 0.03 & 0.07 & 0.38 & \textemdash{} & \textemdash{} & \textemdash{} & 0.44 & 0.27 & 0.13 \\ - \nadamwresnet & 0.03 & 0.07 & 0.38 & 0.43 & 0.77 & \textemdash{} & \textemdash{} & 0.21 & 0.27 \\ - \dilocosw & 0.05 & 0.08 & 0.48 & 0.45 & 0.72 & \textemdash{} & \textemdash{} & 0.16 & 0.19 \\ - \dilocoswii & 0.03 & 0.09 & 0.54 & 0.61 & \textemdash{} & \textemdash{} & \textemdash{} & 0.27 & 0.27 \\ + \sfadamwii & 0.03 & 0.06 & \textbf{0.59} & \textemdash{} & \textemdash{} & 0.61 & 0.42 & 0.16 & 0.06 \\ + \nadamwbase & 0.03 & 0.07 & 0.32 & \textbf{0.43} & \textbf{0.56} & \textemdash{} & \textemdash{} & 0.13 & 0.11 \\ + \muonpt & 0.03 & \textbf{0.08} & 0.22 & \textemdash{} & \textemdash{} & \textbf{0.72} & \textemdash{} & 0.08 & 0.08 \\ + \sfadamw & 0.03 & 0.08 & \textemdash{} & \textemdash{} & \textemdash{} & \textbf{0.72} & 0.42 & 0.13 & 0.08 \\ + \sfadamwjaxii & 0.03 & 0.07 & 0.21 & \textemdash{} & 0.27 & \textemdash{} & \textemdash{} & 0.13 & \textbf{0.08} \\ + \sfadamwjax & \textbf{0.03} & 0.09 & 0.16 & \textemdash{} & 0.27 & \textemdash{} & \textemdash{} & 0.16 & 0.08 \\ + \muonjax & \textbf{0.05} & 0.08 & 0.32 & \textbf{0.40} & 0.48 & \textemdash{} & \textemdash{} & \textbf{0.05} & \textbf{0.27} \\ + \ademamix & 0.03 & 0.08 & 0.32 & \textemdash{} & \textemdash{} & 0.67 & \textbf{0.47} & 0.19 & 0.24 \\ + \cautiousnadamw & \textbf{0.03} & 0.07 & 0.48 & \textbf{0.43} & \textbf{0.59} & \textemdash{} & \textemdash{} & 0.16 & 0.16 \\ + \nadamw & 0.03 & 0.06 & 0.48 & \textbf{0.45} & 0.51 & \textemdash{} & \textemdash{} & \textbf{0.16} & \textbf{0.27} \\ + \lion & \textbf{0.03} & 0.07 & 0.38 & \textemdash{} & \textemdash{} & \textemdash{} & \textbf{0.44} & 0.27 & 0.13 \\ + \nadamwresnet & \textbf{0.03} & 0.07 & 0.38 & \textbf{0.43} & \textbf{0.77} & \textemdash{} & \textemdash{} & 0.21 & \textbf{0.27} \\ + \dilocosw & \textbf{0.05} & 0.08 & 0.48 & \textbf{0.45} & \textbf{0.72} & \textemdash{} & \textemdash{} & \textbf{0.16} & \textbf{0.19} \\ + \dilocoswii & \textbf{0.03} & 0.09 & 0.54 & \textbf{0.61} & \textemdash{} & \textemdash{} & \textemdash{} & \textbf{0.27} & \textbf{0.27} \\ \bottomrule \end{tabular}% } diff --git a/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.pdf b/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.pdf index 291b16c8..9922b318 100644 Binary files a/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.pdf and b/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.pdf differ diff --git a/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.png b/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.png index 28f2a669..d642968d 100644 Binary files a/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.png and b/artifacts/tech_report_v1/section_4_leaderboards/out/results/wallclock_vs_steps.png differ diff --git a/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.ipynb b/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.ipynb index c0d149d6..9d31c6cc 100644 --- a/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.ipynb +++ b/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.ipynb @@ -8832,7 +8832,7 @@ "\n", "_budget_caption = (\n", " r\"Time to target as a fraction of each workload's self-tuning-ruleset \"\n", - " rf'time budget ({SELF_TUNING_RUNTIME_FACTOR:g}\\times the external-tuning maximum allowed runtime).'\n", + " rf'time budget (${SELF_TUNING_RUNTIME_FACTOR:g}\\times$ the external-tuning maximum allowed runtime).'\n", " if SELF_TUNING_RULESET else\n", " r\"Time to target as a fraction of each workload's maximum allowed runtime.\"\n", ")\n", @@ -9814,28 +9814,44 @@ "# before reading the legend. Dark marker edges keep light fills legible on\n", "# white.\n", "_FAMILIES = [\n", - " ('o', ['Schedule-Free AdamW v2 (PyTorch)', 'Schedule-Free AdamW v2 (JAX)',\n", - " 'Schedule-Free AdamW (JAX)', 'Schedule-Free AdamW (PyTorch)']),\n", - " ('s', ['NAdamW (JAX)', 'NAdamW (Baseline AlgoPerf v0.5) (JAX)',\n", - " 'NAdamW (Tuned for ResNet) (JAX)', 'Cautious NAdamW (JAX)']),\n", - " ('D', ['Muon (PyTorch)', 'Muon (JAX)']),\n", - " ('^', ['Single Worker DiLoCo (JAX)', 'Single Worker DiLoCo v2 (JAX)']),\n", - " ('P', ['AdEMAMix (PyTorch)']),\n", - " ('X', ['Lion (PyTorch)']),\n", + " ('Schedule-Free AdamW', 'o', [\n", + " 'Schedule-Free AdamW v2 (PyTorch)',\n", + " 'Schedule-Free AdamW v2 (JAX)',\n", + " 'Schedule-Free AdamW (PyTorch)',\n", + " 'Schedule-Free AdamW (JAX)',\n", + " ]),\n", + " ('NAdamW', 's', [\n", + " 'NAdamW (Baseline AlgoPerf v0.5) (JAX)',\n", + " 'NAdamW (JAX)',\n", + " 'NAdamW (Tuned for ResNet) (JAX)',\n", + " 'Cautious NAdamW (JAX)',\n", + " ]),\n", + " ('Muon', 'D', ['Muon (PyTorch)', 'Muon (JAX)']),\n", + " ('Single Worker DiLoCo', '^', [\n", + " 'Single Worker DiLoCo (JAX)',\n", + " 'Single Worker DiLoCo v2 (JAX)',\n", + " ]),\n", + " ('AdEMAMix', 'P', ['AdEMAMix (PyTorch)']),\n", + " ('Lion', 'X', ['Lion (PyTorch)']),\n", "]\n", - "_FAMILY_MARKER = {name: marker for marker, members in _FAMILIES for name in members}\n", + "_FAMILY_MARKER = {\n", + " name: marker\n", + " for _, marker, members in _FAMILIES\n", + " for name in members\n", + "}\n", "\n", "fig, ax = plt.subplots(figsize=(9.5, 5.6))\n", "\n", "lims = (0.10, 0.60)\n", "ax.plot(lims, lims, linestyle='--', color='#999999', linewidth=1.0, zorder=1)\n", - "ax.text(0.135, 0.575, 'wall-clock advantage\\n(cheap, fast steps)',\n", + "ax.text(0.135, 0.575, 'wall-clock advantage\\n(lower cost per step)',\n", " ha='left', va='top', fontsize=8.5, style='italic', color='#777777')\n", - "ax.text(0.575, 0.135, 'step advantage\\n(expensive steps)',\n", + "ax.text(0.575, 0.135, 'step advantage\\n(fewer steps to target)',\n", " ha='right', va='bottom', fontsize=8.5, style='italic', color='#777777')\n", "\n", + "_scatter_handles = {}\n", "for name, row in cmp.iterrows():\n", - " ax.scatter(\n", + " _scatter_handles[name] = ax.scatter(\n", " row.steps, row.wallclock,\n", " color=SUBMISSION_STYLE[name]['color'],\n", " marker=_FAMILY_MARKER.get(name, 'o'),\n", @@ -9863,7 +9879,18 @@ "ax.set_ylabel('Wall-clock benchmark score')\n", "ax.set_title('Wall-clock vs. step-based benchmark scores', pad=8)\n", "\n", + "_family_legend_order = [\n", + " name\n", + " for _, _, members in _FAMILIES\n", + " for name in members\n", + " if name in _scatter_handles\n", + "]\n", + "_family_legend_order.extend(\n", + " name for name in cmp.index if name not in _family_legend_order\n", + ")\n", "ax.legend(\n", + " [_scatter_handles[name] for name in _family_legend_order],\n", + " _family_legend_order,\n", " loc='center left', bbox_to_anchor=(1.03, 0.5), ncol=1,\n", " borderaxespad=0, frameon=True, handlelength=1.0, labelspacing=0.5,\n", ")\n", @@ -9892,10 +9919,12 @@ "(minimize) targets increase by this fraction; accuracy-style (maximize)\n", "targets decrease by it. Artifacts use a `_relaxed` suffix\n", "(`time_to_targets_relaxed.csv`, `scores_relaxed.csv`,\n", - "`scores_relaxed_table.tex`, `performance_profile_by_score_relaxed.{pdf,png}`)\n", + "`scores_standard_vs_relaxed.csv`, `scores_relaxed_table.tex`,\n", + "`performance_profile_by_score_relaxed.{pdf,png}`)\n", "and this never modifies the frozen `workload_targets*.json` files.\n", "Also emits `time_to_target_table_relaxed.tex`, the relaxed-target analog of\n", - "Section 5b's time-to-target table." + "Section 5b's time-to-target table, plus\n", + "`newly_reached_targets_relaxed.csv` to record which cells should be bolded." ] }, { @@ -10088,6 +10117,8 @@ ], "source": [ "TARGET_RELAXATION_FRACTION = 0.10 # 10% relaxation, applied to every workload\n", + "_RELAXATION_LABEL = f'{TARGET_RELAXATION_FRACTION:.0%}'\n", + "_RELAXATION_LABEL_LATEX = _RELAXATION_LABEL.replace('%', r'\\%')\n", "\n", "RELAXED_WORKLOAD_CONFIG = WORKLOAD_CONFIG.with_target_relaxations(\n", " {'all': TARGET_RELAXATION_FRACTION}\n", @@ -10132,12 +10163,66 @@ "display(scores_relaxed.sort_values('score', ascending=False))\n", "print(f'Saved to {scores_relaxed_path}')\n", "\n", - "# ── LaTeX table ────────────────────────────────────────────────────────────\n", - "scores_relaxed_latex = scores_to_latex(\n", - " scores_relaxed,\n", - " caption=f'AlgoPerf Self-Tuning Leaderboard (targets relaxed {TARGET_RELAXATION_FRACTION:.0%})',\n", - " label='tab:scores_relaxed',\n", + "# ── Standard-vs-relaxed comparison table ──────────────────────────────────\n", + "# Keep the score and rank movement in one auditable DataFrame, then use it for\n", + "# both the CSV byproduct and the report table. Positive rank_shift means that a\n", + "# submission moves up when targets are relaxed.\n", + "relaxed_cmp = pd.DataFrame({\n", + " 'standard_score': scores['score'],\n", + " 'relaxed_score': scores_relaxed['score'],\n", + "})\n", + "relaxed_cmp['standard_rank'] = (\n", + " relaxed_cmp.standard_score.rank(ascending=False, method='min').astype(int)\n", + ")\n", + "relaxed_cmp['relaxed_rank'] = (\n", + " relaxed_cmp.relaxed_score.rank(ascending=False, method='min').astype(int)\n", ")\n", + "relaxed_cmp['rank_shift'] = (\n", + " relaxed_cmp.standard_rank - relaxed_cmp.relaxed_rank\n", + ")\n", + "relaxed_cmp = relaxed_cmp.sort_values('relaxed_rank')\n", + "\n", + "relaxed_cmp_path = os.path.join(\n", + " BYPRODUCTS_DIR, 'scores_standard_vs_relaxed.csv'\n", + ")\n", + "relaxed_cmp.to_csv(relaxed_cmp_path)\n", + "print(f'Saved to {relaxed_cmp_path}')\n", + "\n", + "_best_standard = relaxed_cmp.standard_score.max()\n", + "_best_relaxed = relaxed_cmp.relaxed_score.max()\n", + "_relaxed_cmp_rows = []\n", + "for name, row in relaxed_cmp.iterrows():\n", + " standard_score = f'{row.standard_score:.4f}'\n", + " relaxed_score = f'{row.relaxed_score:.4f}'\n", + " if row.standard_score == _best_standard:\n", + " standard_score = r'\\textbf{' + standard_score + '}'\n", + " if row.relaxed_score == _best_relaxed:\n", + " relaxed_score = r'\\textbf{' + relaxed_score + '}'\n", + " _relaxed_cmp_rows.append(\n", + " f' {latex_name(name)} & {standard_score} & '\n", + " f'{int(row.standard_rank)} & {relaxed_score} & '\n", + " f'{int(row.relaxed_rank)} & {_fmt_shift(int(row.rank_shift))} '\n", + " + r'\\\\'\n", + " )\n", + "\n", + "scores_relaxed_latex = '\\n'.join([\n", + " r'\\begin{table}[htbp]',\n", + " r' \\centering',\n", + " r' \\caption{Standard vs.\\ ' + _RELAXATION_LABEL_LATEX\n", + " + r' relaxed-target AlgoPerf self-tuning leaderboard. $\\Delta$ is the rank'\n", + " + r' change after relaxing targets.}',\n", + " r' \\label{tab:scores_relaxed}',\n", + " r' \\begin{tabular}{lrrrrc}',\n", + " r' \\toprule',\n", + " r' & \\multicolumn{2}{c}{Standard} & \\multicolumn{2}{c}{10\\% relaxed} & \\\\',\n", + " r' \\cmidrule(lr){2-3}\\cmidrule(lr){4-5}',\n", + " r' Submission & Score & Rank & Score & Rank & $\\Delta$ \\\\',\n", + " r' \\midrule',\n", + " *_relaxed_cmp_rows,\n", + " r' \\bottomrule',\n", + " r' \\end{tabular}',\n", + " r'\\end{table}',\n", + "])\n", "print(scores_relaxed_latex)\n", "\n", "scores_relaxed_latex_path = os.path.join(RESULTS_DIR, 'scores_relaxed_table.tex')\n", @@ -10483,24 +10568,50 @@ "_relaxed_col_spec = 'l' + 'r' * len(_relaxed_workloads)\n", "_relaxed_wl_headers = ' & '.join(_WL_MACRO.get(w, w) for w in _relaxed_workloads)\n", "\n", + "# A target is newly reached when its standard time-to-target is non-finite but\n", + "# its relaxed time-to-target is finite. Persist this mask so the bolding in the\n", + "# LaTeX table is mechanically checkable without parsing LaTeX.\n", + "def _target_reached(v):\n", + " return pd.notna(v) and np.isfinite(v)\n", + "\n", + "\n", + "_newly_reached_relaxed = (\n", + " ~ttt.map(_target_reached) & ttt_relaxed.map(_target_reached)\n", + ")\n", + "_newly_reached_path = os.path.join(\n", + " BYPRODUCTS_DIR, 'newly_reached_targets_relaxed.csv'\n", + ")\n", + "_newly_reached_relaxed.to_csv(_newly_reached_path)\n", + "print(\n", + " f'Newly reached targets: {_newly_reached_relaxed.to_numpy().sum()} '\n", + " f'(saved to {_newly_reached_path})'\n", + ")\n", + "\n", "_relaxed_ttt_rows = []\n", "for name, row in ttt_relaxed_display.iterrows():\n", - " cells = [str(v).replace('%', r'\\%') for v in row]\n", + " cells = []\n", + " for workload, value in row.items():\n", + " cell = str(value).replace('%', r'\\%')\n", + " if _newly_reached_relaxed.loc[name, workload]:\n", + " cell = r'\\textbf{' + cell + '}'\n", + " cells.append(cell)\n", " _relaxed_ttt_rows.append(' ' + latex_name(name) + ' & ' + ' & '.join(cells) + r' \\\\')\n", "\n", "_relaxed_budget_caption = (\n", - " rf'Time to target with targets relaxed {TARGET_RELAXATION_FRACTION:.0%}, as a '\n", + " rf'Time to target with targets relaxed {_RELAXATION_LABEL_LATEX}, as a '\n", " r\"fraction of each workload's self-tuning-ruleset time budget \"\n", - " rf'({SELF_TUNING_RUNTIME_FACTOR:g}\\times the external-tuning maximum allowed runtime).'\n", + " rf'(${SELF_TUNING_RUNTIME_FACTOR:g}\\times$ the external-tuning maximum allowed runtime).'\n", " if SELF_TUNING_RULESET else\n", - " rf'Time to target with targets relaxed {TARGET_RELAXATION_FRACTION:.0%}, as a '\n", + " rf'Time to target with targets relaxed {_RELAXATION_LABEL_LATEX}, as a '\n", " r\"fraction of each workload's maximum allowed runtime.\"\n", ")\n", "\n", "ttt_relaxed_latex = '\\n'.join([\n", " r'\\begin{table}[htbp]',\n", " r' \\centering',\n", - " r' \\caption{' + _relaxed_budget_caption + r' \\textemdash{} = target not reached.}',\n", + " r' \\caption{' + _relaxed_budget_caption\n", + " + r' \\textbf{Bold} values mark targets reached only after relaxation;'\n", + " + r' \\textemdash{} = target not reached.}',\n", " r' \\label{tab:time_to_target_relaxed}',\n", " r' \\resizebox{\\textwidth}{!}{%',\n", " r' \\begin{tabular}{' + _relaxed_col_spec + '}',\n", diff --git a/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.py b/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.py index a5e4c470..9e8289ea 100644 --- a/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.py +++ b/artifacts/tech_report_v1/section_4_leaderboards/score_submissions.py @@ -586,7 +586,7 @@ def _fmt(workload, v): _budget_caption = ( r"Time to target as a fraction of each workload's self-tuning-ruleset " - rf'time budget ({SELF_TUNING_RUNTIME_FACTOR:g}\times the external-tuning maximum allowed runtime).' + rf'time budget (${SELF_TUNING_RUNTIME_FACTOR:g}\times$ the external-tuning maximum allowed runtime).' if SELF_TUNING_RULESET else r"Time to target as a fraction of each workload's maximum allowed runtime." ) @@ -805,28 +805,44 @@ def _fmt_steps(v): # before reading the legend. Dark marker edges keep light fills legible on # white. _FAMILIES = [ - ('o', ['Schedule-Free AdamW v2 (PyTorch)', 'Schedule-Free AdamW v2 (JAX)', - 'Schedule-Free AdamW (JAX)', 'Schedule-Free AdamW (PyTorch)']), - ('s', ['NAdamW (JAX)', 'NAdamW (Baseline AlgoPerf v0.5) (JAX)', - 'NAdamW (Tuned for ResNet) (JAX)', 'Cautious NAdamW (JAX)']), - ('D', ['Muon (PyTorch)', 'Muon (JAX)']), - ('^', ['Single Worker DiLoCo (JAX)', 'Single Worker DiLoCo v2 (JAX)']), - ('P', ['AdEMAMix (PyTorch)']), - ('X', ['Lion (PyTorch)']), + ('Schedule-Free AdamW', 'o', [ + 'Schedule-Free AdamW v2 (PyTorch)', + 'Schedule-Free AdamW v2 (JAX)', + 'Schedule-Free AdamW (PyTorch)', + 'Schedule-Free AdamW (JAX)', + ]), + ('NAdamW', 's', [ + 'NAdamW (Baseline AlgoPerf v0.5) (JAX)', + 'NAdamW (JAX)', + 'NAdamW (Tuned for ResNet) (JAX)', + 'Cautious NAdamW (JAX)', + ]), + ('Muon', 'D', ['Muon (PyTorch)', 'Muon (JAX)']), + ('Single Worker DiLoCo', '^', [ + 'Single Worker DiLoCo (JAX)', + 'Single Worker DiLoCo v2 (JAX)', + ]), + ('AdEMAMix', 'P', ['AdEMAMix (PyTorch)']), + ('Lion', 'X', ['Lion (PyTorch)']), ] -_FAMILY_MARKER = {name: marker for marker, members in _FAMILIES for name in members} +_FAMILY_MARKER = { + name: marker + for _, marker, members in _FAMILIES + for name in members +} fig, ax = plt.subplots(figsize=(9.5, 5.6)) lims = (0.10, 0.60) ax.plot(lims, lims, linestyle='--', color='#999999', linewidth=1.0, zorder=1) -ax.text(0.135, 0.575, 'wall-clock advantage\n(cheap, fast steps)', +ax.text(0.135, 0.575, 'wall-clock advantage\n(lower cost per step)', ha='left', va='top', fontsize=8.5, style='italic', color='#777777') -ax.text(0.575, 0.135, 'step advantage\n(expensive steps)', +ax.text(0.575, 0.135, 'step advantage\n(fewer steps to target)', ha='right', va='bottom', fontsize=8.5, style='italic', color='#777777') +_scatter_handles = {} for name, row in cmp.iterrows(): - ax.scatter( + _scatter_handles[name] = ax.scatter( row.steps, row.wallclock, color=SUBMISSION_STYLE[name]['color'], marker=_FAMILY_MARKER.get(name, 'o'), @@ -854,7 +870,18 @@ def _fmt_steps(v): ax.set_ylabel('Wall-clock benchmark score') ax.set_title('Wall-clock vs. step-based benchmark scores', pad=8) +_family_legend_order = [ + name + for _, _, members in _FAMILIES + for name in members + if name in _scatter_handles +] +_family_legend_order.extend( + name for name in cmp.index if name not in _family_legend_order +) ax.legend( + [_scatter_handles[name] for name in _family_legend_order], + _family_legend_order, loc='center left', bbox_to_anchor=(1.03, 0.5), ncol=1, borderaxespad=0, frameon=True, handlelength=1.0, labelspacing=0.5, ) @@ -879,13 +906,17 @@ def _fmt_steps(v): # (minimize) targets increase by this fraction; accuracy-style (maximize) # targets decrease by it. Artifacts use a `_relaxed` suffix # (`time_to_targets_relaxed.csv`, `scores_relaxed.csv`, -# `scores_relaxed_table.tex`, `performance_profile_by_score_relaxed.{pdf,png}`) +# `scores_standard_vs_relaxed.csv`, `scores_relaxed_table.tex`, +# `performance_profile_by_score_relaxed.{pdf,png}`) # and this never modifies the frozen `workload_targets*.json` files. # Also emits `time_to_target_table_relaxed.tex`, the relaxed-target analog of -# Section 5b's time-to-target table. +# Section 5b's time-to-target table, plus +# `newly_reached_targets_relaxed.csv` to record which cells should be bolded. # %% TARGET_RELAXATION_FRACTION = 0.10 # 10% relaxation, applied to every workload +_RELAXATION_LABEL = f'{TARGET_RELAXATION_FRACTION:.0%}' +_RELAXATION_LABEL_LATEX = _RELAXATION_LABEL.replace('%', r'\%') RELAXED_WORKLOAD_CONFIG = WORKLOAD_CONFIG.with_target_relaxations( {'all': TARGET_RELAXATION_FRACTION} @@ -930,12 +961,66 @@ def _fmt_steps(v): display(scores_relaxed.sort_values('score', ascending=False)) print(f'Saved to {scores_relaxed_path}') -# ── LaTeX table ──────────────────────────────────────────────────────────── -scores_relaxed_latex = scores_to_latex( - scores_relaxed, - caption=f'AlgoPerf Self-Tuning Leaderboard (targets relaxed {TARGET_RELAXATION_FRACTION:.0%})', - label='tab:scores_relaxed', +# ── Standard-vs-relaxed comparison table ────────────────────────────────── +# Keep the score and rank movement in one auditable DataFrame, then use it for +# both the CSV byproduct and the report table. Positive rank_shift means that a +# submission moves up when targets are relaxed. +relaxed_cmp = pd.DataFrame({ + 'standard_score': scores['score'], + 'relaxed_score': scores_relaxed['score'], +}) +relaxed_cmp['standard_rank'] = ( + relaxed_cmp.standard_score.rank(ascending=False, method='min').astype(int) +) +relaxed_cmp['relaxed_rank'] = ( + relaxed_cmp.relaxed_score.rank(ascending=False, method='min').astype(int) ) +relaxed_cmp['rank_shift'] = ( + relaxed_cmp.standard_rank - relaxed_cmp.relaxed_rank +) +relaxed_cmp = relaxed_cmp.sort_values('relaxed_rank') + +relaxed_cmp_path = os.path.join( + BYPRODUCTS_DIR, 'scores_standard_vs_relaxed.csv' +) +relaxed_cmp.to_csv(relaxed_cmp_path) +print(f'Saved to {relaxed_cmp_path}') + +_best_standard = relaxed_cmp.standard_score.max() +_best_relaxed = relaxed_cmp.relaxed_score.max() +_relaxed_cmp_rows = [] +for name, row in relaxed_cmp.iterrows(): + standard_score = f'{row.standard_score:.4f}' + relaxed_score = f'{row.relaxed_score:.4f}' + if row.standard_score == _best_standard: + standard_score = r'\textbf{' + standard_score + '}' + if row.relaxed_score == _best_relaxed: + relaxed_score = r'\textbf{' + relaxed_score + '}' + _relaxed_cmp_rows.append( + f' {latex_name(name)} & {standard_score} & ' + f'{int(row.standard_rank)} & {relaxed_score} & ' + f'{int(row.relaxed_rank)} & {_fmt_shift(int(row.rank_shift))} ' + + r'\\' + ) + +scores_relaxed_latex = '\n'.join([ + r'\begin{table}[htbp]', + r' \centering', + r' \caption{Standard vs.\ ' + _RELAXATION_LABEL_LATEX + + r' relaxed-target AlgoPerf self-tuning leaderboard. $\Delta$ is the rank' + + r' change after relaxing targets.}', + r' \label{tab:scores_relaxed}', + r' \begin{tabular}{lrrrrc}', + r' \toprule', + r' & \multicolumn{2}{c}{Standard} & \multicolumn{2}{c}{10\% relaxed} & \\', + r' \cmidrule(lr){2-3}\cmidrule(lr){4-5}', + r' Submission & Score & Rank & Score & Rank & $\Delta$ \\', + r' \midrule', + *_relaxed_cmp_rows, + r' \bottomrule', + r' \end{tabular}', + r'\end{table}', +]) print(scores_relaxed_latex) scores_relaxed_latex_path = os.path.join(RESULTS_DIR, 'scores_relaxed_table.tex') @@ -968,24 +1053,50 @@ def _fmt_steps(v): _relaxed_col_spec = 'l' + 'r' * len(_relaxed_workloads) _relaxed_wl_headers = ' & '.join(_WL_MACRO.get(w, w) for w in _relaxed_workloads) +# A target is newly reached when its standard time-to-target is non-finite but +# its relaxed time-to-target is finite. Persist this mask so the bolding in the +# LaTeX table is mechanically checkable without parsing LaTeX. +def _target_reached(v): + return pd.notna(v) and np.isfinite(v) + + +_newly_reached_relaxed = ( + ~ttt.map(_target_reached) & ttt_relaxed.map(_target_reached) +) +_newly_reached_path = os.path.join( + BYPRODUCTS_DIR, 'newly_reached_targets_relaxed.csv' +) +_newly_reached_relaxed.to_csv(_newly_reached_path) +print( + f'Newly reached targets: {_newly_reached_relaxed.to_numpy().sum()} ' + f'(saved to {_newly_reached_path})' +) + _relaxed_ttt_rows = [] for name, row in ttt_relaxed_display.iterrows(): - cells = [str(v).replace('%', r'\%') for v in row] + cells = [] + for workload, value in row.items(): + cell = str(value).replace('%', r'\%') + if _newly_reached_relaxed.loc[name, workload]: + cell = r'\textbf{' + cell + '}' + cells.append(cell) _relaxed_ttt_rows.append(' ' + latex_name(name) + ' & ' + ' & '.join(cells) + r' \\') _relaxed_budget_caption = ( - rf'Time to target with targets relaxed {TARGET_RELAXATION_FRACTION:.0%}, as a ' + rf'Time to target with targets relaxed {_RELAXATION_LABEL_LATEX}, as a ' r"fraction of each workload's self-tuning-ruleset time budget " - rf'({SELF_TUNING_RUNTIME_FACTOR:g}\times the external-tuning maximum allowed runtime).' + rf'(${SELF_TUNING_RUNTIME_FACTOR:g}\times$ the external-tuning maximum allowed runtime).' if SELF_TUNING_RULESET else - rf'Time to target with targets relaxed {TARGET_RELAXATION_FRACTION:.0%}, as a ' + rf'Time to target with targets relaxed {_RELAXATION_LABEL_LATEX}, as a ' r"fraction of each workload's maximum allowed runtime." ) ttt_relaxed_latex = '\n'.join([ r'\begin{table}[htbp]', r' \centering', - r' \caption{' + _relaxed_budget_caption + r' \textemdash{} = target not reached.}', + r' \caption{' + _relaxed_budget_caption + + r' \textbf{Bold} values mark targets reached only after relaxation;' + + r' \textemdash{} = target not reached.}', r' \label{tab:time_to_target_relaxed}', r' \resizebox{\textwidth}{!}{%', r' \begin{tabular}{' + _relaxed_col_spec + '}', diff --git a/scoring/config.py b/scoring/config.py index 534d6699..426d04b9 100644 --- a/scoring/config.py +++ b/scoring/config.py @@ -182,7 +182,9 @@ def step_hint(self, workload: str) -> int: """Returns the step hint for a workload.""" return self._target(workload).step_hint - def max_runtime_sec(self, workload: str, self_tuning_ruleset: bool = False) -> float: + def max_runtime_sec( + self, workload: str, self_tuning_ruleset: bool = False + ) -> float: """Returns the runtime budget (seconds) for a workload. Returns the external-tuning ruleset's budget, or