From ee0ad4b69b64e349f48841ebe0f8d94e59a4701e Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:18:08 -0700 Subject: [PATCH 1/8] [None][infra] Skip pre-merge perf gating when main has already regressed Both perf pipelines compare against the same post-merge baseline, so once a regression lands on main every PR measures the same regressed value and every PR fails a perf-sanity test it did not break. That blocks the whole queue until someone lands a fix, after which everyone else must rebase. prepare_regressive_test_cases now consults the latest post-merge document for the same test case -- already fetched as latest_history_data_dict, so no new OpenSearch query -- and re-evaluates that value against the same baseline at the same pre-merge threshold used for the pre-merge verdict. When main's own latest value would itself miss that gate, the pre-merge case is reported as a warning instead of raising. The uploaded b_is_regression stays exactly as measured; no new field is written. Notes on the semantics, documented in both READMEs: - The b_is_regression recorded by the post-merge run is deliberately not reused, because it was computed at the tighter post-merge threshold (5%). Re-evaluating at the pre-merge threshold keeps the exemption exactly as wide as the failure it prevents: if main is only 6% down, a PR reproducing that value never fails the 10% gate, so there is nothing to exempt and the gate stays armed. A shared _is_regressive helper makes the two comparisons structurally identical so they cannot drift apart. - Anything unusable in that record -- metric absent, null, non-numeric or non-positive -- does not exempt, so a broken post-merge document leaves the gate armed rather than silently disarming it. - Granularity is the whole test case (cmd_idx), not the individual metric, so within the exempt band the case is exempt entirely: a further regression the PR adds on top of main's still passes. - The exemption self-clears as the rolling baseline absorbs the regression, which also means the regression stops being reported; the gate then re-arms against the lowered bar unless the fix lands. - To force gating back on, mark the offending post-merge document b_is_valid: false so the history query drops it. tests/unittest/others/test_perf_regression_branch.py grows from 9 to 48 tests covering the exemption firing, every way it must not fire (missing key, None, metric absent, null, non-numeric, zero, negative, exactly at the threshold, and regressed only within the pre-merge gate), that a bare recorded verdict with no value does not exempt, minimize as well as maximize metrics, per-cmd_idx keying, both the Tier-A d_baseline_* and the Tier-B rolling-baseline paths, and inertness on post-merge runs and under fail_on_regression=False. Every "does not raise" case is paired with a structurally identical raising case differing only in the history fake, so a degenerate always-true or always-false predicate turns half the matrix red. The module also gains the pytestmark pytest.mark.cpu_only it was missing, without which conftest's pytest_ignore_collect dropped the whole file from the CPU stage and none of its tests ran anywhere. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- .../perf/README_perf_regression_system.md | 46 +- .../defs/perf/README_test_perf_sanity.md | 30 ++ .../defs/perf/perf_regression_utils.py | 129 ++++-- .../others/test_perf_regression_branch.py | 401 +++++++++++++++++- 4 files changed, 561 insertions(+), 45 deletions(-) diff --git a/tests/integration/defs/perf/README_perf_regression_system.md b/tests/integration/defs/perf/README_perf_regression_system.md index 514040d839b7..d33cf2ea12f8 100644 --- a/tests/integration/defs/perf/README_perf_regression_system.md +++ b/tests/integration/defs/perf/README_perf_regression_system.md @@ -35,10 +35,10 @@ The main entry point is `process_and_upload_test_results()`, which orchestrates | 2 | Enrich data | Merges `job_config` and `extra_fields` into each entry, then calls `add_id()` | | 3 | `get_common_values()` | Scans all entries to find match_keys where every entry has the same value (e.g., all share `s_gpu_type=H100`). These become additional query filters | | 4 | `get_history_data()` | Queries OpenSearch with the narrowed filters, matches results back to `cmd_idx` | -| 5 | `prepare_regressive_test_cases()` | Compares each metric's new value against baseline. Baseline comes from `latest_baseline_threshold_dict` (the most recent entry with baseline fields); if missing, falls back to `calculate_baseline_metrics()`. Threshold also comes from `latest_baseline_threshold_dict`; if missing, uses defaults. Sets `b_is_regression=True` if any regression metric exceeds the threshold | +| 5 | `prepare_regressive_test_cases()` | Compares each metric's new value against baseline. Baseline comes from `latest_baseline_threshold_dict` (the most recent entry with baseline fields); if missing, falls back to `calculate_baseline_metrics()`. Threshold also comes from `latest_baseline_threshold_dict`; if missing, uses defaults. Sets `b_is_regression=True` if any regression metric exceeds the threshold. Returns the set of pre-merge `cmd_idx` exempt from gating (empty when the history query failed) | | 6 | `add_baseline_fields_to_post_merge_data()` | Post-merge only: embeds `d_baseline_*` and `d_threshold_*` fields into new data from `latest_baseline_threshold_dict`. Only sets fields when inherited values exist and are > 0; skips otherwise | | 7 | `post_new_perf_data()` | Uploads to OpenSearch | -| 8 | `check_perf_regression()` | Prints regression details. For pre-merge, raises `RuntimeError` if `fail_on_regression=True` (default for pre-merge, auto-detected) | +| 8 | `check_perf_regression()` | Prints regression details. For pre-merge, raises `RuntimeError` if `fail_on_regression=True` (default for pre-merge, auto-detected) and at least one regressive case is **not** exempt. Exempt cases are still printed as warnings | `s_branch` comes from `globalVars["build_branch"]`, which the pipeline resolves once in `resolveBuildBranch()` (`jenkins/L0_MergeRequest.groovy`) and also writes @@ -62,6 +62,48 @@ builds. - A metric is regressive if the new value breaches `baseline * (1 +/- threshold)` - Default thresholds: **5%** for post-merge, **10%** for pre-merge (can be overridden per-metric via embedded `d_threshold_*` fields from history) +**Pre-merge exemption when `main` has already regressed** + +Both pipelines compare against the same baseline, so a regression that lands on +`main` makes every subsequent PR measure the same regressed value and fail a +test it did not break — blocking all PRs until a fix merges and everyone +rebases. To prevent that, a pre-merge case does not fail the stage when the +latest post-merge record for the same case would itself miss the pre-merge gate: + +- The latest post-merge **value** is re-evaluated here, against the same baseline + and the same **pre-merge** threshold (10%) used for the pre-merge verdict. The + `b_is_regression` recorded by the post-merge run is *not* consulted, because it + was computed at the tighter post-merge threshold (5%). +- That makes the exemption exactly as wide as the failure it prevents. If `main` + sits within the pre-merge threshold, a PR reproducing `main`'s value does not + fail the gate, so there is nothing to exempt and the gate stays armed. +- Granularity is the whole test case (per `cmd_idx`), not per metric. +- Anything unusable in that record — metric absent, null, non-numeric, or + non-positive — does **not** exempt, so a broken post-merge document leaves the + gate armed rather than silently disarming it. +- The pre-merge document still uploads `b_is_regression` exactly as measured and + keeps the full metric detail in `s_regression_info`; only the `RuntimeError` is + suppressed, and an explanatory line is appended. No new field is written. +- `latest_history_data_dict` already supplies this record, so no extra + OpenSearch query is issued. + +Consequences worth knowing: + +- Within the exempt band the case is **fully** exempt: once `main` is more than + 10% off baseline, a PR that makes the same case worse still passes. This is + inherent to exempting a whole case rather than diffing against `main`'s value. +- It **self-clears**, which also hides the regression. Nothing seeds + `d_baseline_*` automatically, so the effective baseline is the rolling + P95/P5 window; within a few days it drifts down to the regressed level, the + latest post-merge value comes back inside the threshold, and the gate re-arms + against the *lowered* bar — with no fix ever landing. +- A single flaky post-merge data point disarms the gate until the next + post-merge run: "latest" is one record, not a smoothed trend. +- To force gating back on, mark the offending post-merge document + `b_is_valid: false` in OpenSearch, which drops it from the history query. +- Match keys do not include `s_stage_name`, so the exempting record is whichever + stage most recently measured that case. + ### Layer 3: Test-Specific Data Assembly Each test script (e.g., `test_perf_sanity.py`) is responsible for: diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 7bd773bd28fc..731499261a97 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -329,6 +329,36 @@ Tests are defined in `jenkins/L0_Test.groovy` under the `launchTestJobs` functio By default, `test_perf_sanity.py` fails CI on perf regression for pre-merge stages and only warns for post-merge stages. This is auto-detected from the Jenkins job URL (`PostMerge` substring), not the stage name. +**A pre-merge case does not fail when `main` has already regressed.** Both pipelines +compare against the same post-merge baseline, so once a regression lands on `main` +every PR measures the same regressed value and every PR fails a test it did not +break — blocking the whole queue until a fix merges and everyone rebases. To avoid +that, `prepare_regressive_test_cases` checks the **latest post-merge document** for +the same test case: if that value misses the same baseline by more than the same +pre-merge threshold (10%), the pre-merge case is reported as a warning but does not +raise. Functional failures still fail the stage as usual. + +The comparison deliberately re-evaluates `main`'s value rather than reusing the +`b_is_regression` the post-merge run recorded, because that verdict was computed at +the tighter post-merge threshold (5%). Re-evaluating keeps the exemption exactly as +wide as the failure it prevents: if `main` is only 6% down, a PR reproducing that +value never fails the 10% gate, so the gate stays armed. + +Two things to be aware of when you see that warning: + +- While a case is exempt it is exempt **entirely**: a genuine new regression the PR + adds on top of `main`'s passes silently. The exempt band is only the range where + the gate would have fired on `main`'s own value anyway, but inside it there is no + coverage. +- The exemption clears itself as the rolling baseline absorbs the regression, which + also means the regression stops being reported. If you land the fix, the baseline + recovers and the gate re-arms at the original level; if nobody does, it re-arms at + the lowered one. Check the post-merge trend before assuming a green stage means the + case is healthy. + +See [README_perf_regression_system.md](README_perf_regression_system.md) for the full +rule and how to force gating back on. + **`FUNCTIONAL-ONLY` stage-name flag**: A pre-merge stage whose name contains `FUNCTIONAL-ONLY` (e.g. `GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4`) still runs the full perf harness — benchmarks execute, metrics are uploaded to OpenSearch, dashboards update — but perf regressions **do not fail CI**. Only functional failures (build errors, crashes, empty output) fail the stage. Use this for pre-merge stages whose goal is to catch functional regressions on paths that only had post-merge coverage before. It preserves the data-continuity benefit of running in pre-merge (baselines still update from PR data points) without the flakiness cost of gating on the noisier disagg perf numbers. diff --git a/tests/integration/defs/perf/perf_regression_utils.py b/tests/integration/defs/perf/perf_regression_utils.py index 3f5a4e98e54a..559e4b2d3a0a 100644 --- a/tests/integration/defs/perf/perf_regression_utils.py +++ b/tests/integration/defs/perf/perf_regression_utils.py @@ -264,6 +264,18 @@ def _calculate_diff(metric, new_value, baseline_value, maximize_metrics): return (baseline_value - new_value) / baseline_value * 100 +def _is_regressive(value, baseline_value, threshold, maximize): + """Return True when *value* misses *baseline_value* by more than *threshold*. + + Shared by the pre-merge verdict and by the check on main's own latest value, + so the two can never drift apart. The comparison is strict: a value sitting + exactly on baseline_value * (1 -/+ threshold) is not a regression. + """ + if maximize: + return value < baseline_value * (1 - threshold) + return value > baseline_value * (1 + threshold) + + def prepare_regressive_test_cases( latest_history_data_dict, latest_baseline_threshold_dict, @@ -277,10 +289,26 @@ def prepare_regressive_test_cases( Uses baseline/threshold fields from latest_baseline_threshold_dict when available, otherwise falls back to calculating baseline from history data. + + Returns the set of pre-merge cmd_idx whose regression must not fail the + stage because the latest post-merge record for the same case is itself + beyond the pre-merge threshold. Both pipelines compare against one shared + baseline, so a regression landed on main would otherwise make every + subsequent PR measure the same regressed value and fail for a change it did + not introduce. + + The latest post-merge value is re-evaluated here against the same baseline + and the same pre-merge threshold, rather than reusing the b_is_regression + the post-merge run recorded at its own tighter threshold. That makes the + exemption exactly as wide as the failure it prevents: if main sits within + the pre-merge threshold, a PR reproducing main's value does not fail the + gate, so there is nothing to exempt and the gate stays armed. """ + exempt_cmd_idxs = set() + # If latest_history_data_dict is None (network failure), skip regression check if latest_history_data_dict is None: - return + return exempt_cmd_idxs for cmd_idx in new_data_dict: new_data = new_data_dict[cmd_idx] @@ -292,6 +320,13 @@ def prepare_regressive_test_cases( regressive_metrics = [] info_lines = [] + # The latest post-merge record for this same case, used only to decide + # whether a pre-merge regression predates the change under test. + latest_post_merge = None if is_post_merge else latest_history_data_dict.get(cmd_idx) + if not isinstance(latest_post_merge, dict): + latest_post_merge = None + post_merge_regressive_metrics = [] + # Pre-calculate fallback baseline from history if needed fallback_baseline = None @@ -340,20 +375,43 @@ def prepare_regressive_test_cases( # Check if this metric is regressive (only for key regression metrics) if metric in regression_metrics: - if metric in maximize_metrics: - # Regressive if new_value < baseline_value * (1 - threshold) - if new_value < baseline_value * (1 - threshold): - regressive_metrics.append(metric) - else: - # Regressive if new_value > baseline_value * (1 + threshold) - if new_value > baseline_value * (1 + threshold): - regressive_metrics.append(metric) + maximize = metric in maximize_metrics + if _is_regressive(new_value, baseline_value, threshold, maximize): + regressive_metrics.append(metric) + + # Re-evaluate main's own latest value against this same baseline + # and this same (pre-merge) threshold. Anything unusable -- field + # absent, null, non-numeric, non-positive -- must not exempt, so + # a broken post-merge record leaves the gate armed. + if latest_post_merge is not None: + prior_value = _safe_float(latest_post_merge.get(metric), None) + if ( + prior_value is not None + and prior_value > 0 + and _is_regressive(prior_value, baseline_value, threshold, maximize) + ): + post_merge_regressive_metrics.append(metric) test_case = new_data.get("s_test_case_name", "unknown") header = f"Regression in {test_case}:" new_data["s_regression_info"] = "\n".join([header] + info_lines) new_data["b_is_regression"] = len(regressive_metrics) > 0 + # A pre-merge regression is not this PR's fault when main's own latest + # value already misses the same gate. post_merge_regressive_metrics is + # only ever populated on a pre-merge run, so post-merge is unaffected. + if post_merge_regressive_metrics: + exempt_cmd_idxs.add(cmd_idx) + if new_data["b_is_regression"]: + new_data["s_regression_info"] += ( + "\n Not failing this stage: the latest post-merge run of this test " + "case already misses the same threshold against the same baseline " + f"({', '.join(post_merge_regressive_metrics)}), so this regression " + "predates the change under test." + ) + + return exempt_cmd_idxs + def _safe_float(value, default): """Convert *value* to float, returning *default* on failure. @@ -409,40 +467,54 @@ def add_baseline_fields_to_post_merge_data( new_data[baseline_key] = baseline_val -def check_perf_regression(new_data_dict, fail_on_regression=False): +def check_perf_regression(new_data_dict, fail_on_regression=False, exempt_cmd_idxs=None): """Check performance regression by printing s_regression_info. Post-merge regressions log warnings. Pre-merge regressions raise RuntimeError when fail_on_regression is True. + + Cases in exempt_cmd_idxs are reported as warnings but never raise: their + regression is already present on main, so failing here would block a PR for + a change it did not introduce. See prepare_regressive_test_cases. """ - regressive_data_list = [ - data for data in new_data_dict.values() if data.get("b_is_regression", False) + exempt_cmd_idxs = exempt_cmd_idxs or set() + + regressive_data_items = [ + (cmd_idx, data) + for cmd_idx, data in new_data_dict.items() + if data.get("b_is_regression", False) ] - if not regressive_data_list: + if not regressive_data_items: print_info("No regression data found.") return post_merge_regressions = [ - data for data in regressive_data_list if data.get("b_is_post_merge", False) + data for _, data in regressive_data_items if data.get("b_is_post_merge", False) ] pre_merge_regressions = [ - data for data in regressive_data_list if not data.get("b_is_post_merge", False) + (cmd_idx, data) + for cmd_idx, data in regressive_data_items + if not data.get("b_is_post_merge", False) ] # Print post-merge regression details as warnings for data in post_merge_regressions: print_warning(data.get("s_regression_info", "")) - # Print pre-merge regression details and raise error - if pre_merge_regressions: - error_parts = [] - for data in pre_merge_regressions: - info = data.get("s_regression_info", "") - print_warning(info) + # Print pre-merge regression details and raise error. An exempt case is + # still reported -- the measurement is real -- but withheld from the error. + error_parts = [] + for cmd_idx, data in pre_merge_regressions: + info = data.get("s_regression_info", "") + print_warning(info) + if cmd_idx not in exempt_cmd_idxs: error_parts.append(info) - if fail_on_regression: - raise RuntimeError("\n".join(error_parts)) + + # Guard on error_parts, not on pre_merge_regressions: when every regressive + # case is exempt there is nothing to report and nothing to fail. + if fail_on_regression and error_parts: + raise RuntimeError("\n".join(error_parts)) def process_and_upload_test_results( @@ -511,8 +583,9 @@ def process_and_upload_test_results( lookup_data_dict, match_keys, common_values_dict ) - # Step 6: Compute regression info - prepare_regressive_test_cases( + # Step 6: Compute regression info. Cases whose regression is already present + # on the latest post-merge run are reported but must not fail the stage. + exempt_cmd_idxs = prepare_regressive_test_cases( latest_history_data_dict, latest_baseline_threshold_dict, history_data_dict, @@ -535,4 +608,8 @@ def process_and_upload_test_results( # Step 9: Check regression (auto-detect fail behavior if not specified) if fail_on_regression is None: fail_on_regression = not is_post_merge - check_perf_regression(new_data_dict, fail_on_regression=fail_on_regression) + check_perf_regression( + new_data_dict, + fail_on_regression=fail_on_regression, + exempt_cmd_idxs=exempt_cmd_idxs, + ) diff --git a/tests/unittest/others/test_perf_regression_branch.py b/tests/unittest/others/test_perf_regression_branch.py index e7e89e1adc63..6b39482c458f 100644 --- a/tests/unittest/others/test_perf_regression_branch.py +++ b/tests/unittest/others/test_perf_regression_branch.py @@ -12,10 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for how perf_regression_utils handles s_branch. +"""Tests for perf_regression_utils history routing and pre-merge gating. -Two concerns, both of which fail open (a green run with no regression check) if -they break: +Three concerns. The first two fail open (a green run with no regression check) +if they break; the third fails closed (every PR blocked by a regression it did +not introduce): 1. s_branch is read from globalVars, never scraped from the job URL. The Jenkins folder segment in a job URL (/job/LLM/job//) names the folder the job @@ -28,6 +29,16 @@ records, so a pre-merge run querying its own "github-pr-" branch matches nothing and its regression check silently becomes a no-op. The queries must see the baseline branch while the uploaded document keeps the real one. + +3. A pre-merge regression is exempt when the latest post-merge record for the + same case already misses the same gate. Both pipelines compare against one + shared baseline, so a regression landed on main makes every subsequent PR + measure the same regressed value and fail. The exemption re-evaluates main's + own latest value against that baseline at the pre-merge threshold -- not the + b_is_regression the post-merge run recorded at its own tighter threshold -- + so it is exactly as wide as the failure it prevents. Per test case; the + pre-merge document still uploads b_is_regression as measured, so nothing is + hidden from the DB. """ import importlib.util @@ -38,6 +49,8 @@ import pytest +pytestmark = pytest.mark.cpu_only + _REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] _MODULE_PATH = _REPO_ROOT / "tests" / "integration" / "defs" / "perf" / "perf_regression_utils.py" @@ -205,14 +218,32 @@ def test_unparsable_global_vars_does_not_raise(monkeypatch): _MATCH_KEYS = ["s_test_case_name", "s_gpu_type", "s_runtime", "s_branch"] -def _new_data_dict() -> dict[int, dict[str, object]]: +_METRIC = "d_output_token_throughput" + +# A baseline high enough that _REGRESSED_VALUE falls below the 10% pre-merge +# threshold and _CLEAN_VALUE stays above it. +_BASELINE = 1000.0 +_REGRESSED_VALUE = 800.0 +_CLEAN_VALUE = 950.0 + +# Tier A: baseline supplied directly, so no timestamps or percentile maths are +# involved in deciding whether a case is regressive. Cover several cmd_idx so a +# multi-case test resolves a baseline for every one of them; a case without a +# baseline is silently skipped rather than judged. +_TIER_A_BASELINE = {idx: {"d_baseline_output_token_throughput": _BASELINE} for idx in range(4)} + + +def _new_data_dict(value: float = 1234.5, count: int = 1) -> dict[int, dict[str, object]]: return { - 0: { - "s_test_case_name": "example_model_fp8_tp8-con32_iter10_1k1k", + idx: { + "s_test_case_name": f"example_model_fp8_tp8-con32_iter10_1k1k_{idx}" + if count > 1 + else "example_model_fp8_tp8-con32_iter10_1k1k", "s_gpu_type": "b200", "s_runtime": "aggr_server", - "d_output_token_throughput": 1234.5, + _METRIC: value, } + for idx in range(count) } @@ -221,12 +252,24 @@ def _run_pipeline( build_branch: str, job_url: str, match_keys: list[str] | None = None, + history: tuple[object, object, object] | None = None, + value: float = 1234.5, + count: int = 1, + expect_error: bool = False, + fail_on_regression: bool | None = None, + minimize: bool = False, ) -> dict[str, dict[int, str]]: - """Run the real pipeline, recording the s_branch each seam observes. + """Run the real pipeline, recording what each seam observes. Only the three OpenSearch seams are replaced. Everything between them -- get_job_info, the enrichment loop, the branch routing, the regression pass -- is the production code path, so a regression in the wiring shows up here. + + ``history`` is the (latest, baseline_threshold, history) triple the stubbed + get_history_data returns, letting a test drive the exemption decision. + ``expect_error`` wraps the call in pytest.raises(RuntimeError) and records + the message; it is never a blanket except, because post_new_perf_data raises + RuntimeError too and swallowing that would make these tests vacuous. """ observed: dict[str, dict[int, str]] = {} @@ -245,10 +288,14 @@ def fake_get_history_data( observed["history_query_names"] = { idx: d["s_test_case_name"] for idx, d in data_dict.items() } - return {}, {}, {} + return ({}, {}, {}) if history is None else history def fake_post_new_perf_data(data_dict: dict[int, dict[str, object]]) -> None: observed["uploaded"] = {idx: d["s_branch"] for idx, d in data_dict.items()} + # Snapshot whole documents: the upload (step 8) runs before the + # regression check (step 9), so this is the document as posted even when + # the check goes on to raise. + observed["uploaded_docs"] = {idx: dict(d) for idx, d in data_dict.items()} monkeypatch.setattr(_perf_regression_utils, "get_common_values", fake_get_common_values) monkeypatch.setattr(_perf_regression_utils, "get_history_data", fake_get_history_data) @@ -260,15 +307,28 @@ def fake_post_new_perf_data(data_dict: dict[int, dict[str, object]]) -> None: } monkeypatch.setenv("globalVars", json.dumps(payload)) - new_data_dict = _new_data_dict() - _perf_regression_utils.process_and_upload_test_results( - new_data_dict, - match_keys if match_keys is not None else _MATCH_KEYS, - maximize_metrics=["d_output_token_throughput"], - minimize_metrics=[], - regression_metrics=["d_output_token_throughput"], - ) + new_data_dict = _new_data_dict(value=value, count=count) + + def _call() -> None: + _perf_regression_utils.process_and_upload_test_results( + new_data_dict, + match_keys if match_keys is not None else _MATCH_KEYS, + maximize_metrics=[] if minimize else [_METRIC], + minimize_metrics=[_METRIC] if minimize else [], + regression_metrics=[_METRIC], + fail_on_regression=fail_on_regression, + ) + + observed["error"] = None + if expect_error: + with pytest.raises(RuntimeError) as excinfo: + _call() + observed["error"] = str(excinfo.value) + else: + _call() + observed["new_data_dict"] = {idx: d["s_branch"] for idx, d in new_data_dict.items()} + observed["final_docs"] = {idx: dict(d) for idx, d in new_data_dict.items()} return observed @@ -333,3 +393,310 @@ def test_no_substitution_when_branch_is_not_a_match_key(monkeypatch: pytest.Monk assert observed["history_query"] == {0: "github-pr-18408"} assert observed["uploaded"] == {0: "github-pr-18408"} + + +# --------------------------------------------------------------------------- # +# A pre-merge regression fails the stage (positive controls) +# +# These hold both before and after the exemption exists. Without them, every +# "does not raise" assertion below could pass because nothing ever raises. +# --------------------------------------------------------------------------- # + + +def _history(latest: object, baseline: object = None, history: object = None): + """Build the get_history_data triple, defaulting to the Tier A baseline.""" + return ( + latest, + _TIER_A_BASELINE if baseline is None else baseline, + {} if history is None else history, + ) + + +def _pre_merge(monkeypatch: pytest.MonkeyPatch, **kwargs): + return _run_pipeline(monkeypatch, "github-pr-18408", _GITHUB_PR_JOB_URL, **kwargs) + + +# What the latest post-merge record for case 0 measured. The exemption +# re-evaluates these against the same baseline at the same pre-merge threshold, +# so what matters is the value, not any verdict the post-merge run recorded. +_CLEAN_LATEST = {0: {_METRIC: _BASELINE}} # main on baseline +_REGRESSED_LATEST = {0: {_METRIC: _REGRESSED_VALUE}} # 20% down: misses the gate +_WITHIN_GATE_LATEST = {0: {_METRIC: _CLEAN_VALUE}} # 5% down: still makes the gate + +# The opening words of the sentence prepare_regressive_test_cases appends to +# s_regression_info when it exempts a case. Asserted both present and absent +# below, so a reworded annotation fails the positive test rather than quietly +# making every "unannotated" assertion vacuous. +_EXEMPTION_NOTE = "Not failing this stage" + + +def test_pre_merge_regression_fails_the_stage(monkeypatch: pytest.MonkeyPatch) -> None: + """The baseline case: main is healthy, the PR regresses, the stage fails.""" + observed = _pre_merge( + monkeypatch, + history=_history(_CLEAN_LATEST), + value=_REGRESSED_VALUE, + expect_error=True, + ) + + assert _METRIC in observed["error"] + assert observed["error"].strip() + assert observed["uploaded_docs"][0]["b_is_regression"] is True + + +def test_pre_merge_within_threshold_passes(monkeypatch: pytest.MonkeyPatch) -> None: + observed = _pre_merge(monkeypatch, history=_history(_CLEAN_LATEST), value=_CLEAN_VALUE) + + assert observed["uploaded_docs"][0]["b_is_regression"] is False + + +def test_pre_merge_exactly_at_threshold_passes(monkeypatch: pytest.MonkeyPatch) -> None: + """Regression is a strict inequality: value == baseline*(1-t) is not one.""" + observed = _pre_merge(monkeypatch, history=_history(_CLEAN_LATEST), value=_BASELINE * 0.9) + + assert observed["uploaded_docs"][0]["b_is_regression"] is False + + +# --------------------------------------------------------------------------- # +# A regressed main exempts the pre-merge gate +# --------------------------------------------------------------------------- # + + +def test_regressed_main_exempts_the_pre_merge_gate(monkeypatch: pytest.MonkeyPatch) -> None: + """The whole point: main already misses the gate, so the PR is not blamed.""" + observed = _pre_merge( + monkeypatch, + history=_history(_REGRESSED_LATEST), + value=_REGRESSED_VALUE, + ) + + # No exception, and specifically not RuntimeError("") from an empty message. + assert observed["error"] is None + # The measurement is still recorded truthfully -- nothing is hidden. + assert observed["uploaded_docs"][0]["b_is_regression"] is True + assert _METRIC in observed["uploaded_docs"][0]["s_regression_info"] + assert _EXEMPTION_NOTE in observed["uploaded_docs"][0]["s_regression_info"] + + +def test_main_within_the_gate_still_fails_the_pre_merge_stage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The rule that separates this from a post-merge-threshold exemption. + + main is down 5% -- enough for the post-merge run to have recorded + b_is_regression, but not enough to fail the 10% pre-merge gate. A PR + reproducing main's value would therefore never have been blocked, so there + is nothing to exempt: a PR that goes on to regress 20% must still fail. + """ + observed = _pre_merge( + monkeypatch, + history=_history(_WITHIN_GATE_LATEST), + value=_REGRESSED_VALUE, + expect_error=True, + ) + + assert _METRIC in observed["error"] + + +def test_a_numeric_string_from_opensearch_still_exempts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """History values arrive untrusted; a numeric string is still a number.""" + observed = _pre_merge( + monkeypatch, + history=_history({0: {_METRIC: str(_REGRESSED_VALUE)}}), + value=_REGRESSED_VALUE, + ) + + assert observed["error"] is None + + +@pytest.mark.parametrize( + "latest, expect_error", + [ + # A minimize metric regresses upward, so main must be ABOVE baseline by + # more than the threshold to exempt. 1200 is +20%, 1050 only +5%. + pytest.param({0: {_METRIC: 1200.0}}, False, id="main-above-the-gate-exempt"), + pytest.param({0: {_METRIC: 1050.0}}, True, id="main-within-the-gate-fails"), + pytest.param({0: {_METRIC: 800.0}}, True, id="main-better-than-baseline-fails"), + ], +) +def test_the_exemption_respects_a_minimize_metric( + monkeypatch: pytest.MonkeyPatch, latest: object, expect_error: bool +) -> None: + """A latency metric regresses upward; the direction must not be inverted. + + The 800 row is the one that catches an inverted comparison: for a minimize + metric that is main comfortably BETTER than baseline, which must never exempt. + """ + observed = _pre_merge( + monkeypatch, + history=_history(latest), + value=1200.0, + minimize=True, + expect_error=expect_error, + ) + + assert observed["uploaded_docs"][0]["b_is_regression"] is True + + +def test_exemption_is_per_test_case(monkeypatch: pytest.MonkeyPatch) -> None: + """Case 0's exemption must not cover case 1, and vice versa.""" + observed = _pre_merge( + monkeypatch, + history=_history({0: {_METRIC: _REGRESSED_VALUE}, 1: {_METRIC: _BASELINE}}), + value=_REGRESSED_VALUE, + count=2, + expect_error=True, + ) + + assert "example_model_fp8_tp8-con32_iter10_1k1k_1" in observed["error"] + assert "example_model_fp8_tp8-con32_iter10_1k1k_0" not in observed["error"] + # Both are still recorded as regressive; only the gate differs. + assert observed["uploaded_docs"][0]["b_is_regression"] is True + assert observed["uploaded_docs"][1]["b_is_regression"] is True + + +_TIER_B_HISTORY = { + 0: [ + {"@timestamp": "2026-09-01T00:00:00Z", _METRIC: _BASELINE}, + {"@timestamp": "2026-09-02T00:00:00Z", _METRIC: _BASELINE}, + {"@timestamp": "2026-09-03T00:00:00Z", _METRIC: _BASELINE}, + ] +} + + +@pytest.mark.parametrize( + "latest, expect_error", + [ + pytest.param(_REGRESSED_LATEST, False, id="regressed-main-exempt"), + pytest.param(_CLEAN_LATEST, True, id="clean-main-fails"), + ], +) +def test_exemption_holds_on_the_rolling_baseline_path( + monkeypatch: pytest.MonkeyPatch, latest: object, expect_error: bool +) -> None: + """Nothing in-repo seeds d_baseline_*, so production uses this path. + + The paired clean-main case proves the Tier B fixture really does produce a + regression, rather than the exemption passing because no baseline resolved. + """ + observed = _pre_merge( + monkeypatch, + history=(latest, {0: None}, _TIER_B_HISTORY), + value=_REGRESSED_VALUE, + expect_error=expect_error, + ) + + assert observed["uploaded_docs"][0]["b_is_regression"] is True + + +# --------------------------------------------------------------------------- # +# The exemption must not fire +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "latest", + [ + pytest.param({0: None}, id="null-record"), + pytest.param({}, id="no-record-for-this-case"), + pytest.param({0: {}}, id="metric-absent"), + pytest.param({0: {_METRIC: _BASELINE}}, id="on-baseline"), + pytest.param({0: {_METRIC: _CLEAN_VALUE}}, id="regressed-but-within-the-gate"), + # Strict inequality, matching how the pre-merge verdict itself is taken. + pytest.param({0: {_METRIC: _BASELINE * 0.9}}, id="exactly-at-the-threshold"), + # A broken post-merge record must leave the gate armed, not disarm it. + pytest.param({0: {_METRIC: 0}}, id="zero"), + pytest.param({0: {_METRIC: -5.0}}, id="negative"), + pytest.param({0: {_METRIC: None}}, id="null-value"), + pytest.param({0: {_METRIC: "not-a-number"}}, id="non-numeric"), + pytest.param({0: {_METRIC: []}}, id="empty-list"), + # The recorded verdict is no longer consulted: it was taken at the + # post-merge threshold, and with no value there is nothing to re-evaluate. + pytest.param({0: {"b_is_regression": True}}, id="verdict-without-a-value"), + ], +) +def test_only_a_value_missing_the_same_gate_exempts( + monkeypatch: pytest.MonkeyPatch, latest: object +) -> None: + """Anything else about the latest post-merge record still fails the stage.""" + observed = _pre_merge( + monkeypatch, history=_history(latest), value=_REGRESSED_VALUE, expect_error=True + ) + + assert _METRIC in observed["error"] + + +def test_an_older_regressed_record_does_not_exempt(monkeypatch: pytest.MonkeyPatch) -> None: + """The rule reads the latest post-merge record, not any record.""" + observed = _pre_merge( + monkeypatch, + history=_history( + _CLEAN_LATEST, + history={ + 0: [ + {"@timestamp": "2026-08-01T00:00:00Z", _METRIC: _REGRESSED_VALUE}, + {"@timestamp": "2026-08-02T00:00:00Z", _METRIC: _REGRESSED_VALUE}, + ] + }, + ), + value=_REGRESSED_VALUE, + expect_error=True, + ) + + assert _METRIC in observed["error"] + + +# --------------------------------------------------------------------------- # +# Everything else is untouched +# --------------------------------------------------------------------------- # + + +def test_post_merge_is_unaffected_and_unannotated(monkeypatch: pytest.MonkeyPatch) -> None: + """Post-merge already only warns; it must not gain an exemption note.""" + observed = _run_pipeline( + monkeypatch, + "main", + _RELEASE_BUILD_JOB_URL, + history=_history(_REGRESSED_LATEST), + value=_REGRESSED_VALUE, + ) + + assert observed["error"] is None + assert observed["uploaded_docs"][0]["b_is_regression"] is True + assert _EXEMPTION_NOTE not in observed["uploaded_docs"][0]["s_regression_info"] + + +def test_functional_only_stage_is_unannotated(monkeypatch: pytest.MonkeyPatch) -> None: + """fail_on_regression=False already passes; no note belongs on a healthy main.""" + observed = _pre_merge( + monkeypatch, + history=_history(_CLEAN_LATEST), + value=_REGRESSED_VALUE, + fail_on_regression=False, + ) + + assert observed["error"] is None + assert observed["uploaded_docs"][0]["b_is_regression"] is True + assert _EXEMPTION_NOTE not in observed["uploaded_docs"][0]["s_regression_info"] + + +def test_history_query_failure_still_skips_the_check(monkeypatch: pytest.MonkeyPatch) -> None: + """A None triple means the query failed: soft-fail, never a TypeError.""" + observed = _pre_merge(monkeypatch, history=(None, None, None), value=_REGRESSED_VALUE) + + assert observed["error"] is None + assert "s_regression_info" not in observed["uploaded_docs"][0] + + +def test_exemption_adds_no_field_to_the_document(monkeypatch: pytest.MonkeyPatch) -> None: + """The exemption lives outside new_data_dict, so no new OpenSearch field. + + post_new_perf_data uploads each record dict wholesale, so any key added to + it would silently become a new field in the index. + """ + exempt = _pre_merge(monkeypatch, history=_history(_REGRESSED_LATEST), value=_REGRESSED_VALUE) + clean = _pre_merge(monkeypatch, history=_history(_CLEAN_LATEST), value=_CLEAN_VALUE) + + assert set(exempt["uploaded_docs"][0]) == set(clean["uploaded_docs"][0]) From 4baa1378d92ce3a76ce91131920536e85ecbf954 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:18:09 -0700 Subject: [PATCH 2/8] [None][infra] Refactor the pre-merge perf-sanity case list The pre-merge perf-sanity list was the wrong shape: 3 of its 5 stages carried FUNCTIONAL-ONLY in the stage name, which test_perf_sanity.py reads as a substring to set fail_on_regression = False, so a perf regression on those stages could not fail CI at all. Two of those three ran e2e cases. Replace it with 13 disagg ctx_only / gen_only cases that all gate on perf regression. No 1k1k cases. Per model 2 ctx_only (small and large concurrency, small = con1 where a config exists) plus 1 gen_only; DeepSeek-V4-Pro gets 2 gen_only (con8, con666). Qwen gates on GB200; DeepSeek-R1, GLM-5 and DeepSeek-V4-Pro gate on GB300. The gen_only picks collectively cover small concurrency (qwen con1), large concurrency (glm-5 con1024), PP and long ISL (dsr1 128k8k ctx1_pp4) and MTP, at the smallest node count available per model. FUNCTIONAL-ONLY is dropped from the two GB200/GB300 stage names that keep running, which is the entire mechanism that turns them into gating stages. Points worth knowing about the new list: - The DeepSeek-V4-Pro con8 gen_only case is waived under https://nvbugs/6661856, and its ctx_only sibling under https://nvbugs/6668776. Both are enrolled with their pre-merge stages declared anyway, so the stage runs a skip until the waive is lifted and the gate activates with it. Declaring the stage up front keeps the test-db entry and the stage list in sync; an enrolled case with no stage would silently never run, and nothing in CI flags that. - gb300_deepseek-r1-fp4_8k1k_con1 ctx_only had a config but appeared in no test list, so it is added to the GB300 post_merge block as well; without a post-merge baseline its pre-merge gate would be a silent no-op. - The B200 dsr1 con1536 gen_only case was the only pre-merge entry in its list and B200 is not in the model-to-GPU mapping, but no other list carried that config. Rather than orphan it, it moves into the post_merge block and that stage goes from 2 splits to 3 -- gen_only requires exactly one test per split (submit.py select_test_case_line). - Three of the new picks are regressed on main right now (glm-5 con1, glm-5 con1024, dsr1 8k1k con4096). They are auto-exempt under the gating change in the previous commit and begin gating when main recovers. Declared pre-merge perf-sanity cost goes from 5 stages / 9 nodes / 44 GPUs to 8 stages / 30 nodes / 120 GPUs. The DeepSeek-V4-Pro con8 gen_only stage is 9 nodes / 36 GPUs of that and runs a skip until its waive lifts. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 49 ++++++++++++------- ...sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml | 17 +------ .../l0_gb200_multi_gpus_perf_sanity.yml | 10 +--- ...sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml | 2 +- .../l0_gb300_multi_gpus_perf_sanity.yml | 22 +++++++++ ...sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml | 3 +- ...sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml | 14 ++++++ ...sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml | 14 ++++++ ...anity_ctx6_node1_gpu4_gen1_node4_gpu16.yml | 14 ++++++ 9 files changed, 100 insertions(+), 45 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9fe24463756f..0a2757096870 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6248,23 +6248,13 @@ def launchTestJobs(pipeline, testFilter, globalVars) "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 3, 4, 8, 1, true], "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 4, 4, 8, 1, true], ] - // B200 PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI) - // 2 Nodes - x86SlurmTestConfigs += buildStageConfigs( - "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8", - "auto:dgx-b200-flex", - "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8", - 1, - 16, - 2 - ) // B200 PerfSanity post-merge disaggregated // 2 Nodes x86SlurmTestConfigs += buildStageConfigs( "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8-Post-Merge", "auto:dgx-b200-flex", "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8", - 2, + 3, 16, 2 ) @@ -6323,8 +6313,9 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-4_GPUs-PyTorch-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus", 2, 3, 4, 1, true, false], "GB300-4_GPUs-PyTorch-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus", 3, 3, 4, 1, true, false], // PerfSanity pre-merge tests - "GB200-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 2, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 2, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 1, 4], + "GB300-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 2, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 2, 4, 1, true, false], // PerfSanity post-merge tests "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 4, 4], "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 4, 4], @@ -6364,10 +6355,10 @@ def launchTestJobs(pipeline, testFilter, globalVars) 8, 2 ) - // PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI) + // PerfSanity pre-merge disaggregated // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( - "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4", + "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", 1, @@ -6455,6 +6446,14 @@ def launchTestJobs(pipeline, testFilter, globalVars) 8, 2 ) + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", + 1, + 12, + 3 + ) // GB300 PerfSanity post-merge disaggregated // 3 Nodes multiNodesSBSAConfigs += buildStageConfigs( @@ -6475,9 +6474,9 @@ def launchTestJobs(pipeline, testFilter, globalVars) 5 ) // GB300 GLM-5 disaggregated (ctx DEP2) - // 3 Nodes (pre-merge, functional-only) + // 3 Nodes (pre-merge) multiNodesSBSAConfigs += buildStageConfigs( - "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8", + "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", 1, @@ -6503,6 +6502,14 @@ def launchTestJobs(pipeline, testFilter, globalVars) 9 ) // 9 Nodes: ctx1 (1 node, 4 GPUs) + gen4 (2 nodes, 8 GPUs each) = 36 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8", + "gb300-flex-aws-cmh", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8", + 1, + 36, + 9 + ) multiNodesSBSAConfigs += buildStageConfigs( "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8-Post-Merge", "gb300-flex-aws-cmh", @@ -6512,6 +6519,14 @@ def launchTestJobs(pipeline, testFilter, globalVars) 9 ) // 10 Nodes: ctx6 (1 node, 4 GPUs each) + gen1 (4 nodes, 16 GPUs) = 40 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16", + 1, + 40, + 10 + ) multiNodesSBSAConfigs += buildStageConfigs( "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge", "auto:gb300-flex", diff --git a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml b/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml index 960b9cf6dc37..db2b294994fd 100644 --- a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml @@ -1,21 +1,5 @@ version: 0.0.1 l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8: -- condition: - ranges: - # 1 ctx worker with each 1 node and 4 GPUs - # 1 gen worker with each 1 node and 8 GPUs - system_gpu_count: - gte: 16 - lte: 16 - wildcards: - gpu: - - '*b200*' - terms: - stage: pre_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - condition: ranges: # 1 ctx worker with each 1 node and 4 GPUs @@ -32,6 +16,7 @@ l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8: tests: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml index 96e5235fbe8c..5a1528d5ed70 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml @@ -14,16 +14,8 @@ l0_gb200_multi_gpus_perf_sanity: stage: pre_merge backend: pytorch tests: - # deepseek-r1-fp4-v2 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_8k1k] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k] - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - # gpt-oss-120b-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - # qwen3-235b-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - condition: ranges: diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml index 48dc2a8e9790..7f80a8330670 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml @@ -14,7 +14,7 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4: stage: pre_merge backend: pytorch tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - condition: ranges: diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index 4a6935a5ef68..d179226e305a 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -1,5 +1,26 @@ version: 0.0.1 l0_gb300_multi_gpus_perf_sanity: +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*gb300*' + linux_distribution_name: ubuntu* + cpu: aarch64 + terms: + stage: pre_merge + backend: pytorch + tests: + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) + - condition: ranges: system_gpu_count: @@ -16,6 +37,7 @@ l0_gb300_multi_gpus_perf_sanity: tests: # ctx_only tests (disagg config) # deepseek-r1-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) # glm-5-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml index 304e8b56a98f..8aaa5d499876 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml @@ -14,8 +14,7 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: stage: pre_merge backend: pytorch tests: - # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - condition: ranges: diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml index 009b84866631..67d0a3fc451f 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml @@ -1,5 +1,19 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: +- condition: + ranges: + system_gpu_count: + gte: 12 + lte: 12 + wildcards: + gpu: + - '*gb300*' + terms: + stage: pre_merge + backend: pytorch + tests: + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) + - condition: ranges: # 1 ctx worker with each 1 node and 4 GPUs diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml index 07ad83e0e1e4..ac7160fbafba 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml @@ -1,5 +1,19 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8: +- condition: + ranges: + system_gpu_count: + gte: 36 + lte: 36 + wildcards: + gpu: + - '*gb300*' + terms: + stage: pre_merge + backend: pytorch + tests: + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - condition: ranges: # 1 ctx worker with 1 node and 4 GPUs diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml index 76f981ff6acb..2b1905de9a9c 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml @@ -1,5 +1,19 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16: +- condition: + ranges: + system_gpu_count: + gte: 40 + lte: 40 + wildcards: + gpu: + - '*gb300*' + terms: + stage: pre_merge + backend: pytorch + tests: + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) + - condition: ranges: # 6 ctx workers each with 1 node and 4 GPUs From bbd432d89f803d921cdb04702a17cabe1e40872d Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Mon, 7 Sep 2026 23:30:56 -0700 Subject: [PATCH 3/8] [None][infra] Add the de-gated perf-sanity cases back to the post-merge lists The pre-merge perf-sanity list refactor removed 8 cases from pre_merge blocks without re-listing them anywhere. Because the post-merge pipeline also runs the pre-merge-named stages, those pre_merge entries were the sole source of post-merge baseline data for these cases, so dropping them silently ends their OpenSearch history and baselines rather than just un-gating them. Add them back to the corresponding post_merge blocks: - l0_gb200_multi_gpus_perf_sanity.yml (+6): the three r1_fp4_v2_*_8k1k aggr cases and the deepseek-r1-fp4 con4096, gpt-oss-120b-fp4 con512/dep2 and qwen3-235b-fp4 con64 ctx_only cases. - l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml (+1): gpt-oss-120b-fp4 con1024 e2e. - l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml (+1): glm-5-fp4 con1024 e2e. Re-listing the deepseek-r1-fp4 con4096 case also fixes the "Check Test List" pre-merge failure: its waiver in waives.txt had become orphaned once its only test-list occurrence was deleted, which fails check_test_list.py --waive. Bump the affected split counts accordingly. The two multi-node perf stages require exactly one test per pytest-split group (submit.py select_test_case_line), so GEN1-NODE1-GPU4-Post-Merge goes 5 -> 6 and GB300 GPU2-GEN1-NODE2-GPU8-Post-Merge goes 2 -> 3. The GB200 4-GPU post-merge list grows 11 -> 17 tests, so widen it 4 -> 6 splits to keep per-stage duration roughly unchanged. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 14 ++++++++------ .../test-db/l0_gb200_multi_gpus_perf_sanity.yml | 6 ++++++ ...perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml | 1 + ...perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 0a2757096870..662d401a4a95 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6317,10 +6317,12 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 2, 4, 1, true, false], "GB300-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 2, 4, 1, true, false], // PerfSanity post-merge tests - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 4, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 4, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 3, 4, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 4, 4, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 6, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 6, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 3, 6, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 4, 6, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 5, 6, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 6, 6, 4], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 5, 4, 1, true, false], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 5, 4, 1, true, false], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 5, 4, 1, true, false], @@ -6379,7 +6381,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", - 5, + 6, 8, 2 ) @@ -6488,7 +6490,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", - 2, + 3, 12, 3 ) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml index 5a1528d5ed70..e150961a930f 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml @@ -36,6 +36,9 @@ l0_gb200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_1k8k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_1k8k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_1k8k] + - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_dep4_mtp1_8k1k] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tep4_mtp3_8k1k] + - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_grace_blackwell-r1_fp4_v2_tp4_mtp3_8k1k] # glm-5-fp4 # gpt-oss-120b-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k] @@ -47,8 +50,11 @@ l0_gb200_multi_gpus_perf_sanity: # deepseek-r1-fp4 # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) # gpt-oss-120b-fp4 # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # qwen3-235b-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml index 7f80a8330670..8de8278d24c2 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml @@ -35,3 +35,4 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml index 8aaa5d499876..62f7d0196bd6 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml @@ -33,3 +33,4 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: # glm-5-fp4 - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) From aad1a022f64335bb863a8bd6fdbb21ef3c2ed48a Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 8 Sep 2026 01:29:11 -0700 Subject: [PATCH 4/8] [None][infra] De-duplicate the perf-sanity pre_merge and post_merge blocks The refactor added pre_merge blocks listing cases that were already present in the same file's post_merge block, so 12 cases were listed twice within one test-db yaml. Because the post-merge pipeline adds the pre-merge-named multi-GPU stages back on top of the Post-Merge stages (L0_Test.groovy: parallelJobsFiltered += multiGpuJobs + postMergeJobs), each of those cases would run twice per post-merge cycle. Keep one listing per case. The duplicate is dropped from the post_merge block rather than pre_merge, so the case still gates pre-merge and still runs -- and uploads post-merge baseline data -- in the post-merge pipeline via its pre-merge-named stage. Affected lists (post_merge entries removed): l0_gb200_multi_gpus_perf_sanity -1 (17->16) l0_gb200_multi_nodes_..._ctx1_node1_gpu1_gen1_node1_gpu4 -1 (6->5) l0_gb300_multi_gpus_perf_sanity -6 (19->13) l0_gb300_multi_nodes_..._ctx1_node1_gpu2_gen1_node2_gpu8 -1 (3->2) l0_gb300_multi_nodes_..._ctx1_node1_gpu4_gen1_node2_gpu8 -1 (4->3) l0_gb300_multi_nodes_..._ctx1_node1_gpu4_gen4_node2_gpu8 -1 (2->1) l0_gb300_multi_nodes_..._ctx6_node1_gpu4_gen1_node4_gpu16 -1 (2->1) Re-sync the split counts of the five affected multi-node post-merge stages, which require exactly one test per pytest-split group. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 10 +++++----- .../test-db/l0_gb200_multi_gpus_perf_sanity.yml | 1 - ...des_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml | 1 - .../test-db/l0_gb300_multi_gpus_perf_sanity.yml | 6 ------ ...des_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml | 1 - ...des_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml | 1 - ...des_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml | 1 - ...es_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml | 1 - 8 files changed, 5 insertions(+), 17 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 662d401a4a95..1121c50e5776 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6381,7 +6381,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", - 6, + 5, 8, 2 ) @@ -6462,7 +6462,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 4, + 3, 12, 3 ) @@ -6490,7 +6490,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", - 3, + 2, 12, 3 ) @@ -6516,7 +6516,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8-Post-Merge", "gb300-flex-aws-cmh", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8", - 2, + 1, 36, 9 ) @@ -6533,7 +6533,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16", - 2, + 1, 40, 10 ) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml index e150961a930f..5a02b52747bd 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml @@ -56,5 +56,4 @@ l0_gb200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # qwen3-235b-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml index 8de8278d24c2..766a03e52679 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml @@ -32,7 +32,6 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4: tests: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index d179226e305a..d1da84b19576 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -37,16 +37,10 @@ l0_gb300_multi_gpus_perf_sanity: tests: # ctx_only tests (disagg config) # deepseek-r1-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) # deepseek-v4-pro-fp4 8k1k - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] TIMEOUT (120) # nemotron-ultra-v3-fp4 8k64k (ctx_only) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml index 62f7d0196bd6..064ea88bb19a 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml @@ -32,5 +32,4 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: tests: # glm-5-fp4 - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml index 67d0a3fc451f..5caa4a699838 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml @@ -29,7 +29,6 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: backend: pytorch tests: # deepseek-r1-fp4 128k8k - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) # nemotron-ultra-v3-fp4 8k64k con64 - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_8k64k_con64_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml index ac7160fbafba..325ebbf775f3 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml @@ -29,5 +29,4 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8: backend: pytorch tests: # deepseek-v4-pro-fp4 8k1k con8 (single-user latency) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml index 2b1905de9a9c..dc9d1e885e3c 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml @@ -29,5 +29,4 @@ l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16: backend: pytorch tests: # deepseek-v4-pro-fp4 8k1k con666 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) From cf8f7d092a9f874143f30d84e6243cc0474301de Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:16:00 -0700 Subject: [PATCH 5/8] [None][infra] Drop the never-referenced b200 2-node perf-sanity list and tidy list comments l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml has never been referenced by any Jenkins stage since it was introduced in #13882: the testlist name appears in zero of the 144 revisions of jenkins/L0_Test.groovy authored since, and in neither revision of jenkins/L0_MergeRequest.groovy. Its two post_merge cases are listed in no other test-db yml, so they have never executed in either the pre-merge or the post-merge pipeline. Remove the dead list and repoint the naming-convention example in README_test_perf_sanity.md at a list that a stage actually consumes. Also fix a stray inline '# glm-5-fp4' comment that labelled the preceding deepseek-r1-fp4 entry in l0_gb300_multi_gpus_perf_sanity.yml, and give the new pre_merge block the same per-model comment grouping as the post_merge block. No change to the effective pre-merge or post-merge run list. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- .../defs/perf/README_test_perf_sanity.md | 2 +- ...00_multi_nodes_perf_sanity_node2_gpu16.yml | 21 ------------------- .../l0_gb300_multi_gpus_perf_sanity.yml | 7 ++++++- 3 files changed, 7 insertions(+), 23 deletions(-) delete mode 100644 tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 731499261a97..cefcf2dfe342 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -297,7 +297,7 @@ Test lists are defined in `tests/integration/test_lists/test-db/`. | Test Type | File Pattern | Example | |-----------|--------------|---------| | Single-node aggregated | `l0_{gpu_type}_multi_gpus_perf_sanity.yml` | `l0_b200_multi_gpus_perf_sanity.yml` | -| Multi-node aggregated | `l0_{gpu_type}_multi_nodes_perf_sanity_node{node count}_gpu{gpu count per test}.yml` | `l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml` | +| Multi-node aggregated | `l0_{gpu_type}_multi_nodes_perf_sanity_node{node count}_gpu{gpu count per test}.yml` | `l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml` | | Multi-node disaggregated | `l0_{gpu_type}_multi_gpus_perf_sanity_ctx{ctx worker count}node{node count per ctx worker}_gpu{gpu count per ctx worker}_gen{gen worker count}node{node count per gen worker}_gpu{gen gpus per gen worker}.yml` | `l0_b200_multi_gpus_perf_sanity_ctx1node1_gpu8_gen1node1_gpu8.yml` | ### Jenkins Pipeline Configuration diff --git a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml b/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml deleted file mode 100644 index a8d684360976..000000000000 --- a/tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_node2_gpu16.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 0.0.1 -l0_b200_multi_nodes_perf_sanity_node2_gpu16: -- condition: - ranges: - # 2 nodes with each node has 8 GPUs - system_gpu_count: - gte: 16 - lte: 16 - wildcards: - gpu: - - '*b200*' - linux_distribution_name: ubuntu* - cpu: x86_64 - terms: - stage: post_merge - backend: pytorch - orchestrator: mpi - tests: - # deepseek-r1-fp4-v2 aggregated 2-nodes - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_blackwell-r1_fp4_v2_dep16_mtp1_8k1k] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_blackwell-r1_fp4_v2_tep16_mtp3] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index d1da84b19576..99108c254298 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -14,10 +14,14 @@ l0_gb300_multi_gpus_perf_sanity: stage: pre_merge backend: pytorch tests: + # ctx_only tests (disagg config) + # deepseek-r1-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) + # glm-5-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) + # deepseek-v4-pro-fp4 8k1k - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) @@ -37,7 +41,8 @@ l0_gb300_multi_gpus_perf_sanity: tests: # ctx_only tests (disagg config) # deepseek-r1-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) + # glm-5-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) # deepseek-v4-pro-fp4 8k1k - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) From 1b0ffdbbb0a9936165c1d723f541e58bb45c22f7 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Fri, 11 Sep 2026 04:11:07 -0700 Subject: [PATCH 6/8] [None][infra] Keep only the ctx_only perf-sanity cases in pre-merge The gen_only perf-sanity cases are multi-node disaggregated and each one claims a whole stage of 8-40 GPUs. Enrolling them in pre-merge means every PR pays for all of them, which is more GPU time than the coverage justifies. Drop the pre_merge block from the five multi-node disagg lists and put each gen_only case back in its post_merge block at the position it held before, bumping the paired Post-Merge stage's testCount to match. Three of the five lists become byte-identical to main again. This is the same treatment the earlier commits in this series already applied to the B200 2-node stage. Pre-merge perf-sanity now declares 3 stages / 3 nodes / 12 GPUs for 8 ctx_only cases, down from 8 / 30 / 120 for 13 cases. The post-merge pipeline's run list is unchanged: it runs the pre_merge and post_merge blocks of every list, and the cases only moved between the two. Pre-merge therefore has no multi-node disagg perf coverage, which is the gap the FUNCTIONAL-ONLY stage flag was added to fill. Post-merge keeps that coverage, and every moved case keeps its OpenSearch baseline history because baselines match on test case name, GPU type, runtime and branch rather than on stage name. Note in the perf-sanity README that no stage sets FUNCTIONAL-ONLY today. Co-Authored-By: Claude Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 53 ++----------------- .../defs/perf/README_test_perf_sanity.md | 4 +- ...sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml | 17 +----- ...sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml | 17 +----- ...sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml | 15 +----- ...sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml | 15 +----- ...anity_ctx6_node1_gpu4_gen1_node4_gpu16.yml | 15 +----- 7 files changed, 13 insertions(+), 123 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 1121c50e5776..203ddd3e994c 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6357,16 +6357,6 @@ def launchTestJobs(pipeline, testFilter, globalVars) 8, 2 ) - // PerfSanity pre-merge disaggregated - // 2 Nodes - multiNodesSBSAConfigs += buildStageConfigs( - "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4", - "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", - 1, - 8, - 2 - ) // PerfSanity post-merge disaggregated // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( @@ -6381,7 +6371,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", - 5, + 6, 8, 2 ) @@ -6448,21 +6438,13 @@ def launchTestJobs(pipeline, testFilter, globalVars) 8, 2 ) - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8", - "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 1, - 12, - 3 - ) // GB300 PerfSanity post-merge disaggregated // 3 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 3, + 4, 12, 3 ) @@ -6476,21 +6458,12 @@ def launchTestJobs(pipeline, testFilter, globalVars) 5 ) // GB300 GLM-5 disaggregated (ctx DEP2) - // 3 Nodes (pre-merge) - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8", - "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", - 1, - 12, - 3 - ) // 3 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", - 2, + 3, 12, 3 ) @@ -6504,36 +6477,20 @@ def launchTestJobs(pipeline, testFilter, globalVars) 9 ) // 9 Nodes: ctx1 (1 node, 4 GPUs) + gen4 (2 nodes, 8 GPUs each) = 36 GPUs - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8", - "gb300-flex-aws-cmh", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8", - 1, - 36, - 9 - ) multiNodesSBSAConfigs += buildStageConfigs( "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8-Post-Merge", "gb300-flex-aws-cmh", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8", - 1, + 2, 36, 9 ) // 10 Nodes: ctx6 (1 node, 4 GPUs each) + gen1 (4 nodes, 16 GPUs) = 40 GPUs - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16", - "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16", - 1, - 40, - 10 - ) multiNodesSBSAConfigs += buildStageConfigs( "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16", - 1, + 2, 40, 10 ) diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index cefcf2dfe342..13850ff02486 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -359,10 +359,12 @@ Two things to be aware of when you see that warning: See [README_perf_regression_system.md](README_perf_regression_system.md) for the full rule and how to force gating back on. -**`FUNCTIONAL-ONLY` stage-name flag**: A pre-merge stage whose name contains `FUNCTIONAL-ONLY` (e.g. `GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4`) still runs the full perf harness — benchmarks execute, metrics are uploaded to OpenSearch, dashboards update — but perf regressions **do not fail CI**. Only functional failures (build errors, crashes, empty output) fail the stage. +**`FUNCTIONAL-ONLY` stage-name flag**: A pre-merge stage whose name contains `FUNCTIONAL-ONLY` still runs the full perf harness — benchmarks execute, metrics are uploaded to OpenSearch, dashboards update — but perf regressions **do not fail CI**. Only functional failures (build errors, crashes, empty output) fail the stage. Use this for pre-merge stages whose goal is to catch functional regressions on paths that only had post-merge coverage before. It preserves the data-continuity benefit of running in pre-merge (baselines still update from PR data points) without the flakiness cost of gating on the noisier disagg perf numbers. +**No stage currently sets this flag.** Pre-merge perf-sanity is single-node `ctx_only` only, and every `ctx_only` case gates on perf; the multi-node disagg cases that used it are post-merge only. The mechanism is kept for the next pre-merge stage that needs non-gating coverage. + Detection is by substring match on `os.environ["stageName"]` inside `test_perf_sanity.py`; no changes to `perf_regression_utils.py`. ### GPU Hours Calculation diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml index 766a03e52679..e00bcf789d65 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml @@ -1,21 +1,5 @@ version: 0.0.1 l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4: -- condition: - ranges: - # 1 ctx worker with each 1 node and 1 GPUs - # 1 gen worker with each 1 node and 4 GPUs - system_gpu_count: - gte: 8 - lte: 8 - wildcards: - gpu: - - '*gb200*' - terms: - stage: pre_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - condition: ranges: # 1 ctx worker with each 1 node and 1 GPUs @@ -32,6 +16,7 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4: tests: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml index 064ea88bb19a..647d50a4f3bf 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml @@ -1,21 +1,5 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: -- condition: - ranges: - # 1 ctx worker with each 1 node and 2 GPUs - # 1 gen worker with each 2 node and 8 GPUs - system_gpu_count: - gte: 12 - lte: 12 - wildcards: - gpu: - - '*gb300*' - terms: - stage: pre_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - - condition: ranges: # 1 ctx worker with each 1 node and 2 GPUs @@ -32,4 +16,5 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: tests: # glm-5-fp4 - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml index 5caa4a699838..009b84866631 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml @@ -1,19 +1,5 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: -- condition: - ranges: - system_gpu_count: - gte: 12 - lte: 12 - wildcards: - gpu: - - '*gb300*' - terms: - stage: pre_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) - - condition: ranges: # 1 ctx worker with each 1 node and 4 GPUs @@ -29,6 +15,7 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: backend: pytorch tests: # deepseek-r1-fp4 128k8k + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) # nemotron-ultra-v3-fp4 8k64k con64 - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_8k64k_con64_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml index 325ebbf775f3..07ad83e0e1e4 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8.yml @@ -1,19 +1,5 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8: -- condition: - ranges: - system_gpu_count: - gte: 36 - lte: 36 - wildcards: - gpu: - - '*gb300*' - terms: - stage: pre_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - condition: ranges: # 1 ctx worker with 1 node and 4 GPUs @@ -29,4 +15,5 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8: backend: pytorch tests: # deepseek-v4-pro-fp4 8k1k con8 (single-user latency) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml index dc9d1e885e3c..76f981ff6acb 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16.yml @@ -1,19 +1,5 @@ version: 0.0.1 l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16: -- condition: - ranges: - system_gpu_count: - gte: 40 - lte: 40 - wildcards: - gpu: - - '*gb300*' - terms: - stage: pre_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) - - condition: ranges: # 6 ctx workers each with 1 node and 4 GPUs @@ -29,4 +15,5 @@ l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16: backend: pytorch tests: # deepseek-v4-pro-fp4 8k1k con666 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) From 24ccb6c6c77a11e837f53696f59840c5dc92c4f3 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Sun, 13 Sep 2026 23:26:03 -0700 Subject: [PATCH 7/8] [None][infra] Cut pre-merge perf-sanity to the two DeepSeek-V4-Pro ctx_only cases GB300 x4 capacity cannot carry 8 gating cases across every PR, so keep only the two deepseek-v4-pro-fp4 8k1k ctx_only cases (con8 and con666) in pre-merge and move the other six to post-merge. The six that move are the gb300 deepseek-r1 con1/con4096 and glm-5 con1/con1024 cases, plus the gb200 qwen3-235b con1/con1024 pair. Each lands in the post_merge block of the same list, in its model's section, so the post-merge pipeline's run list is unchanged: it runs the pre_merge and post_merge blocks of every list, and these cases only moved between the two. l0_gb200_multi_gpus_perf_sanity loses its pre_merge block entirely and GB200-4_GPUs-PyTorch-PerfSanity-1 goes with it, since a stage whose test-db context selects nothing would just burn a 4-GPU node. GB300 pre-merge drops from two splits to one: two ctx_only cases fit one stage well inside the 6-per-stage convention, and one node per PR instead of three is the point of the change. Pre-merge perf-sanity is now 1 stage / 1 node / 4 GPUs for 2 cases, down from 8 / 30 / 120 for 13 cases before this series. Post-merge stage counts need no change: gb200 carries 18 cases over 6 splits and gb300 18 over 5. Every moved case keeps its OpenSearch baseline history, because baselines match on test case name, GPU type, runtime and branch rather than on stage name. Co-Authored-By: Claude Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 7 +++---- .../l0_gb200_multi_gpus_perf_sanity.yml | 19 ++----------------- .../l0_gb300_multi_gpus_perf_sanity.yml | 10 ++++------ 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9ddebc96086c..5943c8db76ae 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6297,10 +6297,9 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-4_GPUs-PyTorch-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus", 1, 3, 4, 1, true, false], "GB300-4_GPUs-PyTorch-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus", 2, 3, 4, 1, true, false], "GB300-4_GPUs-PyTorch-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus", 3, 3, 4, 1, true, false], - // PerfSanity pre-merge tests - "GB200-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 1, 4], - "GB300-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 2, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 2, 4, 1, true, false], + // PerfSanity pre-merge tests. GB300 x4 capacity is the binding constraint, so + // pre-merge gating is one stage on one node: the two DeepSeek-V4-Pro ctx_only cases. + "GB300-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 1, 4, 1, true, false], // PerfSanity post-merge tests "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 6, 4], "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 6, 4], diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml index 5a02b52747bd..9a9130d1a53d 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml @@ -1,22 +1,5 @@ version: 0.0.1 l0_gb200_multi_gpus_perf_sanity: -- condition: - ranges: - system_gpu_count: - gte: 4 - lte: 4 - wildcards: - gpu: - - '*gb200*' - linux_distribution_name: ubuntu* - cpu: aarch64 - terms: - stage: pre_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - - condition: ranges: system_gpu_count: @@ -56,4 +39,6 @@ l0_gb200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) # qwen3-235b-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index 10bbeccb6b2c..18ad74340214 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -15,12 +15,6 @@ l0_gb300_multi_gpus_perf_sanity: backend: pytorch tests: # ctx_only tests (disagg config) - # deepseek-r1-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - # glm-5-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) # deepseek-v4-pro-fp4 8k1k - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) @@ -41,9 +35,13 @@ l0_gb300_multi_gpus_perf_sanity: tests: # ctx_only tests (disagg config) # deepseek-r1-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90) # deepseek-v4-pro-fp4 8k1k - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-time_breakdown-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (90) From d2af2122f3da98753adeede6b956cb2b460e3e3b Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Mon, 14 Sep 2026 01:48:34 -0700 Subject: [PATCH 8/8] [None][infra] Drop the pre-merge exemption unit tests The perf-sanity gating change does not need unit-test coverage, so revert tests/unittest/others/test_perf_regression_branch.py to main. That file pre-exists on main with 14 test functions covering s_branch history routing, so it is reverted rather than deleted: only the 384 lines this branch added are removed, main's own coverage is untouched, and the file drops out of this PR's diff entirely. This also drops the one-line `pytestmark = pytest.mark.cpu_only` the branch had added. Main's tests in that file therefore remain uncollected by tests/unittest/conftest.py's pytest_ignore_collect, exactly as they are on main today; enabling them is a separate concern from this PR. Co-Authored-By: Claude Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- .../others/test_perf_regression_branch.py | 401 +----------------- 1 file changed, 17 insertions(+), 384 deletions(-) diff --git a/tests/unittest/others/test_perf_regression_branch.py b/tests/unittest/others/test_perf_regression_branch.py index 6b39482c458f..e7e89e1adc63 100644 --- a/tests/unittest/others/test_perf_regression_branch.py +++ b/tests/unittest/others/test_perf_regression_branch.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for perf_regression_utils history routing and pre-merge gating. +"""Tests for how perf_regression_utils handles s_branch. -Three concerns. The first two fail open (a green run with no regression check) -if they break; the third fails closed (every PR blocked by a regression it did -not introduce): +Two concerns, both of which fail open (a green run with no regression check) if +they break: 1. s_branch is read from globalVars, never scraped from the job URL. The Jenkins folder segment in a job URL (/job/LLM/job//) names the folder the job @@ -29,16 +28,6 @@ records, so a pre-merge run querying its own "github-pr-" branch matches nothing and its regression check silently becomes a no-op. The queries must see the baseline branch while the uploaded document keeps the real one. - -3. A pre-merge regression is exempt when the latest post-merge record for the - same case already misses the same gate. Both pipelines compare against one - shared baseline, so a regression landed on main makes every subsequent PR - measure the same regressed value and fail. The exemption re-evaluates main's - own latest value against that baseline at the pre-merge threshold -- not the - b_is_regression the post-merge run recorded at its own tighter threshold -- - so it is exactly as wide as the failure it prevents. Per test case; the - pre-merge document still uploads b_is_regression as measured, so nothing is - hidden from the DB. """ import importlib.util @@ -49,8 +38,6 @@ import pytest -pytestmark = pytest.mark.cpu_only - _REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] _MODULE_PATH = _REPO_ROOT / "tests" / "integration" / "defs" / "perf" / "perf_regression_utils.py" @@ -218,32 +205,14 @@ def test_unparsable_global_vars_does_not_raise(monkeypatch): _MATCH_KEYS = ["s_test_case_name", "s_gpu_type", "s_runtime", "s_branch"] -_METRIC = "d_output_token_throughput" - -# A baseline high enough that _REGRESSED_VALUE falls below the 10% pre-merge -# threshold and _CLEAN_VALUE stays above it. -_BASELINE = 1000.0 -_REGRESSED_VALUE = 800.0 -_CLEAN_VALUE = 950.0 - -# Tier A: baseline supplied directly, so no timestamps or percentile maths are -# involved in deciding whether a case is regressive. Cover several cmd_idx so a -# multi-case test resolves a baseline for every one of them; a case without a -# baseline is silently skipped rather than judged. -_TIER_A_BASELINE = {idx: {"d_baseline_output_token_throughput": _BASELINE} for idx in range(4)} - - -def _new_data_dict(value: float = 1234.5, count: int = 1) -> dict[int, dict[str, object]]: +def _new_data_dict() -> dict[int, dict[str, object]]: return { - idx: { - "s_test_case_name": f"example_model_fp8_tp8-con32_iter10_1k1k_{idx}" - if count > 1 - else "example_model_fp8_tp8-con32_iter10_1k1k", + 0: { + "s_test_case_name": "example_model_fp8_tp8-con32_iter10_1k1k", "s_gpu_type": "b200", "s_runtime": "aggr_server", - _METRIC: value, + "d_output_token_throughput": 1234.5, } - for idx in range(count) } @@ -252,24 +221,12 @@ def _run_pipeline( build_branch: str, job_url: str, match_keys: list[str] | None = None, - history: tuple[object, object, object] | None = None, - value: float = 1234.5, - count: int = 1, - expect_error: bool = False, - fail_on_regression: bool | None = None, - minimize: bool = False, ) -> dict[str, dict[int, str]]: - """Run the real pipeline, recording what each seam observes. + """Run the real pipeline, recording the s_branch each seam observes. Only the three OpenSearch seams are replaced. Everything between them -- get_job_info, the enrichment loop, the branch routing, the regression pass -- is the production code path, so a regression in the wiring shows up here. - - ``history`` is the (latest, baseline_threshold, history) triple the stubbed - get_history_data returns, letting a test drive the exemption decision. - ``expect_error`` wraps the call in pytest.raises(RuntimeError) and records - the message; it is never a blanket except, because post_new_perf_data raises - RuntimeError too and swallowing that would make these tests vacuous. """ observed: dict[str, dict[int, str]] = {} @@ -288,14 +245,10 @@ def fake_get_history_data( observed["history_query_names"] = { idx: d["s_test_case_name"] for idx, d in data_dict.items() } - return ({}, {}, {}) if history is None else history + return {}, {}, {} def fake_post_new_perf_data(data_dict: dict[int, dict[str, object]]) -> None: observed["uploaded"] = {idx: d["s_branch"] for idx, d in data_dict.items()} - # Snapshot whole documents: the upload (step 8) runs before the - # regression check (step 9), so this is the document as posted even when - # the check goes on to raise. - observed["uploaded_docs"] = {idx: dict(d) for idx, d in data_dict.items()} monkeypatch.setattr(_perf_regression_utils, "get_common_values", fake_get_common_values) monkeypatch.setattr(_perf_regression_utils, "get_history_data", fake_get_history_data) @@ -307,28 +260,15 @@ def fake_post_new_perf_data(data_dict: dict[int, dict[str, object]]) -> None: } monkeypatch.setenv("globalVars", json.dumps(payload)) - new_data_dict = _new_data_dict(value=value, count=count) - - def _call() -> None: - _perf_regression_utils.process_and_upload_test_results( - new_data_dict, - match_keys if match_keys is not None else _MATCH_KEYS, - maximize_metrics=[] if minimize else [_METRIC], - minimize_metrics=[_METRIC] if minimize else [], - regression_metrics=[_METRIC], - fail_on_regression=fail_on_regression, - ) - - observed["error"] = None - if expect_error: - with pytest.raises(RuntimeError) as excinfo: - _call() - observed["error"] = str(excinfo.value) - else: - _call() - + new_data_dict = _new_data_dict() + _perf_regression_utils.process_and_upload_test_results( + new_data_dict, + match_keys if match_keys is not None else _MATCH_KEYS, + maximize_metrics=["d_output_token_throughput"], + minimize_metrics=[], + regression_metrics=["d_output_token_throughput"], + ) observed["new_data_dict"] = {idx: d["s_branch"] for idx, d in new_data_dict.items()} - observed["final_docs"] = {idx: dict(d) for idx, d in new_data_dict.items()} return observed @@ -393,310 +333,3 @@ def test_no_substitution_when_branch_is_not_a_match_key(monkeypatch: pytest.Monk assert observed["history_query"] == {0: "github-pr-18408"} assert observed["uploaded"] == {0: "github-pr-18408"} - - -# --------------------------------------------------------------------------- # -# A pre-merge regression fails the stage (positive controls) -# -# These hold both before and after the exemption exists. Without them, every -# "does not raise" assertion below could pass because nothing ever raises. -# --------------------------------------------------------------------------- # - - -def _history(latest: object, baseline: object = None, history: object = None): - """Build the get_history_data triple, defaulting to the Tier A baseline.""" - return ( - latest, - _TIER_A_BASELINE if baseline is None else baseline, - {} if history is None else history, - ) - - -def _pre_merge(monkeypatch: pytest.MonkeyPatch, **kwargs): - return _run_pipeline(monkeypatch, "github-pr-18408", _GITHUB_PR_JOB_URL, **kwargs) - - -# What the latest post-merge record for case 0 measured. The exemption -# re-evaluates these against the same baseline at the same pre-merge threshold, -# so what matters is the value, not any verdict the post-merge run recorded. -_CLEAN_LATEST = {0: {_METRIC: _BASELINE}} # main on baseline -_REGRESSED_LATEST = {0: {_METRIC: _REGRESSED_VALUE}} # 20% down: misses the gate -_WITHIN_GATE_LATEST = {0: {_METRIC: _CLEAN_VALUE}} # 5% down: still makes the gate - -# The opening words of the sentence prepare_regressive_test_cases appends to -# s_regression_info when it exempts a case. Asserted both present and absent -# below, so a reworded annotation fails the positive test rather than quietly -# making every "unannotated" assertion vacuous. -_EXEMPTION_NOTE = "Not failing this stage" - - -def test_pre_merge_regression_fails_the_stage(monkeypatch: pytest.MonkeyPatch) -> None: - """The baseline case: main is healthy, the PR regresses, the stage fails.""" - observed = _pre_merge( - monkeypatch, - history=_history(_CLEAN_LATEST), - value=_REGRESSED_VALUE, - expect_error=True, - ) - - assert _METRIC in observed["error"] - assert observed["error"].strip() - assert observed["uploaded_docs"][0]["b_is_regression"] is True - - -def test_pre_merge_within_threshold_passes(monkeypatch: pytest.MonkeyPatch) -> None: - observed = _pre_merge(monkeypatch, history=_history(_CLEAN_LATEST), value=_CLEAN_VALUE) - - assert observed["uploaded_docs"][0]["b_is_regression"] is False - - -def test_pre_merge_exactly_at_threshold_passes(monkeypatch: pytest.MonkeyPatch) -> None: - """Regression is a strict inequality: value == baseline*(1-t) is not one.""" - observed = _pre_merge(monkeypatch, history=_history(_CLEAN_LATEST), value=_BASELINE * 0.9) - - assert observed["uploaded_docs"][0]["b_is_regression"] is False - - -# --------------------------------------------------------------------------- # -# A regressed main exempts the pre-merge gate -# --------------------------------------------------------------------------- # - - -def test_regressed_main_exempts_the_pre_merge_gate(monkeypatch: pytest.MonkeyPatch) -> None: - """The whole point: main already misses the gate, so the PR is not blamed.""" - observed = _pre_merge( - monkeypatch, - history=_history(_REGRESSED_LATEST), - value=_REGRESSED_VALUE, - ) - - # No exception, and specifically not RuntimeError("") from an empty message. - assert observed["error"] is None - # The measurement is still recorded truthfully -- nothing is hidden. - assert observed["uploaded_docs"][0]["b_is_regression"] is True - assert _METRIC in observed["uploaded_docs"][0]["s_regression_info"] - assert _EXEMPTION_NOTE in observed["uploaded_docs"][0]["s_regression_info"] - - -def test_main_within_the_gate_still_fails_the_pre_merge_stage( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The rule that separates this from a post-merge-threshold exemption. - - main is down 5% -- enough for the post-merge run to have recorded - b_is_regression, but not enough to fail the 10% pre-merge gate. A PR - reproducing main's value would therefore never have been blocked, so there - is nothing to exempt: a PR that goes on to regress 20% must still fail. - """ - observed = _pre_merge( - monkeypatch, - history=_history(_WITHIN_GATE_LATEST), - value=_REGRESSED_VALUE, - expect_error=True, - ) - - assert _METRIC in observed["error"] - - -def test_a_numeric_string_from_opensearch_still_exempts( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """History values arrive untrusted; a numeric string is still a number.""" - observed = _pre_merge( - monkeypatch, - history=_history({0: {_METRIC: str(_REGRESSED_VALUE)}}), - value=_REGRESSED_VALUE, - ) - - assert observed["error"] is None - - -@pytest.mark.parametrize( - "latest, expect_error", - [ - # A minimize metric regresses upward, so main must be ABOVE baseline by - # more than the threshold to exempt. 1200 is +20%, 1050 only +5%. - pytest.param({0: {_METRIC: 1200.0}}, False, id="main-above-the-gate-exempt"), - pytest.param({0: {_METRIC: 1050.0}}, True, id="main-within-the-gate-fails"), - pytest.param({0: {_METRIC: 800.0}}, True, id="main-better-than-baseline-fails"), - ], -) -def test_the_exemption_respects_a_minimize_metric( - monkeypatch: pytest.MonkeyPatch, latest: object, expect_error: bool -) -> None: - """A latency metric regresses upward; the direction must not be inverted. - - The 800 row is the one that catches an inverted comparison: for a minimize - metric that is main comfortably BETTER than baseline, which must never exempt. - """ - observed = _pre_merge( - monkeypatch, - history=_history(latest), - value=1200.0, - minimize=True, - expect_error=expect_error, - ) - - assert observed["uploaded_docs"][0]["b_is_regression"] is True - - -def test_exemption_is_per_test_case(monkeypatch: pytest.MonkeyPatch) -> None: - """Case 0's exemption must not cover case 1, and vice versa.""" - observed = _pre_merge( - monkeypatch, - history=_history({0: {_METRIC: _REGRESSED_VALUE}, 1: {_METRIC: _BASELINE}}), - value=_REGRESSED_VALUE, - count=2, - expect_error=True, - ) - - assert "example_model_fp8_tp8-con32_iter10_1k1k_1" in observed["error"] - assert "example_model_fp8_tp8-con32_iter10_1k1k_0" not in observed["error"] - # Both are still recorded as regressive; only the gate differs. - assert observed["uploaded_docs"][0]["b_is_regression"] is True - assert observed["uploaded_docs"][1]["b_is_regression"] is True - - -_TIER_B_HISTORY = { - 0: [ - {"@timestamp": "2026-09-01T00:00:00Z", _METRIC: _BASELINE}, - {"@timestamp": "2026-09-02T00:00:00Z", _METRIC: _BASELINE}, - {"@timestamp": "2026-09-03T00:00:00Z", _METRIC: _BASELINE}, - ] -} - - -@pytest.mark.parametrize( - "latest, expect_error", - [ - pytest.param(_REGRESSED_LATEST, False, id="regressed-main-exempt"), - pytest.param(_CLEAN_LATEST, True, id="clean-main-fails"), - ], -) -def test_exemption_holds_on_the_rolling_baseline_path( - monkeypatch: pytest.MonkeyPatch, latest: object, expect_error: bool -) -> None: - """Nothing in-repo seeds d_baseline_*, so production uses this path. - - The paired clean-main case proves the Tier B fixture really does produce a - regression, rather than the exemption passing because no baseline resolved. - """ - observed = _pre_merge( - monkeypatch, - history=(latest, {0: None}, _TIER_B_HISTORY), - value=_REGRESSED_VALUE, - expect_error=expect_error, - ) - - assert observed["uploaded_docs"][0]["b_is_regression"] is True - - -# --------------------------------------------------------------------------- # -# The exemption must not fire -# --------------------------------------------------------------------------- # - - -@pytest.mark.parametrize( - "latest", - [ - pytest.param({0: None}, id="null-record"), - pytest.param({}, id="no-record-for-this-case"), - pytest.param({0: {}}, id="metric-absent"), - pytest.param({0: {_METRIC: _BASELINE}}, id="on-baseline"), - pytest.param({0: {_METRIC: _CLEAN_VALUE}}, id="regressed-but-within-the-gate"), - # Strict inequality, matching how the pre-merge verdict itself is taken. - pytest.param({0: {_METRIC: _BASELINE * 0.9}}, id="exactly-at-the-threshold"), - # A broken post-merge record must leave the gate armed, not disarm it. - pytest.param({0: {_METRIC: 0}}, id="zero"), - pytest.param({0: {_METRIC: -5.0}}, id="negative"), - pytest.param({0: {_METRIC: None}}, id="null-value"), - pytest.param({0: {_METRIC: "not-a-number"}}, id="non-numeric"), - pytest.param({0: {_METRIC: []}}, id="empty-list"), - # The recorded verdict is no longer consulted: it was taken at the - # post-merge threshold, and with no value there is nothing to re-evaluate. - pytest.param({0: {"b_is_regression": True}}, id="verdict-without-a-value"), - ], -) -def test_only_a_value_missing_the_same_gate_exempts( - monkeypatch: pytest.MonkeyPatch, latest: object -) -> None: - """Anything else about the latest post-merge record still fails the stage.""" - observed = _pre_merge( - monkeypatch, history=_history(latest), value=_REGRESSED_VALUE, expect_error=True - ) - - assert _METRIC in observed["error"] - - -def test_an_older_regressed_record_does_not_exempt(monkeypatch: pytest.MonkeyPatch) -> None: - """The rule reads the latest post-merge record, not any record.""" - observed = _pre_merge( - monkeypatch, - history=_history( - _CLEAN_LATEST, - history={ - 0: [ - {"@timestamp": "2026-08-01T00:00:00Z", _METRIC: _REGRESSED_VALUE}, - {"@timestamp": "2026-08-02T00:00:00Z", _METRIC: _REGRESSED_VALUE}, - ] - }, - ), - value=_REGRESSED_VALUE, - expect_error=True, - ) - - assert _METRIC in observed["error"] - - -# --------------------------------------------------------------------------- # -# Everything else is untouched -# --------------------------------------------------------------------------- # - - -def test_post_merge_is_unaffected_and_unannotated(monkeypatch: pytest.MonkeyPatch) -> None: - """Post-merge already only warns; it must not gain an exemption note.""" - observed = _run_pipeline( - monkeypatch, - "main", - _RELEASE_BUILD_JOB_URL, - history=_history(_REGRESSED_LATEST), - value=_REGRESSED_VALUE, - ) - - assert observed["error"] is None - assert observed["uploaded_docs"][0]["b_is_regression"] is True - assert _EXEMPTION_NOTE not in observed["uploaded_docs"][0]["s_regression_info"] - - -def test_functional_only_stage_is_unannotated(monkeypatch: pytest.MonkeyPatch) -> None: - """fail_on_regression=False already passes; no note belongs on a healthy main.""" - observed = _pre_merge( - monkeypatch, - history=_history(_CLEAN_LATEST), - value=_REGRESSED_VALUE, - fail_on_regression=False, - ) - - assert observed["error"] is None - assert observed["uploaded_docs"][0]["b_is_regression"] is True - assert _EXEMPTION_NOTE not in observed["uploaded_docs"][0]["s_regression_info"] - - -def test_history_query_failure_still_skips_the_check(monkeypatch: pytest.MonkeyPatch) -> None: - """A None triple means the query failed: soft-fail, never a TypeError.""" - observed = _pre_merge(monkeypatch, history=(None, None, None), value=_REGRESSED_VALUE) - - assert observed["error"] is None - assert "s_regression_info" not in observed["uploaded_docs"][0] - - -def test_exemption_adds_no_field_to_the_document(monkeypatch: pytest.MonkeyPatch) -> None: - """The exemption lives outside new_data_dict, so no new OpenSearch field. - - post_new_perf_data uploads each record dict wholesale, so any key added to - it would silently become a new field in the index. - """ - exempt = _pre_merge(monkeypatch, history=_history(_REGRESSED_LATEST), value=_REGRESSED_VALUE) - clean = _pre_merge(monkeypatch, history=_history(_CLEAN_LATEST), value=_CLEAN_VALUE) - - assert set(exempt["uploaded_docs"][0]) == set(clean["uploaded_docs"][0])