diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index e271402e6baa..a3716f4ad871 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -98,7 +98,6 @@ examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_go examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-cuda-graph] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-fp8-blockwise] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_wan.py::test_wan_feature_accuracy_against_golden[wan22-nvfp4] SKIP (https://nvbugs/6572800) -full:A10/unittest/scripts/test_perf_sanity_helpers.py::test_add_perf_metric_value_omits_the_family_outside_gen_only SKIP (https://nvbugs/6770503) full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:A100/disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6581064) @@ -292,12 +291,6 @@ unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[cuda_graph_config0] SKIP unittest/llmapi/test_memory_profiling.py::test_profile_kvcache SKIP (https://nvbugs/5580781) unittest/llmapi/test_mpi_session.py::test_remote_mpi_session[flashinfer_workspace] SKIP (https://nvbugs/6758990) unittest/others/test_vendor_sources.py::test_exact_create_check_digest_and_sync SKIP (https://nvbugs/6746167) -unittest/scripts/test_perf_sanity_helpers.py::test_every_device_step_time_metric_is_a_minimize_metric SKIP (https://nvbugs/6770503) -unittest/scripts/test_perf_sanity_helpers.py::test_every_gated_metric_is_actually_emitted SKIP (https://nvbugs/6770503) -unittest/scripts/test_perf_sanity_helpers.py::test_every_written_line_parses_and_none_shadows_another SKIP (https://nvbugs/6770503) -unittest/scripts/test_perf_sanity_helpers.py::test_sentinel_timeout_falls_back_to_current_gen_logs SKIP (https://nvbugs/6770503) -unittest/usage/test_llmapi_config_telemetry_docs.py::test_build_capture_manifest_matches_committed_golden SKIP (https://nvbugs/6786176) -unittest/usage/test_llmapi_config_telemetry_docs.py::test_renderer_emits_table_from_committed_golden SKIP (https://nvbugs/6786197) verl/test_verl_cases.py::test_async_generate SKIP (https://nvbugs/6683838) verl/test_verl_cases.py::test_async_memory_management SKIP (https://nvbugs/6683838) verl/test_verl_cases.py::test_init_without_device_mesh SKIP (https://nvbugs/6683838) diff --git a/tests/unittest/scripts/test_perf_sanity_helpers.py b/tests/unittest/scripts/test_perf_sanity_helpers.py index 074a3d731b5b..14912a3a4455 100644 --- a/tests/unittest/scripts/test_perf_sanity_helpers.py +++ b/tests/unittest/scripts/test_perf_sanity_helpers.py @@ -850,14 +850,15 @@ def test_sentinel_timeout_falls_back_to_current_gen_logs( "wait_for_gen_log_sentinels", lambda self: False, ) - parse_calls: list[tuple[str, int, list[int]]] = [] + parse_calls: list[tuple[str, int, list[int], object]] = [] def parse_device_step_time( output_dir: str, num_gen_servers: int, start_offsets: list[int], + end_offsets: list[int] | None = None, ) -> perf_sanity._DeviceStepTimeStats: - parse_calls.append((output_dir, num_gen_servers, start_offsets)) + parse_calls.append((output_dir, num_gen_servers, start_offsets, end_offsets)) return perf_sanity._DeviceStepTimeStats(mean=7.25, median=7.2, std=0.115, p75=7.3, p99=7.42) monkeypatch.setattr( @@ -868,7 +869,7 @@ def parse_device_step_time( commands._append_gen_worker_device_step_time(pending, outputs) - assert parse_calls == [(str(tmp_path), 2, [10, 20])] + assert parse_calls == [(str(tmp_path), 2, [10, 20], None)] expected = ( "Average Per Iter Device Step Time (ms): 7.25\n" "Median Per Iter Device Step Time (ms): 7.2000\n" @@ -1306,7 +1307,7 @@ def test_every_written_line_parses_and_none_shadows_another( metrics: dict[str, float] = {} for line in outputs[0].split("\n"): - for name, regex in perf_sanity.GEN_ONLY_PERF_METRIC_LOG_QUERIES.items(): + for name, regex in perf_sanity.DEVICE_STEP_TIME_LOG_QUERIES.items(): if name in metrics: continue match = regex.search(line) @@ -1314,37 +1315,44 @@ def test_every_written_line_parses_and_none_shadows_another( metrics[name] = float(match.group(1)) break - assert set(metrics) == set(perf_sanity.GEN_ONLY_DEVICE_STEP_TIME_METRICS) + assert set(metrics) == set(perf_sanity.DEVICE_STEP_TIME_METRICS) assert metrics["mean_gen_worker_per_iter_device_step_time"] == pytest.approx(7.17, abs=0.01) assert metrics["std_gen_worker_per_iter_device_step_time"] > 0.0 def test_every_device_step_time_metric_is_a_minimize_metric() -> None: """A metric absent from both lists raises ValueError in check_regression.""" - for name in perf_sanity.GEN_ONLY_DEVICE_STEP_TIME_METRICS: + for name in perf_sanity.DEVICE_STEP_TIME_METRICS: assert f"d_{name}" in perf_sanity.MINIMIZE_METRICS -def test_add_perf_metric_value_skips_absent_statistics() -> None: - """TypeCheckForOpenSearchDB rejects both None and int for a d_ key.""" +@pytest.mark.parametrize("benchmark_mode", perf_sanity.DEVICE_STEP_TIME_MODES) +def test_add_perf_metric_value_skips_absent_statistics(benchmark_mode: str) -> None: + """TypeCheckForOpenSearchDB rejects both None and int for a d_ key. + + Parametrized over every mode in DEVICE_STEP_TIME_MODES -- not just + gen_only -- so this pins the positive side of the widening: if the tuple + is ever narrowed back to ("gen_only",), this test starts failing for the + dropped mode instead of staying silently green. + """ metrics = dict.fromkeys(perf_sanity.PERF_METRIC_LOG_QUERIES, 1.0) metrics["mean_gen_worker_per_iter_device_step_time"] = 7 new_data: dict = {} - perf_sanity.add_perf_metric_value(new_data, metrics, False, "gen_only") + perf_sanity.add_perf_metric_value(new_data, metrics, False, benchmark_mode) assert new_data["d_mean_gen_worker_per_iter_device_step_time"] == 7.0 assert isinstance(new_data["d_mean_gen_worker_per_iter_device_step_time"], float) assert "d_p99_gen_worker_per_iter_device_step_time" not in new_data -def test_add_perf_metric_value_omits_the_family_outside_gen_only() -> None: - """e2e and ctx_only never emit these lines, so they must not be uploaded.""" +def test_add_perf_metric_value_omits_the_family_outside_device_step_time_modes() -> None: + """ctx_only isn't in DEVICE_STEP_TIME_MODES, so it must not be uploaded.""" metrics = dict.fromkeys(perf_sanity.PERF_METRIC_LOG_QUERIES, 1.0) metrics["mean_gen_worker_per_iter_device_step_time"] = 7.0 new_data: dict = {} - perf_sanity.add_perf_metric_value(new_data, metrics, False, "e2e") + perf_sanity.add_perf_metric_value(new_data, metrics, False, "ctx_only") assert not [key for key in new_data if "gen_worker_per_iter" in key] @@ -1362,5 +1370,5 @@ def test_every_gated_metric_is_checkable() -> None: def test_every_gated_metric_is_actually_emitted() -> None: """A gated metric the log never carries is skipped by 'not in new_data'.""" - emitted = {f"d_{name}" for name in perf_sanity.GEN_ONLY_DEVICE_STEP_TIME_METRICS} + emitted = {f"d_{name}" for name in perf_sanity.DEVICE_STEP_TIME_METRICS} assert set(perf_sanity.GEN_ONLY_REGRESSION_METRICS) <= emitted