From d4a51228c398b47f2aa7283b1f093713bf3ac887 Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:57:46 -0700 Subject: [PATCH 1/5] [NVBUG-6481034][fix] Align perf launcher with pytest shard Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- jenkins/scripts/perf/submit.py | 195 ++++++++++++++++++--- tests/unittest/scripts/test_perf_submit.py | 60 +++++++ 2 files changed, 233 insertions(+), 22 deletions(-) diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py index 561f7425e469..5ee06ee187bf 100755 --- a/jenkins/scripts/perf/submit.py +++ b/jenkins/scripts/perf/submit.py @@ -33,9 +33,12 @@ """ import argparse +import heapq +import json import math import os import re +import shlex import sys import yaml @@ -63,26 +66,166 @@ def _import_precheck_config(llm_src): # --------------------------------------------------------------------------- # # Test list parsing # --------------------------------------------------------------------------- # -def parse_test_case_name(test_list_path, llm_src, split_group=0): - """Parse the selected line of the test list. - - Returns (config_yaml_path, server_name, benchmark_mode, runtime_mode). - See the module docstring for the supported test name shapes. - """ +def _read_test_list_lines(test_list_path): with open(test_list_path, "r") as f: lines = [line.strip() for line in f if line.strip()] - if not lines: raise ValueError(f"Test list is empty: {test_list_path}") + return lines + + +def _pytest_command_tokens(script_prefix_lines): + pytest_command_line = next( + (line for line in script_prefix_lines if "export pytestCommand=" in line), "" + ) + if not pytest_command_line: + return [] + command = pytest_command_line.split("=", 1)[1].strip() + if len(command) >= 2 and command[0] == command[-1] and command[0] in ('"', "'"): + command = command[1:-1] + return shlex.split(command) + + +def _pytest_option(tokens, option): + for index, token in enumerate(tokens): + if token == option: + return tokens[index + 1] if index + 1 < len(tokens) else None + if token.startswith(f"{option}="): + return token.split("=", 1)[1] + return None + + +def _test_nodeid(test_line): + """Strip test-list markers from a line, matching pytest's selected nodeid.""" + return re.split( + r"\s+(?:XFAIL|SKIP|UNSTABLE|TIMEOUT)(?:\s|$)", + test_line, + maxsplit=1, + )[0] + + +def _load_pytest_split_durations(tokens, llm_src): + durations_option = _pytest_option(tokens, "--durations-path") + if durations_option: + durations_path = durations_option + if not os.path.exists(durations_path): + durations_path = os.path.join( + llm_src, + "tests", + "integration", + "defs", + os.path.basename(durations_option), + ) + else: + durations_path = os.path.join(llm_src, "tests", "integration", "defs", ".test_durations") + + try: + with open(durations_path, "r") as durations_file: + durations = json.load(durations_file) + except FileNotFoundError: + durations = {} + if isinstance(durations, list): + durations = dict(durations) + if not isinstance(durations, dict): + raise ValueError(f"Invalid pytest-split durations file: {durations_path}") + return durations, durations_path + + +def _select_least_duration_group(lines, durations, splits, group): + """Mirror pytest-split's LeastDurationAlgorithm exactly.""" + if splits < 1: + raise ValueError(f"pytest --splits must be >= 1, got {splits}") + if group < 1 or group > splits: + raise ValueError(f"pytest --group must be in [1, {splits}], got {group}") + + nodeids = [_test_nodeid(line) for line in lines] + relevant_durations = { + nodeid: float(durations[nodeid]) for nodeid in nodeids if nodeid in durations + } + average_duration = ( + sum(relevant_durations.values()) / len(relevant_durations) if relevant_durations else 1.0 + ) + items = [ + (line, nodeid, relevant_durations.get(nodeid, average_duration), original_index) + for original_index, (line, nodeid) in enumerate(zip(lines, nodeids)) + ] - if split_group > 0: + # pytest-split first sorts by item name, then performs a stable descending + # duration sort. It greedily places each item in the least-loaded group. + items.sort(key=lambda item: item[1]) + items.sort(key=lambda item: item[2], reverse=True) + selected = [[] for _ in range(splits)] + group_heap = [(0.0, group_index) for group_index in range(splits)] + heapq.heapify(group_heap) + for line, _nodeid, duration, original_index in items: + group_duration, group_index = heapq.heappop(group_heap) + selected[group_index].append((original_index, line)) + heapq.heappush(group_heap, (group_duration + duration, group_index)) + + return [line for _original_index, line in sorted(selected[group - 1], key=lambda item: item[0])] + + +def select_test_case_line(test_list_path, llm_src, script_prefix_lines, split_group=0): + """Select the same test as the pytest-split shard in ``pytestCommand``.""" + lines = _read_test_list_lines(test_list_path) + if split_group <= 0: + return lines[0] + + tokens = _pytest_command_tokens(script_prefix_lines) + splits_option = _pytest_option(tokens, "--splits") + group_option = _pytest_option(tokens, "--group") + algorithm = _pytest_option(tokens, "--splitting-algorithm") + if splits_option is None or group_option is None: if split_group > len(lines): raise ValueError( f"split_group {split_group} exceeds number of tests in test list ({len(lines)})" ) - line = lines[split_group - 1] + return lines[split_group - 1] + if algorithm != "least_duration": + raise ValueError( + "Multi-node perf launcher only supports pytest-split's least_duration " + f"algorithm, got {algorithm!r}" + ) + + splits = int(splits_option) + pytest_group = int(group_option) + if pytest_group != split_group: + raise ValueError( + f"submit.py split_group={split_group} disagrees with pytest --group={pytest_group}" + ) + durations, durations_path = _load_pytest_split_durations(tokens, llm_src) + selected = _select_least_duration_group(lines, durations, splits, pytest_group) + if len(selected) != 1: + raise ValueError( + "Multi-node perf launch requires exactly one test in each pytest-split " + f"group, but group {pytest_group}/{splits} selected {len(selected)} tests " + f"using {durations_path}: {selected}" + ) + print( + f"Selected pytest-split group {pytest_group}/{splits} test using " + f"{durations_path}: {_test_nodeid(selected[0])}" + ) + return selected[0] + + +def parse_test_case_name(test_list_path, llm_src, split_group=0, selected_line=None): + """Parse the selected line of the test list. + + Returns (config_yaml_path, server_name, benchmark_mode, runtime_mode). + See the module docstring for the supported test name shapes. + """ + if selected_line is not None: + line = selected_line else: - line = lines[0] + lines = _read_test_list_lines(test_list_path) + if split_group > 0: + if split_group > len(lines): + raise ValueError( + f"split_group {split_group} exceeds number of tests in test list ({len(lines)})" + ) + line = lines[split_group - 1] + else: + line = lines[0] if "[" not in line or "]" not in line: raise ValueError(f"Invalid test list format. Expected name with brackets: {line}") @@ -484,7 +627,9 @@ def main(): "--split-group", type=int, default=0, - help="1-indexed split group id. Selects the N-th test from the test list.", + help=( + "1-indexed pytest-split group id. Selects the same duration-balanced test as pytest." + ), ) parser.add_argument("--stage-name", default="", help="Stage name (for logging / GPU detect)") parser.add_argument( @@ -497,19 +642,29 @@ def main(): args = parser.parse_args() + with open(args.script_prefix, "r") as f: + script_prefix_content = f.read() + script_prefix_lines = script_prefix_content.split("\n") + + selected_test_line = select_test_case_line( + args.test_list, + args.llm_src, + script_prefix_lines, + args.split_group, + ) config_yaml, server_name, benchmark_mode, runtime_mode = parse_test_case_name( - args.test_list, args.llm_src, args.split_group + args.test_list, + args.llm_src, + args.split_group, + selected_line=selected_test_line, ) with open(config_yaml, "r") as f: config = yaml.safe_load(f) - # Recover test_case_name (the bracketed pytest test id) for the per-test - # output dir — same line/split logic as parse_test_case_name. - with open(args.test_list, "r") as f: - lines = [ln.strip() for ln in f if ln.strip()] - sel = lines[args.split_group - 1] if args.split_group > 0 else lines[0] - test_case_name = sel.split("[")[-1].split("]")[0] if "[" in sel else "" + test_case_name = ( + selected_test_line.split("[")[-1].split("]")[0] if "[" in selected_test_line else "" + ) hardware_config = get_hardware_config(config, runtime_mode, benchmark_mode, server_name) env_config = get_env_config(config, runtime_mode, benchmark_mode, server_name) @@ -522,10 +677,6 @@ def main(): print(f"Environment configuration: {env_config}") print(f"Benchmark configuration: {benchmark_config}") - with open(args.script_prefix, "r") as f: - script_prefix_content = f.read() - script_prefix_lines = script_prefix_content.split("\n") - with open(args.srun_args, "r") as f: srun_args_content = f.read() srun_args_lines = srun_args_content.split() diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py index 059b5d792527..0ebc92be3c4d 100644 --- a/tests/unittest/scripts/test_perf_submit.py +++ b/tests/unittest/scripts/test_perf_submit.py @@ -15,6 +15,7 @@ # limitations under the License. import importlib.util +import json from pathlib import Path from types import ModuleType @@ -62,6 +63,11 @@ def test_get_benchmark_config_accepts_positive_integer(submit_module: ModuleType assert benchmark_config["concurrency"] == int(concurrency) +@pytest.fixture +def ci_submit_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + return _load_module(SUBMIT_PATHS[0], monkeypatch) + + @pytest.mark.parametrize( "concurrency", (True, 1.5, [], {}, "0", 0, "-1", -1, "1.5", "not-an-integer", None), @@ -102,3 +108,57 @@ def test_example_worker_environment_exports_positive_concurrency(example_submit_ ) assert worker_environment["TLLM_BENCHMARK_REQ_QUEUES_SIZE"] == "4301" + + +def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_module, tmp_path): + test_lines = [ + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1] TIMEOUT (90)", + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25] TIMEOUT (90)", + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1] TIMEOUT (90)", + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25] TIMEOUT (90)", + ] + test_list_path = tmp_path / "test_list.txt" + test_list_path.write_text("\n".join(test_lines), encoding="utf-8") + + durations_dir = tmp_path / "tests" / "integration" / "defs" + durations_dir.mkdir(parents=True) + durations = { + ci_submit_module._test_nodeid(test_lines[0]): 836.268, + ci_submit_module._test_nodeid(test_lines[1]): 1462.754, + ci_submit_module._test_nodeid(test_lines[2]): 2211.1548, + ci_submit_module._test_nodeid(test_lines[3]): 2548.912, + } + (durations_dir / ".test_durations").write_text(json.dumps(durations), encoding="utf-8") + script_prefix_lines = [ + 'export pytestCommand="pytest --splitting-algorithm least_duration ' + "--splits 4 --group 3 " + '--durations-path /remote/tests/integration/defs/.test_durations"' + ] + + selected = ci_submit_module.select_test_case_line( + test_list_path, + tmp_path, + script_prefix_lines, + split_group=3, + ) + + assert selected == test_lines[1] + + +def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path): + test_list_path = tmp_path / "test_list.txt" + test_list_path.write_text( + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n", + encoding="utf-8", + ) + script_prefix_lines = [ + 'export pytestCommand="pytest --splitting-algorithm least_duration --splits 1 --group 1"' + ] + + with pytest.raises(ValueError, match="disagrees with pytest --group"): + ci_submit_module.select_test_case_line( + test_list_path, + tmp_path, + script_prefix_lines, + split_group=2, + ) From 9b6d78c4eaaa7160bd4bc7fad4bd9d5499fc63d6 Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:27:01 -0700 Subject: [PATCH 2/5] [NVBUG-6481034][fix] Address shard selection review feedback Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- jenkins/scripts/perf/submit.py | 6 ++++-- requirements-dev.txt | 2 +- tests/unittest/scripts/test_perf_submit.py | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py index 5ee06ee187bf..b9fffb125cd5 100755 --- a/jenkins/scripts/perf/submit.py +++ b/jenkins/scripts/perf/submit.py @@ -122,8 +122,10 @@ def _load_pytest_split_durations(tokens, llm_src): try: with open(durations_path, "r") as durations_file: durations = json.load(durations_file) - except FileNotFoundError: - durations = {} + except FileNotFoundError as error: + raise FileNotFoundError( + f"pytest-split durations file not found: {durations_path}" + ) from error if isinstance(durations, list): durations = dict(durations) if not isinstance(durations, dict): diff --git a/requirements-dev.txt b/requirements-dev.txt index d5dcc3a2bf7e..fe640557eb2a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ pytest-env pytest-forked pytest-xdist pytest-timeout -pytest-split +pytest-split==0.10.0 pytest-mock pytest-threadleak pytest-unused-fixtures diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py index 0ebc92be3c4d..3e3d116d7f9a 100644 --- a/tests/unittest/scripts/test_perf_submit.py +++ b/tests/unittest/scripts/test_perf_submit.py @@ -162,3 +162,24 @@ def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path): script_prefix_lines, split_group=2, ) + + +def test_ci_submit_rejects_missing_pytest_split_durations(ci_submit_module, tmp_path): + test_list_path = tmp_path / "test_list.txt" + test_list_path.write_text( + "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n", + encoding="utf-8", + ) + script_prefix_lines = [ + 'export pytestCommand="pytest --splitting-algorithm least_duration ' + '--splits 1 --group 1 --durations-path /remote/.test_durations"' + ] + + expected_path = tmp_path / "tests" / "integration" / "defs" / ".test_durations" + with pytest.raises(FileNotFoundError, match=f"durations file not found: {expected_path}"): + ci_submit_module.select_test_case_line( + test_list_path, + tmp_path, + script_prefix_lines, + split_group=1, + ) From 63f27375eb84c59d12b17099b3d0c7d2860da2a5 Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:56:27 -0700 Subject: [PATCH 3/5] [NVBUG-6481034][test] Guard pytest-split compatibility Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- requirements-dev.txt | 2 +- tests/unittest/scripts/test_perf_submit.py | 38 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index fe640557eb2a..d5dcc3a2bf7e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ pytest-env pytest-forked pytest-xdist pytest-timeout -pytest-split==0.10.0 +pytest-split pytest-mock pytest-threadleak pytest-unused-fixtures diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py index 3e3d116d7f9a..2bf40a8df7a0 100644 --- a/tests/unittest/scripts/test_perf_submit.py +++ b/tests/unittest/scripts/test_perf_submit.py @@ -20,6 +20,7 @@ from types import ModuleType import pytest +from pytest_split.algorithms import LeastDurationAlgorithm REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent SUBMIT_PATHS = ( @@ -29,6 +30,14 @@ EXAMPLE_SUBMIT_PATH = REPO_ROOT / "examples" / "disaggregated" / "slurm" / "benchmark" / "submit.py" +class _FakePytestItem: + def __init__(self, nodeid: str): + self.nodeid = nodeid + + def __str__(self) -> str: + return self.nodeid + + def _load_module(path: Path, monkeypatch: pytest.MonkeyPatch) -> ModuleType: monkeypatch.syspath_prepend(str(path.parent)) spec = importlib.util.spec_from_file_location(f"perf_submit_{path.parent.name}", path) @@ -145,6 +154,35 @@ def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_m assert selected == test_lines[1] +def test_ci_submit_selector_matches_installed_pytest_split(ci_submit_module): + lines = [ + f"perf/test_perf_sanity.py::test_e2e[case-{case_name}] TIMEOUT (90)" + for case_name in ("zeta", "alpha", "gamma", "beta", "epsilon", "delta") + ] + nodeids = [ci_submit_module._test_nodeid(line) for line in lines] + items = [_FakePytestItem(nodeid) for nodeid in nodeids] + duration_sets = ( + {nodeid: float(index + 1) for index, nodeid in enumerate(nodeids)}, + dict.fromkeys(nodeids, 4.0), + {nodeids[1]: 8.0, nodeids[4]: 2.0, "irrelevant::test": 1000.0}, + {}, + ) + + for durations in duration_sets: + for splits in (2, 3, 4): + expected_groups = LeastDurationAlgorithm()(splits, items, durations) + for group, expected_group in enumerate(expected_groups, start=1): + selected = ci_submit_module._select_least_duration_group( + lines, + durations, + splits, + group, + ) + assert [ci_submit_module._test_nodeid(line) for line in selected] == [ + item.nodeid for item in expected_group.selected + ] + + def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path): test_list_path = tmp_path / "test_list.txt" test_list_path.write_text( From 9429527428c2d4dac8213bbcbe2cd54334fc43fc Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:03:38 -0700 Subject: [PATCH 4/5] [NVBUG-6481034][fix] Address selector contract review Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- jenkins/scripts/perf/submit.py | 6 ++- tests/unittest/scripts/test_perf_submit.py | 59 ++++++++++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py index b9fffb125cd5..259ef6c307c4 100755 --- a/jenkins/scripts/perf/submit.py +++ b/jenkins/scripts/perf/submit.py @@ -68,7 +68,11 @@ def _import_precheck_config(llm_src): # --------------------------------------------------------------------------- # def _read_test_list_lines(test_list_path): with open(test_list_path, "r") as f: - lines = [line.strip() for line in f if line.strip()] + lines = [] + for line in f: + stripped_line = line.strip() + if stripped_line and not stripped_line.startswith("#"): + lines.append(stripped_line) if not lines: raise ValueError(f"Test list is empty: {test_list_path}") return lines diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py index 2bf40a8df7a0..ed4fb8dedea7 100644 --- a/tests/unittest/scripts/test_perf_submit.py +++ b/tests/unittest/scripts/test_perf_submit.py @@ -31,7 +31,7 @@ class _FakePytestItem: - def __init__(self, nodeid: str): + def __init__(self, nodeid: str) -> None: self.nodeid = nodeid def __str__(self) -> str: @@ -119,7 +119,10 @@ def test_example_worker_environment_exports_positive_concurrency(example_submit_ assert worker_environment["TLLM_BENCHMARK_REQ_QUEUES_SIZE"] == "4301" -def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_module, tmp_path): +def test_ci_submit_selects_same_least_duration_shard_as_pytest_split( + ci_submit_module: ModuleType, + tmp_path: Path, +) -> None: test_lines = [ "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1] TIMEOUT (90)", "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25] TIMEOUT (90)", @@ -154,7 +157,9 @@ def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_m assert selected == test_lines[1] -def test_ci_submit_selector_matches_installed_pytest_split(ci_submit_module): +def test_ci_submit_selector_matches_installed_pytest_split( + ci_submit_module: ModuleType, +) -> None: lines = [ f"perf/test_perf_sanity.py::test_e2e[case-{case_name}] TIMEOUT (90)" for case_name in ("zeta", "alpha", "gamma", "beta", "epsilon", "delta") @@ -183,7 +188,48 @@ def test_ci_submit_selector_matches_installed_pytest_split(ci_submit_module): ] -def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path): +@pytest.mark.parametrize( + ("splits", "group", "match"), + ( + (0, 1, "--splits"), + (2, 0, "--group"), + (2, 3, "--group"), + ), +) +def test_ci_submit_rejects_invalid_least_duration_groups( + ci_submit_module: ModuleType, + splits: int, + group: int, + match: str, +) -> None: + with pytest.raises(ValueError, match=match): + ci_submit_module._select_least_duration_group( + ["perf/test_perf_sanity.py::test_e2e[case]"], + {}, + splits, + group, + ) + + +def test_ci_submit_ignores_test_list_comments( + ci_submit_module: ModuleType, + tmp_path: Path, +) -> None: + test_list_path = tmp_path / "test_list.txt" + test_list_path.write_text( + "# section comment\n\nperf/test_perf_sanity.py::test_e2e[case]\n", + encoding="utf-8", + ) + + assert ci_submit_module._read_test_list_lines(test_list_path) == [ + "perf/test_perf_sanity.py::test_e2e[case]" + ] + + +def test_ci_submit_rejects_split_group_disagreement( + ci_submit_module: ModuleType, + tmp_path: Path, +) -> None: test_list_path = tmp_path / "test_list.txt" test_list_path.write_text( "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n", @@ -202,7 +248,10 @@ def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path): ) -def test_ci_submit_rejects_missing_pytest_split_durations(ci_submit_module, tmp_path): +def test_ci_submit_rejects_missing_pytest_split_durations( + ci_submit_module: ModuleType, + tmp_path: Path, +) -> None: test_list_path = tmp_path / "test_list.txt" test_list_path.write_text( "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n", From 79f24c6a2b4f04469575e86514e10d38494371ca Mon Sep 17 00:00:00 2001 From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:10:05 -0700 Subject: [PATCH 5/5] [NVBUG-6481034][refactor] Use one shard selection path Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- jenkins/scripts/perf/submit.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py index 259ef6c307c4..72b176df3a95 100755 --- a/jenkins/scripts/perf/submit.py +++ b/jenkins/scripts/perf/submit.py @@ -214,24 +214,13 @@ def select_test_case_line(test_list_path, llm_src, script_prefix_lines, split_gr return selected[0] -def parse_test_case_name(test_list_path, llm_src, split_group=0, selected_line=None): - """Parse the selected line of the test list. +def parse_test_case_name(llm_src, selected_line): + """Parse the selected test-list line. Returns (config_yaml_path, server_name, benchmark_mode, runtime_mode). See the module docstring for the supported test name shapes. """ - if selected_line is not None: - line = selected_line - else: - lines = _read_test_list_lines(test_list_path) - if split_group > 0: - if split_group > len(lines): - raise ValueError( - f"split_group {split_group} exceeds number of tests in test list ({len(lines)})" - ) - line = lines[split_group - 1] - else: - line = lines[0] + line = selected_line if "[" not in line or "]" not in line: raise ValueError(f"Invalid test list format. Expected name with brackets: {line}") @@ -659,10 +648,8 @@ def main(): args.split_group, ) config_yaml, server_name, benchmark_mode, runtime_mode = parse_test_case_name( - args.test_list, args.llm_src, - args.split_group, - selected_line=selected_test_line, + selected_test_line, ) with open(config_yaml, "r") as f: