From 631fb4fd195c064b9a5ee327801ac8803ed4ea4c Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:17:09 -0700 Subject: [PATCH 1/2] [None][feat] serve: default num_serve_frontends to 8 with single-frontend fallback Make the plain `trtllm-serve` subcommand run 8 HTTP frontend processes against one executor by default (`--num_serve_frontends`, added in #16523). At high concurrency a single serving process is host-bound on its asyncio loop; several SO_REUSEPORT frontends remove that ceiling. - The LlmArgs field default stays 1: a bare LLM() has no HTTP frontends to fan out to and only trtllm-serve can spawn them, so trtllm-bench / eval / Python users do not pay for the multi-frontend IPC topology. Because 8 != 1 the CLI value survives get_llm_args's default filter; a YAML `num_serve_frontends: 1` still overrides the untyped CLI default. - When the count comes from the default and the configuration can only run one frontend (--grpc, --port 0 / --report_addr, orchestrator_type, enable_resource_governor), log why and run 1 frontend instead of failing at startup. An explicit --num_serve_frontends > 1 keeps the loud error. - api_stability reference, new CPU tests (l0_cpu), docs section. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- .../commands/trtllm-serve/trtllm-serve.rst | 12 ++ tensorrt_llm/commands/serve.py | 63 +++++++- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../references/trtllm_serve_cli.yaml | 2 +- .../test_serve_num_frontends_default.py | 147 ++++++++++++++++++ 5 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 tests/unittest/llmapi/test_serve_num_frontends_default.py diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst index 36e154c02408..5a58d2ddd063 100644 --- a/docs/source/commands/trtllm-serve/trtllm-serve.rst +++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst @@ -35,6 +35,18 @@ The following abbreviated command syntax shows the commonly used arguments to st For the full syntax and argument descriptions, refer to :ref:`syntax`. +Multiple HTTP Frontends +----------------------- + +At high concurrency a single serving process is host-bound: one asyncio event loop parses every request and writes every streamed chunk, and queuing on that loop rather than GPU time starts to dominate first-token latency. ``trtllm-serve`` therefore runs **8 HTTP frontend processes** against one executor by default (``--num_serve_frontends``, prototype). The launcher process owns the engine; the other frontends attach to it and share the serving port via ``SO_REUSEPORT``, so clients still see a single URL. + +.. note:: + + * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend. + * Multiple frontends only work on the default executor path behind the OpenAI HTTP server on a fixed port. With ``--grpc``, ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``) or ``enable_resource_governor`` the default falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. + * Each request is handled entirely by the frontend that accepted its connection, and per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, and ``/metrics``, ``/perf_metrics``, ``/prometheus/metrics`` and ``/health`` describe the frontend that happened to answer rather than the whole server. + * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly. + Inference Endpoints ------------------- diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index a045382b6256..5618ccba7ada 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -490,6 +490,46 @@ def _init_multi_frontend_mode(llm_args: dict, return mode +# Default for the plain `trtllm-serve` subcommand. At high concurrency one +# serving process is host-bound (a single asyncio loop parses every request +# and writes every streamed chunk), so several frontends per executor pay off +# by default. The LlmArgs field itself keeps default 1: a bare LLM() has no +# HTTP frontends to fan out to, and only trtllm-serve can spawn them. +DEFAULT_NUM_SERVE_FRONTENDS = 8 + + +def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool, + port: int, grpc: bool, + report_addr: Optional[str]) -> None: + """Fall back to one frontend when the *default* count cannot be honored. + + Multi-frontend mode only exists on the classic IPC executor path behind + the OpenAI HTTP server on a fixed port. When the user did not ask for a + specific count (neither on the CLI nor in the --config YAML) and the + configuration can only run a single frontend, run one instead of failing + at startup. An explicit request keeps the loud error from the guard that + owns the incompatibility (gRPC / port 0 / orchestrator_type / governor). + """ + num_frontends = llm_args.get("num_serve_frontends", 1) + if requested or num_frontends <= 1: + return + if grpc: + reason = "--grpc" + elif port == 0 or report_addr: + reason = "--port 0 / --report_addr" + elif llm_args.get("orchestrator_type") is not None: + reason = f"orchestrator_type={llm_args['orchestrator_type']!r}" + elif llm_args.get("enable_resource_governor"): + reason = "enable_resource_governor" + else: + return + logger.info( + f"num_serve_frontends defaults to {num_frontends}, but {reason} " + "supports a single serving frontend only; running 1 frontend. Pass " + "--num_serve_frontends explicitly to override.") + llm_args["num_serve_frontends"] = 1 + + def _spawn_attached_frontends(llm, num_frontends: int) -> list: """Spawn num_frontends - 1 attached serving frontend processes. @@ -653,7 +693,8 @@ def launch_server( logger.warning( "num_serve_frontends > 1: stateful Responses API storage " "(store/previous_response_id) is disabled; the per-frontend " - "in-memory store cannot be shared across frontends.") + "in-memory store cannot be shared across frontends. Pass " + "--num_serve_frontends 1 to keep it.") os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1" addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, @@ -1103,10 +1144,14 @@ def launch_visual_gen_server( status="prototype") @stability_option("--num_serve_frontends", type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS), - default=1, + default=DEFAULT_NUM_SERVE_FRONTENDS, help="Number of HTTP frontend processes serving one " "executor; values > 1 share the serving port via " - "SO_REUSEPORT (classic IPC executor path only).", + "SO_REUSEPORT (classic IPC executor path only). The " + "default falls back to 1 for configurations that support " + "a single frontend only (--grpc, --port 0/--report_addr, " + "orchestrator_type, enable_resource_governor); an explicit " + "value fails instead.", status="prototype") @stability_option("--num_input_processor_workers", type=click.IntRange(min=1), @@ -1468,6 +1513,16 @@ def _serve_llm(): llm_args = update_llm_args_with_extra_dict( llm_args, llm_args_extra_dict, explicit_cli_keys=explicit_cli_keys) + # The multi-frontend default only applies where it can run; an + # explicit CLI flag or YAML key is honored as-is (and may fail loudly). + _resolve_default_num_serve_frontends( + llm_args, + requested=("num_serve_frontends" in explicit_cli_keys + or "num_serve_frontends" in llm_args_extra_dict), + port=port, + grpc=grpc, + report_addr=report_addr) + _apply_effective_telemetry_config(llm_args, telemetry=telemetry) metadata_server_cfg = parse_metadata_server_config_file( @@ -1504,7 +1559,7 @@ def _serve_llm(): media_io_kwargs=parsed_media_io_kwargs) if grpc: - if num_serve_frontends != 1: + if llm_args.get("num_serve_frontends", 1) != 1: raise click.UsageError( "--num_serve_frontends must be 1 when --grpc is enabled.") diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 0e7e5e66f7f1..87d39642ce2e 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -146,6 +146,7 @@ l0_cpu: - unittest/llmapi/test_rl_control_auth.py - unittest/llmapi/test_sampling_params.py - unittest/llmapi/test_serialization.py + - unittest/llmapi/test_serve_num_frontends_default.py - unittest/llmapi/test_serve_report_addr.py - unittest/llmapi/test_session_prefetcher.py - unittest/llmapi/test_tokenizer_aliases.py diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index fa44aa9ee841..4e465b8de0bb 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -314,7 +314,7 @@ commands: - "--num_postprocess_workers" num_serve_frontends: type: int - default: 1 + default: 8 status: prototype required: false multiple: false diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py new file mode 100644 index 000000000000..0eb76520bada --- /dev/null +++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit coverage for trtllm-serve's multi-frontend default. + +``--num_serve_frontends`` defaults to several HTTP frontend processes while +the LlmArgs field keeps default 1. The pieces that make the default reach the +executor -- surviving the CLI-vs-LlmArgs default filter, yielding to the YAML, +and falling back to a single frontend where more cannot run -- are covered +here; spawning the frontends needs a GPU and lives in the integration tests. +""" + +from typing import Optional + +import pytest + +from tensorrt_llm.commands import serve as serve_cmd +from tensorrt_llm.commands.serve import ( + DEFAULT_NUM_SERVE_FRONTENDS, + _init_multi_frontend_mode, + _resolve_default_num_serve_frontends, + get_llm_args, +) +from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS +from tensorrt_llm.llmapi.llm_args import BaseLlmArgs +from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict + +pytestmark = pytest.mark.cpu_only + + +def _resolve( + llm_args: dict, + *, + requested: bool = False, + port: int = 8000, + grpc: bool = False, + report_addr: Optional[str] = None, +) -> int: + _resolve_default_num_serve_frontends( + llm_args, requested=requested, port=port, grpc=grpc, report_addr=report_addr + ) + return llm_args.get("num_serve_frontends", 1) + + +def test_cli_default_is_multi_frontend() -> None: + option = next(p for p in serve_cmd.serve.params if p.name == "num_serve_frontends") + assert option.default == DEFAULT_NUM_SERVE_FRONTENDS + assert 1 < DEFAULT_NUM_SERVE_FRONTENDS <= MAX_NUM_FRONTENDS + + +def test_llm_args_default_stays_single_frontend() -> None: + # A bare LLM() has no HTTP frontends to fan out to; only trtllm-serve + # spawns them, so the executor-side default must stay at one lane. + assert BaseLlmArgs.model_fields["num_serve_frontends"].default == 1 + + +def test_cli_default_survives_the_llm_args_default_filter() -> None: + # get_llm_args drops CLI values equal to the LlmArgs default unless the + # flag was typed. The serve default differs from the LlmArgs default, so + # it must reach the LLM constructor without being typed on the CLI ... + llm_args, _ = get_llm_args( + model="m", + backend="pytorch", + gpus_per_node=1, + num_serve_frontends=DEFAULT_NUM_SERVE_FRONTENDS, + ) + assert llm_args["num_serve_frontends"] == DEFAULT_NUM_SERVE_FRONTENDS + # ... while an untyped 1 collapses onto the LlmArgs default. + llm_args, _ = get_llm_args(model="m", backend="pytorch", gpus_per_node=1, num_serve_frontends=1) + assert "num_serve_frontends" not in llm_args + + +def test_yaml_value_overrides_the_cli_default() -> None: + llm_args = update_llm_args_with_extra_dict( + {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS}, + {"num_serve_frontends": 1}, + explicit_cli_keys=set(), + ) + assert llm_args["num_serve_frontends"] == 1 + + +def test_explicit_cli_value_wins_over_yaml() -> None: + llm_args = update_llm_args_with_extra_dict( + {"num_serve_frontends": 4}, + {"num_serve_frontends": 1}, + explicit_cli_keys={"num_serve_frontends"}, + ) + assert llm_args["num_serve_frontends"] == 4 + + +def test_plain_config_keeps_the_default() -> None: + llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS} + assert _resolve(llm_args) == DEFAULT_NUM_SERVE_FRONTENDS + assert _init_multi_frontend_mode(llm_args, enabled=True).is_launcher + + +@pytest.mark.parametrize( + "extra_llm_args,kwargs", + [ + ({"orchestrator_type": "rpc"}, {}), + ({"orchestrator_type": "ray"}, {}), + ({"enable_resource_governor": True}, {}), + ({}, {"grpc": True}), + ({}, {"port": 0}), + ({}, {"report_addr": "/tmp/bound.addr"}), + ], +) +def test_default_falls_back_to_one_frontend(extra_llm_args: dict, kwargs: dict) -> None: + llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args} + assert _resolve(llm_args, **kwargs) == 1 + # The fallback leaves a consistent single-frontend configuration behind: + # the mode resolver no longer sees a launcher and does not raise. + if "orchestrator_type" in extra_llm_args: + assert not _init_multi_frontend_mode(llm_args, enabled=True).is_launcher + + +def test_explicit_request_is_not_silently_downgraded() -> None: + llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"} + assert _resolve(llm_args, requested=True) == DEFAULT_NUM_SERVE_FRONTENDS + with pytest.raises(ValueError, match="orchestrator_type"): + _init_multi_frontend_mode(llm_args, enabled=True) + + +def test_single_frontend_is_left_alone() -> None: + llm_args = {"num_serve_frontends": 1, "orchestrator_type": "rpc"} + assert _resolve(llm_args) == 1 + assert _resolve({"orchestrator_type": "rpc"}) == 1 + + +def test_fallback_logs_the_reason(monkeypatch: pytest.MonkeyPatch) -> None: + messages = [] + monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg)) + _resolve({"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"}) + assert len(messages) == 1 + assert "orchestrator_type='rpc'" in messages[0] + assert "--num_serve_frontends" in messages[0] From 80ee1206690f7a4e62c4e42f1b63d6fd4f346e62 Mon Sep 17 00:00:00 2001 From: Lance Liao <108499334+lancelly@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:37:29 -0700 Subject: [PATCH 2/2] [None][feat] serve: drop the dedicated CPU test file for the frontend default The default and its fallback are covered by the api_stability CLI gate and the e2e runs; keep the PR to the behaviour change. Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com> --- .../integration/test_lists/test-db/l0_cpu.yml | 1 - .../test_serve_num_frontends_default.py | 147 ------------------ 2 files changed, 148 deletions(-) delete mode 100644 tests/unittest/llmapi/test_serve_num_frontends_default.py diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 87d39642ce2e..0e7e5e66f7f1 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -146,7 +146,6 @@ l0_cpu: - unittest/llmapi/test_rl_control_auth.py - unittest/llmapi/test_sampling_params.py - unittest/llmapi/test_serialization.py - - unittest/llmapi/test_serve_num_frontends_default.py - unittest/llmapi/test_serve_report_addr.py - unittest/llmapi/test_session_prefetcher.py - unittest/llmapi/test_tokenizer_aliases.py diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py deleted file mode 100644 index 0eb76520bada..000000000000 --- a/tests/unittest/llmapi/test_serve_num_frontends_default.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Unit coverage for trtllm-serve's multi-frontend default. - -``--num_serve_frontends`` defaults to several HTTP frontend processes while -the LlmArgs field keeps default 1. The pieces that make the default reach the -executor -- surviving the CLI-vs-LlmArgs default filter, yielding to the YAML, -and falling back to a single frontend where more cannot run -- are covered -here; spawning the frontends needs a GPU and lives in the integration tests. -""" - -from typing import Optional - -import pytest - -from tensorrt_llm.commands import serve as serve_cmd -from tensorrt_llm.commands.serve import ( - DEFAULT_NUM_SERVE_FRONTENDS, - _init_multi_frontend_mode, - _resolve_default_num_serve_frontends, - get_llm_args, -) -from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS -from tensorrt_llm.llmapi.llm_args import BaseLlmArgs -from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict - -pytestmark = pytest.mark.cpu_only - - -def _resolve( - llm_args: dict, - *, - requested: bool = False, - port: int = 8000, - grpc: bool = False, - report_addr: Optional[str] = None, -) -> int: - _resolve_default_num_serve_frontends( - llm_args, requested=requested, port=port, grpc=grpc, report_addr=report_addr - ) - return llm_args.get("num_serve_frontends", 1) - - -def test_cli_default_is_multi_frontend() -> None: - option = next(p for p in serve_cmd.serve.params if p.name == "num_serve_frontends") - assert option.default == DEFAULT_NUM_SERVE_FRONTENDS - assert 1 < DEFAULT_NUM_SERVE_FRONTENDS <= MAX_NUM_FRONTENDS - - -def test_llm_args_default_stays_single_frontend() -> None: - # A bare LLM() has no HTTP frontends to fan out to; only trtllm-serve - # spawns them, so the executor-side default must stay at one lane. - assert BaseLlmArgs.model_fields["num_serve_frontends"].default == 1 - - -def test_cli_default_survives_the_llm_args_default_filter() -> None: - # get_llm_args drops CLI values equal to the LlmArgs default unless the - # flag was typed. The serve default differs from the LlmArgs default, so - # it must reach the LLM constructor without being typed on the CLI ... - llm_args, _ = get_llm_args( - model="m", - backend="pytorch", - gpus_per_node=1, - num_serve_frontends=DEFAULT_NUM_SERVE_FRONTENDS, - ) - assert llm_args["num_serve_frontends"] == DEFAULT_NUM_SERVE_FRONTENDS - # ... while an untyped 1 collapses onto the LlmArgs default. - llm_args, _ = get_llm_args(model="m", backend="pytorch", gpus_per_node=1, num_serve_frontends=1) - assert "num_serve_frontends" not in llm_args - - -def test_yaml_value_overrides_the_cli_default() -> None: - llm_args = update_llm_args_with_extra_dict( - {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS}, - {"num_serve_frontends": 1}, - explicit_cli_keys=set(), - ) - assert llm_args["num_serve_frontends"] == 1 - - -def test_explicit_cli_value_wins_over_yaml() -> None: - llm_args = update_llm_args_with_extra_dict( - {"num_serve_frontends": 4}, - {"num_serve_frontends": 1}, - explicit_cli_keys={"num_serve_frontends"}, - ) - assert llm_args["num_serve_frontends"] == 4 - - -def test_plain_config_keeps_the_default() -> None: - llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS} - assert _resolve(llm_args) == DEFAULT_NUM_SERVE_FRONTENDS - assert _init_multi_frontend_mode(llm_args, enabled=True).is_launcher - - -@pytest.mark.parametrize( - "extra_llm_args,kwargs", - [ - ({"orchestrator_type": "rpc"}, {}), - ({"orchestrator_type": "ray"}, {}), - ({"enable_resource_governor": True}, {}), - ({}, {"grpc": True}), - ({}, {"port": 0}), - ({}, {"report_addr": "/tmp/bound.addr"}), - ], -) -def test_default_falls_back_to_one_frontend(extra_llm_args: dict, kwargs: dict) -> None: - llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args} - assert _resolve(llm_args, **kwargs) == 1 - # The fallback leaves a consistent single-frontend configuration behind: - # the mode resolver no longer sees a launcher and does not raise. - if "orchestrator_type" in extra_llm_args: - assert not _init_multi_frontend_mode(llm_args, enabled=True).is_launcher - - -def test_explicit_request_is_not_silently_downgraded() -> None: - llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"} - assert _resolve(llm_args, requested=True) == DEFAULT_NUM_SERVE_FRONTENDS - with pytest.raises(ValueError, match="orchestrator_type"): - _init_multi_frontend_mode(llm_args, enabled=True) - - -def test_single_frontend_is_left_alone() -> None: - llm_args = {"num_serve_frontends": 1, "orchestrator_type": "rpc"} - assert _resolve(llm_args) == 1 - assert _resolve({"orchestrator_type": "rpc"}) == 1 - - -def test_fallback_logs_the_reason(monkeypatch: pytest.MonkeyPatch) -> None: - messages = [] - monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg)) - _resolve({"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"}) - assert len(messages) == 1 - assert "orchestrator_type='rpc'" in messages[0] - assert "--num_serve_frontends" in messages[0]