Skip to content

[https://nvbugs/6721558][fix] Only record first token time when collecting perf metrics #18746

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
trtllm-agent wants to merge 1 commit into from
Closed

[https://nvbugs/6721558][fix] Only record first token time when collecting perf metrics #18746

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 20 additions & 10 deletions tensorrt_llm/serve/openai_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1756,14 +1756,27 @@ async def get_kv_cache_events(self) -> JSONResponse:
pass
return JSONResponse(content=events)

def _record_server_first_token_time(self,
raw_request: Optional[Request]) -> None:
"""Stamp the first-token time in the reference clock domain, once.

The value is only ever read back by ``build_request_metrics_record``,
which runs under ``_collect_perf_metrics``, so sampling the clock on
the default path would be work nobody consumes.
"""
if not self._collect_perf_metrics or raw_request is None:
return
if not getattr(raw_request.state, "server_first_token_time", None):
raw_request.state.server_first_token_time = (
self._adjusted_steady_clock.now())

async def _extract_metrics(self, res: RequestOutput, raw_request: Request):
if not res.finished:
return
if self._collect_perf_metrics:
if raw_request and not getattr(raw_request.state,
"server_first_token_time", None):
raw_request.state.server_first_token_time = (
self._adjusted_steady_clock.now())
# Fallback for the non-streaming paths, which never reach the
# streaming stamp site.
self._record_server_first_token_time(raw_request)
record = build_request_metrics_record(
res, raw_request, adjusted_clock=self._adjusted_steady_clock)
if record is not None and raw_request is not None:
Expand Down Expand Up @@ -1873,8 +1886,7 @@ async def chat_stream_generator(
if not self.postproc_worker_enabled:
post_processor, args = postproc_params.post_processor, postproc_params.postproc_args
first_response = await anext(promise)
raw_request.state.server_first_token_time = self._adjusted_steady_clock.now(
)
self._record_server_first_token_time(raw_request)
pp_results = first_response.outputs[
0]._postprocess_result if self.postproc_worker_enabled else post_processor(
first_response, args)
Expand Down Expand Up @@ -2503,8 +2515,7 @@ async def producer(generator: AsyncIterator[Any], idx: int):

async def generator_wrapper(generator: AsyncIterator[Any]):
first_response = await anext(generator)
raw_request.state.server_first_token_time = self._adjusted_steady_clock.now(
)
self._record_server_first_token_time(raw_request)
yield first_response
async for output in generator:
yield output
Expand Down Expand Up @@ -2636,8 +2647,7 @@ async def create_streaming_generator(promise: RequestOutput,
if not self.postproc_worker_enabled:
post_processor, args = postproc_params.post_processor, postproc_params.postproc_args
first_response = await anext(promise)
raw_request.state.server_first_token_time = (
self._adjusted_steady_clock.now())
self._record_server_first_token_time(raw_request)
pp_results = (first_response.outputs[0]._postprocess_result if
self.postproc_worker_enabled else post_processor(
first_response, args))
Expand Down
Loading