From 4a59fda0663ec50850ce1750dd54701cc45ec7a1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 10:31:20 -0500 Subject: [PATCH 1/2] Add full-response output token throughput Signed-off-by: Cam Quilici --- docs/metrics-reference.md | 53 ++++++++++++- src/aiperf/common/models/export_models.py | 8 ++ .../metrics/types/decode_duration_metric.py | 37 +++++++++ .../types/output_token_throughput_metrics.py | 37 +++++++++ .../metrics/test_decode_duration_metric.py | 42 ++++++++++- ...output_token_throughput_per_user_metric.py | 75 ++++++++++++++++++- 6 files changed, 247 insertions(+), 5 deletions(-) diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index f84c2bf460..fabf712d31 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -301,6 +301,28 @@ decode_duration_ms = decode_duration_ns / 1e6 --- +### Full Decode Duration + +**Type:** [Record Metric](#record-metrics) + +Measures the client-observed wall-clock interval from the first non-empty parsed +content response until the HTTP response is fully consumed. Unlike Decode +Duration, it includes time after the last parsed content chunk, such as generation +that a structured-output parser suppresses before the terminal usage response. + +**Formula:** +```python +full_decode_duration_ns = request.end_perf_ns - first_content_response.perf_ns +``` + +**Notes:** +- This is a client-observed full-response duration, not server kernel execution time. +- It includes terminal serialization, transport, and client-processing overhead. +- It requires an explicit request-end timestamp and at least one non-empty content response. +- Existing profile exports can reconstruct it as `(request_end_ns - request_start_ns) - time_to_first_token_ns`. + +--- + ### Inter Token Latency (ITL) **Type:** [Record Metric](#record-metrics) @@ -365,6 +387,29 @@ output_token_throughput_per_user = 1.0 / inter_token_latency_seconds - Computes the inverse of ITL to show tokens per second from an individual user's perspective. - Differs from Output Token Throughput (aggregate across all concurrent requests) by focusing on single-request experience. - Useful for understanding the user experience independent of concurrency effects. +- Assumes the output token count describes the content-delivery interval. If a server reports tokens that its response parser suppresses, use Full-Response Output Token Throughput Per User for a full-lifecycle rate. + +--- + +### Full-Response Output Token Throughput Per User + +**Type:** [Record Metric](#record-metrics) + +Measures a per-request output token rate over the full client-observed decode +window, including time after the final parsed content response. + +**Formula:** +```python +full_response_output_token_throughput_per_user = ( + output_sequence_length - 1 +) / full_decode_duration_seconds +``` + +**Notes:** +- Excludes TTFT but measures through full HTTP response completion. +- With server-reported token counting, this approximates raw engine decode TPS when the response parser suppresses generated tokens. +- It remains a client-observed approximation because AIPerf does not have raw engine first/last-token timestamps. +- Compare it with Output Token Throughput Per User to detect a gap between parsed content delivery and full response completion. --- @@ -1681,7 +1726,8 @@ total_error_isl = sum(r.error_isl for r in records if not r.valid) **Type:** [Record Metric](#record-metrics) -Measures the total end-to-end time from sending a request until receiving the final response. For streaming requests with multiple responses, this measures until the last response is received. This is the complete time experienced by the client for a single request. +Measures the time from request start until the final non-empty parsed content +response. Usage-only and terminal responses are excluded. **Formula:** ```python @@ -1689,8 +1735,9 @@ request_latency_ns = request.content_responses[-1].perf_ns - request.start_perf_ ``` **Notes:** -- Includes all components: network time, queuing, prompt processing, token generation, and response transmission. -- For streaming requests, measures from request start to the final chunk received. +- Includes network time, queuing, prompt processing, and content delivery through the final parsed content chunk. +- It can be shorter than the HTTP lifecycle when a response parser suppresses generated content or terminal usage arrives later. +- Use `http_req_duration` for the complete HTTP exchange and Full Decode Duration for the post-TTFT interval through request completion. --- diff --git a/src/aiperf/common/models/export_models.py b/src/aiperf/common/models/export_models.py index c14805f00b..af4f5cf8a2 100644 --- a/src/aiperf/common/models/export_models.py +++ b/src/aiperf/common/models/export_models.py @@ -315,6 +315,14 @@ class JsonExportData(AIPerfBaseModel): inter_token_latency: JsonMetricResult | None = None output_token_throughput: JsonMetricResult | None = None output_token_throughput_per_user: JsonMetricResult | None = None + full_decode_duration: JsonMetricResult | None = Field( + default=None, + description="Client-observed duration from first parsed content through full request completion.", + ) + full_response_output_token_throughput_per_user: JsonMetricResult | None = Field( + default=None, + description="Per-request output token rate over the full decode duration.", + ) output_sequence_length: JsonMetricResult | None = None input_sequence_length: JsonMetricResult | None = None goodput: JsonMetricResult | None = None diff --git a/src/aiperf/metrics/types/decode_duration_metric.py b/src/aiperf/metrics/types/decode_duration_metric.py index 36829b2cc8..119fc579a9 100644 --- a/src/aiperf/metrics/types/decode_duration_metric.py +++ b/src/aiperf/metrics/types/decode_duration_metric.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from aiperf.common.enums import MetricFlags, MetricTimeUnit +from aiperf.common.exceptions import NoMetricValue from aiperf.common.models import ParsedResponseRecord from aiperf.metrics import BaseRecordMetric from aiperf.metrics.metric_dicts import MetricRecordDict @@ -38,3 +39,39 @@ def _parse_record( if decode_duration < 0: raise ValueError("Request latency is less than time to first token.") return decode_duration + + +class FullDecodeDurationMetric(BaseRecordMetric[int]): + """Client-observed interval from first content to full request completion.""" + + tag = "full_decode_duration" + header = "Full Decode Duration" + short_header = "Full Decode Duration" + unit = MetricTimeUnit.NANOSECONDS + display_unit = MetricTimeUnit.MILLISECONDS + display_order = 360 + flags = ( + MetricFlags.STREAMING_TOKENS_ONLY + | MetricFlags.PERCENTILE_INCLUDES_FAILED_REQUESTS + ) + required_metrics = None + + def _parse_record( + self, + record: ParsedResponseRecord, + record_metrics: MetricRecordDict, + ) -> int: + if not record.content_responses: + raise NoMetricValue( + "Full decode duration requires at least 1 non-empty content response." + ) + if record.request.end_perf_ns is None: + raise NoMetricValue( + "Full decode duration requires an explicit request end timestamp." + ) + + first_content_ts = record.content_responses[0].perf_ns + duration = record.request.end_perf_ns - first_content_ts + if duration < 0: + raise ValueError("Request end timestamp is before first content response.") + return duration diff --git a/src/aiperf/metrics/types/output_token_throughput_metrics.py b/src/aiperf/metrics/types/output_token_throughput_metrics.py index a90a560fcf..51a67acbc3 100644 --- a/src/aiperf/metrics/types/output_token_throughput_metrics.py +++ b/src/aiperf/metrics/types/output_token_throughput_metrics.py @@ -7,8 +7,10 @@ from aiperf.metrics import BaseDerivedMetric, BaseRecordMetric from aiperf.metrics.metric_dicts import MetricRecordDict, MetricResultsDict from aiperf.metrics.types.benchmark_duration_metric import BenchmarkDurationMetric +from aiperf.metrics.types.decode_duration_metric import FullDecodeDurationMetric from aiperf.metrics.types.inter_token_latency_metric import InterTokenLatencyMetric from aiperf.metrics.types.output_sequence_length_metric import ( + OutputSequenceLengthMetric, TotalOutputSequenceLengthMetric, ) @@ -76,3 +78,38 @@ def _parse_record( "ITL is zero, cannot calculate output token throughput per user metric" ) return 1 / converted_itl + + +class FullResponseOutputTokenThroughputPerUserMetric(BaseRecordMetric[float]): + """Token rate from first parsed content through full request completion.""" + + tag = "full_response_output_token_throughput_per_user" + header = "Full-Response Output Token Throughput Per User" + short_header = "Full Output TPS/User" + short_header_hide_unit = True + unit = MetricOverTimeUnit.TOKENS_PER_SECOND_PER_USER + display_order = 520 + flags = MetricFlags.STREAMING_TOKENS_ONLY | MetricFlags.LARGER_IS_BETTER + required_metrics = { + FullDecodeDurationMetric.tag, + OutputSequenceLengthMetric.tag, + } + + def _parse_record( + self, + record: ParsedResponseRecord, + record_metrics: MetricRecordDict, + ) -> float: + osl = record_metrics.get_or_raise(OutputSequenceLengthMetric) + if osl < 2: # type: ignore + raise NoMetricValue(f"Output sequence length must be at least 2, got {osl}") + + duration = record_metrics.get_converted_or_raise( + FullDecodeDurationMetric, + self.unit.time_unit, # type: ignore + ) + if duration == 0: + raise NoMetricValue( + "Full decode duration is zero, cannot calculate full-response output token throughput" + ) + return (osl - 1) / duration # type: ignore diff --git a/tests/unit/metrics/test_decode_duration_metric.py b/tests/unit/metrics/test_decode_duration_metric.py index a765be54bd..31a346b92d 100644 --- a/tests/unit/metrics/test_decode_duration_metric.py +++ b/tests/unit/metrics/test_decode_duration_metric.py @@ -5,7 +5,10 @@ from aiperf.common.exceptions import NoMetricValue from aiperf.metrics.metric_dicts import MetricRecordDict -from aiperf.metrics.types.decode_duration_metric import DecodeDurationMetric +from aiperf.metrics.types.decode_duration_metric import ( + DecodeDurationMetric, + FullDecodeDurationMetric, +) from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.ttft_metric import TTFTMetric from tests.unit.metrics.conftest import create_record, run_simple_metrics_pipeline @@ -52,3 +55,40 @@ def test_decode_duration_rejects_negative_interval(self): match="Request latency is less than time to first token", ): DecodeDurationMetric().parse_record(record, record_metrics) + + +class TestFullDecodeDurationMetric: + def test_uses_first_content_and_explicit_request_end(self) -> None: + record = create_record(start_ns=100, responses=[120, 200]) + record.request.end_perf_ns = 300 + + metric_results = run_simple_metrics_pipeline( + [record], + FullDecodeDurationMetric.tag, + ) + + assert metric_results[FullDecodeDurationMetric.tag] == [180] + + def test_requires_content_response(self) -> None: + record = create_record() + record.responses = [] + + with pytest.raises(NoMetricValue): + FullDecodeDurationMetric().parse_record(record, MetricRecordDict()) + + def test_requires_explicit_request_end(self) -> None: + record = create_record(start_ns=100, responses=[120]) + record.request.end_perf_ns = None + + with pytest.raises(NoMetricValue, match="explicit request end timestamp"): + FullDecodeDurationMetric().parse_record(record, MetricRecordDict()) + + def test_rejects_request_end_before_first_content(self) -> None: + record = create_record(start_ns=100, responses=[120]) + record.request.end_perf_ns = 110 + + with pytest.raises( + ValueError, + match="Request end timestamp is before first content response", + ): + FullDecodeDurationMetric().parse_record(record, MetricRecordDict()) diff --git a/tests/unit/metrics/test_output_token_throughput_per_user_metric.py b/tests/unit/metrics/test_output_token_throughput_per_user_metric.py index c2f1d7aac6..b7899be6a6 100644 --- a/tests/unit/metrics/test_output_token_throughput_per_user_metric.py +++ b/tests/unit/metrics/test_output_token_throughput_per_user_metric.py @@ -5,11 +5,16 @@ from aiperf.common.exceptions import NoMetricValue from aiperf.metrics.metric_dicts import MetricRecordDict +from aiperf.metrics.types.decode_duration_metric import FullDecodeDurationMetric from aiperf.metrics.types.inter_token_latency_metric import InterTokenLatencyMetric +from aiperf.metrics.types.output_sequence_length_metric import ( + OutputSequenceLengthMetric, +) from aiperf.metrics.types.output_token_throughput_metrics import ( + FullResponseOutputTokenThroughputPerUserMetric, OutputTokenThroughputPerUserMetric, ) -from tests.unit.metrics.conftest import create_record +from tests.unit.metrics.conftest import create_record, run_simple_metrics_pipeline class TestOutputTokenThroughputPerUserMetric: @@ -49,3 +54,71 @@ def test_output_token_throughput_per_user_none_itl_error(self): with pytest.raises(NoMetricValue): metric.parse_record(record, metric_dict) + + +class TestFullResponseOutputTokenThroughputPerUserMetric: + def test_kimi_parser_gap_uses_full_request_end(self) -> None: + request_start_ns = 1_000_000_000 + record = create_record( + start_ns=request_start_ns, + responses=[1_529_058_811, 1_610_559_573], + ) + record.request.end_perf_ns = request_start_ns + 145_861_451_008 + assert record.token_counts is not None + record.token_counts.output = 26_571 + + metric_results = run_simple_metrics_pipeline( + [record], + OutputTokenThroughputPerUserMetric.tag, + FullResponseOutputTokenThroughputPerUserMetric.tag, + ) + + assert metric_results[OutputTokenThroughputPerUserMetric.tag] == pytest.approx( + [326_009.2218524288] + ) + assert metric_results[ + FullResponseOutputTokenThroughputPerUserMetric.tag + ] == pytest.approx([182.82228456622666]) + + def test_calculates_rate_over_full_decode_duration(self) -> None: + record = create_record() + metric_dict = MetricRecordDict( + { + FullDecodeDurationMetric.tag: 1_000_000_000, + OutputSequenceLengthMetric.tag: 11, + } + ) + + result = FullResponseOutputTokenThroughputPerUserMetric().parse_record( + record, metric_dict + ) + + assert result == 10.0 + + def test_requires_at_least_two_tokens(self) -> None: + record = create_record() + metric_dict = MetricRecordDict( + { + FullDecodeDurationMetric.tag: 1_000_000_000, + OutputSequenceLengthMetric.tag: 1, + } + ) + + with pytest.raises(NoMetricValue, match="at least 2"): + FullResponseOutputTokenThroughputPerUserMetric().parse_record( + record, metric_dict + ) + + def test_rejects_zero_duration(self) -> None: + record = create_record() + metric_dict = MetricRecordDict( + { + FullDecodeDurationMetric.tag: 0, + OutputSequenceLengthMetric.tag: 11, + } + ) + + with pytest.raises(NoMetricValue, match="duration is zero"): + FullResponseOutputTokenThroughputPerUserMetric().parse_record( + record, metric_dict + ) From 8128d08e7cfa31c36254f3c3d8c167488d51dde0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 15:31:40 -0500 Subject: [PATCH 2/2] Add full-response inter-token latency Signed-off-by: Cam Quilici --- docs/metrics-reference.md | 35 +++++++++-- src/aiperf/common/models/export_models.py | 4 ++ .../types/inter_token_latency_metric.py | 32 ++++++++++ .../types/output_token_throughput_metrics.py | 26 ++++---- .../test_inter_token_latency_metric.py | 62 ++++++++++++++++++- ...output_token_throughput_per_user_metric.py | 33 +++------- 6 files changed, 147 insertions(+), 45 deletions(-) diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index fabf712d31..4ac5fe79b8 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -347,6 +347,31 @@ inter_token_latency_ms = inter_token_latency_ns / 1e6 - Compare runs at equivalent output lengths, or inspect Decode Duration and Output Sequence Length alongside ITL. - Streaming chunks can contain multiple tokens. ITL uses token count, while ICL uses chunk arrival timestamps. - Result is in seconds when used for throughput calculations (Output Token Throughput Per User). +- Assumes the output token count describes the parsed-content interval. If a server reports tokens that its response parser suppresses, use Full-Response Inter Token Latency. + +--- + +### Full-Response Inter Token Latency + +**Type:** [Record Metric](#record-metrics) + +Measures the average token interval over the full client-observed decode window, +from the first non-empty parsed content response through explicit HTTP request +completion. + +**Formula:** +```python +full_response_inter_token_latency_ns = ( + full_decode_duration_ns / (output_sequence_length - 1) +) +``` + +**Notes:** +- Requires an output sequence length of at least 2 tokens and a valid Full Decode Duration. +- Uses the same token normalization as Inter Token Latency while extending the interval through HTTP response completion. +- With server-reported token counting, it keeps the duration and token count aligned when a response parser suppresses generated tokens. +- This remains a client-observed average, not a distribution of raw engine token-to-token timestamps. +- Streaming chunks can contain multiple tokens. A response delivered entirely in one content chunk may not expose a meaningful post-first-content decode interval. --- @@ -387,7 +412,7 @@ output_token_throughput_per_user = 1.0 / inter_token_latency_seconds - Computes the inverse of ITL to show tokens per second from an individual user's perspective. - Differs from Output Token Throughput (aggregate across all concurrent requests) by focusing on single-request experience. - Useful for understanding the user experience independent of concurrency effects. -- Assumes the output token count describes the content-delivery interval. If a server reports tokens that its response parser suppresses, use Full-Response Output Token Throughput Per User for a full-lifecycle rate. +- Assumes the output token count describes the content-delivery interval. If a server reports tokens that its response parser suppresses, compare it with the full-response metric pair. --- @@ -401,15 +426,17 @@ window, including time after the final parsed content response. **Formula:** ```python full_response_output_token_throughput_per_user = ( - output_sequence_length - 1 -) / full_decode_duration_seconds + 1.0 / full_response_inter_token_latency_seconds +) ``` **Notes:** +- Computes the inverse of Full-Response Inter Token Latency, mirroring the relationship between Output Token Throughput Per User and Inter Token Latency. - Excludes TTFT but measures through full HTTP response completion. - With server-reported token counting, this approximates raw engine decode TPS when the response parser suppresses generated tokens. - It remains a client-observed approximation because AIPerf does not have raw engine first/last-token timestamps. -- Compare it with Output Token Throughput Per User to detect a gap between parsed content delivery and full response completion. +- Aggregate latency percentiles and throughput percentiles are not interchangeable: `1 / p75(latency)` describes the slow tail, while `p75(throughput)` describes the fast side of the reciprocal distribution. +- Compare the full-response metric pair with the existing metric pair to detect a gap between parsed content delivery and full response completion. --- diff --git a/src/aiperf/common/models/export_models.py b/src/aiperf/common/models/export_models.py index af4f5cf8a2..2cbcb792ec 100644 --- a/src/aiperf/common/models/export_models.py +++ b/src/aiperf/common/models/export_models.py @@ -319,6 +319,10 @@ class JsonExportData(AIPerfBaseModel): default=None, description="Client-observed duration from first parsed content through full request completion.", ) + full_response_inter_token_latency: JsonMetricResult | None = Field( + default=None, + description="Average token interval from first parsed content through full request completion.", + ) full_response_output_token_throughput_per_user: JsonMetricResult | None = Field( default=None, description="Per-request output token rate over the full decode duration.", diff --git a/src/aiperf/metrics/types/inter_token_latency_metric.py b/src/aiperf/metrics/types/inter_token_latency_metric.py index 800e937b9b..2aa0f05242 100644 --- a/src/aiperf/metrics/types/inter_token_latency_metric.py +++ b/src/aiperf/metrics/types/inter_token_latency_metric.py @@ -6,6 +6,7 @@ from aiperf.common.models import ParsedResponseRecord from aiperf.metrics import BaseRecordMetric from aiperf.metrics.metric_dicts import MetricRecordDict +from aiperf.metrics.types.decode_duration_metric import FullDecodeDurationMetric from aiperf.metrics.types.output_sequence_length_metric import ( OutputSequenceLengthMetric, ) @@ -53,3 +54,34 @@ def _parse_record( request_latency = record_metrics.get_or_raise(RequestLatencyMetric) return (request_latency - ttft) / (osl - 1) # type: ignore + + +class FullResponseInterTokenLatencyMetric(BaseRecordMetric[float]): + """Average token interval through explicit HTTP request completion.""" + + tag = "full_response_inter_token_latency" + header = "Full-Response Inter Token Latency" + short_header = "Full-Response ITL" + unit = MetricTimeUnit.NANOSECONDS + display_unit = MetricTimeUnit.MILLISECONDS + display_order = 410 + flags = ( + MetricFlags.STREAMING_TOKENS_ONLY + | MetricFlags.PERCENTILE_INCLUDES_FAILED_REQUESTS + ) + required_metrics = { + FullDecodeDurationMetric.tag, + OutputSequenceLengthMetric.tag, + } + + def _parse_record( + self, + record: ParsedResponseRecord, + record_metrics: MetricRecordDict, + ) -> float: + osl = record_metrics.get_or_raise(OutputSequenceLengthMetric) + if osl < 2: # type: ignore + raise NoMetricValue(f"Output sequence length must be at least 2, got {osl}") + + full_decode_duration = record_metrics.get_or_raise(FullDecodeDurationMetric) + return full_decode_duration / (osl - 1) # type: ignore diff --git a/src/aiperf/metrics/types/output_token_throughput_metrics.py b/src/aiperf/metrics/types/output_token_throughput_metrics.py index 51a67acbc3..231f41e70a 100644 --- a/src/aiperf/metrics/types/output_token_throughput_metrics.py +++ b/src/aiperf/metrics/types/output_token_throughput_metrics.py @@ -7,10 +7,11 @@ from aiperf.metrics import BaseDerivedMetric, BaseRecordMetric from aiperf.metrics.metric_dicts import MetricRecordDict, MetricResultsDict from aiperf.metrics.types.benchmark_duration_metric import BenchmarkDurationMetric -from aiperf.metrics.types.decode_duration_metric import FullDecodeDurationMetric -from aiperf.metrics.types.inter_token_latency_metric import InterTokenLatencyMetric +from aiperf.metrics.types.inter_token_latency_metric import ( + FullResponseInterTokenLatencyMetric, + InterTokenLatencyMetric, +) from aiperf.metrics.types.output_sequence_length_metric import ( - OutputSequenceLengthMetric, TotalOutputSequenceLengthMetric, ) @@ -81,7 +82,7 @@ def _parse_record( class FullResponseOutputTokenThroughputPerUserMetric(BaseRecordMetric[float]): - """Token rate from first parsed content through full request completion.""" + """Inverse of full-response inter-token latency.""" tag = "full_response_output_token_throughput_per_user" header = "Full-Response Output Token Throughput Per User" @@ -91,8 +92,7 @@ class FullResponseOutputTokenThroughputPerUserMetric(BaseRecordMetric[float]): display_order = 520 flags = MetricFlags.STREAMING_TOKENS_ONLY | MetricFlags.LARGER_IS_BETTER required_metrics = { - FullDecodeDurationMetric.tag, - OutputSequenceLengthMetric.tag, + FullResponseInterTokenLatencyMetric.tag, } def _parse_record( @@ -100,16 +100,12 @@ def _parse_record( record: ParsedResponseRecord, record_metrics: MetricRecordDict, ) -> float: - osl = record_metrics.get_or_raise(OutputSequenceLengthMetric) - if osl < 2: # type: ignore - raise NoMetricValue(f"Output sequence length must be at least 2, got {osl}") - - duration = record_metrics.get_converted_or_raise( - FullDecodeDurationMetric, + converted_itl = record_metrics.get_converted_or_raise( + FullResponseInterTokenLatencyMetric, self.unit.time_unit, # type: ignore ) - if duration == 0: + if converted_itl == 0: raise NoMetricValue( - "Full decode duration is zero, cannot calculate full-response output token throughput" + "Full-response ITL is zero, cannot calculate full-response output token throughput" ) - return (osl - 1) / duration # type: ignore + return 1 / converted_itl diff --git a/tests/unit/metrics/test_inter_token_latency_metric.py b/tests/unit/metrics/test_inter_token_latency_metric.py index da01b80a2f..e972efc9ae 100644 --- a/tests/unit/metrics/test_inter_token_latency_metric.py +++ b/tests/unit/metrics/test_inter_token_latency_metric.py @@ -6,7 +6,11 @@ from aiperf.common.exceptions import NoMetricValue from aiperf.metrics.metric_dicts import MetricRecordDict -from aiperf.metrics.types.inter_token_latency_metric import InterTokenLatencyMetric +from aiperf.metrics.types.decode_duration_metric import FullDecodeDurationMetric +from aiperf.metrics.types.inter_token_latency_metric import ( + FullResponseInterTokenLatencyMetric, + InterTokenLatencyMetric, +) from aiperf.metrics.types.output_sequence_length_metric import ( OutputSequenceLengthMetric, ) @@ -81,3 +85,59 @@ def test_inter_token_latency_missing_required_metrics(self): with pytest.raises(NoMetricValue): InterTokenLatencyMetric().parse_record(record, empty_metrics) + + +class TestFullResponseInterTokenLatencyMetric: + def test_calculates_full_decode_duration_per_token_interval(self) -> None: + record = create_record() + metric_dict = MetricRecordDict( + { + FullDecodeDurationMetric.tag: 1_000_000_000, + OutputSequenceLengthMetric.tag: 11, + } + ) + + result = FullResponseInterTokenLatencyMetric().parse_record(record, metric_dict) + + assert result == 100_000_000 + + def test_kimi_parser_gap_uses_full_request_end(self) -> None: + request_start_ns = 1_000_000_000 + record = create_record( + start_ns=request_start_ns, + responses=[1_529_058_811, 1_610_559_573], + ) + record.request.end_perf_ns = request_start_ns + 145_861_451_008 + assert record.token_counts is not None + record.token_counts.output = 26_571 + + metric_results = run_simple_metrics_pipeline( + [record], + InterTokenLatencyMetric.tag, + FullResponseInterTokenLatencyMetric.tag, + ) + + assert metric_results[InterTokenLatencyMetric.tag] == pytest.approx( + [81_500_762 / 26_570] + ) + assert metric_results[FullResponseInterTokenLatencyMetric.tag] == pytest.approx( + [145_332_392_197 / 26_570] + ) + + def test_requires_at_least_two_tokens(self) -> None: + record = create_record() + metric_dict = MetricRecordDict( + { + FullDecodeDurationMetric.tag: 1_000_000_000, + OutputSequenceLengthMetric.tag: 1, + } + ) + + with pytest.raises(NoMetricValue, match="at least 2"): + FullResponseInterTokenLatencyMetric().parse_record(record, metric_dict) + + def test_requires_dependencies(self) -> None: + with pytest.raises(NoMetricValue): + FullResponseInterTokenLatencyMetric().parse_record( + create_record(), MetricRecordDict() + ) diff --git a/tests/unit/metrics/test_output_token_throughput_per_user_metric.py b/tests/unit/metrics/test_output_token_throughput_per_user_metric.py index b7899be6a6..ae64e7376a 100644 --- a/tests/unit/metrics/test_output_token_throughput_per_user_metric.py +++ b/tests/unit/metrics/test_output_token_throughput_per_user_metric.py @@ -5,10 +5,9 @@ from aiperf.common.exceptions import NoMetricValue from aiperf.metrics.metric_dicts import MetricRecordDict -from aiperf.metrics.types.decode_duration_metric import FullDecodeDurationMetric -from aiperf.metrics.types.inter_token_latency_metric import InterTokenLatencyMetric -from aiperf.metrics.types.output_sequence_length_metric import ( - OutputSequenceLengthMetric, +from aiperf.metrics.types.inter_token_latency_metric import ( + FullResponseInterTokenLatencyMetric, + InterTokenLatencyMetric, ) from aiperf.metrics.types.output_token_throughput_metrics import ( FullResponseOutputTokenThroughputPerUserMetric, @@ -80,12 +79,11 @@ def test_kimi_parser_gap_uses_full_request_end(self) -> None: FullResponseOutputTokenThroughputPerUserMetric.tag ] == pytest.approx([182.82228456622666]) - def test_calculates_rate_over_full_decode_duration(self) -> None: + def test_calculates_inverse_of_full_response_itl(self) -> None: record = create_record() metric_dict = MetricRecordDict( { - FullDecodeDurationMetric.tag: 1_000_000_000, - OutputSequenceLengthMetric.tag: 11, + FullResponseInterTokenLatencyMetric.tag: 100_000_000, } ) @@ -95,30 +93,15 @@ def test_calculates_rate_over_full_decode_duration(self) -> None: assert result == 10.0 - def test_requires_at_least_two_tokens(self) -> None: + def test_rejects_zero_full_response_itl(self) -> None: record = create_record() metric_dict = MetricRecordDict( { - FullDecodeDurationMetric.tag: 1_000_000_000, - OutputSequenceLengthMetric.tag: 1, + FullResponseInterTokenLatencyMetric.tag: 0, } ) - with pytest.raises(NoMetricValue, match="at least 2"): - FullResponseOutputTokenThroughputPerUserMetric().parse_record( - record, metric_dict - ) - - def test_rejects_zero_duration(self) -> None: - record = create_record() - metric_dict = MetricRecordDict( - { - FullDecodeDurationMetric.tag: 0, - OutputSequenceLengthMetric.tag: 11, - } - ) - - with pytest.raises(NoMetricValue, match="duration is zero"): + with pytest.raises(NoMetricValue, match="ITL is zero"): FullResponseOutputTokenThroughputPerUserMetric().parse_record( record, metric_dict )