From dcbd942654ef87b357c7722837855dbccedc2353 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 1 Sep 2026 15:33:30 -0500 Subject: [PATCH] fix(agentx): tag automatic warmup metric baselines as warmup Signed-off-by: Cam Quilici --- docs/server-metrics/server-metrics.md | 5 ++- src/aiperf/timing/config.py | 1 + .../test_server_metrics_manager.py | 37 ++++++++++++++++++- .../test_phase_config_agentic_replay.py | 1 + 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/docs/server-metrics/server-metrics.md b/docs/server-metrics/server-metrics.md index 75c7fdc77e..839a69723a 100644 --- a/docs/server-metrics/server-metrics.md +++ b/docs/server-metrics/server-metrics.md @@ -176,6 +176,10 @@ AIPerf automatically: **Time filtering:** Statistics in JSON/CSV exports exclude the warmup period, showing only metrics from the profiling phase. The JSONL file contains all scrapes (including warmup) for complete time-series analysis. +This also applies to AgentX's automatically created warmup phase: both its start +and end baseline scrapes are tagged as warmup, so their counters do not enter the +profiling totals. + **Format selection:** By default, JSON, CSV, and Parquet formats are generated (JSONL is opt-in to avoid large files). To opt out of Parquet, or to include JSONL for time-series analysis: ```bash # Disable Parquet (JSON + CSV only) @@ -659,4 +663,3 @@ with open('server_metrics_export.json') as f: latency = data['metrics']['vllm:e2e_request_latency_seconds']['series'][0]['stats'] assert latency['p99_estimate'] < 5.0, f"P99 latency too high: {latency['p99_estimate']}" ``` - diff --git a/src/aiperf/timing/config.py b/src/aiperf/timing/config.py index 50e662192f..49bf42439d 100644 --- a/src/aiperf/timing/config.py +++ b/src/aiperf/timing/config.py @@ -644,6 +644,7 @@ def _build_agentic_warmup_config(phase: PhaseConfig) -> CreditPhaseConfig | None return CreditPhaseConfig( phase=CreditPhase.WARMUP, timing_mode=TimingMode.AGENTIC_REPLAY, + phase_kind="warmup", # Duration mode is strategy-terminated by its timer. Count mode uses # the generic request-count stop condition as a global backstop while # the agentic strategy independently enforces each lane's quota. diff --git a/tests/unit/server_metrics/test_server_metrics_manager.py b/tests/unit/server_metrics/test_server_metrics_manager.py index 2bec3384e1..4aec0b72d6 100644 --- a/tests/unit/server_metrics/test_server_metrics_manager.py +++ b/tests/unit/server_metrics/test_server_metrics_manager.py @@ -5,6 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from pydantic import TypeAdapter from aiperf.common.enums import BaselineKind, CommandType, CreditPhase from aiperf.common.messages import ( @@ -16,13 +17,15 @@ from aiperf.common.models import CreditPhaseStats, ErrorDetails from aiperf.common.models.server_metrics_models import ServerMetricsRecord from aiperf.config.flags.cli_config import CLIConfig +from aiperf.config.phases import PhaseConfig from aiperf.credit.messages import ( CreditPhaseCompleteMessage, CreditPhaseStartMessage, ) from aiperf.plugin.enums import EndpointType, TimingMode from aiperf.server_metrics.manager import ServerMetricsManager -from aiperf.timing.config import CreditPhaseConfig +from aiperf.timing.config import CreditPhaseConfig, _build_agentic_warmup_config +from aiperf.timing.phase.publisher import PhasePublisher from tests.unit.conftest import make_run_from_cli @@ -498,6 +501,38 @@ async def test_record_callback_handles_send_failure( class TestPhaseTransitionRace: + @pytest.mark.asyncio + @pytest.mark.parametrize("kind", [BaselineKind.START, BaselineKind.END]) + async def test_agentic_auto_warmup_baselines_are_not_profiling( + self, cfg_with_endpoint: CLIConfig, kind: BaselineKind + ) -> None: + phase = TypeAdapter(PhaseConfig).validate_python( + { + "name": "profiling", + "type": "concurrency", + "concurrency": 8, + "duration": 3600, + "timing_mode": TimingMode.AGENTIC_REPLAY, + } + ) + warmup = _build_agentic_warmup_config(phase) + assert warmup is not None + pub_client = AsyncMock() + publisher = PhasePublisher(pub_client=pub_client, service_id="timing-manager") + await publisher.publish_phase_baseline_request(warmup, "auto-warmup", kind) + message = pub_client.publish.call_args.args[0] + + manager = ServerMetricsManager(run=make_run_from_cli(cfg_with_endpoint)) + collector = MagicMock() + manager._collectors = {"http://localhost:8000/metrics": collector} + with patch.object( + manager, "_collect_and_process_metrics_for_phase", new_callable=AsyncMock + ) as collect: + await manager.collect_baseline(message) + + assert message.phase_kind == "warmup" + collect.assert_awaited_once_with(collector, CreditPhase.WARMUP) + """Phase-tagging transitions must be compare-and-set: message handlers run as independent tasks, so CREDIT_PHASE_START(PROFILING) can interleave with the awaited warmup-final scrapes inside _on_credit_phase_complete.""" diff --git a/tests/unit/timing/test_phase_config_agentic_replay.py b/tests/unit/timing/test_phase_config_agentic_replay.py index 7baf752b74..086401c6ac 100644 --- a/tests/unit/timing/test_phase_config_agentic_replay.py +++ b/tests/unit/timing/test_phase_config_agentic_replay.py @@ -35,6 +35,7 @@ def test_warmup_config_uses_agentic_replay_when_top_level_is_agentic_replay() -> assert warmup is not None assert warmup.timing_mode == TimingMode.AGENTIC_REPLAY assert warmup.phase == CreditPhase.WARMUP + assert warmup.phase_kind == "warmup" def test_profiling_config_propagates_cap() -> None: