Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs/server-metrics/server-metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,10 @@ AIPerf automatically:

**Time filtering:** Statistics in JSON/CSV exports exclude the warmup period, showing only metrics from the profiling phase. The JSONL file contains all scrapes (including warmup) for complete time-series analysis.

This also applies to AgentX's automatically created warmup phase: both its start
and end baseline scrapes are tagged as warmup, so their counters do not enter the
profiling totals.

**Format selection:** By default, JSON, CSV, and Parquet formats are generated (JSONL is opt-in to avoid large files). To opt out of Parquet, or to include JSONL for time-series analysis:
```bash
# Disable Parquet (JSON + CSV only)
Expand Down Expand Up @@ -659,4 +663,3 @@ with open('server_metrics_export.json') as f:
latency = data['metrics']['vllm:e2e_request_latency_seconds']['series'][0]['stats']
assert latency['p99_estimate'] < 5.0, f"P99 latency too high: {latency['p99_estimate']}"
```

1 change: 1 addition & 0 deletions src/aiperf/timing/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,7 @@ def _build_agentic_warmup_config(phase: PhaseConfig) -> CreditPhaseConfig | None
return CreditPhaseConfig(
phase=CreditPhase.WARMUP,
timing_mode=TimingMode.AGENTIC_REPLAY,
phase_kind="warmup",
# Duration mode is strategy-terminated by its timer. Count mode uses
# the generic request-count stop condition as a global backstop while
# the agentic strategy independently enforces each lane's quota.
Expand Down
37 changes: 36 additions & 1 deletion tests/unit/server_metrics/test_server_metrics_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from pydantic import TypeAdapter

from aiperf.common.enums import BaselineKind, CommandType, CreditPhase
from aiperf.common.messages import (
Expand All @@ -16,13 +17,15 @@
from aiperf.common.models import CreditPhaseStats, ErrorDetails
from aiperf.common.models.server_metrics_models import ServerMetricsRecord
from aiperf.config.flags.cli_config import CLIConfig
from aiperf.config.phases import PhaseConfig
from aiperf.credit.messages import (
CreditPhaseCompleteMessage,
CreditPhaseStartMessage,
)
from aiperf.plugin.enums import EndpointType, TimingMode
from aiperf.server_metrics.manager import ServerMetricsManager
from aiperf.timing.config import CreditPhaseConfig
from aiperf.timing.config import CreditPhaseConfig, _build_agentic_warmup_config
from aiperf.timing.phase.publisher import PhasePublisher
from tests.unit.conftest import make_run_from_cli


Expand Down Expand Up @@ -498,6 +501,38 @@ async def test_record_callback_handles_send_failure(


class TestPhaseTransitionRace:
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", [BaselineKind.START, BaselineKind.END])
async def test_agentic_auto_warmup_baselines_are_not_profiling(
self, cfg_with_endpoint: CLIConfig, kind: BaselineKind
) -> None:
phase = TypeAdapter(PhaseConfig).validate_python(
{
"name": "profiling",
"type": "concurrency",
"concurrency": 8,
"duration": 3600,
"timing_mode": TimingMode.AGENTIC_REPLAY,
}
)
warmup = _build_agentic_warmup_config(phase)
assert warmup is not None
pub_client = AsyncMock()
publisher = PhasePublisher(pub_client=pub_client, service_id="timing-manager")
await publisher.publish_phase_baseline_request(warmup, "auto-warmup", kind)
message = pub_client.publish.call_args.args[0]

manager = ServerMetricsManager(run=make_run_from_cli(cfg_with_endpoint))
collector = MagicMock()
manager._collectors = {"http://localhost:8000/metrics": collector}
with patch.object(
manager, "_collect_and_process_metrics_for_phase", new_callable=AsyncMock
) as collect:
await manager.collect_baseline(message)

assert message.phase_kind == "warmup"
collect.assert_awaited_once_with(collector, CreditPhase.WARMUP)

"""Phase-tagging transitions must be compare-and-set: message handlers run
as independent tasks, so CREDIT_PHASE_START(PROFILING) can interleave with
the awaited warmup-final scrapes inside _on_credit_phase_complete."""
Expand Down
1 change: 1 addition & 0 deletions tests/unit/timing/test_phase_config_agentic_replay.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def test_warmup_config_uses_agentic_replay_when_top_level_is_agentic_replay() ->
assert warmup is not None
assert warmup.timing_mode == TimingMode.AGENTIC_REPLAY
assert warmup.phase == CreditPhase.WARMUP
assert warmup.phase_kind == "warmup"


def test_profiling_config_propagates_cap() -> None:
Expand Down
Loading