Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/benchmark-modes/semianalysis-agentx-faq.md
Original file line number Diff line number Diff line change
Expand Up @@ -837,9 +837,9 @@ the failed and total request counts, observed failure percentage, configured lim
operator to the inference-server logs.

For profiling phases that meet the AgentX scenario's minimum valid duration, the scenario also
requires TTFT or inter-token-latency observations to extend through at least 98% of the phase. This
catches a server that stops returning responses while allowing a healthy long response to keep
proving global server activity even when no new request starts near the boundary. A stalled run
requires TTFT or inter-token-latency observations to extend through at least 95% of the phase. This
catches a server that stops returning responses while allowing a sparse low-concurrency run to end
with a long request in flight. A stalled run
exits non-zero, the JSON artifact is retained with `submission_valid: false` and reason
`insufficient_profile_metric_coverage`, and the error directs the operator to the server logs.
Warmup observations and intentionally short `--unsafe-override` smoke runs do not count.
Expand Down
2 changes: 1 addition & 1 deletion src/aiperf/common/scenario/inferencex_agentx_mvp.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,5 @@
system_idle_gap_cap_seconds=10.0,
forbid_inter_turn_delay_cap=True,
require_cache_bust=CacheBustTarget.FIRST_TURN_PREFIX,
minimum_profile_metric_coverage_ratio=0.98,
minimum_profile_metric_coverage_ratio=0.95,
)
2 changes: 1 addition & 1 deletion tests/unit/common/scenario/test_scenario_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def test_inferencex_agentx_mvp_registered():
assert spec.inter_turn_delay_cap_seconds is None
assert spec.trace_idle_gap_cap_seconds is None
assert spec.system_idle_gap_cap_seconds == 10.0
assert spec.minimum_profile_metric_coverage_ratio == 0.98
assert spec.minimum_profile_metric_coverage_ratio == 0.95
assert spec.forbid_trace_idle_gap_cap is False
assert spec.forbid_inter_turn_delay_cap is True

Expand Down
4 changes: 2 additions & 2 deletions tests/unit/records/test_records_manager_process_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ async def test_accumulator_summarize_failure_does_not_abort(self) -> None:

@pytest.mark.asyncio
async def test_agentx_metric_coverage_failure_is_fatal(self) -> None:
"""A duration-based AgentX phase below 98% remains exportable but fatal."""
"""A duration-based AgentX phase below 95% remains exportable but fatal."""
acc = _make_summary_accumulator([_STUB_METRIC_RESULT])
acc.profile_metric_duration_coverage.return_value = (
ProfileMetricDurationCoverage(
Expand All @@ -306,7 +306,7 @@ async def test_agentx_metric_coverage_failure_is_fatal(self) -> None:

assert len(result.fatal_errors) == 1
assert result.fatal_errors[0].type == "ProfileMetricCoverageError"
assert "required 98.0%" in result.fatal_errors[0].message
assert "required 95.0%" in result.fatal_errors[0].message
assert "check inference server logs" in result.fatal_errors[0].message
assert result.results.runtime_submission_invalid_reasons == [
"insufficient_profile_metric_coverage"
Expand Down
Loading