From c5f353f0ae120bc4d2531122dbcec90194ffcaa9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 06:36:11 -0500 Subject: [PATCH] fix(agentx): allow sparse low-concurrency profile tails Signed-off-by: Cam Quilici --- docs/benchmark-modes/semianalysis-agentx-faq.md | 6 +++--- src/aiperf/common/scenario/inferencex_agentx_mvp.py | 2 +- tests/unit/common/scenario/test_scenario_registry.py | 2 +- tests/unit/records/test_records_manager_process_results.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/benchmark-modes/semianalysis-agentx-faq.md b/docs/benchmark-modes/semianalysis-agentx-faq.md index a2744f4219..94ab541b7f 100644 --- a/docs/benchmark-modes/semianalysis-agentx-faq.md +++ b/docs/benchmark-modes/semianalysis-agentx-faq.md @@ -837,9 +837,9 @@ the failed and total request counts, observed failure percentage, configured lim operator to the inference-server logs. For profiling phases that meet the AgentX scenario's minimum valid duration, the scenario also -requires TTFT or inter-token-latency observations to extend through at least 98% of the phase. This -catches a server that stops returning responses while allowing a healthy long response to keep -proving global server activity even when no new request starts near the boundary. A stalled run +requires TTFT or inter-token-latency observations to extend through at least 95% of the phase. This +catches a server that stops returning responses while allowing a sparse low-concurrency run to end +with a long request in flight. A stalled run exits non-zero, the JSON artifact is retained with `submission_valid: false` and reason `insufficient_profile_metric_coverage`, and the error directs the operator to the server logs. Warmup observations and intentionally short `--unsafe-override` smoke runs do not count. diff --git a/src/aiperf/common/scenario/inferencex_agentx_mvp.py b/src/aiperf/common/scenario/inferencex_agentx_mvp.py index 0a34079f10..cf5bb28f4b 100644 --- a/src/aiperf/common/scenario/inferencex_agentx_mvp.py +++ b/src/aiperf/common/scenario/inferencex_agentx_mvp.py @@ -36,5 +36,5 @@ system_idle_gap_cap_seconds=10.0, forbid_inter_turn_delay_cap=True, require_cache_bust=CacheBustTarget.FIRST_TURN_PREFIX, - minimum_profile_metric_coverage_ratio=0.98, + minimum_profile_metric_coverage_ratio=0.95, ) diff --git a/tests/unit/common/scenario/test_scenario_registry.py b/tests/unit/common/scenario/test_scenario_registry.py index 3164d73450..2d376c7b6e 100644 --- a/tests/unit/common/scenario/test_scenario_registry.py +++ b/tests/unit/common/scenario/test_scenario_registry.py @@ -36,7 +36,7 @@ def test_inferencex_agentx_mvp_registered(): assert spec.inter_turn_delay_cap_seconds is None assert spec.trace_idle_gap_cap_seconds is None assert spec.system_idle_gap_cap_seconds == 10.0 - assert spec.minimum_profile_metric_coverage_ratio == 0.98 + assert spec.minimum_profile_metric_coverage_ratio == 0.95 assert spec.forbid_trace_idle_gap_cap is False assert spec.forbid_inter_turn_delay_cap is True diff --git a/tests/unit/records/test_records_manager_process_results.py b/tests/unit/records/test_records_manager_process_results.py index 6a1d22ddab..abd09d8053 100644 --- a/tests/unit/records/test_records_manager_process_results.py +++ b/tests/unit/records/test_records_manager_process_results.py @@ -282,7 +282,7 @@ async def test_accumulator_summarize_failure_does_not_abort(self) -> None: @pytest.mark.asyncio async def test_agentx_metric_coverage_failure_is_fatal(self) -> None: - """A duration-based AgentX phase below 98% remains exportable but fatal.""" + """A duration-based AgentX phase below 95% remains exportable but fatal.""" acc = _make_summary_accumulator([_STUB_METRIC_RESULT]) acc.profile_metric_duration_coverage.return_value = ( ProfileMetricDurationCoverage( @@ -306,7 +306,7 @@ async def test_agentx_metric_coverage_failure_is_fatal(self) -> None: assert len(result.fatal_errors) == 1 assert result.fatal_errors[0].type == "ProfileMetricCoverageError" - assert "required 98.0%" in result.fatal_errors[0].message + assert "required 95.0%" in result.fatal_errors[0].message assert "check inference server logs" in result.fatal_errors[0].message assert result.results.runtime_submission_invalid_reasons == [ "insufficient_profile_metric_coverage"