Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/benchmark-modes/semianalysis-agentx-faq.md
Original file line number Diff line number Diff line change
Expand Up @@ -837,8 +837,9 @@ the failed and total request counts, observed failure percentage, configured lim
operator to the inference-server logs.

For profiling phases that meet the AgentX scenario's minimum valid duration, the scenario also
requires TTFT and inter-token-latency observations to extend through at least 98% of the phase. This
catches a server that stops returning responses while AIPerf still has requests in flight: the run
requires TTFT or inter-token-latency observations to extend through at least 98% of the phase. This
catches a server that stops returning responses while allowing a healthy long response to keep
proving global server activity even when no new request starts near the boundary. A stalled run
exits non-zero, the JSON artifact is retained with `submission_valid: false` and reason
`insufficient_profile_metric_coverage`, and the error directs the operator to the server logs.
Warmup observations and intentionally short `--unsafe-override` smoke runs do not count.
Expand Down
6 changes: 4 additions & 2 deletions docs/tutorials/agentx-mvp.md
Original file line number Diff line number Diff line change
Expand Up @@ -654,8 +654,10 @@ the aggregate file — divide it by
to see how close you were to the limit.

**Run exits non-zero with `ProfileMetricCoverageError`**
The server stopped producing TTFT or inter-token-latency observations before 98% of the configured
profiling duration elapsed. AIPerf retains the result artifact, marks it invalid with
The server stopped producing both TTFT and inter-token-latency observations before 98% of the
configured profiling duration elapsed. Either signal proves global server activity, so a long
streaming response remains valid even when no new request starts near the boundary. AIPerf retains
the result artifact, marks it invalid with
`insufficient_profile_metric_coverage`, and reports the observed coverage for both signals. Check
the inference-server logs for a crash or stalled request processing. Warmup metrics and profiling
phases shorter than the scenario's minimum valid duration are excluded.
Expand Down
6 changes: 3 additions & 3 deletions src/aiperf/common/models/record_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,10 +439,10 @@ class ProfileMetricDurationCoverage(AIPerfBaseModel):

@property
def passed(self) -> bool:
"""Return whether every required latency signal met the threshold."""
"""Return whether any latency signal proves late profiling activity."""
return (
self.ttft_ratio >= self.required_ratio
and self.inter_token_latency_ratio >= self.required_ratio
or self.inter_token_latency_ratio >= self.required_ratio
)


Expand Down Expand Up @@ -511,7 +511,7 @@ class ProfileResults(AIPerfBaseModel):
)
metric_duration_coverage: list[ProfileMetricDurationCoverage] = Field(
default_factory=list,
description="Post-run TTFT and ITL duration-coverage checks, when enabled.",
description="Post-run latency-signal duration-coverage checks, when enabled.",
)
runtime_submission_invalid_reasons: list[str] = Field(
default_factory=list,
Expand Down
2 changes: 1 addition & 1 deletion src/aiperf/common/scenario/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ class ScenarioSpec(AIPerfBaseModel):
le=1.0,
description=(
"Minimum fraction of each duration-based profiling phase that must "
"contain both TTFT and inter-token-latency observations. A run that "
"contain TTFT or inter-token-latency observations. A run that "
"falls below the threshold exits non-zero and is not a valid scenario "
"submission. None disables the post-run check."
),
Expand Down
6 changes: 3 additions & 3 deletions src/aiperf/records/records_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -1826,9 +1826,9 @@ def _validate_profile_metric_duration_coverage(
f"for phase {phase_config.name!r}: TTFT={coverage.ttft_ratio:.1%}, "
"inter-token latency="
f"{coverage.inter_token_latency_ratio:.1%} over the configured "
f"{float(phase_config.duration):.1f}s duration. At least one required "
f"metric stopped more than {allowed_tail_seconds:.1f}s before the "
"nominal profiling end; check inference server logs for a stalled "
f"{float(phase_config.duration):.1f}s duration. Neither latency "
f"signal extended into the final {allowed_tail_seconds:.1f}s before "
"the nominal profiling end; check inference server logs for a stalled "
"or unavailable server."
)
self.error(message)
Expand Down
38 changes: 35 additions & 3 deletions tests/unit/post_processors/test_metrics_accumulator.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,10 @@ async def test_profile_metric_duration_coverage_accepts_threshold_boundary(
assert coverage.passed is True

@pytest.mark.asyncio
async def test_profile_metric_duration_coverage_requires_itl(
async def test_profile_metric_duration_coverage_accepts_ttft_without_itl(
self, mock_metric_registry: Mock, mock_run
) -> None:
"""TTFT alone cannot make an interactive streaming run valid."""
"""Late TTFT proves activity when a response has no inter-token interval."""
processor = MetricsAccumulator(mock_run)
phase_start_ns = 100 * NANOS_PER_SECOND
record = create_metric_records_data(
Expand All @@ -183,7 +183,39 @@ async def test_profile_metric_duration_coverage_requires_itl(

assert coverage.ttft_ratio == pytest.approx(0.98)
assert coverage.inter_token_latency_ratio == 0.0
assert coverage.passed is False
assert coverage.passed is True

@pytest.mark.asyncio
async def test_profile_metric_duration_coverage_accepts_late_streaming_activity(
self, mock_metric_registry: Mock, mock_run
) -> None:
"""A long response streaming near the end remains globally live."""
processor = MetricsAccumulator(mock_run)
phase_start_ns = 100 * NANOS_PER_SECOND
record = create_metric_records_data(
session_num=0,
request_start_ns=147 * NANOS_PER_SECOND,
request_end_ns=199 * NANOS_PER_SECOND,
results=[
{"time_to_first_token": NANOS_PER_SECOND},
{"inter_token_latency": 100_000_000},
],
)
await processor.process_record(record)

coverage = processor.profile_metric_duration_coverage(
ExportContext(
start_ns=phase_start_ns,
phase=CreditPhase.PROFILING,
),
phase_name="profiling",
expected_duration_seconds=100.0,
required_ratio=0.98,
)

assert coverage.ttft_ratio == pytest.approx(0.48)
assert coverage.inter_token_latency_ratio == pytest.approx(0.99)
assert coverage.passed is True

@pytest.mark.asyncio
async def test_process_record_record_metric_list_values(
Expand Down
26 changes: 26 additions & 0 deletions tests/unit/records/test_records_manager_process_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,32 @@ async def test_agentx_metric_coverage_passes_at_threshold(self) -> None:
assert result.fatal_errors == []
assert result.results.runtime_submission_invalid_reasons == []

@pytest.mark.asyncio
async def test_agentx_metric_coverage_accepts_late_streaming_activity(self) -> None:
acc = _make_summary_accumulator([_STUB_METRIC_RESULT])
acc.profile_metric_duration_coverage.return_value = (
ProfileMetricDurationCoverage(
phase_name="profiling",
expected_duration_seconds=3600.0,
required_ratio=0.98,
ttft_ratio=0.979504,
inter_token_latency_ratio=1.0,
)
)
mgr = _make_manager_mock(accumulators={AccumulatorType.METRIC_RESULTS: acc})
mgr.run.cfg.scenario = "inferencex-agentx-mvp"
phase_config = MagicMock()
phase_config.name = "profiling"
phase_config.duration = 3600.0
mgr.run.cfg.get_profiling_phases.return_value = [phase_config]

result = await mgr._process_results(
phase=CreditPhase.PROFILING, cancelled=False
)

assert result.fatal_errors == []
assert result.results.runtime_submission_invalid_reasons == []

@pytest.mark.asyncio
async def test_agentx_short_unsafe_smoke_skips_metric_coverage(self) -> None:
acc = _make_summary_accumulator([_STUB_METRIC_RESULT])
Expand Down
Loading