diff --git a/docs/benchmark-modes/semianalysis-agentx-faq.md b/docs/benchmark-modes/semianalysis-agentx-faq.md index 8ae2a5d781..a2744f4219 100644 --- a/docs/benchmark-modes/semianalysis-agentx-faq.md +++ b/docs/benchmark-modes/semianalysis-agentx-faq.md @@ -837,8 +837,9 @@ the failed and total request counts, observed failure percentage, configured lim operator to the inference-server logs. For profiling phases that meet the AgentX scenario's minimum valid duration, the scenario also -requires TTFT and inter-token-latency observations to extend through at least 98% of the phase. This -catches a server that stops returning responses while AIPerf still has requests in flight: the run +requires TTFT or inter-token-latency observations to extend through at least 98% of the phase. This +catches a server that stops returning responses while allowing a healthy long response to keep +proving global server activity even when no new request starts near the boundary. A stalled run exits non-zero, the JSON artifact is retained with `submission_valid: false` and reason `insufficient_profile_metric_coverage`, and the error directs the operator to the server logs. Warmup observations and intentionally short `--unsafe-override` smoke runs do not count. diff --git a/docs/tutorials/agentx-mvp.md b/docs/tutorials/agentx-mvp.md index 0172d077f4..7e0267f7c0 100644 --- a/docs/tutorials/agentx-mvp.md +++ b/docs/tutorials/agentx-mvp.md @@ -654,8 +654,10 @@ the aggregate file — divide it by to see how close you were to the limit. **Run exits non-zero with `ProfileMetricCoverageError`** -The server stopped producing TTFT or inter-token-latency observations before 98% of the configured -profiling duration elapsed. AIPerf retains the result artifact, marks it invalid with +The server stopped producing both TTFT and inter-token-latency observations before 98% of the +configured profiling duration elapsed. Either signal proves global server activity, so a long +streaming response remains valid even when no new request starts near the boundary. AIPerf retains +the result artifact, marks it invalid with `insufficient_profile_metric_coverage`, and reports the observed coverage for both signals. Check the inference-server logs for a crash or stalled request processing. Warmup metrics and profiling phases shorter than the scenario's minimum valid duration are excluded. diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 2a9e06adc7..5dcebb5347 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -439,10 +439,10 @@ class ProfileMetricDurationCoverage(AIPerfBaseModel): @property def passed(self) -> bool: - """Return whether every required latency signal met the threshold.""" + """Return whether any latency signal proves late profiling activity.""" return ( self.ttft_ratio >= self.required_ratio - and self.inter_token_latency_ratio >= self.required_ratio + or self.inter_token_latency_ratio >= self.required_ratio ) @@ -511,7 +511,7 @@ class ProfileResults(AIPerfBaseModel): ) metric_duration_coverage: list[ProfileMetricDurationCoverage] = Field( default_factory=list, - description="Post-run TTFT and ITL duration-coverage checks, when enabled.", + description="Post-run latency-signal duration-coverage checks, when enabled.", ) runtime_submission_invalid_reasons: list[str] = Field( default_factory=list, diff --git a/src/aiperf/common/scenario/base.py b/src/aiperf/common/scenario/base.py index 964be1d88d..6970c9917c 100644 --- a/src/aiperf/common/scenario/base.py +++ b/src/aiperf/common/scenario/base.py @@ -144,7 +144,7 @@ class ScenarioSpec(AIPerfBaseModel): le=1.0, description=( "Minimum fraction of each duration-based profiling phase that must " - "contain both TTFT and inter-token-latency observations. A run that " + "contain TTFT or inter-token-latency observations. A run that " "falls below the threshold exits non-zero and is not a valid scenario " "submission. None disables the post-run check." ), diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 9e5cb94669..b8c2a7c386 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -1826,9 +1826,9 @@ def _validate_profile_metric_duration_coverage( f"for phase {phase_config.name!r}: TTFT={coverage.ttft_ratio:.1%}, " "inter-token latency=" f"{coverage.inter_token_latency_ratio:.1%} over the configured " - f"{float(phase_config.duration):.1f}s duration. At least one required " - f"metric stopped more than {allowed_tail_seconds:.1f}s before the " - "nominal profiling end; check inference server logs for a stalled " + f"{float(phase_config.duration):.1f}s duration. Neither latency " + f"signal extended into the final {allowed_tail_seconds:.1f}s before " + "the nominal profiling end; check inference server logs for a stalled " "or unavailable server." ) self.error(message) diff --git a/tests/unit/post_processors/test_metrics_accumulator.py b/tests/unit/post_processors/test_metrics_accumulator.py index 4a382794d7..b24996dc98 100644 --- a/tests/unit/post_processors/test_metrics_accumulator.py +++ b/tests/unit/post_processors/test_metrics_accumulator.py @@ -157,10 +157,10 @@ async def test_profile_metric_duration_coverage_accepts_threshold_boundary( assert coverage.passed is True @pytest.mark.asyncio - async def test_profile_metric_duration_coverage_requires_itl( + async def test_profile_metric_duration_coverage_accepts_ttft_without_itl( self, mock_metric_registry: Mock, mock_run ) -> None: - """TTFT alone cannot make an interactive streaming run valid.""" + """Late TTFT proves activity when a response has no inter-token interval.""" processor = MetricsAccumulator(mock_run) phase_start_ns = 100 * NANOS_PER_SECOND record = create_metric_records_data( @@ -183,7 +183,39 @@ async def test_profile_metric_duration_coverage_requires_itl( assert coverage.ttft_ratio == pytest.approx(0.98) assert coverage.inter_token_latency_ratio == 0.0 - assert coverage.passed is False + assert coverage.passed is True + + @pytest.mark.asyncio + async def test_profile_metric_duration_coverage_accepts_late_streaming_activity( + self, mock_metric_registry: Mock, mock_run + ) -> None: + """A long response streaming near the end remains globally live.""" + processor = MetricsAccumulator(mock_run) + phase_start_ns = 100 * NANOS_PER_SECOND + record = create_metric_records_data( + session_num=0, + request_start_ns=147 * NANOS_PER_SECOND, + request_end_ns=199 * NANOS_PER_SECOND, + results=[ + {"time_to_first_token": NANOS_PER_SECOND}, + {"inter_token_latency": 100_000_000}, + ], + ) + await processor.process_record(record) + + coverage = processor.profile_metric_duration_coverage( + ExportContext( + start_ns=phase_start_ns, + phase=CreditPhase.PROFILING, + ), + phase_name="profiling", + expected_duration_seconds=100.0, + required_ratio=0.98, + ) + + assert coverage.ttft_ratio == pytest.approx(0.48) + assert coverage.inter_token_latency_ratio == pytest.approx(0.99) + assert coverage.passed is True @pytest.mark.asyncio async def test_process_record_record_metric_list_values( diff --git a/tests/unit/records/test_records_manager_process_results.py b/tests/unit/records/test_records_manager_process_results.py index f58e98bb32..6a1d22ddab 100644 --- a/tests/unit/records/test_records_manager_process_results.py +++ b/tests/unit/records/test_records_manager_process_results.py @@ -341,6 +341,32 @@ async def test_agentx_metric_coverage_passes_at_threshold(self) -> None: assert result.fatal_errors == [] assert result.results.runtime_submission_invalid_reasons == [] + @pytest.mark.asyncio + async def test_agentx_metric_coverage_accepts_late_streaming_activity(self) -> None: + acc = _make_summary_accumulator([_STUB_METRIC_RESULT]) + acc.profile_metric_duration_coverage.return_value = ( + ProfileMetricDurationCoverage( + phase_name="profiling", + expected_duration_seconds=3600.0, + required_ratio=0.98, + ttft_ratio=0.979504, + inter_token_latency_ratio=1.0, + ) + ) + mgr = _make_manager_mock(accumulators={AccumulatorType.METRIC_RESULTS: acc}) + mgr.run.cfg.scenario = "inferencex-agentx-mvp" + phase_config = MagicMock() + phase_config.name = "profiling" + phase_config.duration = 3600.0 + mgr.run.cfg.get_profiling_phases.return_value = [phase_config] + + result = await mgr._process_results( + phase=CreditPhase.PROFILING, cancelled=False + ) + + assert result.fatal_errors == [] + assert result.results.runtime_submission_invalid_reasons == [] + @pytest.mark.asyncio async def test_agentx_short_unsafe_smoke_skips_metric_coverage(self) -> None: acc = _make_summary_accumulator([_STUB_METRIC_RESULT])