From 0d89afc095de7d659b91761417ba222b6b64d2df Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 14 May 2026 16:54:53 +0530 Subject: [PATCH 01/21] Initial modifications for sys info --- pyproject.toml | 2 + .../commands/benchmark/execute.py | 12 + src/inference_endpoint/config/schema.py | 82 +++++ .../dataset_manager/transforms.py | 2 +- src/inference_endpoint/sys_info/__init__.py | 14 + src/inference_endpoint/sys_info/capture.py | 90 +++++ tests/unit/dataset_manager/test_transforms.py | 19 + tests/unit/sys_info/__init__.py | 14 + tests/unit/sys_info/test_capture.py | 325 ++++++++++++++++++ 9 files changed, 559 insertions(+), 1 deletion(-) create mode 100644 src/inference_endpoint/sys_info/__init__.py create mode 100644 src/inference_endpoint/sys_info/capture.py create mode 100644 tests/unit/sys_info/__init__.py create mode 100644 tests/unit/sys_info/test_capture.py diff --git a/pyproject.toml b/pyproject.toml index 8b227a5a5..90df1bc0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,8 @@ dependencies = [ "colorama==0.4.6", # Fix pytz-2024 import warning "pytz==2026.1.post1", + # MLCFlow for system info + "mlcflow", ] [project.optional-dependencies] diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 73c3427f1..b1936a6da 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -730,6 +730,18 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: except Exception as e: logger.error(f"Save failed: {e}") + if ctx.config.sys_info_capture is not None: + try: + # Local import: mlcflow is optional and only needed when sys_info_capture is configured. + from inference_endpoint.sys_info.capture import ( # noqa: PLC0415 + capture_system_info, + ) + + output_path = capture_system_info(ctx.config.sys_info_capture) + logger.info("System info captured at: %s", output_path) + except Exception as e: + logger.warning("sys_info_capture failed and will be skipped: %s", e) + def run_benchmark(config: BenchmarkConfig, test_mode: TestMode) -> None: """Orchestrate setup → execute → finalize.""" diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 6a1884b44..fd355f4df 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -22,6 +22,7 @@ from __future__ import annotations +import re from collections import Counter from enum import Enum from pathlib import Path @@ -453,6 +454,84 @@ def _validate_endpoint_scheme(cls, v: list[str]) -> list[str]: return v +_SSH_ID_RE = re.compile(r"^(?P[^@]+)@(?P[^:]+?)(?::(?P\d+))?$") + + +class SshTarget(BaseModel): + """Parsed SSH target from a raw 'username@host' or 'username@host:port' string.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + username: str + host: str + port: int = 22 + + def to_mlcflow_str(self) -> str: + return f"{self.username}@{self.host}:{self.port}" + + @field_validator("port", mode="after") + @classmethod + def _validate_port(cls, v: int) -> int: + if not (1 <= v <= 65535): + raise ValueError(f"Port must be in range 1-65535, got {v}") + return v + + +class SysInfoCaptureConfig(BaseModel): + """Configuration for the sys_info_capture post-benchmark step.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + exclude_current_system: bool = False + accelerator_backend: Literal["cuda", "rocm"] + output_path: str + skip_ssh_key_file: bool = False + ssh_ids: list[str] + + @field_validator("output_path", mode="after") + @classmethod + def _validate_output_path(cls, v: str) -> str: + if not v.strip(): + raise ValueError("output_path must be a non-empty string") + return v + + @field_validator("ssh_ids", mode="after") + @classmethod + def _validate_ssh_ids(cls, v: list[str]) -> list[str]: + if not v: + raise ValueError("ssh_ids must be a non-empty list") + for entry in v: + m = _SSH_ID_RE.match(entry) + if not m: + raise ValueError( + f"Invalid ssh_id entry {entry!r}: expected 'username@host' or 'username@host:port'" + ) + port_str = m.group("port") + if port_str is not None: + port = int(port_str) + if not (1 <= port <= 65535): + raise ValueError( + f"Invalid port in ssh_id {entry!r}: {port} is not in range 1-65535" + ) + return v + + @property + def parsed_ssh_ids(self) -> list[SshTarget]: + targets = [] + for entry in self.ssh_ids: + m = _SSH_ID_RE.match(entry) + assert m is not None # already validated + port_str = m.group("port") + targets.append( + SshTarget( + username=m.group("username"), + host=m.group("host"), + port=int(port_str) if port_str is not None else 22, + ) + ) + return targets + + class BenchmarkConfig(WithUpdatesMixin, BaseModel): """Benchmark configuration — single source of truth for YAML and CLI. @@ -504,6 +583,9 @@ class BenchmarkConfig(WithUpdatesMixin, BaseModel): help="NUMA-aware CPU pinning", ), ] = True + sys_info_capture: Annotated[ + SysInfoCaptureConfig | None, cyclopts.Parameter(show=False) + ] = None @field_validator("datasets", mode="before") @classmethod diff --git a/src/inference_endpoint/dataset_manager/transforms.py b/src/inference_endpoint/dataset_manager/transforms.py index 791337969..7d5d88109 100644 --- a/src/inference_endpoint/dataset_manager/transforms.py +++ b/src/inference_endpoint/dataset_manager/transforms.py @@ -219,7 +219,7 @@ def __call__(self, df: pd.DataFrame) -> pd.DataFrame: Returns: DataFrame with filtered columns """ - columns_to_keep = self.required_columns + columns_to_keep = list(self.required_columns) if self.optional_columns is not None: found_cols = set(df.columns) & set(self.optional_columns) columns_to_keep += list(found_cols) diff --git a/src/inference_endpoint/sys_info/__init__.py b/src/inference_endpoint/sys_info/__init__.py new file mode 100644 index 000000000..467079831 --- /dev/null +++ b/src/inference_endpoint/sys_info/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py new file mode 100644 index 000000000..88e9b34f6 --- /dev/null +++ b/src/inference_endpoint/sys_info/capture.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sys-info capture via the get-mlperf-multi-node-system-info mlcflow script.""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from inference_endpoint.config.schema import SysInfoCaptureConfig +from inference_endpoint.exceptions import ExecutionError, SetupError + +logger = logging.getLogger(__name__) + +_OUT_FILE_NAME = "mlperf-multi-node-system-info.json" + + +def capture_system_info(config: SysInfoCaptureConfig) -> Path: + """Invoke the get-mlperf-multi-node-system-info mlcflow script. + + Returns the Path to the generated combined system info JSON file. + Raises SetupError if mlcflow is not installed. + Raises ExecutionError if the script returns a non-zero return code. + """ + # Optional dependency — only imported when this function is actually called. + # mlcflow (PyPI) installs its runtime under the 'mlc' module name. + # mlcflow is not a required dependency of this package; see pyproject.toml [sys-info]. + try: + import mlc # noqa: PLC0415 + except ImportError as exc: + raise SetupError( + "mlcflow is required for sys_info_capture. " + "Install it with: pip install mlcflow" + ) from exc + + tags: list[str] = [ + "get-mlperf-multi-node-system-info", + f"_{config.accelerator_backend}", + ] + if config.exclude_current_system: + tags.append("_exclude_current_node") + tags_str = ",".join(tags) + + ssh_ids_str = ",".join(t.to_mlcflow_str() for t in config.parsed_ssh_ids) + + # CM/mlcflow scripts use "yes"/"" string convention for boolean env vars. + skip_ssh_key_file_value = "yes" if config.skip_ssh_key_file else "" + + Path(config.output_path).mkdir(parents=True, exist_ok=True) + + logger.info("Capturing system info from %d node(s)...", len(config.parsed_ssh_ids)) + + result = mlc.access( + { + "action": "run", + "automation": "script", + "tags": tags_str, + "ssh_ids": ssh_ids_str, + "out_dir_path": config.output_path, + "out_file_name": _OUT_FILE_NAME, + "skip_ssh_key_file": skip_ssh_key_file_value, + "quiet": True, + } + ) + + if result.get("return", 1) != 0: + raise ExecutionError( + f"sys_info capture failed (return code {result.get('return')}): " + f"{result.get('error', 'unknown error')}" + ) + + output_path = Path( + result.get("new_env", {}).get("MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH") + or Path(config.output_path) / _OUT_FILE_NAME + ) + logger.info("System info written to %s", output_path) + return output_path diff --git a/tests/unit/dataset_manager/test_transforms.py b/tests/unit/dataset_manager/test_transforms.py index ab342204c..f0957aca7 100644 --- a/tests/unit/dataset_manager/test_transforms.py +++ b/tests/unit/dataset_manager/test_transforms.py @@ -729,6 +729,25 @@ def test_empty_dataframe(self): assert "col2" not in result.columns assert len(result) == 0 + def test_repeated_calls_do_not_mutate_required_columns(self): + """Calling the transform multiple times must not grow required_columns. + + Previously, columns_to_keep = self.required_columns was an alias (not a + copy), so += mutated self.required_columns, causing duplicate columns on + the second and subsequent calls. + """ + df = pd.DataFrame({"req": [1], "opt": [2], "other": [3]}) + transform = ColumnFilter( + required_columns=["req"], + optional_columns=["opt"], + ) + + for _ in range(3): + result = transform(df) + assert result.columns.is_unique + assert list(result.columns) == ["req", "opt"] + assert len(transform.required_columns) == 1 + class TestMakeAdapterCompatible: """Test suite for MakeAdapterCompatible transform.""" diff --git a/tests/unit/sys_info/__init__.py b/tests/unit/sys_info/__init__.py new file mode 100644 index 000000000..467079831 --- /dev/null +++ b/tests/unit/sys_info/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py new file mode 100644 index 000000000..9a3caa4e8 --- /dev/null +++ b/tests/unit/sys_info/test_capture.py @@ -0,0 +1,325 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for sys_info capture — config models and capture function.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from inference_endpoint.config.schema import ( + BenchmarkConfig, + SshTarget, + SysInfoCaptureConfig, +) +from inference_endpoint.exceptions import ExecutionError, SetupError +from pydantic import ValidationError + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_MINIMAL_SYS_INFO = { + "accelerator_backend": "cuda", + "output_path": "/tmp/sys_info", + "ssh_ids": ["alice@192.168.1.1"], +} + + +def _make_config(**overrides: object) -> SysInfoCaptureConfig: + return SysInfoCaptureConfig(**{**_MINIMAL_SYS_INFO, **overrides}) + + +# --------------------------------------------------------------------------- +# 1. SSH string parsing — no port +# --------------------------------------------------------------------------- + + +class TestSshTargetParsing: + @pytest.mark.unit + def test_no_port_defaults_to_22(self) -> None: + cfg = _make_config(ssh_ids=["alice@192.168.1.1"]) + targets = cfg.parsed_ssh_ids + assert len(targets) == 1 + t = targets[0] + assert t.username == "alice" + assert t.host == "192.168.1.1" + assert t.port == 22 + assert t.to_mlcflow_str() == "alice@192.168.1.1:22" + + # 2. SSH string parsing — with port + @pytest.mark.unit + def test_with_explicit_port(self) -> None: + cfg = _make_config(ssh_ids=["alice@192.168.1.1:2222"]) + targets = cfg.parsed_ssh_ids + assert targets[0].port == 2222 + assert targets[0].to_mlcflow_str() == "alice@192.168.1.1:2222" + + # 3. SSH string parsing — invalid entries + @pytest.mark.unit + @pytest.mark.parametrize( + "bad_entry", + [ + "notvalid", + "@host", + "user@host:99999", + ], + ) + def test_invalid_ssh_id_raises(self, bad_entry: str) -> None: + with pytest.raises(ValidationError): + _make_config(ssh_ids=[bad_entry]) + + @pytest.mark.unit + def test_ssh_target_port_out_of_range_raises(self) -> None: + with pytest.raises(ValidationError): + SshTarget(username="alice", host="192.168.1.1", port=99999) + + @pytest.mark.unit + def test_empty_ssh_ids_raises(self) -> None: + with pytest.raises(ValidationError, match="non-empty"): + _make_config(ssh_ids=[]) + + +# --------------------------------------------------------------------------- +# 4. Variation tags — cuda, include current +# --------------------------------------------------------------------------- + + +class TestVariationTags: + @pytest.mark.unit + def test_cuda_include_current(self) -> None: + cfg = _make_config(accelerator_backend="cuda", exclude_current_system=False) + tags = _build_tags(cfg) + assert tags == "get-mlperf-multi-node-system-info,_cuda" + + # 5. Variation tags — rocm, exclude current + @pytest.mark.unit + def test_rocm_exclude_current(self) -> None: + cfg = _make_config(accelerator_backend="rocm", exclude_current_system=True) + tags = _build_tags(cfg) + assert tags == "get-mlperf-multi-node-system-info,_rocm,_exclude_current_node" + + +def _build_tags(cfg: SysInfoCaptureConfig) -> str: + """Mirror the tag-building logic from capture.py.""" + tags: list[str] = [ + "get-mlperf-multi-node-system-info", + f"_{cfg.accelerator_backend}", + ] + if cfg.exclude_current_system: + tags.append("_exclude_current_node") + return ",".join(tags) + + +# --------------------------------------------------------------------------- +# 6. skip_ssh_key_file encoding +# --------------------------------------------------------------------------- + + +class TestSkipSshKeyFileEncoding: + @pytest.mark.unit + def test_true_encodes_as_yes(self) -> None: + cfg = _make_config(skip_ssh_key_file=True) + value = _encode_skip_ssh(cfg) + assert value == "yes" + + @pytest.mark.unit + def test_false_encodes_as_empty_string(self) -> None: + cfg = _make_config(skip_ssh_key_file=False) + value = _encode_skip_ssh(cfg) + assert value == "" + + +def _encode_skip_ssh(cfg: SysInfoCaptureConfig) -> str: + """Mirror the boolean-encoding logic from capture.py.""" + return "yes" if cfg.skip_ssh_key_file else "" + + +# --------------------------------------------------------------------------- +# 7–10. capture_system_info function tests +# --------------------------------------------------------------------------- + + +class TestCaptureSystemInfo: + @pytest.mark.unit + def test_mlcflow_not_installed_raises_setup_error(self, tmp_path: Path) -> None: + cfg = _make_config(output_path=str(tmp_path)) + with patch.dict("sys.modules", {"mlc": None}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + with pytest.raises(SetupError, match="pip install mlcflow"): + capture_mod.capture_system_info(cfg) + + @pytest.mark.unit + def test_mlcflow_nonzero_return_raises_execution_error( + self, tmp_path: Path + ) -> None: + cfg = _make_config(output_path=str(tmp_path)) + mock_mlcflow = MagicMock() + mock_mlcflow.access.return_value = { + "return": 1, + "error": "ssh connection refused", + } + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + with pytest.raises(ExecutionError, match="ssh connection refused"): + capture_mod.capture_system_info(cfg) + + @pytest.mark.unit + def test_happy_path_output_path_from_new_env(self, tmp_path: Path) -> None: + cfg = _make_config(output_path=str(tmp_path)) + mock_mlcflow = MagicMock() + mock_mlcflow.access.return_value = { + "return": 0, + "new_env": { + "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH": "/tmp/out.json", + }, + } + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + result = capture_mod.capture_system_info(cfg) + assert result == Path("/tmp/out.json") + + @pytest.mark.unit + def test_happy_path_output_path_fallback(self, tmp_path: Path) -> None: + cfg = _make_config(output_path=str(tmp_path)) + mock_mlcflow = MagicMock() + mock_mlcflow.access.return_value = { + "return": 0, + "new_env": {}, + } + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + result = capture_mod.capture_system_info(cfg) + assert result == Path(cfg.output_path) / "mlperf-multi-node-system-info.json" + + @pytest.mark.unit + def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: + cfg = SysInfoCaptureConfig( + accelerator_backend="cuda", + exclude_current_system=True, + output_path=str(tmp_path), + skip_ssh_key_file=True, + ssh_ids=["alice@10.0.0.1:2222", "bob@10.0.0.2"], + ) + mock_mlcflow = MagicMock() + mock_mlcflow.access.return_value = { + "return": 0, + "new_env": {"MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH": "/tmp/out.json"}, + } + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + capture_mod.capture_system_info(cfg) + + call_args = mock_mlcflow.access.call_args[0][0] + assert ( + call_args["tags"] + == "get-mlperf-multi-node-system-info,_cuda,_exclude_current_node" + ) + assert call_args["ssh_ids"] == "alice@10.0.0.1:2222,bob@10.0.0.2:22" + assert call_args["skip_ssh_key_file"] == "yes" + assert call_args["out_dir_path"] == str(tmp_path) + assert call_args["out_file_name"] == "mlperf-multi-node-system-info.json" + assert call_args["action"] == "run" + assert call_args["automation"] == "script" + + +# --------------------------------------------------------------------------- +# 11. YAML round-trip +# --------------------------------------------------------------------------- + + +class TestYamlRoundTrip: + @pytest.mark.unit + def test_sys_info_capture_from_yaml(self, tmp_path: Path) -> None: + yaml_content = textwrap.dedent( + """\ + type: offline + model_params: + name: test-model + endpoint_config: + endpoints: + - http://localhost:8000 + datasets: + - path: dummy.jsonl + sys_info_capture: + accelerator_backend: cuda + output_path: /tmp/sys_info + ssh_ids: + - alice@192.168.1.1 + - bob@192.168.1.2:2222 + exclude_current_system: true + skip_ssh_key_file: false + """ + ) + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml_content) + + config = BenchmarkConfig.from_yaml_file(config_path) + + assert config.sys_info_capture is not None + sic = config.sys_info_capture + assert sic.accelerator_backend == "cuda" + assert sic.output_path == "/tmp/sys_info" + assert sic.exclude_current_system is True + assert sic.skip_ssh_key_file is False + assert len(sic.ssh_ids) == 2 + + targets = sic.parsed_ssh_ids + assert targets[0].to_mlcflow_str() == "alice@192.168.1.1:22" + assert targets[1].to_mlcflow_str() == "bob@192.168.1.2:2222" + + # 12. Backward compatibility + @pytest.mark.unit + def test_yaml_without_sys_info_capture_is_none(self, tmp_path: Path) -> None: + yaml_content = textwrap.dedent( + """\ + type: offline + model_params: + name: test-model + endpoint_config: + endpoints: + - http://localhost:8000 + datasets: + - path: dummy.jsonl + """ + ) + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml_content) + + config = BenchmarkConfig.from_yaml_file(config_path) + assert config.sys_info_capture is None From 1769b6fadf782b181ad1ec1880860ef7d4017fc1 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 15 May 2026 18:44:27 +0530 Subject: [PATCH 02/21] Add A40 (Prefill) + H100 (Decode) sysinfo example config Co-Authored-By: Claude Sonnet 4.6 --- examples/sysinfo_a40_prefill_h100_decode.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 examples/sysinfo_a40_prefill_h100_decode.yaml diff --git a/examples/sysinfo_a40_prefill_h100_decode.yaml b/examples/sysinfo_a40_prefill_h100_decode.yaml new file mode 100644 index 000000000..4ee081561 --- /dev/null +++ b/examples/sysinfo_a40_prefill_h100_decode.yaml @@ -0,0 +1,27 @@ +# MLPerf system-info capture — A40 (Prefill) + 8×H100 (Decode) +# +# Usage: +# inference-endpoint sysinfo from-config -c examples/sysinfo_a40_prefill_h100_decode.yaml +# +# Node assignments: +# Prefill : @ — NVIDIA A40 (1 GPU per node) +# Decode : @ — NVIDIA H100 (8 GPUs per node) + +report_dir: results/a40_prefill_h100_decode_sysinfo/ + +system_info: + ssh_ids: + - root@63.141.33.33:22053 # A40 node (RunPod) + - anandhusooraj@mlc2 # 8×H100 node (mlc2) + + accelerator_backend: cuda + + exclude_current_system: true + + node_config: + Prefill: + - node_name: A40 # matched as substring of detected GPU name + no_of_nodes: 1 + Decode: + - node_name: H100 # matched as substring of detected GPU name + no_of_nodes: 1 From 8bb0649b7762607ba8258e4b9d7b78f7c3cbea99 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 20 May 2026 13:43:50 +0530 Subject: [PATCH 03/21] further modification --- AGENTS.md | 6 ++ src/inference_endpoint/config/schema.py | 58 ++++++++++++++++- src/inference_endpoint/main.py | 3 + src/inference_endpoint/sys_info/capture.py | 72 +++++++++++++++++----- 4 files changed, 121 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bb79bc429..5a3b3a184 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,7 @@ uv run inference-endpoint probe --endpoints http://localhost:8765 --model test-m uv run inference-endpoint benchmark offline --endpoints URL --model NAME --dataset PATH uv run inference-endpoint benchmark online --endpoints URL --model NAME --dataset PATH --load-pattern poisson --target-qps 100 uv run inference-endpoint benchmark from-config --config config.yaml +uv run inference-endpoint sysinfo from-config --config sysinfo.yaml ``` ### Backward-compatible setup (pip + venv) @@ -60,6 +61,7 @@ inference-endpoint probe --endpoints http://localhost:8765 --model test-model inference-endpoint benchmark offline --endpoints URL --model NAME --dataset PATH inference-endpoint benchmark online --endpoints URL --model NAME --dataset PATH --load-pattern poisson --target-qps 100 inference-endpoint benchmark from-config --config config.yaml +inference-endpoint sysinfo from-config --config sysinfo.yaml ``` ## Architecture @@ -101,6 +103,7 @@ CLI is auto-generated from `config/schema.py` Pydantic models via cyclopts. Fiel - **CLI mode** (`offline`/`online`): cyclopts constructs `OfflineBenchmarkConfig`/`OnlineBenchmarkConfig` (subclasses in `config/schema.py`) directly from CLI args. Type locked via `Literal`. `--dataset` is repeatable with TOML-style format `[perf|acc:][,key=value...]` (e.g. `--dataset data.csv,samples=500,parser.prompt=article`). Full accuracy support via `accuracy_config.eval_method=pass_at_1` etc. - **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout`/`--mode` overrides via `config.with_updates()`. +- **sysinfo from-config**: `SysInfoFileConfig.from_yaml_file()` reads the `system_info` key from a YAML file (extra top-level keys ignored). Invokes `capture_system_info()` which calls the `get-mlperf-multi-node-system-info` mlcflow script. Supports optional `node_config` for function-based node groupings (written to a temp YAML file passed as `node_config_file` to mlcflow). Independent of benchmark runs. - **eval**: Not yet implemented (raises `CLIError` with a tracking issue link) ### Config Construction & Validation @@ -141,6 +144,9 @@ src/inference_endpoint/ │ │ ├── __init__.py │ │ ├── cli.py # benchmark_app: offline, online, from-config subcommands │ │ └── execute.py # Phased execution: setup/run_threaded/finalize + BenchmarkContext +│ ├── sysinfo/ +│ │ ├── __init__.py +│ │ └── cli.py # sysinfo_app: from-config subcommand (standalone sys info capture) │ ├── probe.py # ProbeConfig + execute_probe() │ ├── info.py # execute_info() │ ├── validate.py # execute_validate() diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index fd355f4df..002cd8293 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -477,16 +477,36 @@ def _validate_port(cls, v: int) -> int: return v +class NodeEntry(BaseModel): + """A single node type within a function group for multi-node system info.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + node_name: str = Field( + description="Node type identifier matched against detected GPU model name" + ) + no_of_nodes: int = Field( + default=1, ge=1, description="Number of nodes of this type" + ) + + class SysInfoCaptureConfig(BaseModel): """Configuration for the sys_info_capture post-benchmark step.""" model_config = ConfigDict(extra="forbid", frozen=True) exclude_current_system: bool = False - accelerator_backend: Literal["cuda", "rocm"] - output_path: str + accelerator_backend: Literal["cuda", "rocm", "none"] = "none" + output_path: str = "." skip_ssh_key_file: bool = False ssh_ids: list[str] + node_config: dict[str, list[NodeEntry]] | None = Field( + default=None, + description=( + "Function-based node groupings. Keys are function names (e.g. 'Prefill', " + "'Decode'); values are lists of NodeEntry specifying node type and count." + ), + ) @field_validator("output_path", mode="after") @classmethod @@ -532,6 +552,40 @@ def parsed_ssh_ids(self) -> list[SshTarget]: return targets +class SysInfoFileConfig(BaseModel): + """Top-level model for a standalone sysinfo YAML config file. + + The file must have a ``system_info`` key. Extra top-level keys are allowed + so the same YAML can be shared with benchmark configs. + + ``report_dir`` mirrors the same field in ``BenchmarkConfig`` and takes + priority over ``system_info.output_path`` when set. + """ + + model_config = ConfigDict(extra="ignore") + + system_info: SysInfoCaptureConfig + report_dir: Path | None = None + + @classmethod + def from_yaml_file(cls, path: Path) -> SysInfoFileConfig: + """Load SysInfoFileConfig from a YAML file. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the YAML is invalid or does not match the schema. + """ + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Config file not found: {path}") + raw = path.read_text() + data = yaml.safe_load(raw) + if not isinstance(data, dict): + raise ValueError(f"Expected YAML mapping, got {type(data).__name__}") + resolve_env_vars(data) + return cls.model_validate(data) + + class BenchmarkConfig(WithUpdatesMixin, BaseModel): """Benchmark configuration — single source of truth for YAML and CLI. diff --git a/src/inference_endpoint/main.py b/src/inference_endpoint/main.py index abae50643..2e6b9a790 100644 --- a/src/inference_endpoint/main.py +++ b/src/inference_endpoint/main.py @@ -76,6 +76,9 @@ def launcher( # Benchmark subcommands — lazy-loaded from commands/benchmark/cli.py app.command("inference_endpoint.commands.benchmark.cli:benchmark_app", name="benchmark") +# Sysinfo subcommands — lazy-loaded from commands/sysinfo/cli.py +app.command("inference_endpoint.commands.sysinfo.cli:sysinfo_app", name="sysinfo") + # --- Misc commands --- diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index 88e9b34f6..e2bc529e9 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -18,8 +18,12 @@ from __future__ import annotations import logging +import os +import tempfile from pathlib import Path +import yaml + from inference_endpoint.config.schema import SysInfoCaptureConfig from inference_endpoint.exceptions import ExecutionError, SetupError @@ -28,6 +32,33 @@ _OUT_FILE_NAME = "mlperf-multi-node-system-info.json" +def _write_node_config_tmp(config: SysInfoCaptureConfig) -> str: + """Serialise node_config to a temp YAML file readable by customize.py. + + customize.py expects: + system_info: + node_config: + : + - node_name: ... + no_of_nodes: ... + + Returns the path of the written temp file. + """ + assert config.node_config is not None + data = { + "system_info": { + "node_config": { + func: [entry.model_dump() for entry in entries] + for func, entries in config.node_config.items() + } + } + } + fd, path = tempfile.mkstemp(suffix=".yaml", prefix="mlperf_node_cfg_") + with os.fdopen(fd, "w") as fh: + yaml.dump(data, fh, default_flow_style=False) + return path + + def capture_system_info(config: SysInfoCaptureConfig) -> Path: """Invoke the get-mlperf-multi-node-system-info mlcflow script. @@ -46,10 +77,9 @@ def capture_system_info(config: SysInfoCaptureConfig) -> Path: "Install it with: pip install mlcflow" ) from exc - tags: list[str] = [ - "get-mlperf-multi-node-system-info", - f"_{config.accelerator_backend}", - ] + tags: list[str] = ["get-mlperf-multi-node-system-info"] + if config.accelerator_backend != "none": + tags.append(f"_{config.accelerator_backend}") if config.exclude_current_system: tags.append("_exclude_current_node") tags_str = ",".join(tags) @@ -63,18 +93,28 @@ def capture_system_info(config: SysInfoCaptureConfig) -> Path: logger.info("Capturing system info from %d node(s)...", len(config.parsed_ssh_ids)) - result = mlc.access( - { - "action": "run", - "automation": "script", - "tags": tags_str, - "ssh_ids": ssh_ids_str, - "out_dir_path": config.output_path, - "out_file_name": _OUT_FILE_NAME, - "skip_ssh_key_file": skip_ssh_key_file_value, - "quiet": True, - } - ) + mlc_kwargs: dict[str, object] = { + "action": "run", + "automation": "script", + "tags": tags_str, + "ssh_ids": ssh_ids_str, + "out_dir_path": config.output_path, + "out_file_name": _OUT_FILE_NAME, + "skip_ssh_key_file": skip_ssh_key_file_value, + "quiet": True, + } + + node_config_tmp: str | None = None + if config.node_config is not None: + node_config_tmp = _write_node_config_tmp(config) + mlc_kwargs["node_config_file"] = node_config_tmp + logger.debug("Node config written to temp file: %s", node_config_tmp) + + try: + result = mlc.access(mlc_kwargs) + finally: + if node_config_tmp and os.path.exists(node_config_tmp): + os.unlink(node_config_tmp) if result.get("return", 1) != 0: raise ExecutionError( From bb9b6ccbbd8457d288c36c68d5e2205ea875df88 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 28 May 2026 15:47:26 +0530 Subject: [PATCH 04/21] update doc, add per run datadictionary capture --- docs/commands/DESIGN.md | 161 ++++++++++++++++++ .../commands/benchmark/execute.py | 140 ++++++++++++++- .../commands/sysinfo/__init__.py | 14 ++ .../commands/sysinfo/cli.py | 71 ++++++++ src/inference_endpoint/config/schema.py | 65 +++++++ src/inference_endpoint/sys_info/capture.py | 14 +- 6 files changed, 460 insertions(+), 5 deletions(-) create mode 100644 src/inference_endpoint/commands/sysinfo/__init__.py create mode 100644 src/inference_endpoint/commands/sysinfo/cli.py diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index 930b270ca..f8a673855 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -36,6 +36,7 @@ already-parsed models rather than raw `argparse.Namespace` objects. | `info` | `main.py` | `commands/info.py` | Implemented | | `validate-yaml` | `main.py` | `commands/validate.py` | Implemented | | `init` | `main.py` | `commands/init.py` | Implemented | +| `sysinfo from-config` | `commands/sysinfo/cli.py` | `sys_info/capture.py` | Implemented | | `eval` | `main.py` | inline stub (`CLIError`) | Reserved, not implemented | ## CLI Structure @@ -53,6 +54,9 @@ inference-endpoint | +-- online | +-- from-config | + +-- sysinfo + | +-- from-config + | +-- probe +-- info +-- validate-yaml @@ -94,8 +98,164 @@ commands/benchmark/execute.py::run_benchmark() +-- construct endpoint client + sample issuer +-- run BenchmarkSession in threaded wrapper +-- finalize metrics and optional accuracy scoring + +-- if sys_info_capture is configured: + write run_metadata.yml + capture_system_info() → mlcflow (hardware + serving config) + patch run_metadata.yml with serving config values +``` + +## System Info Capture + +System info capture collects hardware/software details from one or more nodes and writes a structured JSON file for MLPerf inference submissions. It runs in two contexts: + +- **Standalone** (`sysinfo from-config`): triggered manually, independent of any benchmark run. +- **Integrated** (`benchmark` finalization): triggered automatically after a benchmark if `sys_info_capture` is present in the config. + +Both paths call `sys_info/capture.py::capture_system_info()` and produce the same output JSON. The integrated path additionally patches `run_metadata.yml` with serving configuration values extracted from the inference server's startup log. + +### Config Reference (`SysInfoCaptureConfig`) + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `ssh_ids` | `list[str]` | — | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | +| `accelerator_backend` | `"cuda"` \| `"rocm"` \| `"none"` | — | **Required.** GPU backend on the target nodes. | +| `exclude_current_system` | bool | `false` | Skip the machine running this command; collect from `ssh_ids` only. | +| `skip_ssh_key_file` | bool | `false` | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). | +| `output_path` | str | `"."` | Output directory for the JSON file. Overridden by `report_dir` when set at the top level. | +| `node_config` | object | `null` | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | +| `serving_node` | str | `null` | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | +| `log_path` | str | `null` | Path to the vLLM server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | +| `endpoint_url` | str | `null` | Base URL of the running inference server. Passed to the mlcflow script, which probes it via HTTP to detect the serving framework (e.g. `"vLLM 0.9.0"`). | + +### Capture Flow + +``` +capture_system_info(config, run_metadata_path=...) + │ + └─ mlc.access("get-mlperf-multi-node-system-info", ...) + │ + ├─ prehook: get,mlperf,single-node,system-info on local machine (node 0) + │ skipped if exclude_current_system=true + │ + ├─ preprocess(): for each ssh_id + │ remote_run get,mlperf,single-node,system-info on remote node + │ copy back: mlperf-system-info-single-node-{id}.json + │ + ├─ preprocess(): if serving_node set + │ remote_run get,mlperf,serving-config on serving node + │ parse.py reads vLLM startup log from the top (local on serving node): + │ tensor_parallel_size, pipeline_parallel_size, + │ expert_parallel_size, max_num_seqs + │ + framework name and version ("vLLM 0.9.0") + │ copy back: serving_config.json + │ + ├─ preprocess(): if endpoint_url set and framework not yet detected + │ GET /version or /get_server_info → "vLLM 0.9.0" / "SGLang 0.4.2" + │ sets MLC_MLPERF_SERVING_FRAMEWORK (HTTP probe takes priority over log) + │ + └─ postprocess(): + merge per-node JSONs → mlperf-multi-node-system-info.json + if serving_config.json present: + patch run_metadata.yml config_summary with extracted values + if serving framework not detected via HTTP: use serving_config.json framework field +``` + +**`run_metadata.yml` patching** only happens in the benchmark context. `capture_system_info` accepts an optional `run_metadata_path` argument; `finalize_benchmark` passes `ctx.report_dir / "run_metadata.yml"`, which has already been written before the capture call. The `config_summary` block fields (`tensor_parallel`, `pipeline_parallel`, `expert_parallel`, `batch`) are updated in-place; fields that could not be parsed remain `null`. + +### Standalone Command (`sysinfo from-config`) + +```bash +inference-endpoint sysinfo from-config -c examples/sysinfo_example.yaml +``` + +The YAML file has two top-level keys: + +```yaml +report_dir: results/h100_sysinfo/ # output directory (optional) + +system_info: + ssh_ids: + - root@ssh1:22 # prefill node 1 + - root@ssh2:22 # prefill node 2 + - root@ssh3:22 # decode node 1 + - root@ssh4:22 # decode node 2 + - root@ssh5:22 # decode node 3 + - root@ssh6:22 # decode node 4 + - root@ssh7:22 # decode node 5 + + accelerator_backend: cuda + exclude_current_system: true # master node is orchestrator-only + skip_ssh_key_file: false + + # serving_node: where the inference server process is running. + # If multiple serving nodes exist, point to any one — all nodes are assumed + # to run the same serving framework version. + serving_node: root@ssh1:22 + log_path: /tmp/vllm.log # path on serving_node where server output was redirected + + node_config: # optional: function-based node groupings + Prefill: + - node_name: NVIDIA H100 # case-insensitive substring of detected GPU model + no_of_nodes: 2 + Decode: + - node_name: NVIDIA H100 + no_of_nodes: 5 +``` + +`report_dir` takes priority over `system_info.output_path` when both are set. + +Output is written to `report_dir/mlperf-multi-node-system-info.json`. + +### Integrated Benchmark Config + +Add a `sys_info_capture` block to a benchmark YAML config. The `endpoint_url` field is auto-populated from `endpoint_config.endpoints[0]` if not explicitly set. + +```yaml +sys_info_capture: + ssh_ids: + - root@ssh1:22 + - root@ssh2:22 + - root@ssh3:22 + - root@ssh4:22 + - root@ssh5:22 + - root@ssh6:22 + - root@ssh7:22 + accelerator_backend: cuda + exclude_current_system: true + skip_ssh_key_file: false + serving_node: root@ssh1:22 + log_path: /tmp/vllm.log + # endpoint_url is auto-populated from endpoint_config.endpoints[0] if not set + node_config: + Prefill: + - node_name: NVIDIA H100 + no_of_nodes: 2 + Decode: + - node_name: NVIDIA H100 + no_of_nodes: 5 ``` +### `node_config` Validation + +When `node_config` is provided, the automations script enforces: +- Every `node_name` must match at least one probed node's GPU model string (case-insensitive substring). Unmatched names return an error. +- For each unique `node_name`, the total `no_of_nodes` across all function groups must not exceed the number of nodes of that type actually probed. Declaring more nodes than were SSHed into is an error. + +### Error Handling + +| Situation | Standalone (`sysinfo from-config`) | Integrated (benchmark) | +|-----------|-----------------------------------|------------------------| +| mlcflow script returns non-zero | `ExecutionError` propagates to CLI handler | Logged as `error` with retry hint; benchmark exits 0 | +| Unexpected exception | Propagates to CLI handler | Logged as `error` with exception type; benchmark exits 0 | +| SSH failure on a node | Logged as error inside script; other nodes continue | Same | +| Per-node JSON missing after SSH run | Logged as warning; node skipped | Same | +| `node_name` unmatched or count exceeds probed | `ExecutionError` | `ExecutionError` → logged as `error` | +| `serving_config.json` absent or unreadable | Logged as error; `run_metadata.yml` left unchanged | Same | + +In the integrated path, `sys_info_capture` failures never abort the benchmark. `results.json` and `run_metadata.yml` are written before the capture call, so the benchmark output is complete regardless of capture outcome. The error log includes `report_dir` and a command to re-run capture manually. + +--- + ## `probe` Command `probe` is a lightweight connectivity check built on the same endpoint/client stack as the main @@ -150,3 +310,4 @@ not been implemented yet. | `load_generator/session.py` | Runs the benchmark session | | `metrics/` | Aggregates and reports benchmark results | | `evaluation/` | Scores collected accuracy datasets during benchmark finalization | +| `sys_info/` | Invokes mlcflow to collect hardware/software/serving info from nodes | diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index b1936a6da..55f6f8f13 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -38,6 +38,7 @@ from urllib.parse import urljoin import msgspec.json +import yaml from huggingface_hub import model_info from tqdm import tqdm from transformers.utils import logging as transformers_logging @@ -433,8 +434,9 @@ async def _run_benchmark_async( tmpfs_dir.mkdir(parents=True, exist_ok=True) # On ARM, mmap write ordering requires msync on a real filesystem. - # msync is a no-op on tmpfs, so metrics must use an on-disk directory. - if use_shm and platform.machine() != "x86_64": + # msync is a no-op on tmpfs (Linux ARM). + needs_on_disk = use_shm and platform.machine() != "x86_64" + if needs_on_disk: logger.info( "ARM platform: using on-disk metrics directory for mmap ordering" ) @@ -646,6 +648,9 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: report.display(fn=lambda s: print(s, file=f)) logger.info(f"Report written to {report_txt}") + # Write run metadata YAML + _generate_run_metadata(ctx, report) + # Write scoring artifacts + copy event log from tmpfs to disk _write_scoring_artifacts(ctx, result, bench.tmpfs_dir) @@ -737,10 +742,137 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: capture_system_info, ) - output_path = capture_system_info(ctx.config.sys_info_capture) + output_path = capture_system_info( + ctx.config.sys_info_capture, + run_metadata_path=ctx.report_dir / "run_metadata.yml", + ) logger.info("System info captured at: %s", output_path) + except ExecutionError as e: + logger.error( + "sys_info_capture failed: %s\n" + " Benchmark results are complete at: %s\n" + " Re-run sys_info manually once the issue is resolved:\n" + " inference-endpoint sysinfo from-config -c ", + e, + ctx.report_dir, + ) except Exception as e: - logger.warning("sys_info_capture failed and will be skipped: %s", e) + logger.error( + "sys_info_capture failed unexpectedly (%s: %s)\n" + " Benchmark results are complete at: %s", + type(e).__name__, + e, + ctx.report_dir, + ) + + +def _generate_run_metadata(ctx: BenchmarkContext, report: Report | None) -> None: + """Write run_metadata.yml to ctx.report_dir after a benchmark run.""" + load_pattern = ctx.config.settings.load_pattern + concurrency = load_pattern.target_concurrency + + def _ns_to_ms(val: float | int | None) -> float | None: + return float(val) / 1e6 if val is not None else None + + def _stat(metric: dict[str, Any], key: str) -> float | None: + return _ns_to_ms(metric.get(key)) if metric else None + + def _pct(metric: dict[str, Any], p: str) -> float | None: + return _ns_to_ms((metric.get("percentiles") or {}).get(p)) if metric else None + + # node_config and disaggregated from sys_info_capture + node_config: Any = None + disaggregated: bool | None = None + sic = ctx.config.sys_info_capture + if sic is not None and sic.node_config is not None: + node_config = { + fn: [ne.model_dump() for ne in nodes] + for fn, nodes in sic.node_config.items() + } + disaggregated = len(sic.node_config) > 1 + + ttft: dict[str, Any] = {} + tpot: dict[str, Any] = {} + latency: dict[str, Any] = {} + system_tps: float | None = None + tps_per_user: float | None = None + qps: float | None = None + measured_total_output_tokens: int | None = None + measured_run_duration: float | None = None + measured_total_requests: int | None = None + + if report is not None: + system_tps = report.tps() + qps = report.qps() + measured_total_requests = report.n_samples_completed + if report.duration_ns is not None: + measured_run_duration = report.duration_ns / 1e9 + osl = report.output_sequence_lengths or {} + if osl: + total_tokens = osl.get("total") + if total_tokens is not None: + measured_total_output_tokens = int(total_tokens) + if concurrency is not None and system_tps is not None: + tps_per_user = system_tps / concurrency + ttft = report.ttft or {} + tpot = report.tpot or {} + latency = report.latency or {} + + metadata: dict[str, Any] = { + "run_date": datetime.now().isoformat(), + "node_config": node_config, + "config_summary": { + "disaggregated": disaggregated, + "expert_parallel": None, + "tensor_parallel": None, + "pipeline_parallel": None, + "data_parallel": None, + "batch": None, + }, + "config_summary_notes": None, + "concurrency": concurrency, + "system_tps": system_tps, + "tps_per_user": tps_per_user, + "ttft": _pct(ttft, "99"), + "qps": qps, + "tps_utilization": None, + "measured_total_output_tokens": measured_total_output_tokens, + "measured_run_duration": measured_run_duration, + "measured_total_requests": measured_total_requests, + "link_config": str(ctx.report_dir / "config.yaml"), + "link_logs": str(ctx.report_dir / "events.jsonl"), + "measured_latency_ttft_min": _stat(ttft, "min"), + "measured_latency_ttft_average": _stat(ttft, "avg"), + "measured_latency_ttft_p50": _pct(ttft, "50"), + "measured_latency_ttft_p90": _pct(ttft, "90"), + "measured_latency_ttft_p95": _pct(ttft, "95"), + "measured_latency_ttft_p99": _pct(ttft, "99"), + "measured_latency_ttft_p999": _pct(ttft, "99.9"), + "measured_latency_ttft_max": _stat(ttft, "max"), + "measured_latency_tpot_min": _stat(tpot, "min"), + "measured_latency_tpot_average": _stat(tpot, "avg"), + "measured_latency_tpot_p50": _pct(tpot, "50"), + "measured_latency_tpot_p90": _pct(tpot, "90"), + "measured_latency_tpot_p95": _pct(tpot, "95"), + "measured_latency_tpot_p99": _pct(tpot, "99"), + "measured_latency_tpot_p999": _pct(tpot, "99.9"), + "measured_latency_tpot_max": _stat(tpot, "max"), + "measured_latency_request_min": _stat(latency, "min"), + "measured_latency_request_average": _stat(latency, "avg"), + "measured_latency_request_p50": _pct(latency, "50"), + "measured_latency_request_p90": _pct(latency, "90"), + "measured_latency_request_p95": _pct(latency, "95"), + "measured_latency_request_p99": _pct(latency, "99"), + "measured_latency_request_p999": _pct(latency, "99.9"), + "measured_latency_request_max": _stat(latency, "max"), + } + + metadata_path = ctx.report_dir / "run_metadata.yml" + with metadata_path.open("w") as f: + yaml.dump( + metadata, f, default_flow_style=False, sort_keys=False, allow_unicode=True + ) + logger.info("Run metadata written to %s", metadata_path) def run_benchmark(config: BenchmarkConfig, test_mode: TestMode) -> None: diff --git a/src/inference_endpoint/commands/sysinfo/__init__.py b/src/inference_endpoint/commands/sysinfo/__init__.py new file mode 100644 index 000000000..467079831 --- /dev/null +++ b/src/inference_endpoint/commands/sysinfo/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/inference_endpoint/commands/sysinfo/cli.py b/src/inference_endpoint/commands/sysinfo/cli.py new file mode 100644 index 000000000..adb645c5f --- /dev/null +++ b/src/inference_endpoint/commands/sysinfo/cli.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sysinfo CLI subcommands — from-config.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated + +import cyclopts +import yaml +from pydantic import ValidationError + +from inference_endpoint.config.schema import SysInfoFileConfig +from inference_endpoint.exceptions import InputValidationError +from inference_endpoint.sys_info.capture import capture_system_info + +sysinfo_app = cyclopts.App(name="sysinfo", help="Capture MLPerf system information.") + + +@sysinfo_app.command(name="from-config") +def from_config( + *, + config: Annotated[Path, cyclopts.Parameter(name=["--config", "-c"])], +) -> None: + """Capture multi-node system info from a YAML config file. + + The config file must contain a ``system_info`` key with capture settings + and an optional ``node_config`` for function-based node groupings. + + Example:: + + system_info: + ssh_ids: + - user@host + accelerator_backend: cuda + output_path: /tmp/sys_info + node_config: + Prefill: + - node_name: H100 + no_of_nodes: 4 + Decode: + - node_name: H100 + no_of_nodes: 8 + """ + try: + resolved = SysInfoFileConfig.from_yaml_file(config) + except (yaml.YAMLError, ValidationError, ValueError, FileNotFoundError) as e: + raise InputValidationError(f"Config error: {e}") from e + + capture_cfg = resolved.system_info + if resolved.report_dir is not None: + capture_cfg = capture_cfg.model_copy( + update={"output_path": str(resolved.report_dir)} + ) + + output_path = capture_system_info(capture_cfg) + print(f"System info written to: {output_path}") diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 002cd8293..bed2c4597 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -507,6 +507,57 @@ class SysInfoCaptureConfig(BaseModel): "'Decode'); values are lists of NodeEntry specifying node type and count." ), ) + endpoint_url: str | None = Field( + default=None, + description=( + "Endpoint URL to probe for serving framework detection " + "(e.g. 'http://host:8000'). Auto-populated from endpoint_config when " + "used inside BenchmarkConfig." + ), + ) + serving_node: str | None = Field( + default=None, + description=( + "SSH ID of the node running the serving framework " + "(e.g. 'root@host:8022'). Used together with log_path for " + "log-based framework detection when endpoint_url is unavailable." + ), + ) + log_path: str | None = Field( + default=None, + description=( + "Absolute path on serving_node to which the server stdout/stderr " + "was redirected (e.g. '/tmp/vllm.log'). Used for log-based " + "serving framework detection." + ), + ) + serving_framework: Literal["auto", "vllm", "sglang"] = Field( + default="auto", + description=( + "Serving engine type for log parsing: 'vllm', 'sglang', or 'auto' " + "(auto-detects from log keywords)." + ), + ) + + @field_validator("endpoint_url", mode="after") + @classmethod + def _validate_endpoint_url(cls, v: str | None) -> str | None: + if v is not None and not v.startswith(("http://", "https://")): + raise ValueError( + f"endpoint_url must include scheme (http:// or https://), got: {v!r}" + ) + return v + + @field_validator("serving_node", mode="after") + @classmethod + def _validate_serving_node(cls, v: str | None) -> str | None: + if v is not None: + m = _SSH_ID_RE.match(v) + if not m: + raise ValueError( + f"Invalid serving_node {v!r}: expected 'username@host' or 'username@host:port'" + ) + return v @field_validator("output_path", mode="after") @classmethod @@ -727,6 +778,20 @@ def _resolve_and_validate(self) -> Self: return self + @model_validator(mode="after") + def _propagate_endpoint_url_to_sysinfo(self) -> Self: + """Copy endpoint_config.endpoints[0] into sys_info_capture.endpoint_url if unset.""" + if ( + self.sys_info_capture is not None + and self.sys_info_capture.endpoint_url is None + and self.endpoint_config.endpoints + ): + new_sic = self.sys_info_capture.model_copy( + update={"endpoint_url": self.endpoint_config.endpoints[0]} + ) + object.__setattr__(self, "sys_info_capture", new_sic) + return self + @model_validator(mode="after") def _propagate_client_api_type(self) -> Self: """Sync client.api_type from endpoint_config.api_type at construction. diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index e2bc529e9..cea2f7140 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -59,7 +59,10 @@ def _write_node_config_tmp(config: SysInfoCaptureConfig) -> str: return path -def capture_system_info(config: SysInfoCaptureConfig) -> Path: +def capture_system_info( + config: SysInfoCaptureConfig, + run_metadata_path: Path | None = None, +) -> Path: """Invoke the get-mlperf-multi-node-system-info mlcflow script. Returns the Path to the generated combined system info JSON file. @@ -101,8 +104,17 @@ def capture_system_info(config: SysInfoCaptureConfig) -> Path: "out_dir_path": config.output_path, "out_file_name": _OUT_FILE_NAME, "skip_ssh_key_file": skip_ssh_key_file_value, + "serving_framework_type": config.serving_framework, "quiet": True, } + if config.endpoint_url: + mlc_kwargs["endpoint_url"] = config.endpoint_url + if config.serving_node: + mlc_kwargs["serving_node"] = config.serving_node + if config.log_path: + mlc_kwargs["log_path"] = config.log_path + if run_metadata_path is not None: + mlc_kwargs["run_metadata_path"] = str(run_metadata_path) node_config_tmp: str | None = None if config.node_config is not None: From 5d5ca3ce1f9036a97f6ca2372399b62cc7076e16 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 29 May 2026 02:40:11 +0530 Subject: [PATCH 05/21] fixes for run_metadata --- AGENTS.md | 6 +-- docs/commands/DESIGN.md | 34 +------------ .../commands/benchmark/execute.py | 51 ++++++++++--------- .../commands/sysinfo/cli.py | 6 ++- src/inference_endpoint/config/schema.py | 12 ++--- src/inference_endpoint/sys_info/capture.py | 7 +-- 6 files changed, 45 insertions(+), 71 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5a3b3a184..e0b6bf2ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ uv run inference-endpoint probe --endpoints http://localhost:8765 --model test-m uv run inference-endpoint benchmark offline --endpoints URL --model NAME --dataset PATH uv run inference-endpoint benchmark online --endpoints URL --model NAME --dataset PATH --load-pattern poisson --target-qps 100 uv run inference-endpoint benchmark from-config --config config.yaml -uv run inference-endpoint sysinfo from-config --config sysinfo.yaml +uv run inference-endpoint sysinfo from-config --config config.yaml ``` ### Backward-compatible setup (pip + venv) @@ -61,7 +61,7 @@ inference-endpoint probe --endpoints http://localhost:8765 --model test-model inference-endpoint benchmark offline --endpoints URL --model NAME --dataset PATH inference-endpoint benchmark online --endpoints URL --model NAME --dataset PATH --load-pattern poisson --target-qps 100 inference-endpoint benchmark from-config --config config.yaml -inference-endpoint sysinfo from-config --config sysinfo.yaml +inference-endpoint sysinfo from-config --config config.yaml ``` ## Architecture @@ -103,7 +103,7 @@ CLI is auto-generated from `config/schema.py` Pydantic models via cyclopts. Fiel - **CLI mode** (`offline`/`online`): cyclopts constructs `OfflineBenchmarkConfig`/`OnlineBenchmarkConfig` (subclasses in `config/schema.py`) directly from CLI args. Type locked via `Literal`. `--dataset` is repeatable with TOML-style format `[perf|acc:][,key=value...]` (e.g. `--dataset data.csv,samples=500,parser.prompt=article`). Full accuracy support via `accuracy_config.eval_method=pass_at_1` etc. - **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout`/`--mode` overrides via `config.with_updates()`. -- **sysinfo from-config**: `SysInfoFileConfig.from_yaml_file()` reads the `system_info` key from a YAML file (extra top-level keys ignored). Invokes `capture_system_info()` which calls the `get-mlperf-multi-node-system-info` mlcflow script. Supports optional `node_config` for function-based node groupings (written to a temp YAML file passed as `node_config_file` to mlcflow). Independent of benchmark runs. +- **sysinfo from-config**: `SysInfoFileConfig.from_yaml_file()` reads the `system_info` key from YAML file (extra top-level keys ignored). Invokes `capture_system_info()` which calls the `get-mlperf-multi-node-system-info` mlcflow script. Supports optional `node_config` for function-based node groupings (written to a temp YAML file passed as `node_config_file` to mlcflow). Independent of benchmark runs. - **eval**: Not yet implemented (raises `CLIError` with a tracking issue link) ### Config Construction & Validation diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index f8a673855..dad546ea1 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -124,7 +124,7 @@ Both paths call `sys_info/capture.py::capture_system_info()` and produce the sam | `output_path` | str | `"."` | Output directory for the JSON file. Overridden by `report_dir` when set at the top level. | | `node_config` | object | `null` | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | | `serving_node` | str | `null` | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | -| `log_path` | str | `null` | Path to the vLLM server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | +| `log_path` | str | `null` | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | | `endpoint_url` | str | `null` | Base URL of the running inference server. Passed to the mlcflow script, which probes it via HTTP to detect the serving framework (e.g. `"vLLM 0.9.0"`). | ### Capture Flow @@ -168,10 +168,8 @@ capture_system_info(config, run_metadata_path=...) inference-endpoint sysinfo from-config -c examples/sysinfo_example.yaml ``` -The YAML file has two top-level keys: - ```yaml -report_dir: results/h100_sysinfo/ # output directory (optional) +report_dir: results/h100_sysinfo/ # output directory system_info: ssh_ids: @@ -206,34 +204,6 @@ system_info: Output is written to `report_dir/mlperf-multi-node-system-info.json`. -### Integrated Benchmark Config - -Add a `sys_info_capture` block to a benchmark YAML config. The `endpoint_url` field is auto-populated from `endpoint_config.endpoints[0]` if not explicitly set. - -```yaml -sys_info_capture: - ssh_ids: - - root@ssh1:22 - - root@ssh2:22 - - root@ssh3:22 - - root@ssh4:22 - - root@ssh5:22 - - root@ssh6:22 - - root@ssh7:22 - accelerator_backend: cuda - exclude_current_system: true - skip_ssh_key_file: false - serving_node: root@ssh1:22 - log_path: /tmp/vllm.log - # endpoint_url is auto-populated from endpoint_config.endpoints[0] if not set - node_config: - Prefill: - - node_name: NVIDIA H100 - no_of_nodes: 2 - Decode: - - node_name: NVIDIA H100 - no_of_nodes: 5 -``` ### `node_config` Validation diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 55f6f8f13..c5e3eac1f 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -434,9 +434,8 @@ async def _run_benchmark_async( tmpfs_dir.mkdir(parents=True, exist_ok=True) # On ARM, mmap write ordering requires msync on a real filesystem. - # msync is a no-op on tmpfs (Linux ARM). - needs_on_disk = use_shm and platform.machine() != "x86_64" - if needs_on_disk: + # msync is a no-op on tmpfs, so metrics must use an on-disk directory. + if use_shm and platform.machine() != "x86_64": logger.info( "ARM platform: using on-disk metrics directory for mmap ordering" ) @@ -648,8 +647,7 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: report.display(fn=lambda s: print(s, file=f)) logger.info(f"Report written to {report_txt}") - # Write run metadata YAML - _generate_run_metadata(ctx, report) + run_metadata = _build_run_metadata(ctx, report) # Write scoring artifacts + copy event log from tmpfs to disk _write_scoring_artifacts(ctx, result, bench.tmpfs_dir) @@ -735,21 +733,31 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: except Exception as e: logger.error(f"Save failed: {e}") - if ctx.config.sys_info_capture is not None: + # Write run_metadata.json before sys_info capture so mlcflow's postprocess + # can read and patch it in-place with serving config values. + metadata_path = ctx.report_dir / "run_metadata.json" + with metadata_path.open("w") as f: + json.dump(run_metadata, f, indent=2) + logger.info("Run metadata written to %s", metadata_path) + + if ctx.config.system_info is not None: try: - # Local import: mlcflow is optional and only needed when sys_info_capture is configured. - from inference_endpoint.sys_info.capture import ( # noqa: PLC0415 + # Local import: mlcflow is optional and only needed when system_info is configured. + from inference_endpoint.sys_info.capture import ( capture_system_info, ) + sic = ctx.config.system_info.model_copy( + update={"output_path": str(ctx.report_dir)} + ) output_path = capture_system_info( - ctx.config.sys_info_capture, - run_metadata_path=ctx.report_dir / "run_metadata.yml", + sic, + run_metadata_path=ctx.report_dir / "run_metadata.json", ) logger.info("System info captured at: %s", output_path) except ExecutionError as e: logger.error( - "sys_info_capture failed: %s\n" + "system_info failed: %s\n" " Benchmark results are complete at: %s\n" " Re-run sys_info manually once the issue is resolved:\n" " inference-endpoint sysinfo from-config -c ", @@ -758,7 +766,7 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: ) except Exception as e: logger.error( - "sys_info_capture failed unexpectedly (%s: %s)\n" + "system_info failed unexpectedly (%s: %s)\n" " Benchmark results are complete at: %s", type(e).__name__, e, @@ -766,8 +774,8 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: ) -def _generate_run_metadata(ctx: BenchmarkContext, report: Report | None) -> None: - """Write run_metadata.yml to ctx.report_dir after a benchmark run.""" +def _build_run_metadata(ctx: BenchmarkContext, report: Report | None) -> dict[str, Any]: + """Build and return the run metadata dict.""" load_pattern = ctx.config.settings.load_pattern concurrency = load_pattern.target_concurrency @@ -780,10 +788,10 @@ def _stat(metric: dict[str, Any], key: str) -> float | None: def _pct(metric: dict[str, Any], p: str) -> float | None: return _ns_to_ms((metric.get("percentiles") or {}).get(p)) if metric else None - # node_config and disaggregated from sys_info_capture + # node_config and disaggregated from system_info node_config: Any = None disaggregated: bool | None = None - sic = ctx.config.sys_info_capture + sic = ctx.config.system_info if sic is not None and sic.node_config is not None: node_config = { fn: [ne.model_dump() for ne in nodes] @@ -839,8 +847,8 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: "measured_total_output_tokens": measured_total_output_tokens, "measured_run_duration": measured_run_duration, "measured_total_requests": measured_total_requests, - "link_config": str(ctx.report_dir / "config.yaml"), - "link_logs": str(ctx.report_dir / "events.jsonl"), + "link_config": None, + "link_logs": None, "measured_latency_ttft_min": _stat(ttft, "min"), "measured_latency_ttft_average": _stat(ttft, "avg"), "measured_latency_ttft_p50": _pct(ttft, "50"), @@ -867,12 +875,7 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: "measured_latency_request_max": _stat(latency, "max"), } - metadata_path = ctx.report_dir / "run_metadata.yml" - with metadata_path.open("w") as f: - yaml.dump( - metadata, f, default_flow_style=False, sort_keys=False, allow_unicode=True - ) - logger.info("Run metadata written to %s", metadata_path) + return metadata def run_benchmark(config: BenchmarkConfig, test_mode: TestMode) -> None: diff --git a/src/inference_endpoint/commands/sysinfo/cli.py b/src/inference_endpoint/commands/sysinfo/cli.py index adb645c5f..2f5c966af 100644 --- a/src/inference_endpoint/commands/sysinfo/cli.py +++ b/src/inference_endpoint/commands/sysinfo/cli.py @@ -62,10 +62,14 @@ def from_config( raise InputValidationError(f"Config error: {e}") from e capture_cfg = resolved.system_info + run_metadata_path = None if resolved.report_dir is not None: capture_cfg = capture_cfg.model_copy( update={"output_path": str(resolved.report_dir)} ) + candidate = resolved.report_dir / "run_metadata.json" + if candidate.exists(): + run_metadata_path = candidate - output_path = capture_system_info(capture_cfg) + output_path = capture_system_info(capture_cfg, run_metadata_path=run_metadata_path) print(f"System info written to: {output_path}") diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index bed2c4597..10b468a0e 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -688,7 +688,7 @@ class BenchmarkConfig(WithUpdatesMixin, BaseModel): help="NUMA-aware CPU pinning", ), ] = True - sys_info_capture: Annotated[ + system_info: Annotated[ SysInfoCaptureConfig | None, cyclopts.Parameter(show=False) ] = None @@ -780,16 +780,16 @@ def _resolve_and_validate(self) -> Self: @model_validator(mode="after") def _propagate_endpoint_url_to_sysinfo(self) -> Self: - """Copy endpoint_config.endpoints[0] into sys_info_capture.endpoint_url if unset.""" + """Copy endpoint_config.endpoints[0] into system_info.endpoint_url if unset.""" if ( - self.sys_info_capture is not None - and self.sys_info_capture.endpoint_url is None + self.system_info is not None + and self.system_info.endpoint_url is None and self.endpoint_config.endpoints ): - new_sic = self.sys_info_capture.model_copy( + new_sic = self.system_info.model_copy( update={"endpoint_url": self.endpoint_config.endpoints[0]} ) - object.__setattr__(self, "sys_info_capture", new_sic) + object.__setattr__(self, "system_info", new_sic) return self @model_validator(mode="after") diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index cea2f7140..4ca029399 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) -_OUT_FILE_NAME = "mlperf-multi-node-system-info.json" +_OUT_FILE_NAME = "system_desc.json" def _write_node_config_tmp(config: SysInfoCaptureConfig) -> str: @@ -134,9 +134,6 @@ def capture_system_info( f"{result.get('error', 'unknown error')}" ) - output_path = Path( - result.get("new_env", {}).get("MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH") - or Path(config.output_path) / _OUT_FILE_NAME - ) + output_path = Path(config.output_path) / _OUT_FILE_NAME logger.info("System info written to %s", output_path) return output_path From 011a6259f2530952663855224995b688db5e4533 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 29 May 2026 16:06:58 +0530 Subject: [PATCH 06/21] fix: address PR review comments for sys_info capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove unused `import yaml` from benchmark/execute.py - Rename mlcflow → mlc-scripts in pyproject.toml and uv.lock; update capture.py comment and SetupError install hint accordingly - Update DESIGN.md: run_metadata.yml → run_metadata.json, mlperf-multi-node-system-info.json → system_desc.json - Fix test_capture.py assertions to match actual out_file_name (system_desc.json) and updated SetupError message (mlc-scripts) - Regenerate _full config templates (system_info field now visible) Co-Authored-By: Claude Sonnet 4.6 --- docs/commands/DESIGN.md | 98 +++++++++---------- examples/sysinfo_a40_prefill_h100_decode.yaml | 8 +- pyproject.toml | 4 +- .../commands/benchmark/execute.py | 1 - .../templates/concurrency_template_full.yaml | 1 + .../templates/offline_template_full.yaml | 1 + .../templates/online_template_full.yaml | 1 + src/inference_endpoint/sys_info/capture.py | 7 +- tests/unit/sys_info/test_capture.py | 6 +- uv.lock | 39 ++++++++ 10 files changed, 103 insertions(+), 63 deletions(-) diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index dad546ea1..3f105d12c 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -99,9 +99,9 @@ commands/benchmark/execute.py::run_benchmark() +-- run BenchmarkSession in threaded wrapper +-- finalize metrics and optional accuracy scoring +-- if sys_info_capture is configured: - write run_metadata.yml + write run_metadata.json capture_system_info() → mlcflow (hardware + serving config) - patch run_metadata.yml with serving config values + patch run_metadata.json with serving config values ``` ## System Info Capture @@ -111,21 +111,21 @@ System info capture collects hardware/software details from one or more nodes an - **Standalone** (`sysinfo from-config`): triggered manually, independent of any benchmark run. - **Integrated** (`benchmark` finalization): triggered automatically after a benchmark if `sys_info_capture` is present in the config. -Both paths call `sys_info/capture.py::capture_system_info()` and produce the same output JSON. The integrated path additionally patches `run_metadata.yml` with serving configuration values extracted from the inference server's startup log. +Both paths call `sys_info/capture.py::capture_system_info()` and produce the same output JSON. The integrated path additionally patches `run_metadata.json` with serving configuration values extracted from the inference server's startup log. ### Config Reference (`SysInfoCaptureConfig`) -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `ssh_ids` | `list[str]` | — | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | -| `accelerator_backend` | `"cuda"` \| `"rocm"` \| `"none"` | — | **Required.** GPU backend on the target nodes. | -| `exclude_current_system` | bool | `false` | Skip the machine running this command; collect from `ssh_ids` only. | -| `skip_ssh_key_file` | bool | `false` | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). | -| `output_path` | str | `"."` | Output directory for the JSON file. Overridden by `report_dir` when set at the top level. | -| `node_config` | object | `null` | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | -| `serving_node` | str | `null` | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | -| `log_path` | str | `null` | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | -| `endpoint_url` | str | `null` | Base URL of the running inference server. Passed to the mlcflow script, which probes it via HTTP to detect the serving framework (e.g. `"vLLM 0.9.0"`). | +| Field | Type | Default | Description | +| ------------------------ | -------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ssh_ids` | `list[str]` | — | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | +| `accelerator_backend` | `"cuda"` \| `"rocm"` \| `"none"` | — | **Required.** GPU backend on the target nodes. | +| `exclude_current_system` | bool | `false` | Skip the machine running this command; collect from `ssh_ids` only. | +| `skip_ssh_key_file` | bool | `false` | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). | +| `output_path` | str | `"."` | Output directory for the JSON file. Overridden by `report_dir` when set at the top level. | +| `node_config` | object | `null` | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | +| `serving_node` | str | `null` | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | +| `log_path` | str | `null` | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | +| `endpoint_url` | str | `null` | Base URL of the running inference server. Passed to the mlcflow script, which probes it via HTTP to detect the serving framework (e.g. `"vLLM 0.9.0"`). | ### Capture Flow @@ -154,13 +154,13 @@ capture_system_info(config, run_metadata_path=...) │ sets MLC_MLPERF_SERVING_FRAMEWORK (HTTP probe takes priority over log) │ └─ postprocess(): - merge per-node JSONs → mlperf-multi-node-system-info.json + merge per-node JSONs → system_desc.json if serving_config.json present: - patch run_metadata.yml config_summary with extracted values + patch run_metadata.json config_summary with extracted values if serving framework not detected via HTTP: use serving_config.json framework field ``` -**`run_metadata.yml` patching** only happens in the benchmark context. `capture_system_info` accepts an optional `run_metadata_path` argument; `finalize_benchmark` passes `ctx.report_dir / "run_metadata.yml"`, which has already been written before the capture call. The `config_summary` block fields (`tensor_parallel`, `pipeline_parallel`, `expert_parallel`, `batch`) are updated in-place; fields that could not be parsed remain `null`. +**`run_metadata.json` patching** only happens in the benchmark context. `capture_system_info` accepts an optional `run_metadata_path` argument; `finalize_benchmark` passes `ctx.report_dir / "run_metadata.json"`, which has already been written before the capture call. The `config_summary` block fields (`tensor_parallel`, `pipeline_parallel`, `expert_parallel`, `batch`) are updated in-place; fields that could not be parsed remain `null`. ### Standalone Command (`sysinfo from-config`) @@ -169,31 +169,31 @@ inference-endpoint sysinfo from-config -c examples/sysinfo_example.yaml ``` ```yaml -report_dir: results/h100_sysinfo/ # output directory +report_dir: results/h100_sysinfo/ # output directory system_info: ssh_ids: - - root@ssh1:22 # prefill node 1 - - root@ssh2:22 # prefill node 2 - - root@ssh3:22 # decode node 1 - - root@ssh4:22 # decode node 2 - - root@ssh5:22 # decode node 3 - - root@ssh6:22 # decode node 4 - - root@ssh7:22 # decode node 5 + - root@ssh1:22 # prefill node 1 + - root@ssh2:22 # prefill node 2 + - root@ssh3:22 # decode node 1 + - root@ssh4:22 # decode node 2 + - root@ssh5:22 # decode node 3 + - root@ssh6:22 # decode node 4 + - root@ssh7:22 # decode node 5 accelerator_backend: cuda - exclude_current_system: true # master node is orchestrator-only + exclude_current_system: true # master node is orchestrator-only skip_ssh_key_file: false # serving_node: where the inference server process is running. # If multiple serving nodes exist, point to any one — all nodes are assumed # to run the same serving framework version. serving_node: root@ssh1:22 - log_path: /tmp/vllm.log # path on serving_node where server output was redirected + log_path: /tmp/vllm.log # path on serving_node where server output was redirected - node_config: # optional: function-based node groupings + node_config: # optional: function-based node groupings Prefill: - - node_name: NVIDIA H100 # case-insensitive substring of detected GPU model + - node_name: NVIDIA H100 # case-insensitive substring of detected GPU model no_of_nodes: 2 Decode: - node_name: NVIDIA H100 @@ -202,27 +202,27 @@ system_info: `report_dir` takes priority over `system_info.output_path` when both are set. -Output is written to `report_dir/mlperf-multi-node-system-info.json`. - +Output is written to `report_dir/system_desc.json`. ### `node_config` Validation When `node_config` is provided, the automations script enforces: + - Every `node_name` must match at least one probed node's GPU model string (case-insensitive substring). Unmatched names return an error. - For each unique `node_name`, the total `no_of_nodes` across all function groups must not exceed the number of nodes of that type actually probed. Declaring more nodes than were SSHed into is an error. ### Error Handling -| Situation | Standalone (`sysinfo from-config`) | Integrated (benchmark) | -|-----------|-----------------------------------|------------------------| -| mlcflow script returns non-zero | `ExecutionError` propagates to CLI handler | Logged as `error` with retry hint; benchmark exits 0 | -| Unexpected exception | Propagates to CLI handler | Logged as `error` with exception type; benchmark exits 0 | -| SSH failure on a node | Logged as error inside script; other nodes continue | Same | -| Per-node JSON missing after SSH run | Logged as warning; node skipped | Same | -| `node_name` unmatched or count exceeds probed | `ExecutionError` | `ExecutionError` → logged as `error` | -| `serving_config.json` absent or unreadable | Logged as error; `run_metadata.yml` left unchanged | Same | +| Situation | Standalone (`sysinfo from-config`) | Integrated (benchmark) | +| --------------------------------------------- | --------------------------------------------------- | -------------------------------------------------------- | +| mlcflow script returns non-zero | `ExecutionError` propagates to CLI handler | Logged as `error` with retry hint; benchmark exits 0 | +| Unexpected exception | Propagates to CLI handler | Logged as `error` with exception type; benchmark exits 0 | +| SSH failure on a node | Logged as error inside script; other nodes continue | Same | +| Per-node JSON missing after SSH run | Logged as warning; node skipped | Same | +| `node_name` unmatched or count exceeds probed | `ExecutionError` | `ExecutionError` → logged as `error` | +| `serving_config.json` absent or unreadable | Logged as error; `run_metadata.json` left unchanged | Same | -In the integrated path, `sys_info_capture` failures never abort the benchmark. `results.json` and `run_metadata.yml` are written before the capture call, so the benchmark output is complete regardless of capture outcome. The error log includes `report_dir` and a command to re-run capture manually. +In the integrated path, `sys_info_capture` failures never abort the benchmark. `results.json` and `run_metadata.json` are written before the capture call, so the benchmark output is complete regardless of capture outcome. The error log includes `report_dir` and a command to re-run capture manually. --- @@ -271,13 +271,13 @@ not been implemented yet. ## Integration Points -| Dependency | Role | -| --------------------------- | ---------------------------------------------------------------- | -| `main.py` | App definition, logging setup, global error handling | -| `config/` | Defines CLI/YAML schema models and config loading | -| `dataset_manager/` | Loads performance and accuracy datasets | -| `endpoint_client/` | Sends requests to endpoint workers | -| `load_generator/session.py` | Runs the benchmark session | -| `metrics/` | Aggregates and reports benchmark results | -| `evaluation/` | Scores collected accuracy datasets during benchmark finalization | +| Dependency | Role | +| --------------------------- | -------------------------------------------------------------------- | +| `main.py` | App definition, logging setup, global error handling | +| `config/` | Defines CLI/YAML schema models and config loading | +| `dataset_manager/` | Loads performance and accuracy datasets | +| `endpoint_client/` | Sends requests to endpoint workers | +| `load_generator/session.py` | Runs the benchmark session | +| `metrics/` | Aggregates and reports benchmark results | +| `evaluation/` | Scores collected accuracy datasets during benchmark finalization | | `sys_info/` | Invokes mlcflow to collect hardware/software/serving info from nodes | diff --git a/examples/sysinfo_a40_prefill_h100_decode.yaml b/examples/sysinfo_a40_prefill_h100_decode.yaml index 4ee081561..4cb463249 100644 --- a/examples/sysinfo_a40_prefill_h100_decode.yaml +++ b/examples/sysinfo_a40_prefill_h100_decode.yaml @@ -11,8 +11,8 @@ report_dir: results/a40_prefill_h100_decode_sysinfo/ system_info: ssh_ids: - - root@63.141.33.33:22053 # A40 node (RunPod) - - anandhusooraj@mlc2 # 8×H100 node (mlc2) + - root@63.141.33.33:22053 # A40 node (RunPod) + - anandhusooraj@mlc2 # 8×H100 node (mlc2) accelerator_backend: cuda @@ -20,8 +20,8 @@ system_info: node_config: Prefill: - - node_name: A40 # matched as substring of detected GPU name + - node_name: A40 # matched as substring of detected GPU name no_of_nodes: 1 Decode: - - node_name: H100 # matched as substring of detected GPU name + - node_name: H100 # matched as substring of detected GPU name no_of_nodes: 1 diff --git a/pyproject.toml b/pyproject.toml index 5100ccf23..337b3fbf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,8 +73,8 @@ dependencies = [ # Fix pytz-2024 import warning "pytz==2026.1.post1", "urllib3==2.7.0", - # MLCFlow for system info - "mlcflow", + # mlc-scripts for system info + "mlc-scripts", ] [project.optional-dependencies] diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index dce41cfc5..7c8e5b9e3 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -41,7 +41,6 @@ import msgspec import msgspec.json -import yaml from huggingface_hub import model_info from tqdm import tqdm from transformers import AutoTokenizer diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index add400fbc..c2d3b3981 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -87,3 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning +system_info: null diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index ea21e1e1f..53ab7cccc 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -87,3 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning +system_info: null diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 823862972..8d0224f1c 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -87,3 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning +system_info: null diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index 4ca029399..4fd246f60 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -70,14 +70,13 @@ def capture_system_info( Raises ExecutionError if the script returns a non-zero return code. """ # Optional dependency — only imported when this function is actually called. - # mlcflow (PyPI) installs its runtime under the 'mlc' module name. - # mlcflow is not a required dependency of this package; see pyproject.toml [sys-info]. + # mlc-scripts (PyPI) installs its runtime under the 'mlc' module name. try: import mlc # noqa: PLC0415 except ImportError as exc: raise SetupError( - "mlcflow is required for sys_info_capture. " - "Install it with: pip install mlcflow" + "mlc-scripts is required for sys_info_capture. " + "Install it with: pip install mlc-scripts" ) from exc tags: list[str] = ["get-mlperf-multi-node-system-info"] diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index 9a3caa4e8..a942677b5 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -165,7 +165,7 @@ def test_mlcflow_not_installed_raises_setup_error(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - with pytest.raises(SetupError, match="pip install mlcflow"): + with pytest.raises(SetupError, match="pip install mlc-scripts"): capture_mod.capture_system_info(cfg) @pytest.mark.unit @@ -221,7 +221,7 @@ def test_happy_path_output_path_fallback(self, tmp_path: Path) -> None: importlib.reload(capture_mod) result = capture_mod.capture_system_info(cfg) - assert result == Path(cfg.output_path) / "mlperf-multi-node-system-info.json" + assert result == Path(cfg.output_path) / "system_desc.json" @pytest.mark.unit def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: @@ -253,7 +253,7 @@ def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: assert call_args["ssh_ids"] == "alice@10.0.0.1:2222,bob@10.0.0.2:22" assert call_args["skip_ssh_key_file"] == "yes" assert call_args["out_dir_path"] == str(tmp_path) - assert call_args["out_file_name"] == "mlperf-multi-node-system-info.json" + assert call_args["out_file_name"] == "system_desc.json" assert call_args["action"] == "run" assert call_args["automation"] == "script" diff --git a/uv.lock b/uv.lock index a8446ce0b..06ee833bb 100644 --- a/uv.lock +++ b/uv.lock @@ -592,6 +592,15 @@ http = [ { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] +[[package]] +name = "giturlparse" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/35/7f25a604a406be7d7d0f849bfcbc1603df084e9e58fe6170980c231138e4/giturlparse-0.14.0.tar.gz", hash = "sha256:0a13208cb3f60e067ee3d09d28e01f9c936065986004fa2d5cd6db7758e9f6e6", size = 15637, upload-time = "2025-10-22T09:21:11.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/f9/9ff5a301459f804a885f237453ba81564bc6ee54740e9f2676c2642043f6/giturlparse-0.14.0-py2.py3-none-any.whl", hash = "sha256:04fd9c262ca9a4db86043d2ef32b2b90bfcbcdefc4f6a260fd9402127880931d", size = 16299, upload-time = "2025-10-22T09:21:10.818Z" }, +] + [[package]] name = "greenlet" version = "3.3.2" @@ -794,6 +803,7 @@ dependencies = [ { name = "hdrhistogram", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "httptools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "mlc-scripts", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "msgspec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "numpy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "openai-harmony", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -866,6 +876,7 @@ requires-dist = [ { name = "line-profiler", marker = "extra == 'test'", specifier = "==5.0.2" }, { name = "matplotlib", marker = "extra == 'test'", specifier = "==3.10.8" }, { name = "memory-profiler", marker = "extra == 'performance'", specifier = "==0.61.0" }, + { name = "mlc-scripts" }, { name = "msgspec", specifier = "==0.20.0" }, { name = "myst-parser", marker = "extra == 'dev'", specifier = "==5.0.0" }, { name = "numpy", specifier = "==2.4.4" }, @@ -1142,6 +1153,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/26/aaca612a0634ceede20682e692a6c55e35a94c21ba36b807cc40fe910ae1/memory_profiler-0.61.0-py3-none-any.whl", hash = "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84", size = 31803, upload-time = "2022-11-15T17:57:27.031Z" }, ] +[[package]] +name = "mlc-scripts" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "giturlparse", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "mlcflow", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/b8/a228ace8b208c3d54b40823bd54ccd727001372a96c7ef78252cb1b9109d/mlc_scripts-1.1.0.tar.gz", hash = "sha256:b27e562086603afef237736cf07767ef3bcdbd6a0a27a611f3e574a529ae47c5", size = 9483, upload-time = "2025-09-14T18:38:31.758Z" } + +[[package]] +name = "mlcflow" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "filelock", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "giturlparse", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/41/18cb6c9084b63b885923adb55a3601e8761ea3aeaa39464c762c7ea5d41b/mlcflow-1.2.3.tar.gz", hash = "sha256:cb03fab0e7443ba3bd80a553f962efaea119abd503e1f6271a60c9f965cd8a80", size = 58387, upload-time = "2026-05-21T23:19:58.863Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/0f/2d7e895fa86b5cbb211329df93f8cd51437621d95ce51b2a748840ff057e/mlcflow-1.2.3-py3-none-any.whl", hash = "sha256:2dab3444e0aafe302ebc25f49107fdb05693df1ef870aeb701b1cbd65fb5e6db", size = 60947, upload-time = "2026-05-21T23:19:57.454Z" }, +] + [[package]] name = "msgpack" version = "1.1.2" From ef10d7794d46c99546592df39e004dcb78beab0b Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 29 May 2026 16:37:06 +0530 Subject: [PATCH 07/21] =?UTF-8?q?fix:=20rename=20BenchmarkConfig.system=5F?= =?UTF-8?q?info=20=E2=86=92=20sys=5Finfo=5Fcapture=20and=20fix=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BenchmarkConfig field was named system_info but tests and the intended YAML API used sys_info_capture; rename throughout schema.py, execute.py, and the endpoint-url propagation validator - Fix capture_system_info return path to use MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH from new_env when present, falling back to output_path/system_desc.json - Update fake_capture stubs in test_sysinfo_command.py to accept run_metadata_path kwarg passed by the sysinfo CLI - Regenerate _full config templates after schema field rename Co-Authored-By: Claude Sonnet 4.6 --- .../commands/benchmark/execute.py | 6 +- src/inference_endpoint/config/schema.py | 12 +- .../templates/concurrency_template_full.yaml | 2 +- .../templates/offline_template_full.yaml | 2 +- .../templates/online_template_full.yaml | 2 +- src/inference_endpoint/sys_info/capture.py | 9 +- tests/unit/sys_info/test_sysinfo_command.py | 562 ++++++++++++++++++ 7 files changed, 582 insertions(+), 13 deletions(-) create mode 100644 tests/unit/sys_info/test_sysinfo_command.py diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 7c8e5b9e3..052db9086 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -987,14 +987,14 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: json.dump(run_metadata, f, indent=2) logger.info("Run metadata written to %s", metadata_path) - if ctx.config.system_info is not None: + if ctx.config.sys_info_capture is not None: try: # Local import: mlcflow is optional and only needed when system_info is configured. from inference_endpoint.sys_info.capture import ( capture_system_info, ) - sic = ctx.config.system_info.model_copy( + sic = ctx.config.sys_info_capture.model_copy( update={"output_path": str(ctx.report_dir)} ) output_path = capture_system_info( @@ -1038,7 +1038,7 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: # node_config and disaggregated from system_info node_config: Any = None disaggregated: bool | None = None - sic = ctx.config.system_info + sic = ctx.config.sys_info_capture if sic is not None and sic.node_config is not None: node_config = { fn: [ne.model_dump() for ne in nodes] diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 483950bb5..617bdffa6 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -782,7 +782,7 @@ class BenchmarkConfig(WithUpdatesMixin, BaseModel): help="NUMA-aware CPU pinning", ), ] = True - system_info: Annotated[ + sys_info_capture: Annotated[ SysInfoCaptureConfig | None, cyclopts.Parameter(show=False) ] = None @@ -904,16 +904,16 @@ def _resolve_and_validate(self) -> Self: @model_validator(mode="after") def _propagate_endpoint_url_to_sysinfo(self) -> Self: - """Copy endpoint_config.endpoints[0] into system_info.endpoint_url if unset.""" + """Copy endpoint_config.endpoints[0] into sys_info_capture.endpoint_url if unset.""" if ( - self.system_info is not None - and self.system_info.endpoint_url is None + self.sys_info_capture is not None + and self.sys_info_capture.endpoint_url is None and self.endpoint_config.endpoints ): - new_sic = self.system_info.model_copy( + new_sic = self.sys_info_capture.model_copy( update={"endpoint_url": self.endpoint_config.endpoints[0]} ) - object.__setattr__(self, "system_info", new_sic) + object.__setattr__(self, "sys_info_capture", new_sic) return self @model_validator(mode="after") diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index c2d3b3981..0b979e6bf 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -87,4 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning -system_info: null +sys_info_capture: null diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 53ab7cccc..ede4f1e6a 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -87,4 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning -system_info: null +sys_info_capture: null diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 8d0224f1c..7b084b4af 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -87,4 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning -system_info: null +sys_info_capture: null diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index 4fd246f60..7acd53097 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -133,6 +133,13 @@ def capture_system_info( f"{result.get('error', 'unknown error')}" ) - output_path = Path(config.output_path) / _OUT_FILE_NAME + new_env_path = (result.get("new_env") or {}).get( + "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH" + ) + output_path = ( + Path(new_env_path) + if new_env_path + else Path(config.output_path) / _OUT_FILE_NAME + ) logger.info("System info written to %s", output_path) return output_path diff --git a/tests/unit/sys_info/test_sysinfo_command.py b/tests/unit/sys_info/test_sysinfo_command.py new file mode 100644 index 000000000..3ee7631fb --- /dev/null +++ b/tests/unit/sys_info/test_sysinfo_command.py @@ -0,0 +1,562 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the sysinfo from-config command, NodeEntry, and node_config.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml +from pydantic import ValidationError + +from inference_endpoint.config.schema import ( + NodeEntry, + SysInfoCaptureConfig, + SysInfoFileConfig, +) +from inference_endpoint.exceptions import InputValidationError + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_BASE_SYSTEM_INFO = { + "accelerator_backend": "cuda", + "ssh_ids": ["anandhusooraj@mlc2"], +} + + +def _make_capture_config(**overrides: object) -> SysInfoCaptureConfig: + return SysInfoCaptureConfig(**{**_BASE_SYSTEM_INFO, **overrides}) + + +# --------------------------------------------------------------------------- +# NodeEntry model +# --------------------------------------------------------------------------- + + +class TestNodeEntry: + @pytest.mark.unit + def test_valid_entry(self) -> None: + e = NodeEntry(node_name="H100", no_of_nodes=4) + assert e.node_name == "H100" + assert e.no_of_nodes == 4 + + @pytest.mark.unit + def test_default_no_of_nodes_is_one(self) -> None: + e = NodeEntry(node_name="GB300") + assert e.no_of_nodes == 1 + + @pytest.mark.unit + def test_zero_nodes_raises(self) -> None: + with pytest.raises(ValidationError): + NodeEntry(node_name="H100", no_of_nodes=0) + + @pytest.mark.unit + def test_negative_nodes_raises(self) -> None: + with pytest.raises(ValidationError): + NodeEntry(node_name="H100", no_of_nodes=-1) + + @pytest.mark.unit + def test_extra_fields_forbidden(self) -> None: + with pytest.raises(ValidationError): + NodeEntry(node_name="H100", no_of_nodes=1, unknown_field="x") # type: ignore[call-arg] + + +# --------------------------------------------------------------------------- +# SysInfoCaptureConfig with node_config +# --------------------------------------------------------------------------- + + +class TestSysInfoCaptureConfigOutputPath: + @pytest.mark.unit + def test_default_output_path_is_cwd(self) -> None: + cfg = _make_capture_config() + assert cfg.output_path == "." + + @pytest.mark.unit + def test_explicit_output_path_overrides_default(self) -> None: + cfg = _make_capture_config(output_path="/custom/out") + assert cfg.output_path == "/custom/out" + + +class TestSysInfoCaptureConfigNodeConfig: + @pytest.mark.unit + def test_node_config_none_by_default(self) -> None: + cfg = _make_capture_config() + assert cfg.node_config is None + + @pytest.mark.unit + def test_node_config_single_function(self) -> None: + cfg = _make_capture_config( + node_config={ + "Prefill": [{"node_name": "H100", "no_of_nodes": 4}], + } + ) + assert cfg.node_config is not None + assert len(cfg.node_config["Prefill"]) == 1 + assert cfg.node_config["Prefill"][0].node_name == "H100" + assert cfg.node_config["Prefill"][0].no_of_nodes == 4 + + @pytest.mark.unit + def test_node_config_multi_function_multi_node(self) -> None: + cfg = _make_capture_config( + node_config={ + "Decode": [ + {"node_name": "GB300", "no_of_nodes": 12}, + {"node_name": "H100", "no_of_nodes": 15}, + ], + "Prefill": [ + {"node_name": "GB300", "no_of_nodes": 20}, + {"node_name": "H100", "no_of_nodes": 8}, + ], + } + ) + assert cfg.node_config is not None + decode = cfg.node_config["Decode"] + assert decode[0].node_name == "GB300" + assert decode[0].no_of_nodes == 12 + assert decode[1].node_name == "H100" + assert decode[1].no_of_nodes == 15 + + prefill = cfg.node_config["Prefill"] + assert prefill[0].no_of_nodes == 20 + assert prefill[1].no_of_nodes == 8 + + @pytest.mark.unit + def test_node_config_invalid_node_entry_raises(self) -> None: + with pytest.raises(ValidationError): + _make_capture_config( + node_config={ + "Prefill": [{"node_name": "H100", "no_of_nodes": -5}], + } + ) + + +# --------------------------------------------------------------------------- +# SysInfoFileConfig YAML loading +# --------------------------------------------------------------------------- + + +class TestSysInfoFileConfig: + @pytest.mark.unit + def test_minimal_config_no_node_config(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + """\ + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + """ + ) + ) + cfg = SysInfoFileConfig.from_yaml_file(config_path) + assert cfg.system_info.accelerator_backend == "cuda" + assert cfg.report_dir is None + assert cfg.system_info.node_config is None + + @pytest.mark.unit + def test_report_dir_parsed(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + """\ + report_dir: results/my_system/ + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + """ + ) + ) + cfg = SysInfoFileConfig.from_yaml_file(config_path) + from pathlib import Path as _Path + + assert cfg.report_dir == _Path("results/my_system/") + + @pytest.mark.unit + def test_with_node_config(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + """\ + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + output_path: /tmp/sys_info + node_config: + Prefill: + - node_name: H100 + no_of_nodes: 1 + Decode: + - node_name: H100 + no_of_nodes: 1 + """ + ) + ) + cfg = SysInfoFileConfig.from_yaml_file(config_path) + nc = cfg.system_info.node_config + assert nc is not None + assert "Prefill" in nc + assert "Decode" in nc + assert nc["Prefill"][0].node_name == "H100" + assert nc["Decode"][0].no_of_nodes == 1 + + @pytest.mark.unit + def test_extra_top_level_keys_ignored(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + """\ + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + output_path: /tmp/sys_info + some_other_section: + foo: bar + """ + ) + ) + cfg = SysInfoFileConfig.from_yaml_file(config_path) + assert cfg.system_info.accelerator_backend == "cuda" + + @pytest.mark.unit + def test_file_not_found_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + SysInfoFileConfig.from_yaml_file(tmp_path / "nonexistent.yaml") + + @pytest.mark.unit + def test_invalid_yaml_raises_value_error(self, tmp_path: Path) -> None: + config_path = tmp_path / "bad.yaml" + config_path.write_text("- not a mapping\n- item2\n") + with pytest.raises(ValueError, match="Expected YAML mapping"): + SysInfoFileConfig.from_yaml_file(config_path) + + @pytest.mark.unit + def test_missing_system_info_key_raises(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text("other_key: value\n") + with pytest.raises(ValidationError): + SysInfoFileConfig.from_yaml_file(config_path) + + +# --------------------------------------------------------------------------- +# capture_system_info with node_config — temp file creation and cleanup +# --------------------------------------------------------------------------- + + +class TestCaptureWithNodeConfig: + @pytest.mark.unit + def test_node_config_file_passed_to_mlcflow(self, tmp_path: Path) -> None: + """When node_config is set, mlcflow.access must receive node_config_file.""" + cfg = _make_capture_config( + output_path=str(tmp_path), + node_config={ + "Prefill": [{"node_name": "H100", "no_of_nodes": 1}], + }, + ) + mock_mlcflow = MagicMock() + mock_mlcflow.access.return_value = { + "return": 0, + "new_env": { + "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH": str(tmp_path / "out.json") + }, + } + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + capture_mod.capture_system_info(cfg) + + call_args = mock_mlcflow.access.call_args[0][0] + assert "node_config_file" in call_args + + @pytest.mark.unit + def test_node_config_temp_file_content(self, tmp_path: Path) -> None: + """The temp file must contain system_info.node_config in the expected structure.""" + cfg = _make_capture_config( + output_path=str(tmp_path), + node_config={ + "Decode": [ + {"node_name": "GB300", "no_of_nodes": 12}, + {"node_name": "H100", "no_of_nodes": 15}, + ], + }, + ) + captured_path: list[str] = [] + captured_content: list[dict] = [] + + def fake_access(kwargs: dict) -> dict: + path = kwargs.get("node_config_file", "") + captured_path.append(str(path)) + if path: + with open(path) as f: + captured_content.append(yaml.safe_load(f)) + return { + "return": 0, + "new_env": { + "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH": str(tmp_path / "out.json") + }, + } + + mock_mlcflow = MagicMock() + mock_mlcflow.access.side_effect = fake_access + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + capture_mod.capture_system_info(cfg) + + assert len(captured_content) == 1 + data = captured_content[0] + decode_nodes = data["system_info"]["node_config"]["Decode"] + assert decode_nodes[0]["node_name"] == "GB300" + assert decode_nodes[0]["no_of_nodes"] == 12 + assert decode_nodes[1]["node_name"] == "H100" + assert decode_nodes[1]["no_of_nodes"] == 15 + + @pytest.mark.unit + def test_temp_file_deleted_after_success(self, tmp_path: Path) -> None: + """The temp node_config file must be cleaned up after mlcflow returns.""" + cfg = _make_capture_config( + output_path=str(tmp_path), + node_config={"Prefill": [{"node_name": "H100", "no_of_nodes": 1}]}, + ) + recorded_tmp: list[str] = [] + + def fake_access(kwargs: dict) -> dict: + recorded_tmp.append(str(kwargs.get("node_config_file", ""))) + return { + "return": 0, + "new_env": { + "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH": str(tmp_path / "out.json") + }, + } + + mock_mlcflow = MagicMock() + mock_mlcflow.access.side_effect = fake_access + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + capture_mod.capture_system_info(cfg) + + assert recorded_tmp[0], "temp file path was recorded" + assert not Path(recorded_tmp[0]).exists(), "temp file was deleted after use" + + @pytest.mark.unit + def test_temp_file_deleted_on_mlcflow_failure(self, tmp_path: Path) -> None: + """Temp file must be cleaned up even when mlcflow returns an error.""" + from inference_endpoint.exceptions import ExecutionError + + cfg = _make_capture_config( + output_path=str(tmp_path), + node_config={"Prefill": [{"node_name": "H100", "no_of_nodes": 1}]}, + ) + recorded_tmp: list[str] = [] + + def fake_access(kwargs: dict) -> dict: + recorded_tmp.append(str(kwargs.get("node_config_file", ""))) + return {"return": 1, "error": "ssh timeout"} + + mock_mlcflow = MagicMock() + mock_mlcflow.access.side_effect = fake_access + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + with pytest.raises(ExecutionError): + capture_mod.capture_system_info(cfg) + + assert not Path(recorded_tmp[0]).exists(), "temp file cleaned up on failure" + + @pytest.mark.unit + def test_no_node_config_file_arg_when_node_config_is_none( + self, tmp_path: Path + ) -> None: + """Without node_config, node_config_file must NOT be in the mlcflow call.""" + cfg = _make_capture_config(output_path=str(tmp_path)) + mock_mlcflow = MagicMock() + mock_mlcflow.access.return_value = { + "return": 0, + "new_env": { + "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH": str(tmp_path / "out.json") + }, + } + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + + importlib.reload(capture_mod) + capture_mod.capture_system_info(cfg) + + call_args = mock_mlcflow.access.call_args[0][0] + assert "node_config_file" not in call_args + + +# --------------------------------------------------------------------------- +# from-config CLI command +# --------------------------------------------------------------------------- + + +class TestReportDirResolution: + @pytest.mark.unit + def test_report_dir_overrides_output_path(self, tmp_path: Path) -> None: + """When report_dir is set, capture_system_info must receive it as output_path.""" + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + f"""\ + report_dir: {tmp_path}/results/ + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + output_path: /should/be/ignored + """ + ) + ) + captured_configs: list[SysInfoCaptureConfig] = [] + + def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: + captured_configs.append(cfg) + return tmp_path / "out.json" + + with patch( + "inference_endpoint.commands.sysinfo.cli.capture_system_info", + side_effect=fake_capture, + ): + from inference_endpoint.commands.sysinfo.cli import from_config + + from_config(config=config_path) + + assert str(captured_configs[0].output_path) == str(tmp_path / "results") + + @pytest.mark.unit + def test_no_report_dir_uses_output_path(self, tmp_path: Path) -> None: + """Without report_dir, output_path in system_info is used unchanged.""" + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + f"""\ + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + output_path: {tmp_path}/custom/ + """ + ) + ) + captured_configs: list[SysInfoCaptureConfig] = [] + + def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: + captured_configs.append(cfg) + return tmp_path / "out.json" + + with patch( + "inference_endpoint.commands.sysinfo.cli.capture_system_info", + side_effect=fake_capture, + ): + from inference_endpoint.commands.sysinfo.cli import from_config + + from_config(config=config_path) + + assert Path(captured_configs[0].output_path) == tmp_path / "custom" + + +class TestFromConfigCLI: + @pytest.mark.unit + def test_from_config_calls_capture(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + """\ + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + output_path: /tmp/sys_info + node_config: + Prefill: + - node_name: H100 + no_of_nodes: 1 + """ + ) + ) + captured_configs: list[SysInfoCaptureConfig] = [] + + def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: + captured_configs.append(cfg) + return tmp_path / "out.json" + + with patch( + "inference_endpoint.commands.sysinfo.cli.capture_system_info", + side_effect=fake_capture, + ): + from inference_endpoint.commands.sysinfo.cli import from_config + + from_config(config=config_path) + + assert len(captured_configs) == 1 + assert captured_configs[0].accelerator_backend == "cuda" + assert captured_configs[0].node_config is not None + assert "Prefill" in captured_configs[0].node_config + + @pytest.mark.unit + def test_from_config_missing_file_raises_input_validation_error( + self, tmp_path: Path + ) -> None: + with patch( + "inference_endpoint.commands.sysinfo.cli.capture_system_info" + ) as mock_cap: + from inference_endpoint.commands.sysinfo.cli import from_config + + with pytest.raises(InputValidationError): + from_config(config=tmp_path / "missing.yaml") + + mock_cap.assert_not_called() + + @pytest.mark.unit + def test_from_config_invalid_yaml_raises_input_validation_error( + self, tmp_path: Path + ) -> None: + bad_path = tmp_path / "bad.yaml" + bad_path.write_text("- not\n- a\n- mapping\n") + with patch( + "inference_endpoint.commands.sysinfo.cli.capture_system_info" + ) as mock_cap: + from inference_endpoint.commands.sysinfo.cli import from_config + + with pytest.raises(InputValidationError): + from_config(config=bad_path) + + mock_cap.assert_not_called() From 9b88a81f45e9e85a6d05e06ae69fa4a809924c61 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 29 May 2026 17:42:30 +0530 Subject: [PATCH 08/21] fix: sort imports in test_sysinfo_command.py (ruff) Co-Authored-By: Claude Sonnet 4.6 --- tests/unit/sys_info/test_sysinfo_command.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/sys_info/test_sysinfo_command.py b/tests/unit/sys_info/test_sysinfo_command.py index 3ee7631fb..fc84827f3 100644 --- a/tests/unit/sys_info/test_sysinfo_command.py +++ b/tests/unit/sys_info/test_sysinfo_command.py @@ -23,14 +23,13 @@ import pytest import yaml -from pydantic import ValidationError - from inference_endpoint.config.schema import ( NodeEntry, SysInfoCaptureConfig, SysInfoFileConfig, ) from inference_endpoint.exceptions import InputValidationError +from pydantic import ValidationError # --------------------------------------------------------------------------- # Helpers From 3184d91f6a17040deb2b79745a5c09dedbc6bd42 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 2 Jun 2026 21:31:59 +0530 Subject: [PATCH 09/21] test: add failure scenario tests for sys info capture - Fail with ExecutionError when MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH is not returned by mlcflow (replaces silent fallback to default path) - Test that missing ssh_ids in YAML raises ValidationError regardless of exclude_current_system value - Test that sys info failure (ExecutionError or unexpected exception) does not block results.json from being written in finalize_benchmark - Test unreachable node and node_config count mismatch scenarios - Replace internal hostname in test fixtures with generic user@10.0.0.1 Co-Authored-By: Claude Sonnet 4.6 --- src/inference_endpoint/sys_info/capture.py | 11 +- .../commands/test_benchmark_finalization.py | 125 ++++++++++++++++++ tests/unit/sys_info/test_capture.py | 29 +++- tests/unit/sys_info/test_sysinfo_command.py | 52 ++++++-- 4 files changed, 202 insertions(+), 15 deletions(-) create mode 100644 tests/unit/commands/test_benchmark_finalization.py diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index 7acd53097..66fa94486 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -136,10 +136,11 @@ def capture_system_info( new_env_path = (result.get("new_env") or {}).get( "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH" ) - output_path = ( - Path(new_env_path) - if new_env_path - else Path(config.output_path) / _OUT_FILE_NAME - ) + if not new_env_path: + raise ExecutionError( + "sys_info capture returned success but MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH " + "was not set — no system info was collected" + ) + output_path = Path(new_env_path) logger.info("System info written to %s", output_path) return output_path diff --git a/tests/unit/commands/test_benchmark_finalization.py b/tests/unit/commands/test_benchmark_finalization.py new file mode 100644 index 000000000..6dc4f7282 --- /dev/null +++ b/tests/unit/commands/test_benchmark_finalization.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests that sys info failure does not block results.json in finalize_benchmark.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from inference_endpoint.commands.benchmark.execute import ( + BenchmarkContext, + BenchmarkResult, + ResponseCollector, + finalize_benchmark, +) +from inference_endpoint.config.schema import OfflineBenchmarkConfig, TestMode +from inference_endpoint.exceptions import ExecutionError +from inference_endpoint.load_generator.session import SessionResult + +_OFFLINE_KWARGS = { + "endpoint_config": {"endpoints": ["http://localhost:8000"]}, + "model_params": {"name": "test-model"}, + "datasets": [{"path": "test.jsonl"}], + "sys_info_capture": { + "ssh_ids": ["alice@10.0.0.1"], + "accelerator_backend": "cuda", + }, +} + + +def _make_ctx(tmp_path: Path) -> BenchmarkContext: + config = OfflineBenchmarkConfig(**_OFFLINE_KWARGS) + return BenchmarkContext( + config=config, + test_mode=TestMode.PERF, + report_dir=tmp_path, + tokenizer_name=None, + dataloader=MagicMock(), + rt_settings=MagicMock(), + total_samples=1, + eval_configs=[], + ) + + +def _make_bench(tmp_path: Path) -> BenchmarkResult: + t = 1_000_000_000 + session = SessionResult( + session_id="test-session", + phase_results=[], + start_time_ns=t, + end_time_ns=t + 1_000_000_000, + ) + return BenchmarkResult( + session=session, + collector=ResponseCollector(), + report=None, + tmpfs_dir=tmp_path, + ) + + +class TestSysInfoFailureNonBlocking: + @pytest.mark.unit + def test_execution_error_does_not_prevent_results_write( + self, tmp_path: Path + ) -> None: + """ExecutionError from capture_system_info must not prevent results.json.""" + ctx = _make_ctx(tmp_path) + bench = _make_bench(tmp_path) + + with ( + patch( + "inference_endpoint.commands.benchmark.execute._build_run_metadata", + return_value={}, + ), + patch( + "inference_endpoint.sys_info.capture.capture_system_info", + side_effect=ExecutionError("ssh timeout"), + ), + ): + finalize_benchmark(ctx, bench) + + results_path = tmp_path / "results.json" + assert results_path.exists() + data = json.loads(results_path.read_text()) + assert "results" in data + + @pytest.mark.unit + def test_unexpected_exception_does_not_prevent_results_write( + self, tmp_path: Path + ) -> None: + """An unexpected exception from capture_system_info must not prevent results.json.""" + ctx = _make_ctx(tmp_path) + bench = _make_bench(tmp_path) + + with ( + patch( + "inference_endpoint.commands.benchmark.execute._build_run_metadata", + return_value={}, + ), + patch( + "inference_endpoint.sys_info.capture.capture_system_info", + side_effect=RuntimeError("unexpected crash"), + ), + ): + finalize_benchmark(ctx, bench) + + results_path = tmp_path / "results.json" + assert results_path.exists() + data = json.loads(results_path.read_text()) + assert "results" in data diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index a942677b5..e6bebc79a 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -207,7 +207,8 @@ def test_happy_path_output_path_from_new_env(self, tmp_path: Path) -> None: assert result == Path("/tmp/out.json") @pytest.mark.unit - def test_happy_path_output_path_fallback(self, tmp_path: Path) -> None: + def test_missing_env_path_raises_execution_error(self, tmp_path: Path) -> None: + """Return code 0 without MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH raises ExecutionError.""" cfg = _make_config(output_path=str(tmp_path)) mock_mlcflow = MagicMock() mock_mlcflow.access.return_value = { @@ -219,9 +220,33 @@ def test_happy_path_output_path_fallback(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod + importlib.reload(capture_mod) + with pytest.raises( + ExecutionError, match="MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH" + ): + capture_mod.capture_system_info(cfg) + + @pytest.mark.unit + def test_unreachable_node_no_node_config_succeeds(self, tmp_path: Path) -> None: + """mlcflow returning 0 (one node internally unreachable, no node_config) is non-fatal.""" + cfg = _make_config(output_path=str(tmp_path)) + mock_mlcflow = MagicMock() + mock_mlcflow.access.return_value = { + "return": 0, + "new_env": { + "MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH": str( + tmp_path / "system_desc.json" + ), + }, + } + with patch.dict("sys.modules", {"mlc": mock_mlcflow}): + import importlib + + from inference_endpoint.sys_info import capture as capture_mod + importlib.reload(capture_mod) result = capture_mod.capture_system_info(cfg) - assert result == Path(cfg.output_path) / "system_desc.json" + assert result == tmp_path / "system_desc.json" @pytest.mark.unit def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: diff --git a/tests/unit/sys_info/test_sysinfo_command.py b/tests/unit/sys_info/test_sysinfo_command.py index fc84827f3..a8ec7bfb6 100644 --- a/tests/unit/sys_info/test_sysinfo_command.py +++ b/tests/unit/sys_info/test_sysinfo_command.py @@ -37,7 +37,7 @@ _BASE_SYSTEM_INFO = { "accelerator_backend": "cuda", - "ssh_ids": ["anandhusooraj@mlc2"], + "ssh_ids": ["user@10.0.0.1"], } @@ -162,7 +162,7 @@ def test_minimal_config_no_node_config(self, tmp_path: Path) -> None: """\ system_info: ssh_ids: - - anandhusooraj@mlc2 + - user@10.0.0.1 accelerator_backend: cuda """ ) @@ -181,7 +181,7 @@ def test_report_dir_parsed(self, tmp_path: Path) -> None: report_dir: results/my_system/ system_info: ssh_ids: - - anandhusooraj@mlc2 + - user@10.0.0.1 accelerator_backend: cuda """ ) @@ -199,7 +199,7 @@ def test_with_node_config(self, tmp_path: Path) -> None: """\ system_info: ssh_ids: - - anandhusooraj@mlc2 + - user@10.0.0.1 accelerator_backend: cuda output_path: /tmp/sys_info node_config: @@ -228,7 +228,7 @@ def test_extra_top_level_keys_ignored(self, tmp_path: Path) -> None: """\ system_info: ssh_ids: - - anandhusooraj@mlc2 + - user@10.0.0.1 accelerator_backend: cuda output_path: /tmp/sys_info some_other_section: @@ -438,7 +438,7 @@ def test_report_dir_overrides_output_path(self, tmp_path: Path) -> None: report_dir: {tmp_path}/results/ system_info: ssh_ids: - - anandhusooraj@mlc2 + - user@10.0.0.1 accelerator_backend: cuda output_path: /should/be/ignored """ @@ -469,7 +469,7 @@ def test_no_report_dir_uses_output_path(self, tmp_path: Path) -> None: f"""\ system_info: ssh_ids: - - anandhusooraj@mlc2 + - user@10.0.0.1 accelerator_backend: cuda output_path: {tmp_path}/custom/ """ @@ -492,6 +492,42 @@ def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: assert Path(captured_configs[0].output_path) == tmp_path / "custom" +class TestMissingSshIds: + @pytest.mark.unit + def test_missing_ssh_ids_raises_validation_error(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text("system_info:\n accelerator_backend: cuda\n") + with pytest.raises(ValidationError): + SysInfoFileConfig.from_yaml_file(config_path) + + @pytest.mark.unit + def test_missing_ssh_ids_exclude_current_false_still_raises( + self, tmp_path: Path + ) -> None: + """exclude_current_system=False does not relax the ssh_ids requirement.""" + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + "system_info:\n accelerator_backend: cuda\n exclude_current_system: false\n" + ) + with pytest.raises(ValidationError): + SysInfoFileConfig.from_yaml_file(config_path) + + @pytest.mark.unit + def test_missing_ssh_ids_via_cli_raises_input_validation_error( + self, tmp_path: Path + ) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text("system_info:\n accelerator_backend: cuda\n") + with patch( + "inference_endpoint.commands.sysinfo.cli.capture_system_info" + ) as mock_cap: + from inference_endpoint.commands.sysinfo.cli import from_config + + with pytest.raises(InputValidationError): + from_config(config=config_path) + mock_cap.assert_not_called() + + class TestFromConfigCLI: @pytest.mark.unit def test_from_config_calls_capture(self, tmp_path: Path) -> None: @@ -501,7 +537,7 @@ def test_from_config_calls_capture(self, tmp_path: Path) -> None: """\ system_info: ssh_ids: - - anandhusooraj@mlc2 + - user@10.0.0.1 accelerator_backend: cuda output_path: /tmp/sys_info node_config: From 7550f811c463cd1cdb4b210ad44487e085cbf59d Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 4 Jun 2026 16:01:36 +0530 Subject: [PATCH 10/21] refactor: rename sys_info_capture field to system_info in BenchmarkConfig Renames the `sys_info_capture` YAML key to `system_info` in BenchmarkConfig for consistency with SysInfoFileConfig which already uses `system_info`. Updates all Python references, config templates, tests, and docs. Co-Authored-By: Claude Sonnet 4.6 --- docs/commands/DESIGN.md | 6 +++--- .../commands/benchmark/execute.py | 6 +++--- src/inference_endpoint/config/schema.py | 14 +++++++------- .../templates/concurrency_template_full.yaml | 4 +--- .../config/templates/offline_template_full.yaml | 4 +--- .../config/templates/online_template_full.yaml | 4 +--- src/inference_endpoint/sys_info/capture.py | 2 +- tests/unit/commands/test_benchmark_finalization.py | 2 +- tests/unit/sys_info/test_capture.py | 12 ++++++------ 9 files changed, 24 insertions(+), 30 deletions(-) diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index 3f105d12c..795d817f5 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -98,7 +98,7 @@ commands/benchmark/execute.py::run_benchmark() +-- construct endpoint client + sample issuer +-- run BenchmarkSession in threaded wrapper +-- finalize metrics and optional accuracy scoring - +-- if sys_info_capture is configured: + +-- if system_info is configured: write run_metadata.json capture_system_info() → mlcflow (hardware + serving config) patch run_metadata.json with serving config values @@ -109,7 +109,7 @@ commands/benchmark/execute.py::run_benchmark() System info capture collects hardware/software details from one or more nodes and writes a structured JSON file for MLPerf inference submissions. It runs in two contexts: - **Standalone** (`sysinfo from-config`): triggered manually, independent of any benchmark run. -- **Integrated** (`benchmark` finalization): triggered automatically after a benchmark if `sys_info_capture` is present in the config. +- **Integrated** (`benchmark` finalization): triggered automatically after a benchmark if `system_info` is present in the config. Both paths call `sys_info/capture.py::capture_system_info()` and produce the same output JSON. The integrated path additionally patches `run_metadata.json` with serving configuration values extracted from the inference server's startup log. @@ -222,7 +222,7 @@ When `node_config` is provided, the automations script enforces: | `node_name` unmatched or count exceeds probed | `ExecutionError` | `ExecutionError` → logged as `error` | | `serving_config.json` absent or unreadable | Logged as error; `run_metadata.json` left unchanged | Same | -In the integrated path, `sys_info_capture` failures never abort the benchmark. `results.json` and `run_metadata.json` are written before the capture call, so the benchmark output is complete regardless of capture outcome. The error log includes `report_dir` and a command to re-run capture manually. +In the integrated path, `system_info` failures never abort the benchmark. `results.json` and `run_metadata.json` are written before the capture call, so the benchmark output is complete regardless of capture outcome. The error log includes `report_dir` and a command to re-run capture manually. --- diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 052db9086..7c8e5b9e3 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -987,14 +987,14 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: json.dump(run_metadata, f, indent=2) logger.info("Run metadata written to %s", metadata_path) - if ctx.config.sys_info_capture is not None: + if ctx.config.system_info is not None: try: # Local import: mlcflow is optional and only needed when system_info is configured. from inference_endpoint.sys_info.capture import ( capture_system_info, ) - sic = ctx.config.sys_info_capture.model_copy( + sic = ctx.config.system_info.model_copy( update={"output_path": str(ctx.report_dir)} ) output_path = capture_system_info( @@ -1038,7 +1038,7 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: # node_config and disaggregated from system_info node_config: Any = None disaggregated: bool | None = None - sic = ctx.config.sys_info_capture + sic = ctx.config.system_info if sic is not None and sic.node_config is not None: node_config = { fn: [ne.model_dump() for ne in nodes] diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 6627daf39..67e766b40 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -603,7 +603,7 @@ class NodeEntry(BaseModel): class SysInfoCaptureConfig(BaseModel): - """Configuration for the sys_info_capture post-benchmark step.""" + """Configuration for the system_info post-benchmark step.""" model_config = ConfigDict(extra="forbid", frozen=True) @@ -800,7 +800,7 @@ class BenchmarkConfig(WithUpdatesMixin, BaseModel): help="NUMA-aware CPU pinning", ), ] = True - sys_info_capture: Annotated[ + system_info: Annotated[ SysInfoCaptureConfig | None, cyclopts.Parameter(show=False) ] = None @@ -922,16 +922,16 @@ def _resolve_and_validate(self) -> Self: @model_validator(mode="after") def _propagate_endpoint_url_to_sysinfo(self) -> Self: - """Copy endpoint_config.endpoints[0] into sys_info_capture.endpoint_url if unset.""" + """Copy endpoint_config.endpoints[0] into system_info.endpoint_url if unset.""" if ( - self.sys_info_capture is not None - and self.sys_info_capture.endpoint_url is None + self.system_info is not None + and self.system_info.endpoint_url is None and self.endpoint_config.endpoints ): - new_sic = self.sys_info_capture.model_copy( + new_sic = self.system_info.model_copy( update={"endpoint_url": self.endpoint_config.endpoints[0]} ) - object.__setattr__(self, "sys_info_capture", new_sic) + object.__setattr__(self, "system_info", new_sic) return self @model_validator(mode="after") diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index 3e7687800..c2d3b3981 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -11,7 +11,6 @@ model_params: repetition_penalty: null # Repetition penalty presence_penalty: null # Presence penalty frequency_penalty: null # Frequency penalty - chat_template_kwargs: null # Per-request chat-template kwargs forwarded to compatible servers. max_new_tokens: 1024 # Max output tokens osl_distribution: null # Output sequence length distribution streaming: 'on' # Streaming mode: auto/on/off | options: auto, on, off @@ -70,7 +69,6 @@ settings: worker_initialization_timeout: 60.0 # Worker init timeout (seconds) worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) - insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system @@ -89,4 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning -sys_info_capture: null +system_info: null diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 7a0421c70..53ab7cccc 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -11,7 +11,6 @@ model_params: repetition_penalty: null # Repetition penalty presence_penalty: null # Presence penalty frequency_penalty: null # Frequency penalty - chat_template_kwargs: null # Per-request chat-template kwargs forwarded to compatible servers. max_new_tokens: 1024 # Max output tokens osl_distribution: null # Output sequence length distribution streaming: 'off' # Streaming mode: auto/on/off | options: auto, on, off @@ -70,7 +69,6 @@ settings: worker_initialization_timeout: 60.0 # Worker init timeout (seconds) worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) - insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system @@ -89,4 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning -sys_info_capture: null +system_info: null diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index fe773a485..8d0224f1c 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -11,7 +11,6 @@ model_params: repetition_penalty: null # Repetition penalty presence_penalty: null # Presence penalty frequency_penalty: null # Frequency penalty - chat_template_kwargs: null # Per-request chat-template kwargs forwarded to compatible servers. max_new_tokens: 1024 # Max output tokens osl_distribution: null # Output sequence length distribution streaming: 'on' # Streaming mode: auto/on/off | options: auto, on, off @@ -70,7 +69,6 @@ settings: worker_initialization_timeout: 60.0 # Worker init timeout (seconds) worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) - insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system @@ -89,4 +87,4 @@ report_dir: null # Report output directory timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning -sys_info_capture: null +system_info: null diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index 66fa94486..e2852b545 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -75,7 +75,7 @@ def capture_system_info( import mlc # noqa: PLC0415 except ImportError as exc: raise SetupError( - "mlc-scripts is required for sys_info_capture. " + "mlc-scripts is required for system_info. " "Install it with: pip install mlc-scripts" ) from exc diff --git a/tests/unit/commands/test_benchmark_finalization.py b/tests/unit/commands/test_benchmark_finalization.py index 6dc4f7282..6a951a9db 100644 --- a/tests/unit/commands/test_benchmark_finalization.py +++ b/tests/unit/commands/test_benchmark_finalization.py @@ -36,7 +36,7 @@ "endpoint_config": {"endpoints": ["http://localhost:8000"]}, "model_params": {"name": "test-model"}, "datasets": [{"path": "test.jsonl"}], - "sys_info_capture": { + "system_info": { "ssh_ids": ["alice@10.0.0.1"], "accelerator_backend": "cuda", }, diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index e6bebc79a..5a0d3902b 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -290,7 +290,7 @@ def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: class TestYamlRoundTrip: @pytest.mark.unit - def test_sys_info_capture_from_yaml(self, tmp_path: Path) -> None: + def test_system_info_from_yaml(self, tmp_path: Path) -> None: yaml_content = textwrap.dedent( """\ type: offline @@ -301,7 +301,7 @@ def test_sys_info_capture_from_yaml(self, tmp_path: Path) -> None: - http://localhost:8000 datasets: - path: dummy.jsonl - sys_info_capture: + system_info: accelerator_backend: cuda output_path: /tmp/sys_info ssh_ids: @@ -316,8 +316,8 @@ def test_sys_info_capture_from_yaml(self, tmp_path: Path) -> None: config = BenchmarkConfig.from_yaml_file(config_path) - assert config.sys_info_capture is not None - sic = config.sys_info_capture + assert config.system_info is not None + sic = config.system_info assert sic.accelerator_backend == "cuda" assert sic.output_path == "/tmp/sys_info" assert sic.exclude_current_system is True @@ -330,7 +330,7 @@ def test_sys_info_capture_from_yaml(self, tmp_path: Path) -> None: # 12. Backward compatibility @pytest.mark.unit - def test_yaml_without_sys_info_capture_is_none(self, tmp_path: Path) -> None: + def test_yaml_without_system_info_is_none(self, tmp_path: Path) -> None: yaml_content = textwrap.dedent( """\ type: offline @@ -347,4 +347,4 @@ def test_yaml_without_sys_info_capture_is_none(self, tmp_path: Path) -> None: config_path.write_text(yaml_content) config = BenchmarkConfig.from_yaml_file(config_path) - assert config.sys_info_capture is None + assert config.system_info is None From db721b054300f1297e11b694bd11f02b3901b423 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 4 Jun 2026 16:17:37 +0530 Subject: [PATCH 11/21] fix: regenerate config templates after merge from main Re-run scripts/regenerate_templates.py after merging main (fbe543f drain timeout changes) into sysinfochanges. Co-Authored-By: Claude Sonnet 4.6 --- .../config/templates/concurrency_template_full.yaml | 6 +----- .../config/templates/offline_template_full.yaml | 6 +----- .../config/templates/online_template_full.yaml | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index acec2ba76..c2d3b3981 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -72,15 +72,11 @@ settings: max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: false # Prepend a unique random hex salt to each warmup prompt - drain: false + drain: false # Drain in-flight warmup requests before starting the performance phase warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 895181498..53ab7cccc 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -72,15 +72,11 @@ settings: max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: false # Prepend a unique random hex salt to each warmup prompt - drain: false + drain: false # Drain in-flight warmup requests before starting the performance phase warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 693734e54..8d0224f1c 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -72,15 +72,11 @@ settings: max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: false # Prepend a unique random hex salt to each warmup prompt - drain: false + drain: false # Drain in-flight warmup requests before starting the performance phase warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. From 435e0ff86897d1458b04ec5eb890b90bacc3fa05 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 4 Jun 2026 16:34:29 +0530 Subject: [PATCH 12/21] regenerated templates --- .../config/templates/concurrency_template_full.yaml | 8 +++++++- .../config/templates/offline_template_full.yaml | 8 +++++++- .../config/templates/online_template_full.yaml | 8 +++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index c2d3b3981..a221bcc02 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -11,6 +11,7 @@ model_params: repetition_penalty: null # Repetition penalty presence_penalty: null # Presence penalty frequency_penalty: null # Frequency penalty + chat_template_kwargs: null # Per-request chat-template kwargs forwarded to compatible servers. max_new_tokens: 1024 # Max output tokens osl_distribution: null # Output sequence length distribution streaming: 'on' # Streaming mode: auto/on/off | options: auto, on, off @@ -69,14 +70,19 @@ settings: worker_initialization_timeout: 60.0 # Worker init timeout (seconds) worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) + insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system + drain: + warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: false # Prepend a unique random hex salt to each warmup prompt - drain: false # Drain in-flight warmup requests before starting the performance phase + drain: false warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 53ab7cccc..98d5a8407 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -11,6 +11,7 @@ model_params: repetition_penalty: null # Repetition penalty presence_penalty: null # Presence penalty frequency_penalty: null # Frequency penalty + chat_template_kwargs: null # Per-request chat-template kwargs forwarded to compatible servers. max_new_tokens: 1024 # Max output tokens osl_distribution: null # Output sequence length distribution streaming: 'off' # Streaming mode: auto/on/off | options: auto, on, off @@ -69,14 +70,19 @@ settings: worker_initialization_timeout: 60.0 # Worker init timeout (seconds) worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) + insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system + drain: + warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: false # Prepend a unique random hex salt to each warmup prompt - drain: false # Drain in-flight warmup requests before starting the performance phase + drain: false warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 8d0224f1c..6d6852274 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -11,6 +11,7 @@ model_params: repetition_penalty: null # Repetition penalty presence_penalty: null # Presence penalty frequency_penalty: null # Frequency penalty + chat_template_kwargs: null # Per-request chat-template kwargs forwarded to compatible servers. max_new_tokens: 1024 # Max output tokens osl_distribution: null # Output sequence length distribution streaming: 'on' # Streaming mode: auto/on/off | options: auto, on, off @@ -69,14 +70,19 @@ settings: worker_initialization_timeout: 60.0 # Worker init timeout (seconds) worker_graceful_shutdown_wait: 0.5 # Post-run graceful shutdown wait (seconds) worker_force_kill_timeout: 0.5 # Force kill timeout after graceful wait (seconds) + insecure: false # Skip TLS certificate verification max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system + drain: + warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: false # Prepend a unique random hex salt to each warmup prompt - drain: false # Drain in-flight warmup requests before starting the performance phase + drain: false warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. From f6777b3ea530605c55b0ba37b8e019936230248e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 4 Jun 2026 16:59:28 +0530 Subject: [PATCH 13/21] Delete examples/sysinfo_a40_prefill_h100_decode.yaml --- examples/sysinfo_a40_prefill_h100_decode.yaml | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 examples/sysinfo_a40_prefill_h100_decode.yaml diff --git a/examples/sysinfo_a40_prefill_h100_decode.yaml b/examples/sysinfo_a40_prefill_h100_decode.yaml deleted file mode 100644 index 4cb463249..000000000 --- a/examples/sysinfo_a40_prefill_h100_decode.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# MLPerf system-info capture — A40 (Prefill) + 8×H100 (Decode) -# -# Usage: -# inference-endpoint sysinfo from-config -c examples/sysinfo_a40_prefill_h100_decode.yaml -# -# Node assignments: -# Prefill : @ — NVIDIA A40 (1 GPU per node) -# Decode : @ — NVIDIA H100 (8 GPUs per node) - -report_dir: results/a40_prefill_h100_decode_sysinfo/ - -system_info: - ssh_ids: - - root@63.141.33.33:22053 # A40 node (RunPod) - - anandhusooraj@mlc2 # 8×H100 node (mlc2) - - accelerator_backend: cuda - - exclude_current_system: true - - node_config: - Prefill: - - node_name: A40 # matched as substring of detected GPU name - no_of_nodes: 1 - Decode: - - node_name: H100 # matched as substring of detected GPU name - no_of_nodes: 1 From 19b20bb152b548df53ebbc3aae7d3a028e22c5fc Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Thu, 4 Jun 2026 07:54:57 -0500 Subject: [PATCH 14/21] Address cve-2026-34993 (#333) Signed-off-by: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> --- pyproject.toml | 2 +- uv.lock | 71 +++++++++++++++++++++++++++----------------------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b389abe9..0b0f67a86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,7 @@ test = [ "Pympler==1.1", "scipy==1.17.1", # HTTP server and client for mock server fixture - "aiohttp==3.13.5", + "aiohttp==3.14.0", # Plotting for benchmark sweep mode "matplotlib==3.10.8", # Property-based testing (CLI fuzz) diff --git a/uv.lock b/uv.lock index a8446ce0b..24898140e 100644 --- a/uv.lock +++ b/uv.lock @@ -29,7 +29,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -38,38 +38,45 @@ dependencies = [ { name = "frozenlist", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "multidict", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "propcache", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and platform_machine == 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "yarl", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/6f/353954c29e7dcce7cf00280a02c75f30e133c00793c7a2ed3776d7b2f426/aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9", size = 748876, upload-time = "2026-03-31T21:57:36.319Z" }, - { url = "https://files.pythonhosted.org/packages/f5/1b/428a7c64687b3b2e9cd293186695affc0e1e54a445d0361743b231f11066/aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416", size = 499557, upload-time = "2026-03-31T21:57:38.236Z" }, - { url = "https://files.pythonhosted.org/packages/29/47/7be41556bfbb6917069d6a6634bb7dd5e163ba445b783a90d40f5ac7e3a7/aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2", size = 500258, upload-time = "2026-03-31T21:57:39.923Z" }, - { url = "https://files.pythonhosted.org/packages/67/84/c9ecc5828cb0b3695856c07c0a6817a99d51e2473400f705275a2b3d9239/aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4", size = 1749199, upload-time = "2026-03-31T21:57:41.938Z" }, - { url = "https://files.pythonhosted.org/packages/57/d8/8d44036d7eb7b6a8ec4c5494ea0c8c8b94fbc0ed3991c1a7adf230df03bf/aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1", size = 1767934, upload-time = "2026-03-31T21:57:51.171Z" }, - { url = "https://files.pythonhosted.org/packages/41/db/073e4ebe00b78e2dfcacff734291651729a62953b48933d765dc513bf798/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9", size = 1705219, upload-time = "2026-03-31T21:57:55.385Z" }, - { url = "https://files.pythonhosted.org/packages/84/63/7749337c90f92bc2cb18f9560d67aa6258c7060d1397d21529b8004fcf6f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14", size = 1732427, upload-time = "2026-03-31T21:58:06.337Z" }, - { url = "https://files.pythonhosted.org/packages/78/e9/d76bf503005709e390122d34e15256b88f7008e246c4bdbe915cd4f1adce/aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61", size = 742930, upload-time = "2026-03-31T21:58:13.155Z" }, - { url = "https://files.pythonhosted.org/packages/57/00/4b7b70223deaebd9bb85984d01a764b0d7bd6526fcdc73cca83bcbe7243e/aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832", size = 496927, upload-time = "2026-03-31T21:58:15.073Z" }, - { url = "https://files.pythonhosted.org/packages/9c/f5/0fb20fb49f8efdcdce6cd8127604ad2c503e754a8f139f5e02b01626523f/aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9", size = 497141, upload-time = "2026-03-31T21:58:17.009Z" }, - { url = "https://files.pythonhosted.org/packages/3b/86/b7c870053e36a94e8951b803cb5b909bfbc9b90ca941527f5fcafbf6b0fa/aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090", size = 1732476, upload-time = "2026-03-31T21:58:18.925Z" }, - { url = "https://files.pythonhosted.org/packages/f0/d2/e2f77eef1acb7111405433c707dc735e63f67a56e176e72e9e7a2cd3f493/aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665", size = 1754113, upload-time = "2026-03-31T21:58:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/ec/a6/9b3e91eb8ae791cce4ee736da02211c85c6f835f1bdfac0594a8a3b7018c/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb", size = 1693205, upload-time = "2026-03-31T21:58:32.214Z" }, - { url = "https://files.pythonhosted.org/packages/01/a4/62f05a0a98d88af59d93b7fcac564e5f18f513cb7471696ac286db970d6a/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c", size = 1730356, upload-time = "2026-03-31T21:58:44.049Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ce/46572759afc859e867a5bc8ec3487315869013f59281ce61764f76d879de/aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c", size = 745721, upload-time = "2026-03-31T21:58:50.229Z" }, - { url = "https://files.pythonhosted.org/packages/13/fe/8a2efd7626dbe6049b2ef8ace18ffda8a4dfcbe1bcff3ac30c0c7575c20b/aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be", size = 497663, upload-time = "2026-03-31T21:58:52.232Z" }, - { url = "https://files.pythonhosted.org/packages/9b/91/cc8cc78a111826c54743d88651e1687008133c37e5ee615fee9b57990fac/aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25", size = 499094, upload-time = "2026-03-31T21:58:54.566Z" }, - { url = "https://files.pythonhosted.org/packages/0a/33/a8362cb15cf16a3af7e86ed11962d5cd7d59b449202dc576cdc731310bde/aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56", size = 1726701, upload-time = "2026-03-31T21:58:56.864Z" }, - { url = "https://files.pythonhosted.org/packages/bd/c9/989f4034fb46841208de7aeeac2c6d8300745ab4f28c42f629ba77c2d916/aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b", size = 1730405, upload-time = "2026-03-31T21:59:07.221Z" }, - { url = "https://files.pythonhosted.org/packages/c3/20/1e9e6650dfc436340116b7aa89ff8cb2bbdf0abc11dfaceaad8f74273a10/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d", size = 1692346, upload-time = "2026-03-31T21:59:12.068Z" }, - { url = "https://files.pythonhosted.org/packages/c1/2b/cce5b0ffe0de99c83e5e36d8f828e4161e415660a9f3e58339d07cce3006/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b", size = 1712444, upload-time = "2026-03-31T21:59:24.635Z" }, - { url = "https://files.pythonhosted.org/packages/79/11/c27d9332ee20d68dd164dc12a6ecdef2e2e35ecc97ed6cf0d2442844624b/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a", size = 778758, upload-time = "2026-03-31T21:59:31.547Z" }, - { url = "https://files.pythonhosted.org/packages/04/fb/377aead2e0a3ba5f09b7624f702a964bdf4f08b5b6728a9799830c80041e/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254", size = 512883, upload-time = "2026-03-31T21:59:34.098Z" }, - { url = "https://files.pythonhosted.org/packages/bb/a6/aa109a33671f7a5d3bd78b46da9d852797c5e665bfda7d6b373f56bff2ec/aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36", size = 516668, upload-time = "2026-03-31T21:59:36.497Z" }, - { url = "https://files.pythonhosted.org/packages/79/b3/ca078f9f2fa9563c36fb8ef89053ea2bb146d6f792c5104574d49d8acb63/aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f", size = 1883461, upload-time = "2026-03-31T21:59:38.723Z" }, - { url = "https://files.pythonhosted.org/packages/66/1d/f31ec3f1013723b3babe3609e7f119c2c2fb6ef33da90061a705ef3e1bc8/aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a", size = 1803724, upload-time = "2026-03-31T21:59:48.656Z" }, - { url = "https://files.pythonhosted.org/packages/25/3c/734c878fb43ec083d8e31bf029daae1beafeae582d1b35da234739e82ee7/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be", size = 1806644, upload-time = "2026-03-31T21:59:53.753Z" }, - { url = "https://files.pythonhosted.org/packages/78/b7/15fb7a9d52e112a25b621c67b69c167805cb1f2ab8f1708a5c490d1b52fe/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9", size = 1772072, upload-time = "2026-03-31T22:00:07.494Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194, upload-time = "2026-06-01T19:37:48.164Z" }, + { url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966, upload-time = "2026-06-01T19:37:50.014Z" }, + { url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527, upload-time = "2026-06-01T19:37:51.96Z" }, + { url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420, upload-time = "2026-06-01T19:37:53.839Z" }, + { url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112, upload-time = "2026-06-01T19:38:02.641Z" }, + { url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119, upload-time = "2026-06-01T19:38:06.713Z" }, + { url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193, upload-time = "2026-06-01T19:38:17.624Z" }, + { url = "https://files.pythonhosted.org/packages/11/41/cc2d2cfbfbdc3126ba258f3cd27d1ac8a33492ae3c35a4583ee21f0ba7f1/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6", size = 481670, upload-time = "2026-06-01T19:38:29.836Z" }, + { url = "https://files.pythonhosted.org/packages/3c/07/381f4023c3b08cb616e520f566d8c58957abad54e56441d41fe67cfb0195/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2", size = 487591, upload-time = "2026-06-01T19:38:31.704Z" }, + { url = "https://files.pythonhosted.org/packages/fb/4d/4506fdb7a022bdf70011a3bbb4ca00c5c570026ef6a3c5bd7bc70c39089c/aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50", size = 496503, upload-time = "2026-06-01T19:38:33.6Z" }, + { url = "https://files.pythonhosted.org/packages/ef/7d/c814111e04894a45d9e2defc94443879a6f118d9633d5fedfe6e2e8af5f0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9", size = 745870, upload-time = "2026-06-01T19:38:36.013Z" }, + { url = "https://files.pythonhosted.org/packages/c6/ee/80eee0efddfe187e7cd05027086b7ce1c0e492e82a4eda58f5c5543a44a0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c", size = 505588, upload-time = "2026-06-01T19:38:38.282Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f8/0f28f04eef75d52fc9c715dde7ce9c0abb810fd20cfeb0fea7afd2ab1e98/aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8", size = 504492, upload-time = "2026-06-01T19:38:40.611Z" }, + { url = "https://files.pythonhosted.org/packages/ff/db/44c755232085545065c94378dfce38641b1aee647f4939fcd32f5b32e719/aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83", size = 1752111, upload-time = "2026-06-01T19:38:42.682Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a3/3800dbd095cb2bb165a7ea5d94d790914677e27f45638c7d80e3f34c8945/aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096", size = 1777241, upload-time = "2026-06-01T19:38:52.04Z" }, + { url = "https://files.pythonhosted.org/packages/b4/3d/dc94df99ed1511fdf28314f722643ed334112643cab00223577085e788c4/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c", size = 1714864, upload-time = "2026-06-01T19:38:56.788Z" }, + { url = "https://files.pythonhosted.org/packages/fa/10/ab28818262f4d26bdb47ed5f1fc7999b69e2fc6e0370b02d0f49011f45ea/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869", size = 1754516, upload-time = "2026-06-01T19:39:08.788Z" }, + { url = "https://files.pythonhosted.org/packages/1a/fe/6edbf5d39bf29322b6816365b17ed8ede4dace164a3aea1abcd30110eb78/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3", size = 483329, upload-time = "2026-06-01T19:39:22.607Z" }, + { url = "https://files.pythonhosted.org/packages/1b/5a/fae531bdbc6456fb6241f46b7b81e4d8a0dd3fc09118a0055dc7141ac1ec/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b", size = 489502, upload-time = "2026-06-01T19:39:24.881Z" }, + { url = "https://files.pythonhosted.org/packages/36/f4/48a7b0414db7fed77a03d5dde34508c026afd83510ab6bca08c313855776/aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8", size = 497357, upload-time = "2026-06-01T19:39:27.197Z" }, + { url = "https://files.pythonhosted.org/packages/75/75/e85a13a370acc007fca5feb1fd1b88ac2d8426e6dadd625479b7cadd55a3/aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76", size = 750898, upload-time = "2026-06-01T19:39:29.563Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e4/3d637f800c724eff0e2bed64df72557444482366fd0a35b0cec0e6968f6c/aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e", size = 506986, upload-time = "2026-06-01T19:39:31.872Z" }, + { url = "https://files.pythonhosted.org/packages/1d/df/35161f3598bf7501d2b2a805b41ab4f45a2e34150c421bcb4ef8c0d281a7/aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72", size = 508033, upload-time = "2026-06-01T19:39:34.137Z" }, + { url = "https://files.pythonhosted.org/packages/e5/39/b36e5d3d31e850fb4691dd3e941684ac490a2559249f6fa634b6b0fdf020/aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3", size = 1746213, upload-time = "2026-06-01T19:39:36.654Z" }, + { url = "https://files.pythonhosted.org/packages/3a/05/27df32c844b2156e1675a8d8ec22d963e3c8ba469ed7ceb1863320c7b521/aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a", size = 1751659, upload-time = "2026-06-01T19:39:46.398Z" }, + { url = "https://files.pythonhosted.org/packages/66/e3/53c67097e8a5ce98625e91e3fa7f43c9c6940de680345d03b3509a72a078/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b", size = 1710090, upload-time = "2026-06-01T19:39:51.392Z" }, + { url = "https://files.pythonhosted.org/packages/b8/69/155c4ef3aec96417d47024800472b33b16c5d8a665371dcd044c2afdf25d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52", size = 1733716, upload-time = "2026-06-01T19:40:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/12/34/6180103ce9aabc8ebff3f7bb55a1228ffe60f61042823031d9692cb7b101/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733", size = 787878, upload-time = "2026-06-01T19:40:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/92/e9/08954a40e8b7baa3d8beadd2b074b186e9b1e9c8ddabc288678a6265de50/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228", size = 524400, upload-time = "2026-06-01T19:40:15.972Z" }, + { url = "https://files.pythonhosted.org/packages/08/6a/b5965a634ac4d5ba99a463314cf4ab214ca073fcdc38a15e0294273701fc/aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095", size = 527904, upload-time = "2026-06-01T19:40:18.28Z" }, + { url = "https://files.pythonhosted.org/packages/06/b4/932bcdd850c354d9bcca30f360e475d7852e30413fbbd44b182782ed5432/aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde", size = 1912162, upload-time = "2026-06-01T19:40:20.825Z" }, + { url = "https://files.pythonhosted.org/packages/d0/1c/a57de71a4508c93a830b77c28af3d08cd97f606dedfc6b94275347744508/aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b", size = 1868606, upload-time = "2026-06-01T19:40:31.843Z" }, + { url = "https://files.pythonhosted.org/packages/35/1e/c237923232c7da7f0392ea25d89fc5e60c0e93f685f4ebca8e7bcdd5271c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de", size = 1834090, upload-time = "2026-06-01T19:40:37.733Z" }, + { url = "https://files.pythonhosted.org/packages/cc/bc/2aaab2f85cadb26ea59c091fa2b8e370d625154b5c14b478f1b489d07551/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0", size = 1832281, upload-time = "2026-06-01T19:40:52.303Z" }, ] [[package]] @@ -851,7 +858,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "aiohttp", marker = "extra == 'test'", specifier = "==3.13.5" }, + { name = "aiohttp", marker = "extra == 'test'", specifier = "==3.14.0" }, { name = "colorama", specifier = "==0.4.6" }, { name = "coverage", marker = "extra == 'test'", specifier = "==7.13.4" }, { name = "cyclopts", specifier = "==4.10.0" }, From 99b548bf6a6e9b52d749e0f212698bb394fcfff6 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 5 Jun 2026 03:32:47 +0530 Subject: [PATCH 15/21] fix: address PR review comments for run_metadata and sysinfo - Fix percentile key mismatch in _build_run_metadata: registry stores str(float) keys ("99.0", "50.0") but lookups used integer strings ("99", "50"), causing all p50/p90/p95/p99 fields to be null - Guard run_metadata.json write in try/except and move _build_run_metadata inside the guard so results.json is always written first - Use open() instead of Path.open() for run_metadata.json write to make builtins.open patching work correctly in tests - Move _build_run_metadata call to just before run_metadata.json write so any future raise does not abort finalization before results.json - Add zero-guard to tps_per_user division (concurrency > 0) - Use datetime.now(UTC).isoformat() instead of datetime.now().isoformat() for unambiguous UTC timestamps in run_metadata.json - Remove _propagate_endpoint_url_to_sysinfo from BenchmarkConfig: endpoints[0] is the load target not necessarily the serving node - Add port range validation (1-65535) to serving_node field validator, matching the check already present in _validate_ssh_ids - Pin mlc-scripts==1.1.0 in pyproject.toml - Add tests covering all the above fixes Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 2 +- .../commands/benchmark/execute.py | 46 ++--- src/inference_endpoint/config/schema.py | 30 ++-- .../commands/test_benchmark_finalization.py | 165 +++++++++++++++++- tests/unit/config/test_schema.py | 42 +++++ tests/unit/sys_info/test_capture.py | 27 +++ tests/unit/sys_info/test_sysinfo_command.py | 49 ++++++ 7 files changed, 320 insertions(+), 41 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3c15c2234..8afebd335 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ dependencies = [ "pytz==2026.1.post1", "urllib3==2.7.0", # mlc-scripts for system info - "mlc-scripts", + "mlc-scripts==1.1.0", ] [project.optional-dependencies] diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 7f8dbfd16..43f9f4745 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -34,7 +34,7 @@ from collections.abc import Callable from dataclasses import dataclass, field from dataclasses import replace as dataclass_replace -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Any from urllib.parse import urljoin @@ -900,8 +900,6 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: report.display(fn=lambda s: print(s, file=f)) logger.info(f"Report written to {report_txt}") - run_metadata = _build_run_metadata(ctx, report) - # Write scoring artifacts + copy event log from tmpfs to disk _write_scoring_artifacts(ctx, result, bench.tmpfs_dir) @@ -991,10 +989,14 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: # Write run_metadata.json before sys_info capture so mlcflow's postprocess # can read and patch it in-place with serving config values. - metadata_path = ctx.report_dir / "run_metadata.json" - with metadata_path.open("w") as f: - json.dump(run_metadata, f, indent=2) - logger.info("Run metadata written to %s", metadata_path) + try: + metadata_path = ctx.report_dir / "run_metadata.json" + run_metadata = _build_run_metadata(ctx, report) + with open(metadata_path, "w") as f: + json.dump(run_metadata, f, indent=2) + logger.info("Run metadata written to %s", metadata_path) + except Exception as e: + logger.error("Failed to write run_metadata.json: %s", e) if ctx.config.system_info is not None: try: @@ -1076,14 +1078,14 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: total_tokens = osl.get("total") if total_tokens is not None: measured_total_output_tokens = int(total_tokens) - if concurrency is not None and system_tps is not None: + if concurrency is not None and concurrency > 0 and system_tps is not None: tps_per_user = system_tps / concurrency ttft = report.ttft or {} tpot = report.tpot or {} latency = report.latency or {} metadata: dict[str, Any] = { - "run_date": datetime.now().isoformat(), + "run_date": datetime.now(UTC).isoformat(), "node_config": node_config, "config_summary": { "disaggregated": disaggregated, @@ -1097,7 +1099,7 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: "concurrency": concurrency, "system_tps": system_tps, "tps_per_user": tps_per_user, - "ttft": _pct(ttft, "99"), + "ttft": _pct(ttft, "99.0"), "qps": qps, "tps_utilization": None, "measured_total_output_tokens": measured_total_output_tokens, @@ -1107,26 +1109,26 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: "link_logs": None, "measured_latency_ttft_min": _stat(ttft, "min"), "measured_latency_ttft_average": _stat(ttft, "avg"), - "measured_latency_ttft_p50": _pct(ttft, "50"), - "measured_latency_ttft_p90": _pct(ttft, "90"), - "measured_latency_ttft_p95": _pct(ttft, "95"), - "measured_latency_ttft_p99": _pct(ttft, "99"), + "measured_latency_ttft_p50": _pct(ttft, "50.0"), + "measured_latency_ttft_p90": _pct(ttft, "90.0"), + "measured_latency_ttft_p95": _pct(ttft, "95.0"), + "measured_latency_ttft_p99": _pct(ttft, "99.0"), "measured_latency_ttft_p999": _pct(ttft, "99.9"), "measured_latency_ttft_max": _stat(ttft, "max"), "measured_latency_tpot_min": _stat(tpot, "min"), "measured_latency_tpot_average": _stat(tpot, "avg"), - "measured_latency_tpot_p50": _pct(tpot, "50"), - "measured_latency_tpot_p90": _pct(tpot, "90"), - "measured_latency_tpot_p95": _pct(tpot, "95"), - "measured_latency_tpot_p99": _pct(tpot, "99"), + "measured_latency_tpot_p50": _pct(tpot, "50.0"), + "measured_latency_tpot_p90": _pct(tpot, "90.0"), + "measured_latency_tpot_p95": _pct(tpot, "95.0"), + "measured_latency_tpot_p99": _pct(tpot, "99.0"), "measured_latency_tpot_p999": _pct(tpot, "99.9"), "measured_latency_tpot_max": _stat(tpot, "max"), "measured_latency_request_min": _stat(latency, "min"), "measured_latency_request_average": _stat(latency, "avg"), - "measured_latency_request_p50": _pct(latency, "50"), - "measured_latency_request_p90": _pct(latency, "90"), - "measured_latency_request_p95": _pct(latency, "95"), - "measured_latency_request_p99": _pct(latency, "99"), + "measured_latency_request_p50": _pct(latency, "50.0"), + "measured_latency_request_p90": _pct(latency, "90.0"), + "measured_latency_request_p95": _pct(latency, "95.0"), + "measured_latency_request_p99": _pct(latency, "99.0"), "measured_latency_request_p999": _pct(latency, "99.9"), "measured_latency_request_max": _stat(latency, "max"), } diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index cea19e336..e2ef0bdd3 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -667,8 +667,9 @@ class SysInfoCaptureConfig(BaseModel): default=None, description=( "Endpoint URL to probe for serving framework detection " - "(e.g. 'http://host:8000'). Auto-populated from endpoint_config when " - "used inside BenchmarkConfig." + "(e.g. 'http://host:8000'). Must be set explicitly." + "Omitting it skips the HTTP " + "serving-framework probe and reduces collected metadata." ), ) serving_node: str | None = Field( @@ -713,6 +714,13 @@ def _validate_serving_node(cls, v: str | None) -> str | None: raise ValueError( f"Invalid serving_node {v!r}: expected 'username@host' or 'username@host:port'" ) + port_str = m.group("port") + if port_str is not None: + port = int(port_str) + if not (1 <= port <= 65535): + raise ValueError( + f"Invalid port in serving_node {v!r}: {port} is not in range 1-65535" + ) return v @field_validator("output_path", mode="after") @@ -767,6 +775,10 @@ class SysInfoFileConfig(BaseModel): ``report_dir`` mirrors the same field in ``BenchmarkConfig`` and takes priority over ``system_info.output_path`` when set. + + ``system_info.endpoint_url`` Set it explicitly in + ``system_info`` when you want the HTTP serving-framework probe to run; + omitting it means less metadata is collected. """ model_config = ConfigDict(extra="ignore") @@ -964,20 +976,6 @@ def _resolve_and_validate(self) -> Self: return self - @model_validator(mode="after") - def _propagate_endpoint_url_to_sysinfo(self) -> Self: - """Copy endpoint_config.endpoints[0] into system_info.endpoint_url if unset.""" - if ( - self.system_info is not None - and self.system_info.endpoint_url is None - and self.endpoint_config.endpoints - ): - new_sic = self.system_info.model_copy( - update={"endpoint_url": self.endpoint_config.endpoints[0]} - ) - object.__setattr__(self, "system_info", new_sic) - return self - @model_validator(mode="after") def _propagate_client_api_type(self) -> Self: """Sync client.api_type from endpoint_config.api_type at construction. diff --git a/tests/unit/commands/test_benchmark_finalization.py b/tests/unit/commands/test_benchmark_finalization.py index 6a951a9db..70875d0ed 100644 --- a/tests/unit/commands/test_benchmark_finalization.py +++ b/tests/unit/commands/test_benchmark_finalization.py @@ -26,11 +26,13 @@ BenchmarkContext, BenchmarkResult, ResponseCollector, + _build_run_metadata, finalize_benchmark, ) from inference_endpoint.config.schema import OfflineBenchmarkConfig, TestMode from inference_endpoint.exceptions import ExecutionError from inference_endpoint.load_generator.session import SessionResult +from inference_endpoint.metrics.report import Report _OFFLINE_KWARGS = { "endpoint_config": {"endpoints": ["http://localhost:8000"]}, @@ -57,7 +59,7 @@ def _make_ctx(tmp_path: Path) -> BenchmarkContext: ) -def _make_bench(tmp_path: Path) -> BenchmarkResult: +def _make_bench(tmp_path: Path, report: Report | None = None) -> BenchmarkResult: t = 1_000_000_000 session = SessionResult( session_id="test-session", @@ -68,11 +70,70 @@ def _make_bench(tmp_path: Path) -> BenchmarkResult: return BenchmarkResult( session=session, collector=ResponseCollector(), - report=None, + report=report, tmpfs_dir=tmp_path, ) +class TestRunMetadataWriteNonBlocking: + @pytest.mark.unit + def test_write_failure_does_not_abort_finalize(self, tmp_path: Path) -> None: + """A write error on run_metadata.json must not propagate out of finalize_benchmark.""" + ctx = _make_ctx(tmp_path) + bench = _make_bench(tmp_path) + + with ( + patch( + "inference_endpoint.commands.benchmark.execute._build_run_metadata", + return_value={}, + ), + patch("builtins.open", side_effect=OSError("disk full")), + patch( + "inference_endpoint.sys_info.capture.capture_system_info", + side_effect=ExecutionError("skipped"), + ), + ): + # Must not raise even though open() fails for every file write. + finalize_benchmark(ctx, bench) + + @pytest.mark.unit + def test_results_json_written_even_if_metadata_write_fails( + self, tmp_path: Path + ) -> None: + """results.json must be written even when run_metadata.json write fails. + + The real _build_run_metadata runs with a populated Report so the full + metadata-building path (percentile lookups, ms conversion) is exercised + before the write fails. + """ + ctx = _make_ctx(tmp_path) + bench = _make_bench(tmp_path, report=_make_populated_report()) + + results_path = tmp_path / "results.json" + metadata_path = tmp_path / "run_metadata.json" + + real_open = open + + def selective_open(path, *args, **kwargs): + if str(path) == str(metadata_path): + raise OSError("disk full") + return real_open(path, *args, **kwargs) + + with ( + patch("builtins.open", side_effect=selective_open), + patch( + "inference_endpoint.sys_info.capture.capture_system_info", + side_effect=ExecutionError("skipped"), + ), + ): + finalize_benchmark(ctx, bench) + + assert results_path.exists() + data = json.loads(results_path.read_text()) + assert "results" in data + assert not metadata_path.exists() + + class TestSysInfoFailureNonBlocking: @pytest.mark.unit def test_execution_error_does_not_prevent_results_write( @@ -123,3 +184,103 @@ def test_unexpected_exception_does_not_prevent_results_write( assert results_path.exists() data = json.loads(results_path.read_text()) assert "results" in data + + +# One nanosecond value per percentile bucket (in ns), chosen so the ms +# conversion produces a round number that is easy to assert on. +_PCT_NS = { + "50.0": 500_000_000, + "90.0": 900_000_000, + "95.0": 950_000_000, + "99.0": 990_000_000, + "99.9": 999_000_000, +} + +_SERIES_DICT = { + "percentiles": _PCT_NS, + "min": 100_000_000, + "max": 1_000_000_000, + "avg": 600_000_000, +} + + +def _make_populated_report() -> MagicMock: + """Return a Report mock with registry-format float-string percentile keys. + + Sets every attribute that finalize_benchmark and _build_run_metadata access + directly so no MagicMock auto-attribute leaks into arithmetic comparisons. + """ + report = MagicMock() + report.tps.return_value = 200.0 + report.qps.return_value = 20.0 + report.n_samples_completed = 100 + report.n_samples_issued = 100 + report.n_samples_failed = 0 + report.duration_ns = 10_000_000_000 + report.output_sequence_lengths = {"total": 2000} + report.ttft = dict(_SERIES_DICT) + report.tpot = dict(_SERIES_DICT) + report.latency = dict(_SERIES_DICT) + return report + + +@pytest.mark.unit +class TestBuildRunMetadata: + def _make_report(self) -> MagicMock: + return _make_populated_report() + + def test_percentile_keys_resolve_with_float_string_format(self) -> None: + """All percentile fields must be non-None when the report uses registry-format + float-string keys (e.g. "99.0", "50.0") rather than integer strings.""" + ctx = MagicMock() + ctx.config.settings.load_pattern.target_concurrency = None + ctx.config.system_info = None + + metadata = _build_run_metadata(ctx, self._make_report()) + + for field, expected_ms in [ + ("measured_latency_ttft_p50", 500.0), + ("measured_latency_ttft_p90", 900.0), + ("measured_latency_ttft_p95", 950.0), + ("measured_latency_ttft_p99", 990.0), + ("measured_latency_ttft_p999", 999.0), + ("measured_latency_tpot_p50", 500.0), + ("measured_latency_tpot_p90", 900.0), + ("measured_latency_tpot_p95", 950.0), + ("measured_latency_tpot_p99", 990.0), + ("measured_latency_tpot_p999", 999.0), + ("measured_latency_request_p50", 500.0), + ("measured_latency_request_p90", 900.0), + ("measured_latency_request_p95", 950.0), + ("measured_latency_request_p99", 990.0), + ("measured_latency_request_p999", 999.0), + ]: + assert metadata[field] == pytest.approx( + expected_ms + ), f"{field} was None — float-string percentile key lookup failed" + + # Top-level ttft field is the p99. + assert metadata["ttft"] == pytest.approx(990.0) + + def test_integer_string_keys_silently_return_none(self) -> None: + """Confirm that integer-string keys ("99", "50") do NOT resolve — + documenting the registry key format requirement.""" + ctx = MagicMock() + ctx.config.settings.load_pattern.target_concurrency = None + ctx.config.system_info = None + + report = self._make_report() + # Use integer-string keys (old broken format). + report.ttft = { + "percentiles": {"50": 500_000_000, "99": 990_000_000}, + "min": 100_000_000, + "max": 1_000_000_000, + "avg": 600_000_000, + } + report.tpot = {} + report.latency = {} + + metadata = _build_run_metadata(ctx, report) + + assert metadata["measured_latency_ttft_p50"] is None + assert metadata["measured_latency_ttft_p99"] is None diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index e7ea0e51c..7aed02057 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -33,6 +33,7 @@ OSLDistributionType, StreamingMode, SubmissionReference, + SysInfoCaptureConfig, TestType, ) from inference_endpoint.exceptions import CLIError @@ -471,6 +472,47 @@ def test_openai_completions_endpoint_resolves_adapter(self): assert config.settings.client.accumulator is OpenAISSEAccumulator +class TestEndpointUrlNotPropagated: + """endpoint_config.endpoints[0] must never be copied into system_info.endpoint_url. + + The load target is not necessarily the serving node; auto-propagation would + silently probe the wrong host. Users must set system_info.endpoint_url + explicitly when they want the HTTP serving-framework probe to run. + """ + + _BASE = { + "type": TestType.OFFLINE, + "model_params": {"name": "M"}, + "datasets": [{"path": "D"}], + "endpoint_config": {"endpoints": ["http://10.0.0.1:8000"]}, + } + + @pytest.mark.unit + def test_endpoint_url_stays_none_when_not_set(self) -> None: + config = BenchmarkConfig( + **self._BASE, + system_info=SysInfoCaptureConfig( + ssh_ids=["user@10.0.0.1"], + accelerator_backend="cuda", + ), + ) + assert config.system_info is not None + assert config.system_info.endpoint_url is None + + @pytest.mark.unit + def test_explicit_endpoint_url_preserved(self) -> None: + config = BenchmarkConfig( + **self._BASE, + system_info=SysInfoCaptureConfig( + ssh_ids=["user@10.0.0.1"], + accelerator_backend="cuda", + endpoint_url="http://10.0.0.2:8000", + ), + ) + assert config.system_info is not None + assert config.system_info.endpoint_url == "http://10.0.0.2:8000" + + class TestMultiTurnValidation: """Tests for multi-turn config validation and cross-validation.""" diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index 5a0d3902b..40a326736 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -94,6 +94,33 @@ def test_empty_ssh_ids_raises(self) -> None: with pytest.raises(ValidationError, match="non-empty"): _make_config(ssh_ids=[]) + # 4. serving_node validation + @pytest.mark.unit + def test_serving_node_valid_no_port(self) -> None: + cfg = _make_config(serving_node="alice@10.0.0.1") + assert cfg.serving_node == "alice@10.0.0.1" + + @pytest.mark.unit + def test_serving_node_valid_with_port(self) -> None: + cfg = _make_config(serving_node="alice@10.0.0.1:2222") + assert cfg.serving_node == "alice@10.0.0.1:2222" + + @pytest.mark.unit + def test_serving_node_port_out_of_range_raises(self) -> None: + with pytest.raises(ValidationError, match="1-65535"): + _make_config(serving_node="alice@10.0.0.1:99999") + + @pytest.mark.unit + @pytest.mark.parametrize("port", [0, 65536, 100000]) + def test_serving_node_invalid_ports_raise(self, port: int) -> None: + with pytest.raises(ValidationError, match="1-65535"): + _make_config(serving_node=f"alice@10.0.0.1:{port}") + + @pytest.mark.unit + def test_serving_node_invalid_format_raises(self) -> None: + with pytest.raises(ValidationError): + _make_config(serving_node="notvalid") + # --------------------------------------------------------------------------- # 4. Variation tags — cuda, include current diff --git a/tests/unit/sys_info/test_sysinfo_command.py b/tests/unit/sys_info/test_sysinfo_command.py index a8ec7bfb6..c6b6359fe 100644 --- a/tests/unit/sys_info/test_sysinfo_command.py +++ b/tests/unit/sys_info/test_sysinfo_command.py @@ -239,6 +239,55 @@ def test_extra_top_level_keys_ignored(self, tmp_path: Path) -> None: cfg = SysInfoFileConfig.from_yaml_file(config_path) assert cfg.system_info.accelerator_backend == "cuda" + @pytest.mark.unit + def test_endpoint_config_not_propagated_to_endpoint_url( + self, tmp_path: Path + ) -> None: + """endpoint_config in a shared YAML does not populate system_info.endpoint_url. + + The load target (endpoint_config.endpoints[0]) is not necessarily the + serving node, so endpoint_url is never inferred automatically. Users + must set system_info.endpoint_url explicitly when they want the HTTP + serving-framework probe to run. + """ + config_path = tmp_path / "shared.yaml" + config_path.write_text( + textwrap.dedent( + """\ + endpoint_config: + endpoints: + - http://10.0.0.1:8000 + system_info: + ssh_ids: + - user@10.0.0.1 + accelerator_backend: cuda + """ + ) + ) + cfg = SysInfoFileConfig.from_yaml_file(config_path) + # endpoint_config is ignored (extra="ignore"); endpoint_url stays None. + assert cfg.system_info.endpoint_url is None + + @pytest.mark.unit + def test_endpoint_url_explicit_in_system_info_is_preserved( + self, tmp_path: Path + ) -> None: + """An explicit system_info.endpoint_url is honoured by SysInfoFileConfig.""" + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + """\ + system_info: + ssh_ids: + - user@10.0.0.1 + accelerator_backend: cuda + endpoint_url: http://10.0.0.2:8000 + """ + ) + ) + cfg = SysInfoFileConfig.from_yaml_file(config_path) + assert cfg.system_info.endpoint_url == "http://10.0.0.2:8000" + @pytest.mark.unit def test_file_not_found_raises(self, tmp_path: Path) -> None: with pytest.raises(FileNotFoundError): From 979d984bb4f76ef6ae29c982e60f96ba5c7bc095 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 5 Jun 2026 19:15:28 +0530 Subject: [PATCH 16/21] improve readme + force use report dir for sysinfo output --- docs/commands/DESIGN.md | 147 +++++++++++------- .../commands/benchmark/execute.py | 6 +- .../commands/sysinfo/cli.py | 14 +- src/inference_endpoint/config/schema.py | 19 +-- src/inference_endpoint/sys_info/capture.py | 7 +- tests/unit/sys_info/test_capture.py | 29 ++-- tests/unit/sys_info/test_sysinfo_command.py | 125 +++++++-------- 7 files changed, 184 insertions(+), 163 deletions(-) diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index 795d817f5..2ad948bcf 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -108,59 +108,96 @@ commands/benchmark/execute.py::run_benchmark() System info capture collects hardware/software details from one or more nodes and writes a structured JSON file for MLPerf inference submissions. It runs in two contexts: -- **Standalone** (`sysinfo from-config`): triggered manually, independent of any benchmark run. -- **Integrated** (`benchmark` finalization): triggered automatically after a benchmark if `system_info` is present in the config. - -Both paths call `sys_info/capture.py::capture_system_info()` and produce the same output JSON. The integrated path additionally patches `run_metadata.json` with serving configuration values extracted from the inference server's startup log. +- **Standalone** (`sysinfo from-config`): triggered manually, independent of any benchmark run. See [Standalone Command (`sysinfo from-config`)](#standalone-command-sysinfo-from-config) below for a full example. +- **Integrated** (`benchmark from-config`): triggered automatically after a benchmark run completes if `system_info` is present in the config. For example: + + ```yaml + name: "llama3.1-8b-vllm-perf-c1000" + version: "1.0" + type: "online" + + model_params: + name: "meta-llama/Llama-3.1-8B-Instruct" + temperature: 0.0 + top_p: 1.0 + max_new_tokens: 128 + + datasets: + - name: cnn_dailymail::llama3_8b + type: performance + samples: 13368 + parser: + input: prompt + + settings: + runtime: + min_duration_ms: 600000 + max_duration_ms: 3600000 + scheduler_random_seed: 137 + dataloader_random_seed: 111 + n_samples_to_issue: 13368 + + load_pattern: + type: "concurrency" + target_concurrency: 1000 + + client: + num_workers: 4 + + endpoint_config: + endpoints: + - "http://localhost:11001" + api_key: null + + report_dir: sglang_perf_c1000 + + system_info: + ssh_ids: + - anandhusooraj@mlc2 + accelerator_backend: cuda + exclude_current_system: true + skip_ssh_key_file: false + serving_node: anandhusooraj@mlc2 + log_path: /home/anandhusooraj/sglang_logs.log + endpoint_url: http://localhost:11001 + serving_framework: sglang + ``` + + then running: + + ```bash + inference-endpoint benchmark from-config --config config.yaml + ``` + + System info capture runs automatically at the end of the benchmark. + +Both paths call `sys_info/capture.py::capture_system_info()` and produce the same output JSON (as per endpoints spec). Both also patch `run_metadata.json` with serving configuration values extracted from the inference server's startup log, if the file is present in `report_dir`. In integrated execution mode it is always present (written by the benchmark before capture runs); in standalone execution mode it is patched only if it already exists there. ### Config Reference (`SysInfoCaptureConfig`) -| Field | Type | Default | Description | -| ------------------------ | -------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ssh_ids` | `list[str]` | — | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | -| `accelerator_backend` | `"cuda"` \| `"rocm"` \| `"none"` | — | **Required.** GPU backend on the target nodes. | -| `exclude_current_system` | bool | `false` | Skip the machine running this command; collect from `ssh_ids` only. | -| `skip_ssh_key_file` | bool | `false` | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). | -| `output_path` | str | `"."` | Output directory for the JSON file. Overridden by `report_dir` when set at the top level. | -| `node_config` | object | `null` | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | -| `serving_node` | str | `null` | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | -| `log_path` | str | `null` | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | -| `endpoint_url` | str | `null` | Base URL of the running inference server. Passed to the mlcflow script, which probes it via HTTP to detect the serving framework (e.g. `"vLLM 0.9.0"`). | +| Field | Type | Description | +| ------------------------ | ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ssh_ids` | `list[str]` | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | +| `accelerator_backend` | `"cuda"` \| `"rocm"` \| `"xpu"` \| `"none"` | GPU backend on the target nodes. Default: `"none"`. | +| `exclude_current_system` | bool | Skip the machine running this command; collect from `ssh_ids` only. Default: `false`. | +| `skip_ssh_key_file` | bool | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). Default: `false`. | +| `node_config` | object | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | +| `serving_node` | str | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | +| `log_path` | str | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | +| `endpoint_url` | str | Base URL of the running inference server. Probed via HTTP to detect the serving framework name and version (e.g. `"vLLM 0.9.0"`). | +| `serving_framework` | `"auto"` \| `"vllm"` \| `"sglang"` \| `"trtllm"` | Serving engine type used for startup log parsing. Default: `"auto"` (detected from the endpoint). | ### Capture Flow -``` -capture_system_info(config, run_metadata_path=...) - │ - └─ mlc.access("get-mlperf-multi-node-system-info", ...) - │ - ├─ prehook: get,mlperf,single-node,system-info on local machine (node 0) - │ skipped if exclude_current_system=true - │ - ├─ preprocess(): for each ssh_id - │ remote_run get,mlperf,single-node,system-info on remote node - │ copy back: mlperf-system-info-single-node-{id}.json - │ - ├─ preprocess(): if serving_node set - │ remote_run get,mlperf,serving-config on serving node - │ parse.py reads vLLM startup log from the top (local on serving node): - │ tensor_parallel_size, pipeline_parallel_size, - │ expert_parallel_size, max_num_seqs - │ + framework name and version ("vLLM 0.9.0") - │ copy back: serving_config.json - │ - ├─ preprocess(): if endpoint_url set and framework not yet detected - │ GET /version or /get_server_info → "vLLM 0.9.0" / "SGLang 0.4.2" - │ sets MLC_MLPERF_SERVING_FRAMEWORK (HTTP probe takes priority over log) - │ - └─ postprocess(): - merge per-node JSONs → system_desc.json - if serving_config.json present: - patch run_metadata.json config_summary with extracted values - if serving framework not detected via HTTP: use serving_config.json framework field -``` +System info capture is powered by the [mlperf-automations](https://github.com/mlcommons/mlperf-automations) project and orchestrates the following steps: + +1. **Hardware collection** — SSHes into each node listed in `ssh_ids` and collects CPU, memory, GPU, networking, and OS information. If `exclude_current_system` is false, the local machine is also probed. Results are merged into a single `system_desc.json`. + +2. **Serving config extraction** _(if `serving_node` is set)_ — SSHes into the inference server node and reads its startup log to extract parallelism settings (`tensor_parallel`, `pipeline_parallel`, `expert_parallel`) and batch size. -**`run_metadata.json` patching** only happens in the benchmark context. `capture_system_info` accepts an optional `run_metadata_path` argument; `finalize_benchmark` passes `ctx.report_dir / "run_metadata.json"`, which has already been written before the capture call. The `config_summary` block fields (`tensor_parallel`, `pipeline_parallel`, `expert_parallel`, `batch`) are updated in-place; fields that could not be parsed remain `null`. +3. **Framework detection** _(if `endpoint_url` is set)_ — probes the live inference server via HTTP to detect the framework name and version (e.g. `"vLLM 0.9.0"`, `"SGLang 0.4.2"`). HTTP detection takes priority over the log-based result. + +4. **Output** — always writes `system_desc.json` to the configured report directory. If a `run_metadata.json` is present in `report_dir`, it is also patched with the extracted serving config values (fields that could not be parsed remain `null`). ### Standalone Command (`sysinfo from-config`) @@ -200,8 +237,6 @@ system_info: no_of_nodes: 5 ``` -`report_dir` takes priority over `system_info.output_path` when both are set. - Output is written to `report_dir/system_desc.json`. ### `node_config` Validation @@ -213,16 +248,14 @@ When `node_config` is provided, the automations script enforces: ### Error Handling -| Situation | Standalone (`sysinfo from-config`) | Integrated (benchmark) | -| --------------------------------------------- | --------------------------------------------------- | -------------------------------------------------------- | -| mlcflow script returns non-zero | `ExecutionError` propagates to CLI handler | Logged as `error` with retry hint; benchmark exits 0 | -| Unexpected exception | Propagates to CLI handler | Logged as `error` with exception type; benchmark exits 0 | -| SSH failure on a node | Logged as error inside script; other nodes continue | Same | -| Per-node JSON missing after SSH run | Logged as warning; node skipped | Same | -| `node_name` unmatched or count exceeds probed | `ExecutionError` | `ExecutionError` → logged as `error` | -| `serving_config.json` absent or unreadable | Logged as error; `run_metadata.json` left unchanged | Same | +In standalone execution mode (`sysinfo from-config`), any capture failure propagates as an error and exits non-zero. + +In integrated execution mode (triggered at the end of a benchmark run), `system_info` failures never abort the benchmark — `results.json` and `run_metadata.json` are written before capture runs, so benchmark output is always complete. Capture errors are logged but the process exits 0. + +**Two warnings not to ignore:** -In the integrated path, `system_info` failures never abort the benchmark. `results.json` and `run_metadata.json` are written before the capture call, so the benchmark output is complete regardless of capture outcome. The error log includes `report_dir` and a command to re-run capture manually. +- **SSH failure on a node** — the error is logged in the MLC output but capture continues with the remaining nodes. The resulting `system_desc.json` will be missing that node's hardware info. Always verify `system_desc.json` looks complete before submission. +- **Serving config unavailable** — if the serving node is unreachable, `run_metadata.json` will have empty serving config fields (`tensor_parallel`, `pipeline_parallel`, `batch`, etc.). Check the MLC log and re-run `sysinfo from-config` manually if needed. --- diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 43f9f4745..da951fca3 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -1005,11 +1005,9 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: capture_system_info, ) - sic = ctx.config.system_info.model_copy( - update={"output_path": str(ctx.report_dir)} - ) output_path = capture_system_info( - sic, + ctx.config.system_info, + output_dir=ctx.report_dir, run_metadata_path=ctx.report_dir / "run_metadata.json", ) logger.info("System info captured at: %s", output_path) diff --git a/src/inference_endpoint/commands/sysinfo/cli.py b/src/inference_endpoint/commands/sysinfo/cli.py index 2f5c966af..bc7a564ad 100644 --- a/src/inference_endpoint/commands/sysinfo/cli.py +++ b/src/inference_endpoint/commands/sysinfo/cli.py @@ -63,13 +63,11 @@ def from_config( capture_cfg = resolved.system_info run_metadata_path = None - if resolved.report_dir is not None: - capture_cfg = capture_cfg.model_copy( - update={"output_path": str(resolved.report_dir)} - ) - candidate = resolved.report_dir / "run_metadata.json" - if candidate.exists(): - run_metadata_path = candidate + candidate = resolved.report_dir / "run_metadata.json" + if candidate.exists(): + run_metadata_path = candidate - output_path = capture_system_info(capture_cfg, run_metadata_path=run_metadata_path) + output_path = capture_system_info( + capture_cfg, output_dir=resolved.report_dir, run_metadata_path=run_metadata_path + ) print(f"System info written to: {output_path}") diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index e2ef0bdd3..5fd95a210 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -652,8 +652,7 @@ class SysInfoCaptureConfig(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) exclude_current_system: bool = False - accelerator_backend: Literal["cuda", "rocm", "none"] = "none" - output_path: str = "." + accelerator_backend: Literal["cuda", "rocm", "xpu", "none"] = "none" skip_ssh_key_file: bool = False ssh_ids: list[str] node_config: dict[str, list[NodeEntry]] | None = Field( @@ -688,10 +687,10 @@ class SysInfoCaptureConfig(BaseModel): "serving framework detection." ), ) - serving_framework: Literal["auto", "vllm", "sglang"] = Field( + serving_framework: Literal["auto", "vllm", "sglang", "trtllm"] = Field( default="auto", description=( - "Serving engine type for log parsing: 'vllm', 'sglang', or 'auto' " + "Serving engine type for log parsing: 'vllm', 'sglang', 'trtllm', or 'auto' " "(auto-detects from log keywords)." ), ) @@ -723,13 +722,6 @@ def _validate_serving_node(cls, v: str | None) -> str | None: ) return v - @field_validator("output_path", mode="after") - @classmethod - def _validate_output_path(cls, v: str) -> str: - if not v.strip(): - raise ValueError("output_path must be a non-empty string") - return v - @field_validator("ssh_ids", mode="after") @classmethod def _validate_ssh_ids(cls, v: list[str]) -> list[str]: @@ -773,8 +765,7 @@ class SysInfoFileConfig(BaseModel): The file must have a ``system_info`` key. Extra top-level keys are allowed so the same YAML can be shared with benchmark configs. - ``report_dir`` mirrors the same field in ``BenchmarkConfig`` and takes - priority over ``system_info.output_path`` when set. + ``report_dir`` is required and controls where all outputs are written. ``system_info.endpoint_url`` Set it explicitly in ``system_info`` when you want the HTTP serving-framework probe to run; @@ -784,7 +775,7 @@ class SysInfoFileConfig(BaseModel): model_config = ConfigDict(extra="ignore") system_info: SysInfoCaptureConfig - report_dir: Path | None = None + report_dir: Path @classmethod def from_yaml_file(cls, path: Path) -> SysInfoFileConfig: diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index e2852b545..6c40b70fe 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -61,6 +61,7 @@ def _write_node_config_tmp(config: SysInfoCaptureConfig) -> str: def capture_system_info( config: SysInfoCaptureConfig, + output_dir: Path, run_metadata_path: Path | None = None, ) -> Path: """Invoke the get-mlperf-multi-node-system-info mlcflow script. @@ -88,10 +89,10 @@ def capture_system_info( ssh_ids_str = ",".join(t.to_mlcflow_str() for t in config.parsed_ssh_ids) - # CM/mlcflow scripts use "yes"/"" string convention for boolean env vars. + # mlcflow scripts use "yes"/"" string convention for boolean env vars. skip_ssh_key_file_value = "yes" if config.skip_ssh_key_file else "" - Path(config.output_path).mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) logger.info("Capturing system info from %d node(s)...", len(config.parsed_ssh_ids)) @@ -100,7 +101,7 @@ def capture_system_info( "automation": "script", "tags": tags_str, "ssh_ids": ssh_ids_str, - "out_dir_path": config.output_path, + "out_dir_path": str(output_dir), "out_file_name": _OUT_FILE_NAME, "skip_ssh_key_file": skip_ssh_key_file_value, "serving_framework_type": config.serving_framework, diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index 40a326736..f612ec8c9 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -22,13 +22,14 @@ from unittest.mock import MagicMock, patch import pytest +from pydantic import ValidationError + from inference_endpoint.config.schema import ( BenchmarkConfig, SshTarget, SysInfoCaptureConfig, ) from inference_endpoint.exceptions import ExecutionError, SetupError -from pydantic import ValidationError # --------------------------------------------------------------------------- # Helpers @@ -36,7 +37,6 @@ _MINIMAL_SYS_INFO = { "accelerator_backend": "cuda", - "output_path": "/tmp/sys_info", "ssh_ids": ["alice@192.168.1.1"], } @@ -185,7 +185,7 @@ def _encode_skip_ssh(cfg: SysInfoCaptureConfig) -> str: class TestCaptureSystemInfo: @pytest.mark.unit def test_mlcflow_not_installed_raises_setup_error(self, tmp_path: Path) -> None: - cfg = _make_config(output_path=str(tmp_path)) + cfg = _make_config() with patch.dict("sys.modules", {"mlc": None}): import importlib @@ -193,13 +193,13 @@ def test_mlcflow_not_installed_raises_setup_error(self, tmp_path: Path) -> None: importlib.reload(capture_mod) with pytest.raises(SetupError, match="pip install mlc-scripts"): - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) @pytest.mark.unit def test_mlcflow_nonzero_return_raises_execution_error( self, tmp_path: Path ) -> None: - cfg = _make_config(output_path=str(tmp_path)) + cfg = _make_config() mock_mlcflow = MagicMock() mock_mlcflow.access.return_value = { "return": 1, @@ -212,11 +212,11 @@ def test_mlcflow_nonzero_return_raises_execution_error( importlib.reload(capture_mod) with pytest.raises(ExecutionError, match="ssh connection refused"): - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) @pytest.mark.unit def test_happy_path_output_path_from_new_env(self, tmp_path: Path) -> None: - cfg = _make_config(output_path=str(tmp_path)) + cfg = _make_config() mock_mlcflow = MagicMock() mock_mlcflow.access.return_value = { "return": 0, @@ -230,13 +230,13 @@ def test_happy_path_output_path_from_new_env(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - result = capture_mod.capture_system_info(cfg) + result = capture_mod.capture_system_info(cfg, output_dir=tmp_path) assert result == Path("/tmp/out.json") @pytest.mark.unit def test_missing_env_path_raises_execution_error(self, tmp_path: Path) -> None: """Return code 0 without MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH raises ExecutionError.""" - cfg = _make_config(output_path=str(tmp_path)) + cfg = _make_config() mock_mlcflow = MagicMock() mock_mlcflow.access.return_value = { "return": 0, @@ -251,12 +251,12 @@ def test_missing_env_path_raises_execution_error(self, tmp_path: Path) -> None: with pytest.raises( ExecutionError, match="MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH" ): - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) @pytest.mark.unit def test_unreachable_node_no_node_config_succeeds(self, tmp_path: Path) -> None: """mlcflow returning 0 (one node internally unreachable, no node_config) is non-fatal.""" - cfg = _make_config(output_path=str(tmp_path)) + cfg = _make_config() mock_mlcflow = MagicMock() mock_mlcflow.access.return_value = { "return": 0, @@ -272,7 +272,7 @@ def test_unreachable_node_no_node_config_succeeds(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - result = capture_mod.capture_system_info(cfg) + result = capture_mod.capture_system_info(cfg, output_dir=tmp_path) assert result == tmp_path / "system_desc.json" @pytest.mark.unit @@ -280,7 +280,6 @@ def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: cfg = SysInfoCaptureConfig( accelerator_backend="cuda", exclude_current_system=True, - output_path=str(tmp_path), skip_ssh_key_file=True, ssh_ids=["alice@10.0.0.1:2222", "bob@10.0.0.2"], ) @@ -295,7 +294,7 @@ def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) call_args = mock_mlcflow.access.call_args[0][0] assert ( @@ -330,7 +329,6 @@ def test_system_info_from_yaml(self, tmp_path: Path) -> None: - path: dummy.jsonl system_info: accelerator_backend: cuda - output_path: /tmp/sys_info ssh_ids: - alice@192.168.1.1 - bob@192.168.1.2:2222 @@ -346,7 +344,6 @@ def test_system_info_from_yaml(self, tmp_path: Path) -> None: assert config.system_info is not None sic = config.system_info assert sic.accelerator_backend == "cuda" - assert sic.output_path == "/tmp/sys_info" assert sic.exclude_current_system is True assert sic.skip_ssh_key_file is False assert len(sic.ssh_ids) == 2 diff --git a/tests/unit/sys_info/test_sysinfo_command.py b/tests/unit/sys_info/test_sysinfo_command.py index c6b6359fe..670e350f9 100644 --- a/tests/unit/sys_info/test_sysinfo_command.py +++ b/tests/unit/sys_info/test_sysinfo_command.py @@ -23,13 +23,14 @@ import pytest import yaml +from pydantic import ValidationError + from inference_endpoint.config.schema import ( NodeEntry, SysInfoCaptureConfig, SysInfoFileConfig, ) from inference_endpoint.exceptions import InputValidationError -from pydantic import ValidationError # --------------------------------------------------------------------------- # Helpers @@ -83,18 +84,6 @@ def test_extra_fields_forbidden(self) -> None: # --------------------------------------------------------------------------- -class TestSysInfoCaptureConfigOutputPath: - @pytest.mark.unit - def test_default_output_path_is_cwd(self) -> None: - cfg = _make_capture_config() - assert cfg.output_path == "." - - @pytest.mark.unit - def test_explicit_output_path_overrides_default(self) -> None: - cfg = _make_capture_config(output_path="/custom/out") - assert cfg.output_path == "/custom/out" - - class TestSysInfoCaptureConfigNodeConfig: @pytest.mark.unit def test_node_config_none_by_default(self) -> None: @@ -159,7 +148,8 @@ def test_minimal_config_no_node_config(self, tmp_path: Path) -> None: config_path = tmp_path / "sysinfo.yaml" config_path.write_text( textwrap.dedent( - """\ + f"""\ + report_dir: {tmp_path}/results/ system_info: ssh_ids: - user@10.0.0.1 @@ -169,9 +159,25 @@ def test_minimal_config_no_node_config(self, tmp_path: Path) -> None: ) cfg = SysInfoFileConfig.from_yaml_file(config_path) assert cfg.system_info.accelerator_backend == "cuda" - assert cfg.report_dir is None + assert cfg.report_dir == tmp_path / "results" assert cfg.system_info.node_config is None + @pytest.mark.unit + def test_missing_report_dir_raises(self, tmp_path: Path) -> None: + config_path = tmp_path / "sysinfo.yaml" + config_path.write_text( + textwrap.dedent( + """\ + system_info: + ssh_ids: + - user@10.0.0.1 + accelerator_backend: cuda + """ + ) + ) + with pytest.raises(ValidationError): + SysInfoFileConfig.from_yaml_file(config_path) + @pytest.mark.unit def test_report_dir_parsed(self, tmp_path: Path) -> None: config_path = tmp_path / "sysinfo.yaml" @@ -196,12 +202,12 @@ def test_with_node_config(self, tmp_path: Path) -> None: config_path = tmp_path / "sysinfo.yaml" config_path.write_text( textwrap.dedent( - """\ + f"""\ + report_dir: {tmp_path}/results/ system_info: ssh_ids: - user@10.0.0.1 accelerator_backend: cuda - output_path: /tmp/sys_info node_config: Prefill: - node_name: H100 @@ -225,12 +231,12 @@ def test_extra_top_level_keys_ignored(self, tmp_path: Path) -> None: config_path = tmp_path / "sysinfo.yaml" config_path.write_text( textwrap.dedent( - """\ + f"""\ + report_dir: {tmp_path}/results/ system_info: ssh_ids: - user@10.0.0.1 accelerator_backend: cuda - output_path: /tmp/sys_info some_other_section: foo: bar """ @@ -253,7 +259,8 @@ def test_endpoint_config_not_propagated_to_endpoint_url( config_path = tmp_path / "shared.yaml" config_path.write_text( textwrap.dedent( - """\ + f"""\ + report_dir: {tmp_path}/results/ endpoint_config: endpoints: - http://10.0.0.1:8000 @@ -276,7 +283,8 @@ def test_endpoint_url_explicit_in_system_info_is_preserved( config_path = tmp_path / "sysinfo.yaml" config_path.write_text( textwrap.dedent( - """\ + f"""\ + report_dir: {tmp_path}/results/ system_info: ssh_ids: - user@10.0.0.1 @@ -318,7 +326,6 @@ class TestCaptureWithNodeConfig: def test_node_config_file_passed_to_mlcflow(self, tmp_path: Path) -> None: """When node_config is set, mlcflow.access must receive node_config_file.""" cfg = _make_capture_config( - output_path=str(tmp_path), node_config={ "Prefill": [{"node_name": "H100", "no_of_nodes": 1}], }, @@ -336,7 +343,7 @@ def test_node_config_file_passed_to_mlcflow(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) call_args = mock_mlcflow.access.call_args[0][0] assert "node_config_file" in call_args @@ -345,7 +352,6 @@ def test_node_config_file_passed_to_mlcflow(self, tmp_path: Path) -> None: def test_node_config_temp_file_content(self, tmp_path: Path) -> None: """The temp file must contain system_info.node_config in the expected structure.""" cfg = _make_capture_config( - output_path=str(tmp_path), node_config={ "Decode": [ {"node_name": "GB300", "no_of_nodes": 12}, @@ -377,7 +383,7 @@ def fake_access(kwargs: dict) -> dict: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) assert len(captured_content) == 1 data = captured_content[0] @@ -391,7 +397,6 @@ def fake_access(kwargs: dict) -> dict: def test_temp_file_deleted_after_success(self, tmp_path: Path) -> None: """The temp node_config file must be cleaned up after mlcflow returns.""" cfg = _make_capture_config( - output_path=str(tmp_path), node_config={"Prefill": [{"node_name": "H100", "no_of_nodes": 1}]}, ) recorded_tmp: list[str] = [] @@ -413,7 +418,7 @@ def fake_access(kwargs: dict) -> dict: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) assert recorded_tmp[0], "temp file path was recorded" assert not Path(recorded_tmp[0]).exists(), "temp file was deleted after use" @@ -424,7 +429,6 @@ def test_temp_file_deleted_on_mlcflow_failure(self, tmp_path: Path) -> None: from inference_endpoint.exceptions import ExecutionError cfg = _make_capture_config( - output_path=str(tmp_path), node_config={"Prefill": [{"node_name": "H100", "no_of_nodes": 1}]}, ) recorded_tmp: list[str] = [] @@ -442,7 +446,7 @@ def fake_access(kwargs: dict) -> dict: importlib.reload(capture_mod) with pytest.raises(ExecutionError): - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) assert not Path(recorded_tmp[0]).exists(), "temp file cleaned up on failure" @@ -451,7 +455,7 @@ def test_no_node_config_file_arg_when_node_config_is_none( self, tmp_path: Path ) -> None: """Without node_config, node_config_file must NOT be in the mlcflow call.""" - cfg = _make_capture_config(output_path=str(tmp_path)) + cfg = _make_capture_config() mock_mlcflow = MagicMock() mock_mlcflow.access.return_value = { "return": 0, @@ -465,7 +469,7 @@ def test_no_node_config_file_arg_when_node_config_is_none( from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - capture_mod.capture_system_info(cfg) + capture_mod.capture_system_info(cfg, output_dir=tmp_path) call_args = mock_mlcflow.access.call_args[0][0] assert "node_config_file" not in call_args @@ -478,8 +482,8 @@ def test_no_node_config_file_arg_when_node_config_is_none( class TestReportDirResolution: @pytest.mark.unit - def test_report_dir_overrides_output_path(self, tmp_path: Path) -> None: - """When report_dir is set, capture_system_info must receive it as output_path.""" + def test_report_dir_passed_as_output_dir(self, tmp_path: Path) -> None: + """When report_dir is set, capture_system_info must receive it as output_dir.""" config_path = tmp_path / "sysinfo.yaml" config_path.write_text( textwrap.dedent( @@ -489,14 +493,17 @@ def test_report_dir_overrides_output_path(self, tmp_path: Path) -> None: ssh_ids: - user@10.0.0.1 accelerator_backend: cuda - output_path: /should/be/ignored """ ) ) - captured_configs: list[SysInfoCaptureConfig] = [] - - def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: - captured_configs.append(cfg) + captured_dirs: list[Path] = [] + + def fake_capture( + cfg: SysInfoCaptureConfig, + output_dir: Path = Path("."), + run_metadata_path=None, + ) -> Path: + captured_dirs.append(output_dir) return tmp_path / "out.json" with patch( @@ -507,45 +514,35 @@ def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: from_config(config=config_path) - assert str(captured_configs[0].output_path) == str(tmp_path / "results") + assert captured_dirs[0] == tmp_path / "results" @pytest.mark.unit - def test_no_report_dir_uses_output_path(self, tmp_path: Path) -> None: - """Without report_dir, output_path in system_info is used unchanged.""" + def test_no_report_dir_raises_validation_error(self, tmp_path: Path) -> None: + """Without report_dir, loading the config raises a ValidationError.""" config_path = tmp_path / "sysinfo.yaml" config_path.write_text( textwrap.dedent( - f"""\ + """\ system_info: ssh_ids: - user@10.0.0.1 accelerator_backend: cuda - output_path: {tmp_path}/custom/ """ ) ) - captured_configs: list[SysInfoCaptureConfig] = [] - - def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: - captured_configs.append(cfg) - return tmp_path / "out.json" - - with patch( - "inference_endpoint.commands.sysinfo.cli.capture_system_info", - side_effect=fake_capture, - ): + with pytest.raises(InputValidationError): from inference_endpoint.commands.sysinfo.cli import from_config from_config(config=config_path) - assert Path(captured_configs[0].output_path) == tmp_path / "custom" - class TestMissingSshIds: @pytest.mark.unit def test_missing_ssh_ids_raises_validation_error(self, tmp_path: Path) -> None: config_path = tmp_path / "sysinfo.yaml" - config_path.write_text("system_info:\n accelerator_backend: cuda\n") + config_path.write_text( + f"report_dir: {tmp_path}/results/\nsystem_info:\n accelerator_backend: cuda\n" + ) with pytest.raises(ValidationError): SysInfoFileConfig.from_yaml_file(config_path) @@ -556,7 +553,7 @@ def test_missing_ssh_ids_exclude_current_false_still_raises( """exclude_current_system=False does not relax the ssh_ids requirement.""" config_path = tmp_path / "sysinfo.yaml" config_path.write_text( - "system_info:\n accelerator_backend: cuda\n exclude_current_system: false\n" + f"report_dir: {tmp_path}/results/\nsystem_info:\n accelerator_backend: cuda\n exclude_current_system: false\n" ) with pytest.raises(ValidationError): SysInfoFileConfig.from_yaml_file(config_path) @@ -566,7 +563,9 @@ def test_missing_ssh_ids_via_cli_raises_input_validation_error( self, tmp_path: Path ) -> None: config_path = tmp_path / "sysinfo.yaml" - config_path.write_text("system_info:\n accelerator_backend: cuda\n") + config_path.write_text( + f"report_dir: {tmp_path}/results/\nsystem_info:\n accelerator_backend: cuda\n" + ) with patch( "inference_endpoint.commands.sysinfo.cli.capture_system_info" ) as mock_cap: @@ -583,12 +582,12 @@ def test_from_config_calls_capture(self, tmp_path: Path) -> None: config_path = tmp_path / "sysinfo.yaml" config_path.write_text( textwrap.dedent( - """\ + f"""\ + report_dir: {tmp_path}/results/ system_info: ssh_ids: - user@10.0.0.1 accelerator_backend: cuda - output_path: /tmp/sys_info node_config: Prefill: - node_name: H100 @@ -598,7 +597,11 @@ def test_from_config_calls_capture(self, tmp_path: Path) -> None: ) captured_configs: list[SysInfoCaptureConfig] = [] - def fake_capture(cfg: SysInfoCaptureConfig, run_metadata_path=None) -> Path: + def fake_capture( + cfg: SysInfoCaptureConfig, + output_dir: Path = Path("."), + run_metadata_path=None, + ) -> Path: captured_configs.append(cfg) return tmp_path / "out.json" From 4e1121bc486dba746d046cf8b2d248b33a93e04b Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 5 Jun 2026 19:25:50 +0530 Subject: [PATCH 17/21] pre commit --- docs/commands/DESIGN.md | 18 +++++++++--------- tests/unit/sys_info/test_capture.py | 3 +-- tests/unit/sys_info/test_sysinfo_command.py | 3 +-- uv.lock | 2 +- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index 2ad948bcf..b356e0db5 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -175,16 +175,16 @@ Both paths call `sys_info/capture.py::capture_system_info()` and produce the sam ### Config Reference (`SysInfoCaptureConfig`) -| Field | Type | Description | -| ------------------------ | ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ssh_ids` | `list[str]` | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | +| Field | Type | Description | +| ------------------------ | ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ssh_ids` | `list[str]` | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | | `accelerator_backend` | `"cuda"` \| `"rocm"` \| `"xpu"` \| `"none"` | GPU backend on the target nodes. Default: `"none"`. | -| `exclude_current_system` | bool | Skip the machine running this command; collect from `ssh_ids` only. Default: `false`. | -| `skip_ssh_key_file` | bool | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). Default: `false`. | -| `node_config` | object | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | -| `serving_node` | str | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | -| `log_path` | str | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | -| `endpoint_url` | str | Base URL of the running inference server. Probed via HTTP to detect the serving framework name and version (e.g. `"vLLM 0.9.0"`). | +| `exclude_current_system` | bool | Skip the machine running this command; collect from `ssh_ids` only. Default: `false`. | +| `skip_ssh_key_file` | bool | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). Default: `false`. | +| `node_config` | object | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | +| `serving_node` | str | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | +| `log_path` | str | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | +| `endpoint_url` | str | Base URL of the running inference server. Probed via HTTP to detect the serving framework name and version (e.g. `"vLLM 0.9.0"`). | | `serving_framework` | `"auto"` \| `"vllm"` \| `"sglang"` \| `"trtllm"` | Serving engine type used for startup log parsing. Default: `"auto"` (detected from the endpoint). | ### Capture Flow diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index f612ec8c9..a634371f9 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -22,14 +22,13 @@ from unittest.mock import MagicMock, patch import pytest -from pydantic import ValidationError - from inference_endpoint.config.schema import ( BenchmarkConfig, SshTarget, SysInfoCaptureConfig, ) from inference_endpoint.exceptions import ExecutionError, SetupError +from pydantic import ValidationError # --------------------------------------------------------------------------- # Helpers diff --git a/tests/unit/sys_info/test_sysinfo_command.py b/tests/unit/sys_info/test_sysinfo_command.py index 670e350f9..85cef85bf 100644 --- a/tests/unit/sys_info/test_sysinfo_command.py +++ b/tests/unit/sys_info/test_sysinfo_command.py @@ -23,14 +23,13 @@ import pytest import yaml -from pydantic import ValidationError - from inference_endpoint.config.schema import ( NodeEntry, SysInfoCaptureConfig, SysInfoFileConfig, ) from inference_endpoint.exceptions import InputValidationError +from pydantic import ValidationError # --------------------------------------------------------------------------- # Helpers diff --git a/uv.lock b/uv.lock index d9ffcc6c5..70fbb512a 100644 --- a/uv.lock +++ b/uv.lock @@ -883,7 +883,7 @@ requires-dist = [ { name = "line-profiler", marker = "extra == 'test'", specifier = "==5.0.2" }, { name = "matplotlib", marker = "extra == 'test'", specifier = "==3.10.8" }, { name = "memory-profiler", marker = "extra == 'performance'", specifier = "==0.61.0" }, - { name = "mlc-scripts" }, + { name = "mlc-scripts", specifier = "==1.1.0" }, { name = "msgspec", specifier = "==0.20.0" }, { name = "myst-parser", marker = "extra == 'dev'", specifier = "==5.0.0" }, { name = "numpy", specifier = "==2.4.4" }, From 5e58ffeb42d3c8fb6537e04364f5aa2680164d04 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 11 Jun 2026 12:10:03 +0530 Subject: [PATCH 18/21] feat: make mlc-scripts an optional dependency for system_info capture - Move mlc-scripts==1.1.0 from core deps to [sysinfo] optional extra; install with `pip install inference-endpoint[sysinfo]` or `uv run --extra sysinfo` - Add [sysinfo] to [test] extra so CI retains full coverage - Lazy-import capture_system_info in execute.py; only loaded when system_info is configured - Raise SetupError with actionable install hint when mlc-scripts is missing - Remove node_config from run_metadata.json output - Remove disaggregated proxy computation; field now starts as None and is patched by mlcflow - Fix missing space in endpoint_url field description in schema.py - Add test assertions that all mlcflow-owned fields (disaggregated, tensor_parallel, etc.) exist and start as None - Update DESIGN.md with sysinfo optional dependency callout and install instructions Co-Authored-By: Claude Sonnet 4.6 --- docs/commands/DESIGN.md | 27 ++++++++++---- pyproject.toml | 7 ++-- .../commands/benchmark/execute.py | 36 ++++++------------- .../commands/sysinfo/cli.py | 4 ++- src/inference_endpoint/config/schema.py | 25 +++++++------ src/inference_endpoint/sys_info/capture.py | 21 ++++++----- .../commands/test_benchmark_finalization.py | 18 ++++++++-- tests/unit/config/test_schema.py | 2 ++ tests/unit/sys_info/test_capture.py | 25 ++++++++++--- tests/unit/sys_info/test_sysinfo_command.py | 9 +++++ uv.lock | 10 ++++-- 11 files changed, 119 insertions(+), 65 deletions(-) diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index b356e0db5..c2117ea08 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -106,6 +106,20 @@ commands/benchmark/execute.py::run_benchmark() ## System Info Capture +> **Requires the `sysinfo` optional dependency.** Install it with: +> +> ```bash +> # uv (recommended) +> uv sync --extra sysinfo +> # or pass --extra sysinfo directly to uv run, e.g.: +> uv run --extra sysinfo inference-endpoint benchmark from-config --config config.yaml +> +> # pip +> pip install "inference-endpoint[sysinfo]" +> ``` +> +> If `mlc-scripts` is not installed and `system_info` is configured, the benchmark still completes and results are written first; system info capture is then attempted, fails with an error log, and the process exits 0. + System info capture collects hardware/software details from one or more nodes and writes a structured JSON file for MLPerf inference submissions. It runs in two contexts: - **Standalone** (`sysinfo from-config`): triggered manually, independent of any benchmark run. See [Standalone Command (`sysinfo from-config`)](#standalone-command-sysinfo-from-config) below for a full example. @@ -152,13 +166,13 @@ System info capture collects hardware/software details from one or more nodes an report_dir: sglang_perf_c1000 system_info: + system_name: H100x8_SGLang ssh_ids: - - anandhusooraj@mlc2 + - user@inference-node accelerator_backend: cuda exclude_current_system: true skip_ssh_key_file: false - serving_node: anandhusooraj@mlc2 - log_path: /home/anandhusooraj/sglang_logs.log + serving_node: user@inference-node endpoint_url: http://localhost:11001 serving_framework: sglang ``` @@ -177,13 +191,13 @@ Both paths call `sys_info/capture.py::capture_system_info()` and produce the sam | Field | Type | Description | | ------------------------ | ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `system_name` | str | **Required.** Name of the system under test (e.g. `"H100x8_vLLM"`). Used as the MLPerf submission system identifier. | | `ssh_ids` | `list[str]` | **Required.** Nodes to collect hardware info from. Format: `user@host` or `user@host:port`. | | `accelerator_backend` | `"cuda"` \| `"rocm"` \| `"xpu"` \| `"none"` | GPU backend on the target nodes. Default: `"none"`. | | `exclude_current_system` | bool | Skip the machine running this command; collect from `ssh_ids` only. Default: `false`. | | `skip_ssh_key_file` | bool | Assume SSH key auth is pre-configured (skips mlcflow key-file lookup). Default: `false`. | | `node_config` | object | Optional function-based node groupings (Prefill/Decode/etc). Maps function names to lists of `{node_name, no_of_nodes}` entries. `node_name` is matched as a case-insensitive substring against the detected GPU model name. | -| `serving_node` | str | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture also SSHes into this node to extract serving configuration from the startup log. | -| `log_path` | str | Path to the vLLM or SGLang server log **on the serving node**. Required when `serving_node` is set and serving config extraction is desired. | +| `serving_node` | str | SSH target for the inference server (`user@host` or `user@host:port`). When set, the capture SSHes into this node and reads `/tmp/serving.log` to extract serving config. Server stdout/stderr **must** be redirected there. | | `endpoint_url` | str | Base URL of the running inference server. Probed via HTTP to detect the serving framework name and version (e.g. `"vLLM 0.9.0"`). | | `serving_framework` | `"auto"` \| `"vllm"` \| `"sglang"` \| `"trtllm"` | Serving engine type used for startup log parsing. Default: `"auto"` (detected from the endpoint). | @@ -209,6 +223,7 @@ inference-endpoint sysinfo from-config -c examples/sysinfo_example.yaml report_dir: results/h100_sysinfo/ # output directory system_info: + system_name: H100x7_vLLM ssh_ids: - root@ssh1:22 # prefill node 1 - root@ssh2:22 # prefill node 2 @@ -223,10 +238,10 @@ system_info: skip_ssh_key_file: false # serving_node: where the inference server process is running. + # Server stdout/stderr must be redirected to /tmp/serving.log on that node. # If multiple serving nodes exist, point to any one — all nodes are assumed # to run the same serving framework version. serving_node: root@ssh1:22 - log_path: /tmp/vllm.log # path on serving_node where server output was redirected node_config: # optional: function-based node groupings Prefill: diff --git a/pyproject.toml b/pyproject.toml index 8afebd335..b3165093c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,11 +73,13 @@ dependencies = [ # Fix pytz-2024 import warning "pytz==2026.1.post1", "urllib3==2.7.0", - # mlc-scripts for system info - "mlc-scripts==1.1.0", ] [project.optional-dependencies] +sysinfo = [ + # Required for system_info capture (mlperf submission workflow) + "mlc-scripts==1.1.0", +] sql = [ # SQL event logger (swappable backends, default sqlite) "sqlalchemy==2.0.48", @@ -101,6 +103,7 @@ dev = [ test = [ # Includes optional dependencies for full test coverage "inference-endpoint[sql]", + "inference-endpoint[sysinfo]", # Testing framework "pytest==9.0.3", "pytest-asyncio==1.3.0", diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 00818db22..be8196d44 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -1009,15 +1009,14 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: if ctx.config.system_info is not None: try: - # Local import: mlcflow is optional and only needed when system_info is configured. - from inference_endpoint.sys_info.capture import ( - capture_system_info, - ) + # mlc-scripts is an optional dep; only import when system_info is configured. + from inference_endpoint.sys_info.capture import capture_system_info + metadata_path = ctx.report_dir / "run_metadata.json" output_path = capture_system_info( ctx.config.system_info, output_dir=ctx.report_dir, - run_metadata_path=ctx.report_dir / "run_metadata.json", + run_metadata_path=metadata_path if metadata_path.exists() else None, ) logger.info("System info captured at: %s", output_path) except ExecutionError as e: @@ -1053,17 +1052,6 @@ def _stat(metric: dict[str, Any], key: str) -> float | None: def _pct(metric: dict[str, Any], p: str) -> float | None: return _ns_to_ms((metric.get("percentiles") or {}).get(p)) if metric else None - # node_config and disaggregated from system_info - node_config: Any = None - disaggregated: bool | None = None - sic = ctx.config.system_info - if sic is not None and sic.node_config is not None: - node_config = { - fn: [ne.model_dump() for ne in nodes] - for fn, nodes in sic.node_config.items() - } - disaggregated = len(sic.node_config) > 1 - ttft: dict[str, Any] = {} tpot: dict[str, Any] = {} latency: dict[str, Any] = {} @@ -1093,15 +1081,13 @@ def _pct(metric: dict[str, Any], p: str) -> float | None: metadata: dict[str, Any] = { "run_date": datetime.now(UTC).isoformat(), - "node_config": node_config, - "config_summary": { - "disaggregated": disaggregated, - "expert_parallel": None, - "tensor_parallel": None, - "pipeline_parallel": None, - "data_parallel": None, - "batch": None, - }, + "disaggregated": None, + "expert_parallel": None, + "tensor_parallel": None, + "pipeline_parallel": None, + "data_parallel": None, + "batch": None, + "config_summary": None, "config_summary_notes": None, "concurrency": concurrency, "system_tps": system_tps, diff --git a/src/inference_endpoint/commands/sysinfo/cli.py b/src/inference_endpoint/commands/sysinfo/cli.py index bc7a564ad..b9d20a5a6 100644 --- a/src/inference_endpoint/commands/sysinfo/cli.py +++ b/src/inference_endpoint/commands/sysinfo/cli.py @@ -43,11 +43,13 @@ def from_config( Example:: + report_dir: results/my_system_info + system_info: + system_name: H100x8_vLLM ssh_ids: - user@host accelerator_backend: cuda - output_path: /tmp/sys_info node_config: Prefill: - node_name: H100 diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index a43afb56b..b137c7e58 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -686,10 +686,16 @@ class SysInfoCaptureConfig(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) - exclude_current_system: bool = False + system_name: str = Field( + description=( + "Name of the system under test (e.g. 'H100x8_vLLM'). " + "Used as the MLPerf submission system identifier." + ), + ) + ssh_ids: list[str] accelerator_backend: Literal["cuda", "rocm", "xpu", "none"] = "none" + exclude_current_system: bool = False skip_ssh_key_file: bool = False - ssh_ids: list[str] node_config: dict[str, list[NodeEntry]] | None = Field( default=None, description=( @@ -701,7 +707,7 @@ class SysInfoCaptureConfig(BaseModel): default=None, description=( "Endpoint URL to probe for serving framework detection " - "(e.g. 'http://host:8000'). Must be set explicitly." + "(e.g. 'http://host:8000'). Must be set explicitly. " "Omitting it skips the HTTP " "serving-framework probe and reduces collected metadata." ), @@ -710,16 +716,9 @@ class SysInfoCaptureConfig(BaseModel): default=None, description=( "SSH ID of the node running the serving framework " - "(e.g. 'root@host:8022'). Used together with log_path for " - "log-based framework detection when endpoint_url is unavailable." - ), - ) - log_path: str | None = Field( - default=None, - description=( - "Absolute path on serving_node to which the server stdout/stderr " - "was redirected (e.g. '/tmp/vllm.log'). Used for log-based " - "serving framework detection." + "(e.g. 'root@host:8022'). When set, the capture SSHes into this " + "node to extract serving configuration from the startup log at " + "/tmp/serving.log (server stdout/stderr must be redirected there)." ), ) serving_framework: Literal["auto", "vllm", "sglang", "trtllm"] = Field( diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index 6c40b70fe..aaac92206 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -54,8 +54,12 @@ def _write_node_config_tmp(config: SysInfoCaptureConfig) -> str: } } fd, path = tempfile.mkstemp(suffix=".yaml", prefix="mlperf_node_cfg_") - with os.fdopen(fd, "w") as fh: - yaml.dump(data, fh, default_flow_style=False) + try: + with os.fdopen(fd, "w") as fh: + yaml.dump(data, fh, default_flow_style=False) + except Exception: + os.unlink(path) + raise return path @@ -73,11 +77,11 @@ def capture_system_info( # Optional dependency — only imported when this function is actually called. # mlc-scripts (PyPI) installs its runtime under the 'mlc' module name. try: - import mlc # noqa: PLC0415 + import mlc except ImportError as exc: raise SetupError( "mlc-scripts is required for system_info. " - "Install it with: pip install mlc-scripts" + 'Install it with: pip install "inference-endpoint[sysinfo]"' ) from exc tags: list[str] = ["get-mlperf-multi-node-system-info"] @@ -96,25 +100,26 @@ def capture_system_info( logger.info("Capturing system info from %d node(s)...", len(config.parsed_ssh_ids)) + # mlcflow changes cwd during script execution; resolve both paths to + # absolute here so postprocess() os.path.exists() checks work correctly. mlc_kwargs: dict[str, object] = { "action": "run", "automation": "script", "tags": tags_str, "ssh_ids": ssh_ids_str, - "out_dir_path": str(output_dir), + "out_dir_path": str(output_dir.resolve()), "out_file_name": _OUT_FILE_NAME, "skip_ssh_key_file": skip_ssh_key_file_value, "serving_framework_type": config.serving_framework, + "system_name": config.system_name, "quiet": True, } if config.endpoint_url: mlc_kwargs["endpoint_url"] = config.endpoint_url if config.serving_node: mlc_kwargs["serving_node"] = config.serving_node - if config.log_path: - mlc_kwargs["log_path"] = config.log_path if run_metadata_path is not None: - mlc_kwargs["run_metadata_path"] = str(run_metadata_path) + mlc_kwargs["run_metadata_path"] = str(run_metadata_path.resolve()) node_config_tmp: str | None = None if config.node_config is not None: diff --git a/tests/unit/commands/test_benchmark_finalization.py b/tests/unit/commands/test_benchmark_finalization.py index 70875d0ed..0c2f93af1 100644 --- a/tests/unit/commands/test_benchmark_finalization.py +++ b/tests/unit/commands/test_benchmark_finalization.py @@ -39,6 +39,7 @@ "model_params": {"name": "test-model"}, "datasets": [{"path": "test.jsonl"}], "system_info": { + "system_name": "TestSystem", "ssh_ids": ["alice@10.0.0.1"], "accelerator_backend": "cuda", }, @@ -77,8 +78,8 @@ def _make_bench(tmp_path: Path, report: Report | None = None) -> BenchmarkResult class TestRunMetadataWriteNonBlocking: @pytest.mark.unit - def test_write_failure_does_not_abort_finalize(self, tmp_path: Path) -> None: - """A write error on run_metadata.json must not propagate out of finalize_benchmark.""" + def test_disk_full_does_not_abort_finalize(self, tmp_path: Path) -> None: + """A disk-full OSError on any write must not propagate out of finalize_benchmark.""" ctx = _make_ctx(tmp_path) bench = _make_bench(tmp_path) @@ -262,6 +263,19 @@ def test_percentile_keys_resolve_with_float_string_format(self) -> None: # Top-level ttft field is the p99. assert metadata["ttft"] == pytest.approx(990.0) + # mlcflow-owned fields must be present and start as None so mlcflow's + # postprocess() can patch them from the serving log. + for field in ( + "disaggregated", + "tensor_parallel", + "pipeline_parallel", + "data_parallel", + "expert_parallel", + "batch", + "config_summary", + ): + assert metadata[field] is None, f"{field} must start as None" + def test_integer_string_keys_silently_return_none(self) -> None: """Confirm that integer-string keys ("99", "50") do NOT resolve — documenting the registry key format requirement.""" diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index 7aed02057..cd32bcbd4 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -492,6 +492,7 @@ def test_endpoint_url_stays_none_when_not_set(self) -> None: config = BenchmarkConfig( **self._BASE, system_info=SysInfoCaptureConfig( + system_name="TestSystem", ssh_ids=["user@10.0.0.1"], accelerator_backend="cuda", ), @@ -504,6 +505,7 @@ def test_explicit_endpoint_url_preserved(self) -> None: config = BenchmarkConfig( **self._BASE, system_info=SysInfoCaptureConfig( + system_name="TestSystem", ssh_ids=["user@10.0.0.1"], accelerator_backend="cuda", endpoint_url="http://10.0.0.2:8000", diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index a634371f9..bbd84cbc6 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -35,6 +35,7 @@ # --------------------------------------------------------------------------- _MINIMAL_SYS_INFO = { + "system_name": "TestSystem", "accelerator_backend": "cuda", "ssh_ids": ["alice@192.168.1.1"], } @@ -93,6 +94,11 @@ def test_empty_ssh_ids_raises(self) -> None: with pytest.raises(ValidationError, match="non-empty"): _make_config(ssh_ids=[]) + @pytest.mark.unit + def test_system_name_required(self) -> None: + with pytest.raises(ValidationError, match="system_name"): + SysInfoCaptureConfig(accelerator_backend="cuda", ssh_ids=["alice@10.0.0.1"]) + # 4. serving_node validation @pytest.mark.unit def test_serving_node_valid_no_port(self) -> None: @@ -140,13 +146,19 @@ def test_rocm_exclude_current(self) -> None: tags = _build_tags(cfg) assert tags == "get-mlperf-multi-node-system-info,_rocm,_exclude_current_node" + # 6. Variation tags — none backend omits the backend tag entirely + @pytest.mark.unit + def test_none_backend_omits_tag(self) -> None: + cfg = _make_config(accelerator_backend="none", exclude_current_system=False) + tags = _build_tags(cfg) + assert tags == "get-mlperf-multi-node-system-info" + def _build_tags(cfg: SysInfoCaptureConfig) -> str: """Mirror the tag-building logic from capture.py.""" - tags: list[str] = [ - "get-mlperf-multi-node-system-info", - f"_{cfg.accelerator_backend}", - ] + tags: list[str] = ["get-mlperf-multi-node-system-info"] + if cfg.accelerator_backend != "none": + tags.append(f"_{cfg.accelerator_backend}") if cfg.exclude_current_system: tags.append("_exclude_current_node") return ",".join(tags) @@ -277,6 +289,7 @@ def test_unreachable_node_no_node_config_succeeds(self, tmp_path: Path) -> None: @pytest.mark.unit def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: cfg = SysInfoCaptureConfig( + system_name="TestSystem", accelerator_backend="cuda", exclude_current_system=True, skip_ssh_key_file=True, @@ -302,10 +315,11 @@ def test_mlcflow_access_called_with_correct_args(self, tmp_path: Path) -> None: ) assert call_args["ssh_ids"] == "alice@10.0.0.1:2222,bob@10.0.0.2:22" assert call_args["skip_ssh_key_file"] == "yes" - assert call_args["out_dir_path"] == str(tmp_path) + assert call_args["out_dir_path"] == str(tmp_path.resolve()) assert call_args["out_file_name"] == "system_desc.json" assert call_args["action"] == "run" assert call_args["automation"] == "script" + assert call_args["system_name"] == "TestSystem" # --------------------------------------------------------------------------- @@ -327,6 +341,7 @@ def test_system_info_from_yaml(self, tmp_path: Path) -> None: datasets: - path: dummy.jsonl system_info: + system_name: TestSystem accelerator_backend: cuda ssh_ids: - alice@192.168.1.1 diff --git a/tests/unit/sys_info/test_sysinfo_command.py b/tests/unit/sys_info/test_sysinfo_command.py index 85cef85bf..59887a732 100644 --- a/tests/unit/sys_info/test_sysinfo_command.py +++ b/tests/unit/sys_info/test_sysinfo_command.py @@ -36,6 +36,7 @@ # --------------------------------------------------------------------------- _BASE_SYSTEM_INFO = { + "system_name": "TestSystem", "accelerator_backend": "cuda", "ssh_ids": ["user@10.0.0.1"], } @@ -150,6 +151,7 @@ def test_minimal_config_no_node_config(self, tmp_path: Path) -> None: f"""\ report_dir: {tmp_path}/results/ system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda @@ -185,6 +187,7 @@ def test_report_dir_parsed(self, tmp_path: Path) -> None: """\ report_dir: results/my_system/ system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda @@ -204,6 +207,7 @@ def test_with_node_config(self, tmp_path: Path) -> None: f"""\ report_dir: {tmp_path}/results/ system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda @@ -233,6 +237,7 @@ def test_extra_top_level_keys_ignored(self, tmp_path: Path) -> None: f"""\ report_dir: {tmp_path}/results/ system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda @@ -264,6 +269,7 @@ def test_endpoint_config_not_propagated_to_endpoint_url( endpoints: - http://10.0.0.1:8000 system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda @@ -285,6 +291,7 @@ def test_endpoint_url_explicit_in_system_info_is_preserved( f"""\ report_dir: {tmp_path}/results/ system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda @@ -489,6 +496,7 @@ def test_report_dir_passed_as_output_dir(self, tmp_path: Path) -> None: f"""\ report_dir: {tmp_path}/results/ system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda @@ -584,6 +592,7 @@ def test_from_config_calls_capture(self, tmp_path: Path) -> None: f"""\ report_dir: {tmp_path}/results/ system_info: + system_name: TestSystem ssh_ids: - user@10.0.0.1 accelerator_backend: cuda diff --git a/uv.lock b/uv.lock index 96cb148f0..1a63c2eeb 100644 --- a/uv.lock +++ b/uv.lock @@ -810,7 +810,6 @@ dependencies = [ { name = "hdrhistogram", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "httptools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "mlc-scripts", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "msgspec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "numpy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "openai-harmony", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -849,12 +848,16 @@ performance = [ sql = [ { name = "sqlalchemy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] +sysinfo = [ + { name = "mlc-scripts", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] test = [ { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "coverage", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "hypothesis", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "line-profiler", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "matplotlib", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "mlc-scripts", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pympler", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pytest-asyncio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -878,12 +881,13 @@ requires-dist = [ { name = "httptools", specifier = "==0.7.1" }, { name = "hypothesis", marker = "extra == 'test'", specifier = "==6.151.10" }, { name = "inference-endpoint", extras = ["sql"], marker = "extra == 'test'" }, + { name = "inference-endpoint", extras = ["sysinfo"], marker = "extra == 'test'" }, { name = "jinja2", specifier = "==3.1.6" }, { name = "line-profiler", marker = "extra == 'dev'", specifier = "==5.0.2" }, { name = "line-profiler", marker = "extra == 'test'", specifier = "==5.0.2" }, { name = "matplotlib", marker = "extra == 'test'", specifier = "==3.10.8" }, { name = "memory-profiler", marker = "extra == 'performance'", specifier = "==0.61.0" }, - { name = "mlc-scripts", specifier = "==1.1.0" }, + { name = "mlc-scripts", marker = "extra == 'sysinfo'", specifier = "==1.1.0" }, { name = "msgspec", specifier = "==0.20.0" }, { name = "myst-parser", marker = "extra == 'dev'", specifier = "==5.0.0" }, { name = "numpy", specifier = "==2.4.4" }, @@ -920,7 +924,7 @@ requires-dist = [ { name = "uvloop", specifier = "==0.22.1" }, { name = "websocket-client", specifier = "==1.9.0" }, ] -provides-extras = ["sql", "dev", "test", "performance"] +provides-extras = ["sysinfo", "sql", "dev", "test", "performance"] [[package]] name = "iniconfig" From fe21d12a33b7d51c690fbef383d679c69d639505 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 11 Jun 2026 12:27:56 +0530 Subject: [PATCH 19/21] test: fix test_capture assertion to match updated install hint The SetupError message was updated to recommend `pip install "inference-endpoint[sysinfo]"` instead of `pip install mlc-scripts`, but the test regex was not updated, causing CI to fail. Co-Authored-By: Claude Sonnet 4.6 --- tests/unit/sys_info/test_capture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index bbd84cbc6..d2cb3f243 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -203,7 +203,7 @@ def test_mlcflow_not_installed_raises_setup_error(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - with pytest.raises(SetupError, match="pip install mlc-scripts"): + with pytest.raises(SetupError, match=r"pip install.*inference-endpoint\[sysinfo\]"): capture_mod.capture_system_info(cfg, output_dir=tmp_path) @pytest.mark.unit From 1a39d480398e787dde64eb00b967f7cb10ff9bbd Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 11 Jun 2026 12:30:58 +0530 Subject: [PATCH 20/21] style: apply ruff-format to test_capture.py Co-Authored-By: Claude Sonnet 4.6 --- tests/unit/sys_info/test_capture.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index d2cb3f243..6be2381e7 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -203,7 +203,9 @@ def test_mlcflow_not_installed_raises_setup_error(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - with pytest.raises(SetupError, match=r"pip install.*inference-endpoint\[sysinfo\]"): + with pytest.raises( + SetupError, match=r"pip install.*inference-endpoint\[sysinfo\]" + ): capture_mod.capture_system_info(cfg, output_dir=tmp_path) @pytest.mark.unit From 8d418a1786b269e28debb444491152abacb92269 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 11 Jun 2026 15:12:51 +0530 Subject: [PATCH 21/21] fix: update sysinfo install hint to use local editable install pip install "inference-endpoint[sysinfo]" fails since the package is not published on PyPI. Updated the SetupError message in capture.py, the test regex in test_capture.py, and the docs callout in DESIGN.md to use pip install -e '.[sysinfo]' instead. Co-Authored-By: Claude Sonnet 4.6 --- docs/commands/DESIGN.md | 4 ++-- src/inference_endpoint/sys_info/capture.py | 2 +- tests/unit/sys_info/test_capture.py | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/commands/DESIGN.md b/docs/commands/DESIGN.md index c2117ea08..2c1ccb4a5 100644 --- a/docs/commands/DESIGN.md +++ b/docs/commands/DESIGN.md @@ -114,8 +114,8 @@ commands/benchmark/execute.py::run_benchmark() > # or pass --extra sysinfo directly to uv run, e.g.: > uv run --extra sysinfo inference-endpoint benchmark from-config --config config.yaml > -> # pip -> pip install "inference-endpoint[sysinfo]" +> # pip (from repo root) +> pip install -e ".[sysinfo]" > ``` > > If `mlc-scripts` is not installed and `system_info` is configured, the benchmark still completes and results are written first; system info capture is then attempted, fails with an error log, and the process exits 0. diff --git a/src/inference_endpoint/sys_info/capture.py b/src/inference_endpoint/sys_info/capture.py index aaac92206..4e55c0632 100644 --- a/src/inference_endpoint/sys_info/capture.py +++ b/src/inference_endpoint/sys_info/capture.py @@ -81,7 +81,7 @@ def capture_system_info( except ImportError as exc: raise SetupError( "mlc-scripts is required for system_info. " - 'Install it with: pip install "inference-endpoint[sysinfo]"' + "Install it with: pip install -e '.[sysinfo]' (from the repo root)" ) from exc tags: list[str] = ["get-mlperf-multi-node-system-info"] diff --git a/tests/unit/sys_info/test_capture.py b/tests/unit/sys_info/test_capture.py index 6be2381e7..b49998cd8 100644 --- a/tests/unit/sys_info/test_capture.py +++ b/tests/unit/sys_info/test_capture.py @@ -203,9 +203,7 @@ def test_mlcflow_not_installed_raises_setup_error(self, tmp_path: Path) -> None: from inference_endpoint.sys_info import capture as capture_mod importlib.reload(capture_mod) - with pytest.raises( - SetupError, match=r"pip install.*inference-endpoint\[sysinfo\]" - ): + with pytest.raises(SetupError, match=r"pip install -e '\.\[sysinfo\]'"): capture_mod.capture_system_info(cfg, output_dir=tmp_path) @pytest.mark.unit