diff --git a/.github/workflows/check-test-list-probe.yml b/.github/workflows/check-test-list-probe.yml new file mode 100644 index 000000000000..c9409ee54f92 --- /dev/null +++ b/.github/workflows/check-test-list-probe.yml @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Probe how long the Jenkins "Check Test List" stage takes on a GitHub-hosted +# runner with CUDA torch, the full requirements-dev set and a pip cache. +name: Check Test List Probe +on: + pull_request: + paths: + - '.github/workflows/check-test-list-probe.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + TORCH_VERSION: '2.14.0' + TORCH_INDEX: https://download.pytorch.org/whl/cu132 + +jobs: + collect: + name: Collect (full-cuda deps, ubuntu-latest) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + # Same layout as the Jenkins stage: the source lives in a subdirectory and + # check_test_list.py is invoked by absolute path from the workspace root. + - uses: actions/checkout@v6 + with: + path: llm + + - uses: actions/setup-python@v6 + id: python + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: | + llm/requirements.txt + llm/requirements-dev.txt + llm/constraints.txt + + - name: Install torch + run: | + start=$SECONDS + python -m pip install --upgrade pip + python -m pip install --index-url "${TORCH_INDEX}" "torch==${TORCH_VERSION}" torchvision + echo "TORCH_SECONDS=$((SECONDS - start))" >> "$GITHUB_ENV" + + # mpi4py loads libmpi when mpi4py.MPI is imported. The CI image ships + # HPC-X OpenMPI; on a plain runner the openmpi wheel provides it. + - name: Install requirements + working-directory: llm + run: | + printf 'torch==%s\n' "${TORCH_VERSION}" > "${RUNNER_TEMP}/torch-pin.txt" + start=$SECONDS + python -m pip install --extra-index-url "${TORCH_INDEX}" \ + -c "${RUNNER_TEMP}/torch-pin.txt" -r requirements-dev.txt openmpi + echo "REQ_SECONDS=$((SECONDS - start))" >> "$GITHUB_ENV" + + # urm.nvidia.com does not resolve from GitHub-hosted runners. check_test_list + # only runs `trt-test-db -d DIR --test-names --output FILE`, which lists every + # test name in DIR/*.yml; this stand-in produces the same set from the YAML. + - name: Install a trt-test-db stand-in + run: | + mkdir -p "${RUNNER_TEMP}/bin" + cat > "${RUNNER_TEMP}/bin/trt-test-db" <<'EOF' + #!/usr/bin/env python3 + import argparse + import glob + import os + + import yaml + + parser = argparse.ArgumentParser() + parser.add_argument("-d", required=True) + parser.add_argument("--test-names", action="store_true", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + names = set() + for path in glob.glob(os.path.join(args.d, "*.yml")): + with open(path) as f: + for key, blocks in (yaml.safe_load(f) or {}).items(): + if isinstance(blocks, list): + for block in blocks: + names.update(str(t) for t in block.get("tests") or []) + with open(args.output, "w") as f: + f.write("\n".join(sorted(names)) + "\n") + EOF + chmod +x "${RUNNER_TEMP}/bin/trt-test-db" + echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH" + + - name: Run check_test_list + env: + NVIDIA_TRITON_SERVER_VERSION: '26.09' + run: | + export LLM_ROOT="$PWD/llm" LLM_BACKEND_ROOT="$PWD/llm/triton_backend" + start=$SECONDS + # Dependencies were installed by the steps above; skip the script's own + # pip install so the measured set is exactly the one under test. + python - <<'EOF' + import importlib.util + import os + import sys + + path = os.path.join(os.environ["LLM_ROOT"], "scripts", "check_test_list.py") + spec = importlib.util.spec_from_file_location("check_test_list", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.install_python_dependencies = lambda llm_src: None + sys.argv = [path, "--l0", "--qa", "--waive", "--validate", "--parity", + "--check-duplicate-waives"] + module.main() + EOF + echo "COLLECT_SECONDS=$((SECONDS - start))" >> "$GITHUB_ENV" + + - name: Summary + if: always() + run: | + { + echo "## Collect with full-cuda deps on ubuntu-latest" + echo "pip cache restored: ${{ steps.python.outputs.cache-hit }}; size now: $(du -sh "$(python -m pip cache dir)" | cut -f1)" + echo + echo "| Step | Seconds |" + echo "|---|---|" + echo "| torch | ${TORCH_SECONDS:-} |" + echo "| requirements | ${REQ_SECONDS:-} |" + echo "| check_test_list | ${COLLECT_SECONDS:-} |" + } | tee -a "$GITHUB_STEP_SUMMARY"