ci-pixi-source-test.yml

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

# Validates the `pixi run test` developer workflow (source build + Cython
# codegen + test suite via the pixi-managed environment).
#
# WHY THIS EXISTS: the main CI (ci.yml / test-wheel-*.yml) tests prebuilt
# *wheels*; it never exercises the pixi *source build*. That path rots silently
# whenever the CUDA pin, generated bindings, conda-forge packages, or the
# cython-test build mechanics drift (see #2182, #2183). This workflow is the
# only thing that runs the pixi source build end to end.
#
# Two tiers, to spend GPU minutes deliberately (GPUs are scarce):
#   - build-smoke (PRs): CPU-only. Source-builds bindings + core, imports them,
#     builds the cython test extensions and checks placement. Catches the
#     compile / ABI / .so-placement regressions WITHOUT a GPU.
#   - build-identity-roundtrip (nightly + manual): CPU-only. cu13 -> cu12 ->
#     cu13 in one checkout, so build artifacts cannot be reused across CUDA
#     majors. Kept off PRs to avoid adding another job to the org's shared
#     concurrent-job quota; the fast unit tests in
#     cuda_core/tests/test_build_hooks.py cover the same intent per-PR.
#   - full-test (nightly + manual): GPU runner, full `pixi run test`.

name: "CI: pixi run test (source build)"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

on:
  schedule:
    # 2:37 AM UTC daily — offset from ci-nightly.yml (2:17) to avoid contending
    # for the same scheduled-workflow window.
    - cron: "37 2 * * *"
  pull_request:
    # Only the files that can break the source-build path (see #2182/#2183).
    paths:
      - "**/pixi.toml"
      - "**/pixi.lock"
      # Path-dependency metadata (including cuda_python_test_helpers) feeds the
      # solve, so it can break the source build without a pixi.toml edit.
      - "**/pyproject.toml"
      - "cuda_bindings_13/build_hooks.py"
      - "cuda_core/build_hooks.py"
      - "cuda_bindings_13/cuda/bindings/**" # generated bindings sources
      - "cuda_bindings_13/tests/cython/**"
      - "cuda_core/tests/cython/**"
      - "ci/versions.yml"
      - "ci/pixi-version.env"
      - ".github/actions/setup-pixi/action.yml"
      - ".github/workflows/ci-pixi-source-test.yml"
  workflow_dispatch:
    inputs:
      cuda-env:
        description: "pixi environment to test (cu13 or cu12)."
        type: string
        default: "cu13"

defaults:
  run:
    shell: bash --noprofile --norc -xeuo pipefail {0}

env:
  # Temporarily bypass Pixi's workspace-wide freshness check: the unrelated
  # samples environment oscillates because excluded PyPI dependencies are
  # still checked. Frozen mode keeps the committed solution and source builds,
  # including nested `pixi run` calls. Freshness enforcement and automated
  # refreshes are temporarily suspended; dependency changes need manual review.
  # Restore PIXI_LOCKED after the released-fix validation in ci/README.md.
  PIXI_FROZEN: "true"

jobs:
  # ── PR guard: CPU-only build + import + placement smoke ──
  build-smoke:
    name: "build smoke (cu13, linux-64, CPU)"
    if: >-
      github.repository_owner == 'nvidia' &&
      (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Full history + tags so setuptools-scm derives the real (13.x)
          # package version; a shallow checkout yields 0.1.dev1, which trips
          # cuda.core's "cuda.bindings 12.x or 13.x must be installed" guard.
          fetch-depth: 0
          filter: blob:none
          # third_party/cccl provides the CCCL headers cuda.core compiles against.
          submodules: true

      - name: Setup pixi
        uses: ./.github/actions/setup-pixi

      - name: Source-build + import + cython-placement smoke
        env:
          CUDA_ENV: ${{ inputs.cuda-env || 'cu13' }}
        run: |
          # pathfinder: pure-Python, no GPU.
          pixi run -e "${CUDA_ENV}" test-pathfinder

          # bindings + core: force the source build (catches nvrtc/driver
          # compile errors like #2182) and import them (catches ABI mismatches).
          pixi run --manifest-path cuda_bindings_13 -e "${CUDA_ENV}" \
            python -c "import cuda.bindings.driver, cuda.bindings.nvrtc, cuda.bindings.runtime; print('bindings import OK')"
          pixi run --manifest-path cuda_core -e "${CUDA_ENV}" \
            python -c "import cuda.core; print('core import OK')"

          # cython test extensions: build them and confirm each .so landed next
          # to its .pyx in tests/cython (catches the placement regression #2180).
          pixi run --manifest-path cuda_bindings_13 -e "${CUDA_ENV}" build-cython-tests
          pixi run --manifest-path cuda_core -e "${CUDA_ENV}" build-cython-tests
          for d in cuda_bindings_13/tests/cython cuda_core/tests/cython; do
            if ! compgen -G "${d}/*.cpython-*.so" > /dev/null; then
              echo "::error::no compiled cython test .so in ${d} (placement regression)"
              exit 1
            fi
          done
          echo "cython test extensions placed correctly"

  # ── Nightly guard: build artifacts must be CUDA-major aware ──
  #
  # Neither Cython nor setuptools tracks the CUDA major in its own up-to-date
  # check: Cython does not hash `compile_time_env`, and an editable install's
  # .so is named by the Python ABI tag alone. Before build_hooks keyed them,
  # a cu13 build followed by a cu12 build in the same checkout failed while
  # compiling cu13-generated C++ against CUDA 12 headers.
  #
  # The round trip (not just cu13 -> cu12) is what catches the second half:
  # coming back to cu13 must not silently reuse the cu12 extension.
  #
  # cuda_core only: cuda-bindings cannot be source-built in its cu12
  # environment at all, for reasons unrelated to stale artifacts.
  build-identity-roundtrip:
    name: "cu13 -> cu12 -> cu13 round trip (linux-64, CPU)"
    if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.repository_owner == 'nvidia' }}
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Full history + tags so setuptools-scm derives the real (13.x)
          # package version; a shallow checkout yields 0.1.dev1, which trips
          # cuda.core's "cuda.bindings 12.x or 13.x must be installed" guard.
          fetch-depth: 0
          filter: blob:none
          # third_party/cccl provides the CCCL headers cuda.core compiles against.
          submodules: true

      - name: Setup pixi
        uses: ./.github/actions/setup-pixi

      - name: Build cu13, then cu12, then cu13 again in one checkout
        run: |
          for cuda_env in cu13 cu12 cu13; do
            echo "::group::${cuda_env}"
            pixi run --manifest-path cuda_core -e "${cuda_env}" \
              python -c "import cuda.core; print('core import OK')"
            echo "::endgroup::"
          done
          # The last build was cu13, and each major must have kept its own
          # generated sources rather than overwriting the other's.
          # Stamp is ABI-scoped (.build-config.<EXT_SUFFIX>). Unset and gnu
          # both record as "gnu"; llvm records as "llvm".
          shopt -s nullglob
          stamps=(cuda_core/build/.build-config.*)
          if (( ${#stamps[@]} != 1 )); then
            echo "::error::expected one ABI stamp, found: ${stamps[*]:-none}"
            exit 1
          fi
          stamp=$(<"${stamps[0]}")
          if [[ ! "${stamp}" =~ ^cu13-(gnu|llvm)-(debug|opt)(-cov)?$ ]]; then
            echo "::error::build stamp is '${stamp}', expected cu13-{gnu|llvm}-{debug|opt}"
            exit 1
          fi
          suffix="${stamp#cu13-}"
          for major in cu12 cu13; do
            if [ ! -d "cuda_core/build/cython/${major}-${suffix}" ]; then
              echo "::error::no ${major}-${suffix} generated-source directory"
              exit 1
            fi
          done

  # ── Nightly: full `pixi run test` on a GPU runner ──
  full-test:
    name: "pixi run test (${{ inputs.cuda-env || 'cu13' }}, linux-64, GPU)"
    if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.repository_owner == 'nvidia' }}
    runs-on: "linux-amd64-gpu-l4-latest-1" # same label scheme as test-wheel-linux.yml
    timeout-minutes: 90
    container:
      options: -u root --security-opt seccomp=unconfined --shm-size 16g
      image: ubuntu:24.04
      env:
        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
    steps:
      - name: Ensure GPU is working
        run: nvidia-smi

      - name: Install system packages
        run: |
          apt-get update
          # ca-certificates + git for checkout; libgl1/libegl1 for pyglet
          # (pulled in by the test env). pixi itself is installed by setup-pixi.
          apt-get install -y --no-install-recommends \
            ca-certificates git libgl1 libegl1

      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Full history + tags so setuptools-scm derives the real (13.x)
          # package version; a shallow checkout yields 0.1.dev1, which trips
          # cuda.core's "cuda.bindings 12.x or 13.x must be installed" guard.
          fetch-depth: 0
          filter: blob:none
          # third_party/cccl provides the CCCL headers cuda.core compiles against.
          submodules: true

      - name: Setup proxy cache
        uses: nv-gha-runners/setup-proxy-cache@main
        continue-on-error: true

      - name: Setup pixi
        uses: ./.github/actions/setup-pixi

      - name: pixi run test
        run: pixi run -e "${{ inputs.cuda-env || 'cu13' }}" test