From 0c04d6b726a2eef009b20c275165006835362cfb Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Tue, 8 Sep 2026 03:17:43 +0400 Subject: [PATCH 1/2] [infra] select the evaluation suite per pipeline The suite mechanism merged in #146 but no workflow chose a suite, so an unset EVAL_SUITE left every job evaluating every record. Pick one per pipeline: nightly runs full, pre-merge runs smoke. Forward the variable into the two containerised eval steps; the third step runs eval_cuvslam_in_docker.sh on the host and inherits it from the workflow environment. Nothing changes yet. Both suites select complete KITTI and EuRoC, so the first behavioral difference arrives with the first full-only dataset, which is what this makes possible: without it, marking a dataset full-only would have no effect and pre-merge would keep paying for nightly's coverage. PR KPI values stay comparable to the nightly history this job diffs against, because a dataset in both suites keeps one reporter config and therefore one KPI prefix. Timeouts are unchanged. A nightly x86 job measures about 30 minutes against a 120 minute budget, and smoke equals full today, so there is nothing to re-measure until a large corpus is enabled. --- .github/workflows/nightly.yml | 7 +++++-- .github/workflows/pr-verify.yml | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 25f8e478..47f9e401 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,6 +23,9 @@ env: EVAL_REPORT_ARTIFACT: eval-reports-staging AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }} S3_DATASETS_BUCKET: ${{ vars.S3_DATASETS_BUCKET }} + # Nightlies carry every evaluation record. The Jetson configurations inherit + # this but never read it: their eval steps are gated on matrix.eval. + EVAL_SUITE: full jobs: check-changes: @@ -197,7 +200,7 @@ jobs: run: | docker run --rm --network host \ -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e S3_DATASETS_BUCKET \ - -e RUNNER_STORAGE_ROOT -e RUNNER_LOCAL_DATASETS_ROOT \ + -e RUNNER_STORAGE_ROOT -e RUNNER_LOCAL_DATASETS_ROOT -e EVAL_SUITE \ -v "$(pwd):/cuvslam:ro" \ -v "$RUNNER_LOCAL_DATASETS_ROOT:$RUNNER_LOCAL_DATASETS_ROOT" \ -w /cuvslam \ @@ -212,7 +215,7 @@ jobs: mkdir -p "$RUNNER_LOCAL_DATASETS_ROOT" docker run --rm --network host \ -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e S3_DATASETS_BUCKET \ - -e RUNNER_LOCAL_DATASETS_ROOT \ + -e RUNNER_LOCAL_DATASETS_ROOT -e EVAL_SUITE \ -v "$(pwd):/cuvslam:ro" \ -v "$RUNNER_LOCAL_DATASETS_ROOT:$RUNNER_LOCAL_DATASETS_ROOT" \ -w /cuvslam \ diff --git a/.github/workflows/pr-verify.yml b/.github/workflows/pr-verify.yml index a076a4cb..3056da56 100644 --- a/.github/workflows/pr-verify.yml +++ b/.github/workflows/pr-verify.yml @@ -19,6 +19,11 @@ env: S3_DATASETS_BUCKET: ${{ vars.S3_DATASETS_BUCKET }} # PR eval runs on x86_64 with the build script's default CUDA/Ubuntu. EVAL_CONFIG: x86_64-cuda12.6.3-ubuntu24.04 + # Pre-merge runs the cheap suite. It selects the same records as full until a + # full-only dataset is enabled, and its KPI values stay comparable to the + # nightly history this job diffs against because the shared datasets keep one + # reporter config, and therefore one KPI prefix, in both suites. + EVAL_SUITE: smoke jobs: lint: @@ -149,7 +154,7 @@ jobs: run: | docker run --rm --network host \ -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e S3_DATASETS_BUCKET \ - -e RUNNER_STORAGE_ROOT -e RUNNER_LOCAL_DATASETS_ROOT \ + -e RUNNER_STORAGE_ROOT -e RUNNER_LOCAL_DATASETS_ROOT -e EVAL_SUITE \ -v "$(pwd):/cuvslam:ro" \ -v "$RUNNER_LOCAL_DATASETS_ROOT:$RUNNER_LOCAL_DATASETS_ROOT" \ -w /cuvslam \ @@ -163,7 +168,7 @@ jobs: mkdir -p "$RUNNER_LOCAL_DATASETS_ROOT" docker run --rm --network host \ -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e S3_DATASETS_BUCKET \ - -e RUNNER_LOCAL_DATASETS_ROOT \ + -e RUNNER_LOCAL_DATASETS_ROOT -e EVAL_SUITE \ -v "$(pwd):/cuvslam:ro" \ -v "$RUNNER_LOCAL_DATASETS_ROOT:$RUNNER_LOCAL_DATASETS_ROOT" \ -w /cuvslam \ From a7bfa3fa4aa2242ea1c72782a1b75f01c6ca38f6 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Tue, 8 Sep 2026 19:03:07 +0400 Subject: [PATCH 2/2] [clean] trim suite comments that restate the value The pr-verify note also claimed smoke and full select the same records, which stops being true as soon as a full-only dataset is enabled. The constraint it was really guarding, that a dataset in both suites must keep one KPI prefix, belongs next to the registry entry that could break it rather than here. --- .github/workflows/nightly.yml | 4 ++-- .github/workflows/pr-verify.yml | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 47f9e401..96d9d515 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,8 +23,8 @@ env: EVAL_REPORT_ARTIFACT: eval-reports-staging AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }} S3_DATASETS_BUCKET: ${{ vars.S3_DATASETS_BUCKET }} - # Nightlies carry every evaluation record. The Jetson configurations inherit - # this but never read it: their eval steps are gated on matrix.eval. + # Jetson configurations inherit this but never read it: their eval steps are + # gated on matrix.eval. EVAL_SUITE: full jobs: diff --git a/.github/workflows/pr-verify.yml b/.github/workflows/pr-verify.yml index 3056da56..889e9646 100644 --- a/.github/workflows/pr-verify.yml +++ b/.github/workflows/pr-verify.yml @@ -19,10 +19,6 @@ env: S3_DATASETS_BUCKET: ${{ vars.S3_DATASETS_BUCKET }} # PR eval runs on x86_64 with the build script's default CUDA/Ubuntu. EVAL_CONFIG: x86_64-cuda12.6.3-ubuntu24.04 - # Pre-merge runs the cheap suite. It selects the same records as full until a - # full-only dataset is enabled, and its KPI values stay comparable to the - # nightly history this job diffs against because the shared datasets keep one - # reporter config, and therefore one KPI prefix, in both suites. EVAL_SUITE: smoke jobs: