[feat] enable TUM in full and ICL-NUIM in both suites

cuvslam-skills/cuvslam-ci/reference.md

@@ -67,7 +67,7 @@ Repository secrets, split read from write so fork-reachable jobs never hold a ke
 
 ## Dataset registry and layout
 
-- `DATASETS` in `tools/python_tools/cuvslam_tools/dataset_registry.py` is the single source of truth. A `DatasetSpec` holds the ID, the preparation module, and its `EvalSpec` records; an `EvalSpec` holds the reporter config filename, the `cuvslam_app` flags, suite membership, and gating. Provisionable means the dataset is present; eval-enabled means it has at least one `EvalSpec`. KITTI and EuRoC are eval-enabled; `tum` and `tartan` are provisionable only.
+- `DATASETS` in `tools/python_tools/cuvslam_tools/dataset_registry.py` is the single source of truth. A `DatasetSpec` holds the ID, the preparation module, and its `EvalSpec` records; an `EvalSpec` holds the reporter config filename, the `cuvslam_app` flags, suite membership, and gating. Provisionable means the dataset is present; eval-enabled means it has at least one `EvalSpec`. KITTI, EuRoC, TUM and ICL-NUIM are eval-enabled; `tartan` and `coda` are provisionable only. Smoke runs KITTI, EuRoC and ICL-NUIM, which covers stereo, stereo-inertial and RGB-D; TUM is full-only because it is the larger RGB-D corpus and ICL-NUIM already covers the modality pre-merge.
 - The module is standard library only and imports converters lazily, so shell wrappers call it with `PYTHONPATH=tools/python_tools` inside `cuvslam-ci:local` before anything is installed. `datasets_config.sh` wraps it as `dataset_registry`; `run_eval.sh` defines its own shim because the S3 variables `datasets_config.sh` requires are absent in the eval container.
 - Subcommands: `validate [--dataset] [--suite]`, `list [--eval] [--suite]`, `eval-records [--suite]` (tab-separated `id`, KPI prefix, config path, flags), `kpi-keys [--suite]`, `prepare-module`, `prepare --root-file`, `verify-staged --root`.
 - Suites: `EVAL_SUITE` selects records in `stage_eval_datasets.sh`, `check_eval_prerequisites.sh`, and `run_eval.sh`, and `eval_cuvslam_in_docker.sh` forwards it into the container. Unset means every record; validation requires every `EvalSpec` to belong to `full`, so unset and `full` agree. `validate --suite` additionally rejects a suite that would select nothing.

scripts/kpi_baseline_ranges.json

@@ -31,6 +31,26 @@
         "EUROC_ARE_VIO_SLAM": {"expected": null, "tol_pct": 10},
         "EUROC_Kabsch_VIO_SLAM": {"expected": null, "tol_pct": 10},
         "EUROC_TrackingLosts_VIO_SLAM": {"expected": null, "tol_abs": 1},
-        "EUROC_FPS_VIO_SLAM": {"expected": null, "tol_pct": 15}
+        "EUROC_FPS_VIO_SLAM": {"expected": null, "tol_pct": 15},
+        "TUM_ATE_RGBD_ODOM": {"expected": null, "tol_pct": 10},
+        "TUM_ARE_RGBD_ODOM": {"expected": null, "tol_pct": 10},
+        "TUM_Kabsch_RGBD_ODOM": {"expected": null, "tol_pct": 10},
+        "TUM_TrackingLosts_RGBD_ODOM": {"expected": null, "tol_abs": 1},
+        "TUM_FPS_RGBD_ODOM": {"expected": null, "tol_pct": 15},
+        "TUM_ATE_RGBD_SLAM": {"expected": null, "tol_pct": 10},
+        "TUM_ARE_RGBD_SLAM": {"expected": null, "tol_pct": 10},
+        "TUM_Kabsch_RGBD_SLAM": {"expected": null, "tol_pct": 10},
+        "TUM_TrackingLosts_RGBD_SLAM": {"expected": null, "tol_abs": 1},
+        "TUM_FPS_RGBD_SLAM": {"expected": null, "tol_pct": 15},
+        "ICL_NUIM_ATE_RGBD_ODOM": {"expected": null, "tol_pct": 10},
+        "ICL_NUIM_ARE_RGBD_ODOM": {"expected": null, "tol_pct": 10},
+        "ICL_NUIM_Kabsch_RGBD_ODOM": {"expected": null, "tol_pct": 10},
+        "ICL_NUIM_TrackingLosts_RGBD_ODOM": {"expected": null, "tol_abs": 1},
+        "ICL_NUIM_FPS_RGBD_ODOM": {"expected": null, "tol_pct": 15},
+        "ICL_NUIM_ATE_RGBD_SLAM": {"expected": null, "tol_pct": 10},
+        "ICL_NUIM_ARE_RGBD_SLAM": {"expected": null, "tol_pct": 10},
+        "ICL_NUIM_Kabsch_RGBD_SLAM": {"expected": null, "tol_pct": 10},
+        "ICL_NUIM_TrackingLosts_RGBD_SLAM": {"expected": null, "tol_abs": 1},
+        "ICL_NUIM_FPS_RGBD_SLAM": {"expected": null, "tol_pct": 15}
     }
 }

tools/python_tools/cuvslam_tools/dataset_registry.py

@@ -172,6 +172,10 @@ def _stereo_args(*extra: str) -> tuple[str, ...]:
     return ("--odometry_mode=multicamera", *extra, "--async_sba=false", "--multicam_mode=moderate", "--use_segments")
 
 
+def _rgbd_args() -> tuple[str, ...]:
+    return ("--odometry_mode=rgbd", "--async_sba=false", "--use_segments")
+
+
 DATASETS: dict[str, DatasetSpec] = {
     "kitti": DatasetSpec(
         dataset_id="kitti",
@@ -201,15 +205,31 @@ def _stereo_args(*extra: str) -> tuple[str, ...]:
             ),
         ),
     ),
-    # Provisionable but not yet evaluated: no reporter configs are produced and no
-    # validated tarball exists. Adding an EvalSpec is what enables a dataset.
+    # Kept out of smoke deliberately: ICL-NUIM already covers RGB-D pre-merge for
+    # a fraction of the staging cost.
     "tum": DatasetSpec(
         dataset_id="tum",
         prepare_module="cuvslam_tools.dataset_preparation.tum.prepare",
+        evals=(
+            EvalSpec(
+                config="tum-rgbd_slam.cfg",
+                args=_rgbd_args(),
+                suites=frozenset({FULL_SUITE}),
+            ),
+        ),
     ),
+    # One config in both suites, not one per suite: a second would derive its own
+    # KPI prefix and make a PR's diff against the nightly history meaningless.
     "icl_nuim": DatasetSpec(
         dataset_id="icl_nuim",
         prepare_module="cuvslam_tools.dataset_preparation.icl_nuim.prepare",
+        evals=(
+            EvalSpec(
+                config="icl_nuim-rgbd_slam.cfg",
+                args=_rgbd_args(),
+                suites=frozenset(SUITES),
+            ),
+        ),
     ),
     "tartan": DatasetSpec(
         dataset_id="tartan",

tools/python_tools/cuvslam_tools/tests/test_dataset_registry.py

@@ -58,6 +58,18 @@ def test_active_eval_records_are_exact(self):
                 "--odometry_mode=inertial --rectified_stereo_camera=false "
                 "--async_sba=false --multicam_mode=moderate --use_segments",
             ),
+            (
+                "tum",
+                "TUM",
+                "tum/tum-rgbd_slam.cfg",
+                "--odometry_mode=rgbd --async_sba=false --use_segments",
+            ),
+            (
+                "icl_nuim",
+                "ICL_NUIM",
+                "icl_nuim/icl_nuim-rgbd_slam.cfg",
+                "--odometry_mode=rgbd --async_sba=false --use_segments",
+            ),
         ]
 
         actual = [
@@ -78,9 +90,9 @@ def test_archive_and_mount_derive_from_the_dataset_id(self):
             self.assertEqual(spec.archive_name, f"{spec.dataset_id}.tar")
             self.assertEqual(spec.mount_name, spec.dataset_id)
 
-    def test_only_kitti_and_euroc_are_eval_enabled(self):
+    def test_eval_enabled_datasets_are_exact(self):
         enabled = [spec.dataset_id for spec in dataset_registry.eval_datasets()]
-        self.assertEqual(enabled, ["kitti", "euroc"])
+        self.assertEqual(enabled, ["kitti", "euroc", "tum", "icl_nuim"])
 
     def test_listing_the_registry_imports_no_converter_dependencies(self):
         # In a subprocess, because sibling test modules import converters and
@@ -101,7 +113,7 @@ def test_registry_is_runnable_as_a_module(self):
             check=True,
         )
         rows = [line.split("\t") for line in completed.stdout.strip().splitlines()]
-        self.assertEqual([row[0] for row in rows], ["kitti", "euroc"])
+        self.assertEqual([row[0] for row in rows], ["kitti", "euroc", "tum", "icl_nuim"])
         self.assertTrue(all(len(row) == 4 for row in rows), rows)
 
     def test_unknown_dataset_is_rejected_with_the_known_ids(self):
@@ -292,10 +304,15 @@ def test_record_outside_full_is_rejected(self):
         finally:
             dataset_registry.DATASETS = original
 
-    def test_both_suites_select_kitti_and_euroc_today(self):
-        for suite in dataset_registry.SUITES:
-            selected = [spec.dataset_id for spec in dataset_registry.eval_datasets(suite)]
-            self.assertEqual(selected, ["kitti", "euroc"], suite)
+    def test_smoke_covers_every_modality(self):
+        # Pre-merge has to exercise stereo, stereo-inertial and RGB-D. Which
+        # datasets provide that is free to change; the coverage is not.
+        modes = {
+            record.kpi_type
+            for spec in dataset_registry.eval_datasets("smoke")
+            for record in dataset_registry.eval_records(spec, "smoke")
+        }
+        self.assertEqual(modes, {"STEREO", "VIO", "RGBD"})
 
     def test_omitted_suite_matches_full(self):
         self.assertEqual(