diff --git a/cuvslam-skills/cuvslam-ci/reference.md b/cuvslam-skills/cuvslam-ci/reference.md index 8898b29c..e6d35339 100644 --- a/cuvslam-skills/cuvslam-ci/reference.md +++ b/cuvslam-skills/cuvslam-ci/reference.md @@ -67,7 +67,7 @@ Repository secrets, split read from write so fork-reachable jobs never hold a ke ## Dataset registry and layout -- `DATASETS` in `tools/python_tools/cuvslam_tools/dataset_registry.py` is the single source of truth. A `DatasetSpec` holds the ID, the preparation module, and its `EvalSpec` records; an `EvalSpec` holds the reporter config filename, the `cuvslam_app` flags, suite membership, and gating. Provisionable means the dataset is present; eval-enabled means it has at least one `EvalSpec`. KITTI and EuRoC are eval-enabled; `tum` and `tartan` are provisionable only. +- `DATASETS` in `tools/python_tools/cuvslam_tools/dataset_registry.py` is the single source of truth. A `DatasetSpec` holds the ID, the preparation module, and its `EvalSpec` records; an `EvalSpec` holds the reporter config filename, the `cuvslam_app` flags, suite membership, and gating. Provisionable means the dataset is present; eval-enabled means it has at least one `EvalSpec`. KITTI, EuRoC, TUM and ICL-NUIM are eval-enabled; `tartan` and `coda` are provisionable only. Smoke runs KITTI, EuRoC and ICL-NUIM, which covers stereo, stereo-inertial and RGB-D; TUM is full-only because it is the larger RGB-D corpus and ICL-NUIM already covers the modality pre-merge. - The module is standard library only and imports converters lazily, so shell wrappers call it with `PYTHONPATH=tools/python_tools` inside `cuvslam-ci:local` before anything is installed. `datasets_config.sh` wraps it as `dataset_registry`; `run_eval.sh` defines its own shim because the S3 variables `datasets_config.sh` requires are absent in the eval container. - Subcommands: `validate [--dataset] [--suite]`, `list [--eval] [--suite]`, `eval-records [--suite]` (tab-separated `id`, KPI prefix, config path, flags), `kpi-keys [--suite]`, `prepare-module`, `prepare --root-file`, `verify-staged --root`. - Suites: `EVAL_SUITE` selects records in `stage_eval_datasets.sh`, `check_eval_prerequisites.sh`, and `run_eval.sh`, and `eval_cuvslam_in_docker.sh` forwards it into the container. Unset means every record; validation requires every `EvalSpec` to belong to `full`, so unset and `full` agree. `validate --suite` additionally rejects a suite that would select nothing. diff --git a/scripts/kpi_baseline_ranges.json b/scripts/kpi_baseline_ranges.json index 8f082635..afffdb1e 100644 --- a/scripts/kpi_baseline_ranges.json +++ b/scripts/kpi_baseline_ranges.json @@ -31,6 +31,26 @@ "EUROC_ARE_VIO_SLAM": {"expected": null, "tol_pct": 10}, "EUROC_Kabsch_VIO_SLAM": {"expected": null, "tol_pct": 10}, "EUROC_TrackingLosts_VIO_SLAM": {"expected": null, "tol_abs": 1}, - "EUROC_FPS_VIO_SLAM": {"expected": null, "tol_pct": 15} + "EUROC_FPS_VIO_SLAM": {"expected": null, "tol_pct": 15}, + "TUM_ATE_RGBD_ODOM": {"expected": null, "tol_pct": 10}, + "TUM_ARE_RGBD_ODOM": {"expected": null, "tol_pct": 10}, + "TUM_Kabsch_RGBD_ODOM": {"expected": null, "tol_pct": 10}, + "TUM_TrackingLosts_RGBD_ODOM": {"expected": null, "tol_abs": 1}, + "TUM_FPS_RGBD_ODOM": {"expected": null, "tol_pct": 15}, + "TUM_ATE_RGBD_SLAM": {"expected": null, "tol_pct": 10}, + "TUM_ARE_RGBD_SLAM": {"expected": null, "tol_pct": 10}, + "TUM_Kabsch_RGBD_SLAM": {"expected": null, "tol_pct": 10}, + "TUM_TrackingLosts_RGBD_SLAM": {"expected": null, "tol_abs": 1}, + "TUM_FPS_RGBD_SLAM": {"expected": null, "tol_pct": 15}, + "ICL_NUIM_ATE_RGBD_ODOM": {"expected": null, "tol_pct": 10}, + "ICL_NUIM_ARE_RGBD_ODOM": {"expected": null, "tol_pct": 10}, + "ICL_NUIM_Kabsch_RGBD_ODOM": {"expected": null, "tol_pct": 10}, + "ICL_NUIM_TrackingLosts_RGBD_ODOM": {"expected": null, "tol_abs": 1}, + "ICL_NUIM_FPS_RGBD_ODOM": {"expected": null, "tol_pct": 15}, + "ICL_NUIM_ATE_RGBD_SLAM": {"expected": null, "tol_pct": 10}, + "ICL_NUIM_ARE_RGBD_SLAM": {"expected": null, "tol_pct": 10}, + "ICL_NUIM_Kabsch_RGBD_SLAM": {"expected": null, "tol_pct": 10}, + "ICL_NUIM_TrackingLosts_RGBD_SLAM": {"expected": null, "tol_abs": 1}, + "ICL_NUIM_FPS_RGBD_SLAM": {"expected": null, "tol_pct": 15} } } diff --git a/tools/python_tools/cuvslam_tools/dataset_registry.py b/tools/python_tools/cuvslam_tools/dataset_registry.py index 7d5aa0de..7a934cea 100644 --- a/tools/python_tools/cuvslam_tools/dataset_registry.py +++ b/tools/python_tools/cuvslam_tools/dataset_registry.py @@ -172,6 +172,10 @@ def _stereo_args(*extra: str) -> tuple[str, ...]: return ("--odometry_mode=multicamera", *extra, "--async_sba=false", "--multicam_mode=moderate", "--use_segments") +def _rgbd_args() -> tuple[str, ...]: + return ("--odometry_mode=rgbd", "--async_sba=false", "--use_segments") + + DATASETS: dict[str, DatasetSpec] = { "kitti": DatasetSpec( dataset_id="kitti", @@ -201,15 +205,31 @@ def _stereo_args(*extra: str) -> tuple[str, ...]: ), ), ), - # Provisionable but not yet evaluated: no reporter configs are produced and no - # validated tarball exists. Adding an EvalSpec is what enables a dataset. + # Kept out of smoke deliberately: ICL-NUIM already covers RGB-D pre-merge for + # a fraction of the staging cost. "tum": DatasetSpec( dataset_id="tum", prepare_module="cuvslam_tools.dataset_preparation.tum.prepare", + evals=( + EvalSpec( + config="tum-rgbd_slam.cfg", + args=_rgbd_args(), + suites=frozenset({FULL_SUITE}), + ), + ), ), + # One config in both suites, not one per suite: a second would derive its own + # KPI prefix and make a PR's diff against the nightly history meaningless. "icl_nuim": DatasetSpec( dataset_id="icl_nuim", prepare_module="cuvslam_tools.dataset_preparation.icl_nuim.prepare", + evals=( + EvalSpec( + config="icl_nuim-rgbd_slam.cfg", + args=_rgbd_args(), + suites=frozenset(SUITES), + ), + ), ), "tartan": DatasetSpec( dataset_id="tartan", diff --git a/tools/python_tools/cuvslam_tools/tests/test_dataset_registry.py b/tools/python_tools/cuvslam_tools/tests/test_dataset_registry.py index 4c38204c..75ab40d7 100644 --- a/tools/python_tools/cuvslam_tools/tests/test_dataset_registry.py +++ b/tools/python_tools/cuvslam_tools/tests/test_dataset_registry.py @@ -58,6 +58,18 @@ def test_active_eval_records_are_exact(self): "--odometry_mode=inertial --rectified_stereo_camera=false " "--async_sba=false --multicam_mode=moderate --use_segments", ), + ( + "tum", + "TUM", + "tum/tum-rgbd_slam.cfg", + "--odometry_mode=rgbd --async_sba=false --use_segments", + ), + ( + "icl_nuim", + "ICL_NUIM", + "icl_nuim/icl_nuim-rgbd_slam.cfg", + "--odometry_mode=rgbd --async_sba=false --use_segments", + ), ] actual = [ @@ -78,9 +90,9 @@ def test_archive_and_mount_derive_from_the_dataset_id(self): self.assertEqual(spec.archive_name, f"{spec.dataset_id}.tar") self.assertEqual(spec.mount_name, spec.dataset_id) - def test_only_kitti_and_euroc_are_eval_enabled(self): + def test_eval_enabled_datasets_are_exact(self): enabled = [spec.dataset_id for spec in dataset_registry.eval_datasets()] - self.assertEqual(enabled, ["kitti", "euroc"]) + self.assertEqual(enabled, ["kitti", "euroc", "tum", "icl_nuim"]) def test_listing_the_registry_imports_no_converter_dependencies(self): # In a subprocess, because sibling test modules import converters and @@ -101,7 +113,7 @@ def test_registry_is_runnable_as_a_module(self): check=True, ) rows = [line.split("\t") for line in completed.stdout.strip().splitlines()] - self.assertEqual([row[0] for row in rows], ["kitti", "euroc"]) + self.assertEqual([row[0] for row in rows], ["kitti", "euroc", "tum", "icl_nuim"]) self.assertTrue(all(len(row) == 4 for row in rows), rows) def test_unknown_dataset_is_rejected_with_the_known_ids(self): @@ -292,10 +304,15 @@ def test_record_outside_full_is_rejected(self): finally: dataset_registry.DATASETS = original - def test_both_suites_select_kitti_and_euroc_today(self): - for suite in dataset_registry.SUITES: - selected = [spec.dataset_id for spec in dataset_registry.eval_datasets(suite)] - self.assertEqual(selected, ["kitti", "euroc"], suite) + def test_smoke_covers_every_modality(self): + # Pre-merge has to exercise stereo, stereo-inertial and RGB-D. Which + # datasets provide that is free to change; the coverage is not. + modes = { + record.kpi_type + for spec in dataset_registry.eval_datasets("smoke") + for record in dataset_registry.eval_records(spec, "smoke") + } + self.assertEqual(modes, {"STEREO", "VIO", "RGBD"}) def test_omitted_suite_matches_full(self): self.assertEqual(