diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..32097586 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Pre-commit hook: keep notebook outputs clear and bootstrap cells in sync. + +Activated once per machine via `git config core.hooksPath .githooks` +(see docs/notebooks/README.md for the full explanation). Fixes any affected +notebook in place and re-stages it, so the commit proceeds with the fix already +applied -- no interruption, no separate re-commit step. +""" + +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +NOTEBOOKS_DIR = REPO_ROOT / "docs" / "notebooks" + + +def staged_files(pattern: str) -> list[str]: + result = subprocess.run( + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR", "--", pattern], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def main() -> int: + if not NOTEBOOKS_DIR.exists(): + return 0 + + staged_notebooks = staged_files("docs/notebooks/*.ipynb") + template_staged = bool(staged_files("docs/notebooks/_mvtb_nb_bootstrap.py")) + + # Output-clearing only ever touches notebooks actually staged for this commit. + output_targets = staged_notebooks + + # Bootstrap-cell regeneration also has to cover every notebook when the + # template itself changed -- otherwise the other notebooks go stale and CI + # fails on files this commit never touched. + if template_staged: + bootstrap_targets = sorted( + str(p.relative_to(REPO_ROOT)) for p in NOTEBOOKS_DIR.glob("*.ipynb") + ) + else: + bootstrap_targets = staged_notebooks + + if not output_targets and not bootstrap_targets: + return 0 + + changed_paths: set[str] = set() + + try: + if output_targets: + paths = [str(REPO_ROOT / t) for t in output_targets] + subprocess.run( + [sys.executable, str(NOTEBOOKS_DIR / "clear_outputs.py"), *paths], + cwd=REPO_ROOT, + check=True, + ) + changed_paths.update(paths) + + if bootstrap_targets: + paths = [str(REPO_ROOT / t) for t in bootstrap_targets] + subprocess.run( + [sys.executable, str(NOTEBOOKS_DIR / "sync_bootstrap.py"), *paths], + cwd=REPO_ROOT, + check=True, + ) + changed_paths.update(paths) + except subprocess.CalledProcessError: + print("pre-commit hook: notebook cleanup failed unexpectedly", file=sys.stderr) + return 1 + + if changed_paths: + subprocess.run(["git", "add", "--", *sorted(changed_paths)], cwd=REPO_ROOT, check=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f780e8c8..193c7529 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -40,6 +40,11 @@ jobs: with: python-version: "3.12" + - name: Check notebook bootstrap cells and outputs are in sync + run: | + python docs/notebooks/sync_bootstrap.py --check + python docs/notebooks/clear_outputs.py --check + - name: Install system dependencies run: sudo apt-get install -y graphviz diff --git a/docs/notebooks/README.md b/docs/notebooks/README.md index 9e651f93..9a9d32df 100644 --- a/docs/notebooks/README.md +++ b/docs/notebooks/README.md @@ -81,4 +81,55 @@ Moving beyond pixels to robust features and 3D reasoning. Created by Peter Corke | QUT Centre for Robotics

+--- + +## Maintainer notes: how these notebooks install themselves + +*(Written 2026-08 - only relevant if you're editing the install machinery itself.)* + +Every notebook here has to work in three different environments, each of which needs the toolbox installed a different way: + +- **Locally** (VS Code, `jupyter notebook`, `nbmake` in CI) — the toolbox is assumed already installed (dev env or `pip install`); nothing to do. +- **Google Colab** — Colab's "Open in Colab" link fetches *only that one `.ipynb` file* from GitHub, no sibling files, so the toolbox has to be `pip install`ed fresh into the Colab VM each session. +- **JupyterLite** (the in-browser "Try it Now" version published alongside the Sphinx docs) — runs on Pyodide/WASM, where packages install via `micropip` rather than `pip`, and the toolbox wheel is installed with `deps=False` (to stop micropip re-resolving compiled packages like numpy/scipy/opencv that Pyodide already provides its own WASM builds of) — which means every *other* runtime dependency (tqdm, requests, ...) has to be listed and installed explicitly. + +**All three cases are handled by one file: `_mvtb_nb_bootstrap.py`.** It detects which environment it's in and does whatever that environment needs, ending with a one-line sanity check so a broken environment-detection is obvious at a glance rather than failing mysteriously three cells later: +``` +Running locally using MVTB v2.2.0 +Running on Colab using MVTB v2.2.0 +Running in browser using MVTB v2.2.0 +``` + +Every notebook's first code cell is a copy of that file's content, marked with a `# MVTB_BOOTSTRAP_CELL` comment. It's a *copy*, not an import — Colab can't see sibling files, so the cell has to be fully self-contained, the same way Colab notebooks in this space normally are (this is deliberate; a fancier design that fetched the shared code over the network at runtime was considered and rejected — it would have made every notebook depend on GitHub being reachable just to install itself, which is a worse failure mode than a bit of duplication). + +**The copy is machine-generated, not hand-maintained**, which is the actual point: edit `_mvtb_nb_bootstrap.py` once, then either let the commit hook regenerate every notebook's marked cell for you automatically (silently, as part of `git commit`), or run it by hand: +``` +python docs/notebooks/sync_bootstrap.py +``` +A second hook also clears notebook outputs on the way in (`clear_notebook_outputs.sh`'s logic), so committed notebooks stay clean without having to remember that step either. + +**Enforcement happens via a plain git hook, not the `pre-commit` framework.** We wanted here a fully mechanical, deterministic regeneration shouldn't need a manual re-commit every single time. So this repo uses a plain git hook instead, versioned at `.githooks/pre-commit`. + +### One-time setup (per machine) + +This is a *local* git setting — it doesn't come along when you clone the repo, and isn't shared by git config, so it needs doing once on every machine you commit from: +``` +git config core.hooksPath .githooks +``` +That's the only step — `.githooks/pre-commit` is already committed with its executable bit set (git tracks that), and it only needs the Python already on your `PATH` plus the stdlib (no extra `pip install`, no `pre-commit` package). To check it's active: +``` +git config --get core.hooksPath # should print .githooks +``` +Nothing is enforced on a machine where this hasn't been set — CI is the backstop for that case, not the primary mechanism. + +The hook stays quiet when there's nothing to do, and only speaks up when it actually changes something: +``` +docs/notebooks/gamma.ipynb: found output, clearing it +docs/notebooks/camera.ipynb: bootstrap cell out of date, regenerating +``` +so an ordinary commit scrolls past without any noise, but one that got silently fixed for you still leaves a visible trace — a lightweight way to notice the safety net firing without it ever stopping you. + +If a notebook's bootstrap cell falls out of sync anyway (hook not installed on this machine, notebook edited elsewhere), CI catches it on the PR — a job re-runs the generator in check-only mode across every notebook and fails if anything doesn't match. + +**Why this exists at all:** before this, 13 of the 16 notebooks each had their own hand-pasted, slightly-drifted copy of the Colab-install snippet, and the JupyterLite demo notebook had its own bespoke version with a manually-maintained dependency list. That's exactly how machinevision-toolbox-python 2.2.0 shipped a broken "Try it Now" demo — `tqdm` was added as a real dependency but nobody updated the one notebook's hand-typed install list, and nothing tested it before release. One template, generated everywhere, checked by CI, closes that gap. diff --git a/docs/notebooks/_mvtb_nb_bootstrap.py b/docs/notebooks/_mvtb_nb_bootstrap.py new file mode 100644 index 00000000..a4170470 --- /dev/null +++ b/docs/notebooks/_mvtb_nb_bootstrap.py @@ -0,0 +1,70 @@ +"""Environment bootstrap for machinevision-toolbox-python's Jupyter notebooks. + +Installs the toolbox (and reports how) across the three environments a notebook in +this folder might run in: a local Jupyter/VS Code install, Google Colab, and +JupyterLite (Pyodide/WASM, in-browser). + +This file is the single source of truth for that logic. Every notebook's own +bootstrap cell is a generated copy of this file's content, produced by +sync_bootstrap.py -- see docs/notebooks/README.md for the full explanation. +""" + +import subprocess +import sys +from pathlib import Path + + +async def ensure_installed() -> bool: + """Install machinevision-toolbox-python if needed, and report the environment. + + :returns: True if running on Google Colab, False otherwise. + """ + if sys.platform == "emscripten": + import micropip + + await micropip.install( + [ + "opencv-python", + "spatialmath-python", + "pgraph-python", + "ansitable", + "mvtb-data", + "tqdm", + "requests", + ] + ) + import cv2 # noqa: F401 - force cv2 into module registry before toolbox import + + wheels = sorted(Path("/pypi").glob("machinevision_toolbox_python-*.whl")) + if wheels: + # Prefer the wheel bundled with this JupyterLite site. + await micropip.install(wheels[-1].as_posix(), deps=False) + else: + # Fall back to PyPI when running outside the published site layout. + await micropip.install("machinevision-toolbox-python", deps=False) + where, colab = "in browser", False + else: + try: + import google.colab # noqa: F401 + except ImportError: + where, colab = "locally", False + else: + print("Installing machinevision-toolbox-python...") + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "-q", + "machinevision-toolbox-python", + ], + check=True, + ) + where, colab = "on Colab", True + + import machinevisiontoolbox + + version = getattr(machinevisiontoolbox, "__version__", "unknown") + print(f"Running {where} using MVTB v{version}") + return colab diff --git a/docs/notebooks/clear_outputs.py b/docs/notebooks/clear_outputs.py new file mode 100644 index 00000000..a23f1d6e --- /dev/null +++ b/docs/notebooks/clear_outputs.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +"""Clear cell outputs and execution counts from every notebook in this folder. + +Pure stdlib (no dependency on jupyter/nbconvert being installed), so it can run as +a commit hook on any machine with just Python. Equivalent to +`jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace`, which is what +clear_notebook_outputs.sh does for ad-hoc/manual use. + +Usage: + python clear_outputs.py clear any notebook that has output + python clear_outputs.py --check report only, exit 1 if anything has output + python clear_outputs.py FILE ... only consider the given notebook(s) +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def clear_cell(cell: dict) -> bool: + """Clear one cell's outputs/execution count in place. + + :returns: True if the cell was (or would be) changed. + """ + if cell.get("cell_type") != "code": + return False + + changed = False + if cell.get("outputs"): + changed = True + cell["outputs"] = [] + if cell.get("execution_count") is not None: + changed = True + cell["execution_count"] = None + if "execution" in cell.get("metadata", {}): + changed = True + del cell["metadata"]["execution"] + + return changed + + +def process_notebook(path: Path, fix: bool) -> bool: + """Check (and optionally clear) one notebook's outputs. + + :returns: True if the notebook was (or would be) changed. + """ + with path.open("r", encoding="utf-8") as f: + nb = json.load(f) + + changed = any(clear_cell(cell) for cell in nb.get("cells", [])) + + if changed and fix: + with path.open("w", encoding="utf-8") as f: + json.dump(nb, f, indent=1) + f.write("\n") + + return changed + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="report notebooks with output without modifying them; exit 1 if any found", + ) + parser.add_argument( + "files", + nargs="*", + type=Path, + help="only consider these notebooks (default: every *.ipynb in this folder)", + ) + args = parser.parse_args() + + fix = not args.check + notebooks = [p.resolve() for p in args.files] if args.files else sorted(HERE.glob("*.ipynb")) + + any_changed = False + for nb_path in notebooks: + changed = process_notebook(nb_path, fix=fix) + if changed: + any_changed = True + rel = nb_path.relative_to(HERE.parent.parent) + if fix: + print(f"{rel}: found output, clearing it") + else: + print(f"{rel}: found output") + + if args.check and any_changed: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/notebooks/sync_bootstrap.py b/docs/notebooks/sync_bootstrap.py new file mode 100644 index 00000000..cce8ff26 --- /dev/null +++ b/docs/notebooks/sync_bootstrap.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +"""Regenerate the bootstrap cell in every notebook from _mvtb_nb_bootstrap.py. + +Every notebook in this folder carries its own copy of the environment-bootstrap +logic, since Google Colab only fetches the single .ipynb file it's opened from and +can't see sibling files. That copy is machine-generated from _mvtb_nb_bootstrap.py, +the single source of truth -- see README.md for the full explanation. + +Usage: + python sync_bootstrap.py regenerate any notebook that's out of date + python sync_bootstrap.py --check report only, exit 1 if anything's out of date + python sync_bootstrap.py FILE ... only consider the given notebook(s) +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +MARKER = "# MVTB_BOOTSTRAP_CELL" +GENERATED_HEADER = ( + f"{MARKER} - generated from docs/notebooks/_mvtb_nb_bootstrap.py " + "by sync_bootstrap.py; do not hand-edit\n" +) + +HERE = Path(__file__).resolve().parent +TEMPLATE_PATH = HERE / "_mvtb_nb_bootstrap.py" + + +def generated_source(template: str) -> list[str]: + """Build the full source (as nbformat-style lines) for the bootstrap cell.""" + body = f"{GENERATED_HEADER}\n{template.rstrip()}\n\nCOLAB = await ensure_installed()\n" + lines = body.splitlines(keepends=True) + return lines + + +def cell_source_text(cell: dict) -> str: + source = cell.get("source", []) + return "".join(source) if isinstance(source, list) else source + + +def is_bootstrap_cell(cell: dict) -> bool: + if cell.get("cell_type") != "code": + return False + text = cell_source_text(cell) + first_line = text.splitlines()[0].strip() if text.strip() else "" + return first_line.startswith(MARKER) + + +def process_notebook(path: Path, template: str, fix: bool) -> bool: + """Check (and optionally fix) one notebook's bootstrap cell. + + :returns: True if the notebook's bootstrap cell was (or would be) changed. + """ + with path.open("r", encoding="utf-8") as f: + nb = json.load(f) + + changed = False + for cell in nb.get("cells", []): + if not is_bootstrap_cell(cell): + continue + + desired = generated_source(template) + current = cell.get("source", []) + if not isinstance(current, list): + current = [current] + + if current != desired: + changed = True + if fix: + cell["source"] = desired + break + + if changed and fix: + with path.open("w", encoding="utf-8") as f: + json.dump(nb, f, indent=1) + f.write("\n") + + return changed + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="report out-of-date notebooks without modifying them; exit 1 if any found", + ) + parser.add_argument( + "files", + nargs="*", + type=Path, + help="only consider these notebooks (default: every *.ipynb in this folder)", + ) + args = parser.parse_args() + + template = TEMPLATE_PATH.read_text(encoding="utf-8") + fix = not args.check + notebooks = [p.resolve() for p in args.files] if args.files else sorted(HERE.glob("*.ipynb")) + + any_changed = False + for nb_path in notebooks: + changed = process_notebook(nb_path, template, fix=fix) + if changed: + any_changed = True + rel = nb_path.relative_to(HERE.parent.parent) + if fix: + print(f"{rel}: bootstrap cell out of date, regenerating") + else: + print(f"{rel}: bootstrap cell out of date") + + if args.check and any_changed: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_notebook_bootstrap.py b/tests/test_notebook_bootstrap.py new file mode 100644 index 00000000..806dd302 --- /dev/null +++ b/tests/test_notebook_bootstrap.py @@ -0,0 +1,165 @@ +"""Tests for the notebook bootstrap-cell generator and output-clearing scripts. + +These exercise the pure notebook-JSON manipulation logic in +docs/notebooks/sync_bootstrap.py and docs/notebooks/clear_outputs.py directly, not +via subprocess -- what can't be tested this way (whether the generated cell's +*content* actually installs correctly in Colab/JupyterLite/locally) needs a real +run in each of those environments, which is a separate, non-pytest concern. +""" + +import importlib.util +import json +import sys +from pathlib import Path + +NOTEBOOKS_DIR = Path(__file__).resolve().parent.parent / "docs" / "notebooks" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +sync_bootstrap = _load_module("_sync_bootstrap", NOTEBOOKS_DIR / "sync_bootstrap.py") +clear_outputs = _load_module("_clear_outputs", NOTEBOOKS_DIR / "clear_outputs.py") + + +def _write_notebook(path: Path, cells: list[dict]) -> None: + nb = { + "cells": cells, + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + } + path.write_text(json.dumps(nb, indent=1) + "\n", encoding="utf-8") + + +def _code_cell(source: str, outputs=None, execution_count=None) -> dict: + return { + "cell_type": "code", + "metadata": {}, + "execution_count": execution_count, + "outputs": outputs or [], + "source": source.splitlines(keepends=True), + } + + +# --- sync_bootstrap.py ------------------------------------------------------- + + +def test_is_bootstrap_cell_detects_marker(): + marked = _code_cell(f"{sync_bootstrap.MARKER}\nprint('hi')\n") + unmarked = _code_cell("print('hi')\n") + markdown = {"cell_type": "markdown", "source": [f"{sync_bootstrap.MARKER}\n"]} + + assert sync_bootstrap.is_bootstrap_cell(marked) + assert not sync_bootstrap.is_bootstrap_cell(unmarked) + assert not sync_bootstrap.is_bootstrap_cell(markdown) + + +def test_generated_source_embeds_template_and_call(): + template = "async def ensure_installed():\n return False\n" + lines = sync_bootstrap.generated_source(template) + text = "".join(lines) + + assert text.startswith(sync_bootstrap.MARKER) + assert "async def ensure_installed():" in text + assert text.rstrip().endswith("COLAB = await ensure_installed()") + + +def test_process_notebook_check_mode_reports_without_writing(tmp_path): + nb_path = tmp_path / "demo.ipynb" + stale_cell = _code_cell(f"{sync_bootstrap.MARKER}\n# stale content\n") + _write_notebook(nb_path, [stale_cell]) + before = nb_path.read_text(encoding="utf-8") + template = "async def ensure_installed():\n return False\n" + + changed = sync_bootstrap.process_notebook(nb_path, template=template, fix=False) + + assert changed is True + assert nb_path.read_text(encoding="utf-8") == before # untouched + + +def test_process_notebook_fix_mode_rewrites_and_is_idempotent(tmp_path): + nb_path = tmp_path / "demo.ipynb" + stale_cell = _code_cell(f"{sync_bootstrap.MARKER}\n# stale content\n") + _write_notebook(nb_path, [stale_cell]) + template = "async def ensure_installed():\n return False\n" + + changed = sync_bootstrap.process_notebook(nb_path, template=template, fix=True) + assert changed is True + + # A second pass over the now-regenerated file should find nothing to do. + changed_again = sync_bootstrap.process_notebook(nb_path, template=template, fix=True) + assert changed_again is False + + nb = json.loads(nb_path.read_text(encoding="utf-8")) + cell_text = "".join(nb["cells"][0]["source"]) + assert "async def ensure_installed():" in cell_text + + +def test_process_notebook_without_marker_cell_is_a_no_op(tmp_path): + nb_path = tmp_path / "demo.ipynb" + _write_notebook(nb_path, [_code_cell("import numpy as np\n")]) + + changed = sync_bootstrap.process_notebook(nb_path, template="whatever", fix=True) + + assert changed is False + + +# --- clear_outputs.py --------------------------------------------------------- + + +def test_clear_cell_strips_output_and_execution_count(): + cell = _code_cell("print(1)\n", outputs=[{"output_type": "stream", "text": "1\n"}], execution_count=3) + + changed = clear_outputs.clear_cell(cell) + + assert changed is True + assert cell["outputs"] == [] + assert cell["execution_count"] is None + + +def test_clear_cell_leaves_clean_cell_untouched(): + cell = _code_cell("print(1)\n") + + changed = clear_outputs.clear_cell(cell) + + assert changed is False + + +def test_clear_cell_ignores_markdown_cells(): + cell = {"cell_type": "markdown", "source": ["# Title\n"]} + + assert clear_outputs.clear_cell(cell) is False + + +def test_process_notebook_fix_mode_clears_and_is_idempotent(tmp_path): + nb_path = tmp_path / "demo.ipynb" + dirty_cell = _code_cell("print(1)\n", outputs=[{"output_type": "stream", "text": "1\n"}], execution_count=3) + _write_notebook(nb_path, [dirty_cell]) + + changed = clear_outputs.process_notebook(nb_path, fix=True) + assert changed is True + + changed_again = clear_outputs.process_notebook(nb_path, fix=True) + assert changed_again is False + + nb = json.loads(nb_path.read_text(encoding="utf-8")) + assert nb["cells"][0]["outputs"] == [] + assert nb["cells"][0]["execution_count"] is None + + +def test_process_notebook_check_mode_does_not_write(tmp_path): + nb_path = tmp_path / "demo.ipynb" + dirty_cell = _code_cell("print(1)\n", outputs=[{"output_type": "stream", "text": "1\n"}], execution_count=3) + _write_notebook(nb_path, [dirty_cell]) + before = nb_path.read_text(encoding="utf-8") + + changed = clear_outputs.process_notebook(nb_path, fix=False) + + assert changed is True + assert nb_path.read_text(encoding="utf-8") == before