diff --git a/LIBLCMS2_DYLIB_COLLISION.md b/LIBLCMS2_DYLIB_COLLISION.md index e584a25a..ae139c91 100644 --- a/LIBLCMS2_DYLIB_COLLISION.md +++ b/LIBLCMS2_DYLIB_COLLISION.md @@ -243,9 +243,23 @@ Fixed and hardened on this branch: a future regression in this area is diagnosable from `negpy.log` alone instead of requiring a multi-day investigation like this one. -Follow-up, not done here: generalize the symbol-closure check to every -same-basename dylib collision in the bundle (not just liblcms2), which -would also catch the pre-existing `libomp.dylib`/numba gap found while -scoping this fix. `libjpeg`/`libpng`/`libz`/`libtiff` are vendored by -multiple of the same packages and carry the same theoretical risk — no -evidence any are currently broken, but untested. +Follow-up, done: `check_bundled_dylib_collisions()` generalizes the +symbol-closure check to every same-basename dylib collision under +`Contents/Frameworks`, not just liblcms2. It scans the whole bundle rather +than a curated list of risky basenames — a manual pass guessing which +libraries might collide (by comparing vendored version strings across +`cv2`/`PIL`/`rawpy`/`imagecodecs`) missed a real one: three packages vendor +byte-different copies of `libjpeg.8.3.2.dylib`, invisible unless you check +for repeated exact filenames rather than reasoning about version numbers. +Also found real (currently benign — no missing symbols today) collisions on +`libpng16.16.dylib` and `libtiff.6.dylib` (`cv2` vs `PIL`). Confirmed on the +real local build: passes clean, and correctly raises when a canonical +symlink is forced to point at a copy missing a symbol a sibling consumer +needs. + +Still not done, deliberately: the `libomp.dylib`/numba gap. It's a +different failure mode (PyInstaller not bundling the library at all, not a +collision between multiple bundled copies), stays outside what this check +looks for, and there is still no evidence it affects NegPy (numba's +OpenMP-parallel pool is optional with a runtime fallback) — not worth +fixing speculatively. diff --git a/build.py b/build.py index 43e8846d..cdb15bf8 100644 --- a/build.py +++ b/build.py @@ -1,3 +1,4 @@ +import functools import glob import os import platform @@ -381,13 +382,81 @@ def fix_lcms2_dylib_collision(): raise RuntimeError(f"imagecodecs' liblcms2.2.dylib is missing symbols other consumers need:\n{detail}") -def _nm_symbols(path: str, *, defined: bool) -> set[str]: +@functools.lru_cache(maxsize=None) +def _nm_symbols(path: str, *, defined: bool) -> frozenset[str]: """Return a Mach-O file's defined-exported (-gU) or undefined (-u) symbol names.""" flag = "-gU" if defined else "-u" out = subprocess.run(["nm", flag, path], capture_output=True, text=True, check=True).stdout if defined: - return {line.split()[-1] for line in out.splitlines() if line.strip()} - return {line.strip() for line in out.splitlines() if line.strip()} + return frozenset(line.split()[-1] for line in out.splitlines() if line.strip()) + return frozenset(line.strip() for line in out.splitlines() if line.strip()) + + +@functools.lru_cache(maxsize=None) +def _dylib_load_basenames(path: str) -> frozenset[str]: + """Basenames a Mach-O file references via LC_LOAD_DYLIB (its real, linked-in dependencies).""" + out = subprocess.run(["otool", "-L", path], capture_output=True, text=True, check=True).stdout + return frozenset(os.path.basename(line.split()[0]) for line in out.splitlines()[1:] if line.strip()) + + +def check_bundled_dylib_collisions(): + """Verify every same-basename dylib PyInstaller collapsed to one canonical + copy still satisfies every real consumer's symbols. + + Generalizes fix_lcms2_dylib_collision() to the whole bundle: cv2, PIL, + rawpy and imagecodecs each vendor their own copies of common libraries + (libjpeg, libpng, libtiff, ...) under identical filenames, and PyInstaller + keeps only one arbitrary pick as the canonical @rpath target for all of + them -- the exact bug class in LIBLCMS2_DYLIB_COLLISION.md, which this + scans for instead of relying on a curated list of known-risky basenames + (that fix's own consumer glob missed a real libjpeg collision on first + pass -- see the doc). Raises rather than warns, so a future dependency + bump that silently drops a symbol fails the build instead of shipping + broken silently. + + Does not check libraries PyInstaller fails to bundle at all (e.g. numba's + optional libomp.dylib) -- that is a different failure mode (absence, not + a collision) with no evidence of user impact; see LIBLCMS2_DYLIB_COLLISION.md. + """ + frameworks = os.path.join("dist", f"{APP_NAME}.app", "Contents", "Frameworks") + all_files = [ + os.path.join(dirpath, name) + for dirpath, _, filenames in os.walk(frameworks) + for name in filenames + if name.endswith((".dylib", ".so")) + ] + by_basename: dict[str, list[str]] = {} + for path in all_files: + if not os.path.islink(path): + by_basename.setdefault(os.path.basename(path), []).append(path) + + gaps = [] + for basename, copies in by_basename.items(): + canonical = os.path.join(frameworks, basename) + if not os.path.islink(canonical): + continue # no top-level collapse for this basename -- nothing collided + canonical_real = os.path.realpath(canonical) + distinct = {p for p in copies if os.path.realpath(p) != canonical_real} + if not distinct: + continue # every copy is byte-identical -- an arbitrary pick can't lose symbols + exports = _nm_symbols(canonical_real, defined=True) + provided_anywhere: set[str] = set(exports) + for p in distinct: + provided_anywhere |= _nm_symbols(p, defined=True) + + for consumer in all_files: + if os.path.islink(consumer) or os.path.realpath(consumer) == canonical_real or consumer in distinct: + continue + if basename not in _dylib_load_basenames(consumer): + continue + needed = _nm_symbols(consumer, defined=False) & provided_anywhere + missing = needed - exports + if missing: + gaps.append((consumer, basename, sorted(missing))) + + if gaps: + detail = "\n".join(f" {os.path.relpath(c, frameworks)} needs {b}: missing {m}" for c, b, m in gaps) + raise RuntimeError(f"a bundled dylib collision leaves a consumer missing symbols:\n{detail}") def codesign_macos_app(): @@ -468,6 +537,7 @@ def build(): package_windows() elif is_macos: fix_lcms2_dylib_collision() + check_bundled_dylib_collisions() codesign_macos_app() package_macos()